diff --git a/src/containers/sound-editor.jsx b/src/containers/sound-editor.jsx
index 94fca95243f500e4d1c27ee38a8c4025bf44635e..3089b3dc344cbe2f26dc0683ad123539bb802251 100644
--- a/src/containers/sound-editor.jsx
+++ b/src/containers/sound-editor.jsx
@@ -6,7 +6,12 @@ import VM from 'scratch-vm';
import {connect} from 'react-redux';
-import {computeChunkedRMS, encodeAndAddSoundToVM, SOUND_BYTE_LIMIT} from '../lib/audio/audio-util.js';
+import {
+ computeChunkedRMS,
+ encodeAndAddSoundToVM,
+ downsampleIfNeeded,
+ dropEveryOtherSample
+} from '../lib/audio/audio-util.js';
import AudioEffects from '../lib/audio/audio-effects.js';
import SoundEditorComponent from '../components/sound-editor/sound-editor.jsx';
import AudioBufferPlayer from '../lib/audio/audio-buffer-player.js';
@@ -39,7 +44,8 @@ class SoundEditor extends React.Component {
'paste',
'handleKeyPress',
'handleContainerClick',
- 'setRef'
+ 'setRef',
+ 'resampleBufferToRate'
]);
this.state = {
copyBuffer: null,
@@ -132,45 +138,32 @@ class SoundEditor extends React.Component {
});
}
submitNewSamples (samples, sampleRate, skipUndo) {
- // Encode the new sound into a wav so that it can be stored
- let wavBuffer = null;
- try {
- wavBuffer = WavEncoder.encode.sync({
- sampleRate: sampleRate,
- channelData: [samples]
+ return downsampleIfNeeded({samples, sampleRate}, this.resampleBufferToRate)
+ .then(({samples: newSamples, sampleRate: newSampleRate}) =>
+ WavEncoder.encode({
+ sampleRate: newSampleRate,
+ channelData: [newSamples]
+ }).then(wavBuffer => {
+ if (!skipUndo) {
+ this.redoStack = [];
+ if (this.undoStack.length >= UNDO_STACK_SIZE) {
+ this.undoStack.shift(); // Drop the first element off the array
+ }
+ this.undoStack.push(this.getUndoItem());
+ }
+ this.resetState(newSamples, newSampleRate);
+ this.props.vm.updateSoundBuffer(
+ this.props.soundIndex,
+ this.audioBufferPlayer.buffer,
+ new Uint8Array(wavBuffer));
+ return true; // Edit was successful
+ })
+ )
+ .catch(e => {
+ // Encoding failed, or the sound was too large to save so edit is rejected
+ log.error(`Encountered error while trying to encode sound update: ${e}`);
+ return false; // Edit was not applied
});
-
- if (wavBuffer.byteLength > SOUND_BYTE_LIMIT) {
- // Cancel the sound update by setting to null
- wavBuffer = null;
- log.error(`Refusing to encode sound larger than ${SOUND_BYTE_LIMIT} bytes`);
- }
- } catch (e) {
- // This error state is mostly for the mock sounds used during testing.
- // Any incorrect sound buffer trying to get interpretd as a Wav file
- // should yield this error.
- // This can also happen if the sound is too be allocated in memory.
- log.error(`Encountered error while trying to encode sound update: ${e}`);
- }
-
- // Do not submit sound if it could not be encoded (i.e. if too large)
- if (wavBuffer) {
- if (!skipUndo) {
- this.redoStack = [];
- if (this.undoStack.length >= UNDO_STACK_SIZE) {
- this.undoStack.shift(); // Drop the first element off the array
- }
- this.undoStack.push(this.getUndoItem());
- }
- this.resetState(samples, sampleRate);
- this.props.vm.updateSoundBuffer(
- this.props.soundIndex,
- this.audioBufferPlayer.buffer,
- new Uint8Array(wavBuffer));
-
- return true; // Update succeeded
- }
- return false; // Update failed
}
handlePlay () {
this.audioBufferPlayer.stop();
@@ -209,13 +202,15 @@ class SoundEditor extends React.Component {
newSamples.set(firstPart, 0);
newSamples.set(secondPart, firstPart.length);
}
- this.submitNewSamples(newSamples, sampleRate);
- this.setState({
- trimStart: null,
- trimEnd: null
+ this.submitNewSamples(newSamples, sampleRate).then(() => {
+ this.setState({
+ trimStart: null,
+ trimEnd: null
+ });
});
}
handleDeleteInverse () {
+ // Delete everything outside of the trimmers
const {samples, sampleRate} = this.copyCurrentBuffer();
const sampleCount = samples.length;
const startIndex = Math.floor(this.state.trimStart * sampleCount);
@@ -224,10 +219,13 @@ class SoundEditor extends React.Component {
if (clippedSamples.length === 0) {
clippedSamples = new Float32Array(1);
}
- this.submitNewSamples(clippedSamples, sampleRate);
- this.setState({
- trimStart: null,
- trimEnd: null
+ this.submitNewSamples(clippedSamples, sampleRate).then(success => {
+ if (success) {
+ this.setState({
+ trimStart: null,
+ trimEnd: null
+ });
+ }
});
}
handleUpdateTrim (trimStart, trimEnd) {
@@ -257,14 +255,15 @@ class SoundEditor extends React.Component {
effects.process((renderedBuffer, adjustedTrimStart, adjustedTrimEnd) => {
const samples = renderedBuffer.getChannelData(0);
const sampleRate = renderedBuffer.sampleRate;
- const success = this.submitNewSamples(samples, sampleRate);
- if (success) {
- if (this.state.trimStart === null) {
- this.handlePlay();
- } else {
- this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay);
+ this.submitNewSamples(samples, sampleRate).then(success => {
+ if (success) {
+ if (this.state.trimStart === null) {
+ this.handlePlay();
+ } else {
+ this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay);
+ }
}
- }
+ });
});
}
tooLoud () {
@@ -287,16 +286,22 @@ class SoundEditor extends React.Component {
this.redoStack.push(this.getUndoItem());
const {samples, sampleRate, trimStart, trimEnd} = this.undoStack.pop();
if (samples) {
- this.submitNewSamples(samples, sampleRate, true);
- this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
+ return this.submitNewSamples(samples, sampleRate, true).then(success => {
+ if (success) {
+ this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
+ }
+ });
}
}
handleRedo () {
const {samples, sampleRate, trimStart, trimEnd} = this.redoStack.pop();
if (samples) {
this.undoStack.push(this.getUndoItem());
- this.submitNewSamples(samples, sampleRate, true);
- this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
+ return this.submitNewSamples(samples, sampleRate, true).then(success => {
+ if (success) {
+ this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
+ }
+ });
}
}
handleCopy () {
@@ -322,25 +327,39 @@ class SoundEditor extends React.Component {
});
}
resampleBufferToRate (buffer, newRate) {
- return new Promise(resolve => {
- if (window.OfflineAudioContext) {
- const sampleRateRatio = newRate / buffer.sampleRate;
- const newLength = sampleRateRatio * buffer.samples.length;
- const offlineContext = new window.OfflineAudioContext(1, newLength, newRate);
- const source = offlineContext.createBufferSource();
- const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate);
- audioBuffer.getChannelData(0).set(buffer.samples);
- source.buffer = audioBuffer;
- source.connect(offlineContext.destination);
- source.start();
- offlineContext.startRendering();
- offlineContext.oncomplete = ({renderedBuffer}) => {
- resolve({
- samples: renderedBuffer.getChannelData(0),
- sampleRate: newRate
- });
- };
+ return new Promise((resolve, reject) => {
+ const sampleRateRatio = newRate / buffer.sampleRate;
+ const newLength = sampleRateRatio * buffer.samples.length;
+ let offlineContext;
+ // Try to use either OfflineAudioContext or webkitOfflineAudioContext to resample
+ // The constructors will throw if trying to resample at an unsupported rate
+ // (e.g. Safari/webkitOAC does not support lower than 44khz).
+ try {
+ if (window.OfflineAudioContext) {
+ offlineContext = new window.OfflineAudioContext(1, newLength, newRate);
+ } else if (window.webkitOfflineAudioContext) {
+ offlineContext = new window.webkitOfflineAudioContext(1, newLength, newRate);
+ }
+ } catch {
+ // If no OAC available and downsampling by 2, downsample by dropping every other sample.
+ if (newRate === buffer.sampleRate / 2) {
+ return resolve(dropEveryOtherSample(buffer));
+ }
+ return reject('Could not resample');
}
+ const source = offlineContext.createBufferSource();
+ const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate);
+ audioBuffer.getChannelData(0).set(buffer.samples);
+ source.buffer = audioBuffer;
+ source.connect(offlineContext.destination);
+ source.start();
+ offlineContext.startRendering();
+ offlineContext.oncomplete = ({renderedBuffer}) => {
+ resolve({
+ samples: renderedBuffer.getChannelData(0),
+ sampleRate: newRate
+ });
+ };
});
}
paste () {
@@ -351,8 +370,11 @@ class SoundEditor extends React.Component {
const newSamples = new Float32Array(newLength);
newSamples.set(samples, 0);
newSamples.set(this.state.copyBuffer.samples, samples.length);
- this.submitNewSamples(newSamples, this.props.sampleRate, false);
- this.handlePlay();
+ this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => {
+ if (success) {
+ this.handlePlay();
+ }
+ });
} else {
// else replace the selection with the pasted sound
const trimStartSamples = this.state.trimStart * samples.length;
@@ -371,11 +393,14 @@ class SoundEditor extends React.Component {
const newDurationSeconds = newSamples.length / this.state.copyBuffer.sampleRate;
const adjustedTrimStart = trimStartSeconds / newDurationSeconds;
const adjustedTrimEnd = trimEndSeconds / newDurationSeconds;
- this.submitNewSamples(newSamples, this.props.sampleRate, false);
- this.setState({
- trimStart: adjustedTrimStart,
- trimEnd: adjustedTrimEnd
- }, this.handlePlay);
+ this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => {
+ if (success) {
+ this.setState({
+ trimStart: adjustedTrimStart,
+ trimEnd: adjustedTrimEnd
+ }, this.handlePlay);
+ }
+ });
}
}
handlePaste () {
diff --git a/src/lib/audio/audio-util.js b/src/lib/audio/audio-util.js
index a01c30de811f9c4ec9dab47175c51e9fc96653a5..fba6be96beff35d02abcf025a987473c0bbea074 100644
--- a/src/lib/audio/audio-util.js
+++ b/src/lib/audio/audio-util.js
@@ -59,9 +59,57 @@ const encodeAndAddSoundToVM = function (vm, samples, sampleRate, name, callback)
});
};
+/**
+ @typedef SoundBuffer
+ @type {Object}
+ @property {Float32Array} samples Array of audio samples
+ @property {number} sampleRate Audio sample rate
+ */
+
+/**
+ * Downsample the given buffer to try to reduce file size below SOUND_BYTE_LIMIT
+ * @param {SoundBuffer} buffer - Buffer to resample
+ * @param {function(SoundBuffer):Promise<SoundBuffer>} resampler - resampler function
+ * @returns {SoundBuffer} Downsampled buffer with half the sample rate
+ */
+const downsampleIfNeeded = (buffer, resampler) => {
+ const {samples, sampleRate} = buffer;
+ const duration = samples.length / sampleRate;
+ const encodedByteLength = samples.length * 2; /* bitDepth 16 bit */
+ // Resolve immediately if already within byte limit
+ if (encodedByteLength < SOUND_BYTE_LIMIT) {
+ return Promise.resolve({samples, sampleRate});
+ }
+ // If encodeable at 22khz, resample and call submitNewSamples again
+ if (duration * 22050 * 2 < SOUND_BYTE_LIMIT) {
+ return resampler({samples, sampleRate}, 22050);
+ }
+ // Cannot save this sound at 22khz, refuse to edit
+ // In the future we could introduce further compression here
+ return Promise.reject('Sound too large to save, refusing to edit');
+};
+
+/**
+ * Drop every other sample of an audio buffer as a last-resort way of downsampling.
+ * @param {SoundBuffer} buffer - Buffer to resample
+ * @returns {SoundBuffer} Downsampled buffer with half the sample rate
+ */
+const dropEveryOtherSample = buffer => {
+ const newLength = Math.floor(buffer.samples.length / 2);
+ const newSamples = new Float32Array(newLength);
+ for (let i = 0; i < newLength; i++) {
+ newSamples[i] = buffer.samples[i * 2];
+ }
+ return {
+ samples: newSamples,
+ sampleRate: buffer.rate / 2
+ };
+};
+
export {
computeRMS,
computeChunkedRMS,
encodeAndAddSoundToVM,
- SOUND_BYTE_LIMIT
+ downsampleIfNeeded,
+ dropEveryOtherSample
};
diff --git a/test/__mocks__/audio-effects.js b/test/__mocks__/audio-effects.js
index 06c36c7e84359a3bac2b4431f21900ff15a3c0f8..9597eec1be0156abfde2e22ce9ead9fc1a520970 100644
--- a/test/__mocks__/audio-effects.js
+++ b/test/__mocks__/audio-effects.js
@@ -14,7 +14,10 @@ export default class MockAudioEffects {
this.buffer = buffer;
this.name = name;
this.process = jest.fn(done => {
- this._finishProcessing = renderedBuffer => done(renderedBuffer, 0, 1);
+ this._finishProcessing = renderedBuffer => {
+ done(renderedBuffer, 0, 1);
+ return new Promise(resolve => setTimeout(resolve));
+ };
});
MockAudioEffects.instance = this;
}
diff --git a/test/unit/containers/sound-editor.test.jsx b/test/unit/containers/sound-editor.test.jsx
index b0a1a8399f64133dfa0de9328593165289b59a06..4e4c967ec9cc71dc6631410ce5b250f4347d7891 100644
--- a/test/unit/containers/sound-editor.test.jsx
+++ b/test/unit/containers/sound-editor.test.jsx
@@ -97,7 +97,7 @@ describe('Sound Editor Container', () => {
expect(vm.renameSound).toHaveBeenCalledWith(soundIndex, 'hello');
});
- test('it handles an effect by submitting the result and playing', () => {
+ test('it handles an effect by submitting the result and playing', async () => {
const wrapper = mountWithIntl(
<SoundEditor
soundIndex={soundIndex}
@@ -106,7 +106,7 @@ describe('Sound Editor Container', () => {
);
const component = wrapper.find(SoundEditorComponent);
component.props().onReverse(); // Could be any of the effects, just testing the end result
- mockAudioEffects.instance._finishProcessing(soundBuffer);
+ await mockAudioEffects.instance._finishProcessing(soundBuffer);
expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled();
expect(vm.updateSoundBuffer).toHaveBeenCalled();
});
@@ -202,7 +202,7 @@ describe('Sound Editor Container', () => {
expect(mockAudioEffects.instance.process).toHaveBeenCalled();
});
- test('undo/redo stack state', () => {
+ test('undo/redo stack state', async () => {
const wrapper = mountWithIntl(
<SoundEditor
soundIndex={soundIndex}
@@ -216,40 +216,40 @@ describe('Sound Editor Container', () => {
// Submitting new samples should make it possible to undo
component.props().onFaster();
- mockAudioEffects.instance._finishProcessing(soundBuffer);
+ await mockAudioEffects.instance._finishProcessing(soundBuffer);
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canUndo')).toEqual(true);
expect(component.prop('canRedo')).toEqual(false);
// Undoing should make it possible to redo and not possible to undo again
- component.props().onUndo();
+ await component.props().onUndo();
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canUndo')).toEqual(false);
expect(component.prop('canRedo')).toEqual(true);
// Redoing should make it possible to undo and not possible to redo again
- component.props().onRedo();
+ await component.props().onRedo();
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canUndo')).toEqual(true);
expect(component.prop('canRedo')).toEqual(false);
// New submission should clear the redo stack
- component.props().onUndo(); // Undo to go back to a state where redo is enabled
+ await component.props().onUndo(); // Undo to go back to a state where redo is enabled
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canRedo')).toEqual(true);
component.props().onFaster();
- mockAudioEffects.instance._finishProcessing(soundBuffer);
+ await mockAudioEffects.instance._finishProcessing(soundBuffer);
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canRedo')).toEqual(false);
});
- test('undo and redo submit new samples and play the sound', () => {
+ test('undo and redo submit new samples and play the sound', async () => {
const wrapper = mountWithIntl(
<SoundEditor
soundIndex={soundIndex}
@@ -260,12 +260,12 @@ describe('Sound Editor Container', () => {
// Set up an undoable state
component.props().onFaster();
- mockAudioEffects.instance._finishProcessing(soundBuffer);
+ await mockAudioEffects.instance._finishProcessing(soundBuffer);
wrapper.update();
component = wrapper.find(SoundEditorComponent);
// Undo should update the sound buffer and play the new samples
- component.props().onUndo();
+ await component.props().onUndo();
expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled();
expect(vm.updateSoundBuffer).toHaveBeenCalled();
@@ -274,7 +274,7 @@ describe('Sound Editor Container', () => {
mockAudioBufferPlayer.instance.play.mockClear();
// Undo should update the sound buffer and play the new samples
- component.props().onRedo();
+ await component.props().onRedo();
expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled();
expect(vm.updateSoundBuffer).toHaveBeenCalled();
});
diff --git a/test/unit/util/audio-util.test.js b/test/unit/util/audio-util.test.js
index 24ddcd4559567dce0fd1677a11a809c365233699..79ff55e8b5551ce355672ae60de67c146c445900 100644
--- a/test/unit/util/audio-util.test.js
+++ b/test/unit/util/audio-util.test.js
@@ -1,4 +1,9 @@
-import {computeRMS, computeChunkedRMS} from '../../../src/lib/audio/audio-util';
+import {
+ computeRMS,
+ computeChunkedRMS,
+ downsampleIfNeeded,
+ dropEveryOtherSample
+} from '../../../src/lib/audio/audio-util';
describe('computeRMS', () => {
test('returns 0 when given no samples', () => {
@@ -49,3 +54,44 @@ describe('computeChunkedRMS', () => {
expect(chunkedLevels).toEqual([Math.sqrt(1 / 0.55), Math.sqrt(1 / 0.55)]);
});
});
+
+describe('downsampleIfNeeded', () => {
+ const samples = {length: 1};
+ const sampleRate = 44100;
+ test('returns given data when no downsampling needed', async () => {
+ samples.length = 1;
+ const res = await downsampleIfNeeded({samples, sampleRate}, null);
+ expect(res.samples).toEqual(samples);
+ expect(res.sampleRate).toEqual(sampleRate);
+ });
+ test('downsamples to 22050 if that puts it under the limit', async () => {
+ samples.length = 44100 * 3 * 60;
+ const resampler = jest.fn(() => 'TEST');
+ const res = await downsampleIfNeeded({samples, sampleRate}, resampler);
+ expect(resampler).toHaveBeenCalledWith({samples, sampleRate}, 22050);
+ expect(res).toEqual('TEST');
+ });
+ test('fails if resampling would not put it under the limit', async () => {
+ samples.length = 44100 * 4 * 60;
+ try {
+ await downsampleIfNeeded({samples, sampleRate}, null);
+ } catch (e) {
+ expect(e).toEqual('Sound too large to save, refusing to edit');
+ }
+ });
+});
+
+describe('dropEveryOtherSample', () => {
+ const buffer = {
+ samples: [1, 0, 2, 0, 3, 0],
+ sampleRate: 2
+ };
+ test('result is half the length', () => {
+ const {samples} = dropEveryOtherSample(buffer, 1);
+ expect(samples.length).toEqual(Math.floor(buffer.samples.length / 2));
+ });
+ test('result contains only even-index items', () => {
+ const {samples} = dropEveryOtherSample(buffer, 1);
+ expect(samples).toEqual(new Float32Array([1, 2, 3]));
+ });
+});