Skip to content

[AUDIO_WORKLET] Optimised output buffer copy #24891

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 133 additions & 47 deletions src/audio_worklet.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,51 +28,119 @@ function createWasmAudioWorkletProcessor(audioParams) {
this.callback = {{{ makeDynCall('iipipipp', 'opts.callback') }}};
this.userData = opts.userData;
// Then the samples per channel to process, fixed for the lifetime of the
// context that created this processor. Note for when moving to Web Audio
// 1.1: the typed array passed to process() should be the same size as this
// 'render quantum size', and this exercise of passing in the value
// shouldn't be required (to be verified)
// context that created this processor. Even though this 'render quantum
// size' is fixed at 128 samples in the 1.0 spec, it will be variable in
// the 1.1 spec. It's passed in now, just to prove it's settable, but will
// eventually be a property of the AudioWorkletGlobalScope (globalThis).
this.samplesPerChannel = opts.samplesPerChannel;
this.bytesPerChannel = this.samplesPerChannel * {{{ getNativeTypeSize('float') }}};

// Creates the output views (see createOutputViews() docs)
this.maxBuffers = Math.min(((wwParams.stackSize - /*minimum alloc*/ 16) / this.bytesPerChannel) | 0, /*sensible limit*/ 64);
this.outputViews = [];
#if ASSERTIONS
console.assert(this.maxBuffers > 0, `AudioWorklet needs more stack allocating (at least ${this.bytesPerChannel})`);
#endif
this.createOutputViews();

#if ASSERTIONS
// Explicitly verify this later in process()
this.ctorOldStackPtr = stackSave();
#endif
}

/**
* Create up-front as many typed views for marshalling the output data as
* may be required (with an arbitrary maximum of 64, for the case where a
* multi-MB stack is passed), allocated at the *top* of the worklet's stack
* (and whose addresses are fixed). The 'minimum alloc' firstly stops
* STACK_OVERFLOW_CHECK failing (since the stack will be full, and 16 bytes
* being the minimum allocation size due to alignments) and leaves room for
* a single AudioSampleFrame as a minumum.
*/
createOutputViews() {
// These are still alloc'd to take advantage of the overflow checks, etc.
var oldStackPtr = stackSave();
var viewDataIdx = {{{ getHeapOffset('stackAlloc(this.maxBuffers * this.bytesPerChannel)', 'float') }}};
#if WEBAUDIO_DEBUG
console.log(`AudioWorklet creating ${this.maxBuffers} buffer one-time views (for a stack size of ${wwParams.stackSize} at address ${ptrToString(viewDataIdx * 4)})`);
#endif
this.outputViews.length = 0;
for (var n = this.maxBuffers; n > 0; n--) {
// Added in reverse so the lowest indices are closest to the stack top
this.outputViews.unshift(
HEAPF32.subarray(viewDataIdx, viewDataIdx += this.samplesPerChannel)
);
}
stackRestore(oldStackPtr);
}

static get parameterDescriptors() {
return audioParams;
}

/**
* Marshals all inputs and parameters to the Wasm memory on the thread's
* stack, then performs the wasm audio worklet call, and finally marshals
* audio output data back.
*
* @param {Object} parameters
*/
process(inputList, outputList, parameters) {
// Marshal all inputs and parameters to the Wasm memory on the thread stack,
// then perform the wasm audio worklet call,
// and finally marshal audio output data back.
#if ALLOW_MEMORY_GROWTH
// recreate the output views if the heap has changed
if (HEAPF32.buffer != this.outputViews[0].buffer) {
this.createOutputViews();
}
#endif

var numInputs = inputList.length;
var numOutputs = outputList.length;

var entry; // reused list entry or index
var subentry; // reused channel or other array in each list entry or index

// Calculate how much stack space is needed.
var bytesPerChannel = this.samplesPerChannel * {{{ getNativeTypeSize('float') }}};
var stackMemoryNeeded = (numInputs + numOutputs) * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}};
// Calculate the required stack and output buffer views (stack is further
// split into aligned structs and the raw float data).
var stackMemoryStruct = (numInputs + numOutputs) * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}};
var stackMemoryData = 0;
for (entry of inputList) {
stackMemoryData += entry.length * this.bytesPerChannel;
}
// Collect the total number of output channels (mapped to array views)
var outputViewsNeeded = 0;
for (entry of outputList) {
outputViewsNeeded += entry.length;
}
stackMemoryData += outputViewsNeeded * this.bytesPerChannel;
var numParams = 0;
for (entry of inputList) stackMemoryNeeded += entry.length * bytesPerChannel;
for (entry of outputList) stackMemoryNeeded += entry.length * bytesPerChannel;
for (entry in parameters) {
stackMemoryNeeded += parameters[entry].byteLength + {{{ C_STRUCTS.AudioParamFrame.__size__ }}};
++numParams;
stackMemoryStruct += {{{ C_STRUCTS.AudioParamFrame.__size__ }}};
stackMemoryData += parameters[entry].byteLength;
}
#if ASSERTIONS
console.assert(outputViewsNeeded <= this.outputViews.length, `Too many AudioWorklet outputs (need ${outputViewsNeeded} but have stack space for ${this.outputViews.length})`);
#endif

// Allocate the necessary stack space.
var oldStackPtr = stackSave();
var inputsPtr = stackAlloc(stackMemoryNeeded);
// Allocate the necessary stack space. All pointer variables are in bytes;
// 'structPtr' starts at the first struct entry (all run sequentially)
// and is the working start to each record; 'dataPtr' is the same for the
// audio/params data, starting after *all* the structs.
// 'structPtr' begins 16-byte aligned, allocated from the internal
// _emscripten_stack_alloc(), as are the output views, and so to ensure
// the views fall on the correct addresses (and we finish at stacktop) we
// request additional bytes, taking this alignment into account, then
// offset `dataPtr` by the difference.
var stackMemoryAligned = (stackMemoryStruct + stackMemoryData + 15) & ~15;
var structPtr = stackAlloc(stackMemoryAligned);
var dataPtr = structPtr + (stackMemoryAligned - stackMemoryData);

// Copy input audio descriptor structs and data to Wasm ('structPtr' is
// reused as the working start to each struct record, 'dataPtr' start of
// the data section, usually after all structs).
var structPtr = inputsPtr;
var dataPtr = inputsPtr + numInputs * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}};
// Copy input audio descriptor structs and data to Wasm (recall, structs
// first, audio data after). 'inputsPtr' is the start of the C callback's
// input AudioSampleFrame.
var /*const*/ inputsPtr = structPtr;
for (entry of inputList) {
// Write the AudioSampleFrame struct instance
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.numberOfChannels, 'entry.length', 'u32') }}};
Expand All @@ -82,28 +150,13 @@ function createWasmAudioWorkletProcessor(audioParams) {
// Marshal the input audio sample data for each audio channel of this input
for (subentry of entry) {
HEAPF32.set(subentry, {{{ getHeapOffset('dataPtr', 'float') }}});
dataPtr += bytesPerChannel;
dataPtr += this.bytesPerChannel;
}
}

// Copy output audio descriptor structs to Wasm
var outputsPtr = dataPtr;
structPtr = outputsPtr;
var outputDataPtr = (dataPtr += numOutputs * {{{ C_STRUCTS.AudioSampleFrame.__size__ }}});
for (entry of outputList) {
// Write the AudioSampleFrame struct instance
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.numberOfChannels, 'entry.length', 'u32') }}};
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.samplesPerChannel, 'this.samplesPerChannel', 'u32') }}};
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.data, 'dataPtr', '*') }}};
structPtr += {{{ C_STRUCTS.AudioSampleFrame.__size__ }}};
// Reserve space for the output data
dataPtr += bytesPerChannel * entry.length;
}

// Copy parameters descriptor structs and data to Wasm
var paramsPtr = dataPtr;
structPtr = paramsPtr;
dataPtr += numParams * {{{ C_STRUCTS.AudioParamFrame.__size__ }}};
// Copy parameters descriptor structs and data to Wasm. 'paramsPtr' is the
// start of the C callback's input AudioParamFrame.
var /*const*/ paramsPtr = structPtr;
for (entry = 0; subentry = parameters[entry++];) {
// Write the AudioParamFrame struct instance
{{{ makeSetValue('structPtr', C_STRUCTS.AudioParamFrame.length, 'subentry.length', 'u32') }}};
Expand All @@ -114,20 +167,53 @@ function createWasmAudioWorkletProcessor(audioParams) {
dataPtr += subentry.length * {{{ getNativeTypeSize('float') }}};
}

// Copy output audio descriptor structs to Wasm. 'outputsPtr' is the start
// of the C callback's output AudioSampleFrame. 'dataPtr' will now be
// aligned with the output views, ending at stacktop.
var /*const*/ outputsPtr = structPtr;
for (entry of outputList) {
// Write the AudioSampleFrame struct instance
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.numberOfChannels, 'entry.length', 'u32') }}};
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.samplesPerChannel, 'this.samplesPerChannel', 'u32') }}};
{{{ makeSetValue('structPtr', C_STRUCTS.AudioSampleFrame.data, 'dataPtr', '*') }}};
structPtr += {{{ C_STRUCTS.AudioSampleFrame.__size__ }}};
// Advance the output pointer to the next output (matching the pre-allocated views)
dataPtr += this.bytesPerChannel * entry.length;
}

#if ASSERTIONS
// If all the maths worked out, we arrived at the original stack address
console.assert(dataPtr == oldStackPtr, `AudioWorklet stack missmatch (audio data finishes at ${dataPtr} instead of ${oldStackPtr})`);

// Sanity checks. If these trip the most likely cause, beyond unforeseen
// stack shenanigans, is that the 'render quantum size' changed after
// construction (which shouldn't be possible).
if (numOutputs) {
// First that the output view addresses match the stack positions
dataPtr -= this.bytesPerChannel;
for (entry = 0; entry < outputViewsNeeded; entry++) {
console.assert(dataPtr == this.outputViews[entry].byteOffset, 'AudioWorklet internal error in addresses of the output array views');
dataPtr -= this.bytesPerChannel;
}
// And that the views' size match the passed in output buffers
for (entry of outputList) {
for (subentry of entry) {
console.assert(subentry.byteLength == this.bytesPerChannel, `AudioWorklet unexpected output buffer size (expected ${this.bytesPerChannel} got ${subentry.byteLength})`);
}
}
}
#endif

// Call out to Wasm callback to perform audio processing
var didProduceAudio = this.callback(numInputs, inputsPtr, numOutputs, outputsPtr, numParams, paramsPtr, this.userData);
if (didProduceAudio) {
// Read back the produced audio data to all outputs and their channels.
// (A garbage-free function TypedArray.copy(dstTypedArray, dstOffset,
// srcTypedArray, srcOffset, count) would sure be handy.. but web does
// not have one, so manually copy all bytes in)
outputDataPtr = {{{ getHeapOffset('outputDataPtr', 'float') }}};
// The preallocated 'outputViews' already have the correct offsets and
// sizes into the stack (recall from createOutputViews() that they run
// backwards).
for (entry of outputList) {
for (subentry of entry) {
// repurposing structPtr for now
for (structPtr = 0; structPtr < this.samplesPerChannel; ++structPtr) {
subentry[structPtr] = HEAPF32[outputDataPtr++];
}
subentry.set(this.outputViews[--outputViewsNeeded]);
}
}
}
Expand Down
Loading