JS: Audio Data Processing

Ingvar StepanyanWeb Developer, MSP@RReverser (http://rreverser.com/)

Native implementations:[Mozilla] Audio Data API (low-level processing)

[WebKit] Web Audio API (+ some high-level sugar)

[Flash] Sound API (low-level processing)

Unifying wrappers:dynamicaudio.js (Audio Data API + Flash)

XAudioJS (Audio Data API + Web Audio API + Flash)

sink.js & co. (Audio Data API + Web Audio API)

Current status

LowLevel

<audio id=“audio” ... />....................................var audio = document.getElementById(‘audio’), channels, rate, bufferLength;

audio.addEventListener('loadedmetadata', function () { channels = audio.mozChannels; rate = audio.mozSampleRate; bufferLength = audio.mozFrameBufferLength; // ...});

Reading audio data

audio.addEventListener('MozAudioAvailable', function () { var buffer = event.frameBuffer, time = event.time;

for (var i = 0; i < bufferLength / channels; i++ ) { // use buffer[i * channels + channelNumber] }});

Reading audio data

0.409862111297023, 0.259322117331362, -0.008777887762307, -0.984780922682458, -0.739880137997811, -0.393993360534229, -0.698508825498069, -0.393512581774658, 0.073465971474621, -0.774029495474732, -0.068926253092157, 0.698052197232118, 0.450745893413682, 0.731625352383412, -0.633931102362129, 0.882738533608791, -0.493848309462582, -0.668645515465273, 0.271953939580002, 0.246314079650827, -0.645711462859856, 0.736913698005765, 0.872459503967142, -0.482284551106221, 0.661431378511110, -0.291951234239424, 0.669216884142705, -0.341110225783700, -0.165754738005617, -0.986556649848787, 0.803597140689702, -0.813939097068497, 0.297290844923532, -0.434373616658307, -0.820145776933725, -0.351005880456336, 0.380713105392354, -0.721515454501945, 0.066484189654290, 0.642101001764223, -0.597205735590506, -0.200460217995576, 0.491272422617811, 0.480988687186248, 0.227128301563507, 0.685168802023231, 0.075880282193273, -0.749673188605156, 0.702223692201174, -0.184544172051190, -0.506130812375945, -0.686331843733017, -0.791173541823365, 0.886386883125611, 0.042702665517532, 0.573126318821646, 0.576012637739739, -0.329392317046949, -0.560594547513062, -0.223420931279321, 0.652487040645743, 0.403985003945232, 0.881201502984618, 0.992330609078359, 0.157417714187382, 0.133725348839930, 0.343408728622162, 0.005933505310489, -0.914475827505964, 0.154820823526593, 0.148971641643792, 0.841135222645066, 0.095968894389784, -0.234891692775066, -0.055960466199662, -0.133320485223315, -0.945514113185999, 0.485242740877991, 0.933864235656065, 0.022960704882891, -0.139667183659487, 0.618401839172406, -0.195349340573667, 0.587291626042564, 0.158475194222853, 0.565554351499445, 0.101859803696333, 0.593813833398595, -0.349654793639986, 0.665640345371317, 0.814509967148215, 0.533843529155366, -0.631997214437279, -0.503973088121093, -0.180033876163095, -0.715287970025233, 0.739453218066262, -0.452880703839526, -0.112778265491663, -0.650074534874509, -0.925708286102557, 0.976021772837429, 0.768202886746198, -0.559265634778471, 0.004451442257993, 0.747659266125223, -0.692840193163558, -0.811031246931423, 0.064833012476722, -0.112213849236344, -0.045264398268911, 0.353260515196007, 0.150881652218966, -0.199176332305722, 0.680000507648084, 0.265540158726514, -0.692481536406930, -0.353516025899132, 0.548221585427517, 0.461960884430420, 0.983277137292909, 0.622653444743655, 0.480297639150934, 0.452168428463840, -0.823544922845037, -0.331975377216007, 0.358569546837979, -0.596899627797886, 0.301496127117938, 0.576480504549117, 0.150743674405126, -0.851734411492563, 0.127175797857987, 0.806251938739738, 0.360201361643523, 0.380717789492329, 0.759847187611911, -0.629616985016364, 0.179681096205048, 0.147705467636446, 0.793359439243774, -0.888943630149254, -0.768200353474536, 0.837243925007694, 0.601330942636268, -0.611155192762695, 0.176371159373866, 0.731621908979866, 0.294121659309740, 0.758972133758613, 0.226735505828943, -0.099879270352797, 0.716827038775917, -0.439818898014638, -0.845310989782625, 0.712989203634548, 0.040939664050535, -0.492177661587544, -0.611892969959734, -0.419016380589113, 0.526549046576405, -0.638706291747430, 0.611630316089939, -0.340452683930807, 0.833273571181310, 0.526974765301767, 0.462530036158461, -0.796029844105523, 0.604456871305559, 0.061317570172945, 0.642369077263937, -0.978365729885714, -0.468053997163355, 0.954769747262937, 0.446717706860744, -0.960283148953476, -0.751094692845392, 0.805721468166348, -0.290676951057867, 0.391488310207055, 0.099343420640492, 0.034684411420110, 0.463771493683684, -0.378392748975848, -0.830120263808777, -0.357574609519144, 0.755175701837563, -0.379417718802803, 0.295501263679325, 0.181793810178941, -0.116752805641118, -0.279166790263324, 0.938534135910232, 0.091496919686446, -0.107904426032210, -0.978447352332630, -0.180865072441111, 0.831093166130408, 0.099057141660298, 0.456832343000489, -0.652587456595693, 0.359989313981682, -0.951405226971930, 0.381285879196237, 0.182679971280062, -0.580757635269123, 0.879272667221741, -0.480336023774189, 0.833228235340136, .......................................

Low-level Data Format

WTF???

Samples ( not “examples” :) )

𝑡=𝑁𝑠𝑎𝑚𝑝𝑙𝑒𝑠

𝑓 𝑠𝑎𝑚𝑝𝑙𝑒

Sample analysis

https://github.com/corbanbrook/dsp.js/

Nyquist–Shannon sampling theorem:

Sample rate

http://bartus.org/akustyk/adc.html

audio.mozSetup(channels, sampleRate);// …audio.mozWriteAudio(samples1);// …audio.mozWriteAudio(samples2);

Writing audio

function AudioStream(readFn, bufSize) { var audio = new Audio(); audio.mozSetup(1, sampleRate); readFn = readFn.bind(audio, bufSize); audio.writeAudio = function(soundData, callback) { var written = this.mozWriteAudio(soundData), length = soundData.length; written < length ? setTimeout(this.writeAudio.bind(this, soundData.subarray(written), callback), written ? 1000 * written / sampleRate : 0) : callback(readFn()); } readFn();

return audio;}

Buffered writing

Main Thread

Write Audio

setTimeout

Read Function

Calculation Worker

Concurrency

Demonstration

http://rreverser.com/dev/piano/