From 4cc4fb91bb560a02d0d44db9a397ba90ae8657a4 Mon Sep 17 00:00:00 2001 From: Rekai Musuka Date: Tue, 4 Sep 2018 11:18:39 +0100 Subject: [PATCH] Working but broken communication with nodejs server --- app/javascript/packs/nodeserver.js | 77 +++++++++++++++ app/javascript/packs/webrtc.js | 4 + package.json | 3 +- yarn.lock | 153 ++++++++++++++++++++++++++++- 4 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 app/javascript/packs/nodeserver.js diff --git a/app/javascript/packs/nodeserver.js b/app/javascript/packs/nodeserver.js new file mode 100644 index 0000000..eef44b0 --- /dev/null +++ b/app/javascript/packs/nodeserver.js @@ -0,0 +1,77 @@ +import io from 'socket.io-client'; + + /** + * The Class Responsible for Talking with Polyglot-Node + * @param {string} url - URL of NodeJS Server. + */ +export default class NodeSocket { + constructor(url) { + this.io = io(url); + this.listen(); + } + + /** + * Create socket.io event listeners. + */ + listen() { + this.io.on('disconnect', (err) => { + console.error("Disconnected from Server."); + }) + + this.io.on('connect', (data) => { + this.io.emit("join", "Hi There NodeJS!"); + console.log("Connected to NodeJS Server"); + }); + } + /** + * This Class Grabs the Audio from the User. + * + * DEPRECAION WARNING. This method makes vital use of createScriptProcessor which will be scrapped at some point + * in the future. TODO: Switch from createScriptProcessor to AudioWorklet (No documentation as of 2018-08-30) + * @deprecated + * @param {MediaStream} stream - Global MediaStream + * @async + */ + async grabAudio(stream) { + this.io.emit('startRecognition', "en-US", "fr"); + + AudioContext = window.AudioContext || window.webkitAudioContext; + const ctx = new AudioContext(); + const processor = ctx.createScriptProcessor(2048, 1, 1); + processor.connect(ctx.destination); + processor.onaudioprocess = e => this.handleBuffer(e); + + // const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + ctx.createMediaStreamSource(stream).connect(processor); + ctx.resume(); + } + + /** + * Takes the Float32Array Array from the ScriptProcessor Event + * and converts it to Int16 for Google Speech to Text use. + * @param {AudioProcessingEvent} e Chunk of Audio w/ Audio F32Array + */ + handleBuffer(e) { + let l = e.inputBuffer.getChannelData(0); + this.io.emit('binary', convertF32ToI16(l)); + + /** + * 32Khz to 16Khz Audio + * Converts Float32 Buffer to Int16Buffer + * @param {Float32Array} buffer - Float32 Buffer; + * @returns {ArrayBuffer} + */ + function convertF32ToI16(buffer) { + let l = buffer.length; + + let buf = new Int16Array(l/3); + + while (l--) { + if (l % 3 == 0) { + buf[l / 3] = buffer[l] * 0xFFFF; + } + } + return buf.buffer; + } + } +} diff --git a/app/javascript/packs/webrtc.js b/app/javascript/packs/webrtc.js index 5bf7376..638aa38 100644 --- a/app/javascript/packs/webrtc.js +++ b/app/javascript/packs/webrtc.js @@ -1,3 +1,4 @@ +import NodeSocket from './nodeserver'; // Broadcast Types const JOIN_ROOM = "JOIN_ROOM"; @@ -12,6 +13,7 @@ let remoteVideoContainer; // Objects let pcPeers = {}; // peer connection let localstream; +let nodeSocket = new NodeSocket("http://localhost:1337"); window.onload = () => { currentUser = document.getElementById("current-user").innerHTML; @@ -33,7 +35,9 @@ document.onreadystatechange = async () => { localstream = stream; localVideo.srcObject = stream + nodeSocket.grabAudio(stream); localVideo.muted = true + } catch (e) { console.error(e); } } }; diff --git a/package.json b/package.json index 1bd8414..2769fde 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "actioncable": "^5.2.1", "babel-plugin-transform-runtime": "^6.23.0", "bootstrap": "3", - "jquery": "^3.3.1" + "jquery": "^3.3.1", + "socket.io-client": "^2.1.1" }, "devDependencies": { "webpack-dev-server": "2.11.2" diff --git a/yarn.lock b/yarn.lock index 2d8018a..d954e15 100644 --- a/yarn.lock +++ b/yarn.lock @@ -61,6 +61,10 @@ actioncable@^5.2.1: version "5.2.1" resolved "https://registry.yarnpkg.com/actioncable/-/actioncable-5.2.1.tgz#615428a8a302cec55117d9049f0fd7952011c962" +after@0.8.2: + version "0.8.2" + resolved "https://registry.yarnpkg.com/after/-/after-0.8.2.tgz#fedb394f9f0e02aa9768e702bda23b505fae7e1f" + ajv-keywords@^3.1.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.2.0.tgz#e86b819c602cf8821ad637413698f1dec021847a" @@ -200,6 +204,10 @@ array-unique@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" +arraybuffer.slice@~0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675" + asn1.js@^4.0.0: version "4.10.1" resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-4.10.1.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0" @@ -236,6 +244,10 @@ async-foreach@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/async-foreach/-/async-foreach-0.1.3.tgz#36121f845c0578172de419a97dbeb1d16ec34542" +async-limiter@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8" + async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" @@ -792,6 +804,10 @@ babylon@^6.18.0: version "6.18.0" resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3" +backo2@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947" + balanced-match@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.1.0.tgz#b504bd05869b39259dd0c5efc35d843176dccc4a" @@ -804,6 +820,10 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" +base64-arraybuffer@0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz#73926771923b5a19747ad666aa5cd4bf9c6e9ce8" + base64-js@^1.0.2: version "1.3.0" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.0.tgz#cab1e6118f051095e58b5281aea8c1cd22bfc0e3" @@ -830,6 +850,12 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +better-assert@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522" + dependencies: + callsite "1.0.0" + big.js@^3.1.3: version "3.2.0" resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e" @@ -838,6 +864,10 @@ binary-extensions@^1.0.0: version "1.11.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205" +blob@0.0.4: + version "0.0.4" + resolved "https://registry.yarnpkg.com/blob/-/blob-0.0.4.tgz#bcf13052ca54463f30f9fc7e95b9a47630a94921" + block-stream@*: version "0.0.9" resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" @@ -1054,6 +1084,10 @@ cache-base@^1.0.1: union-value "^1.0.0" unset-value "^1.0.0" +callsite@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/callsite/-/callsite-1.0.0.tgz#280398e5d664bd74038b6f0905153e6e8af1bc20" + camelcase-keys@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-2.1.0.tgz#308beeaffdf28119051efa1d932213c91b8f92e7" @@ -1306,10 +1340,18 @@ commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" -component-emitter@^1.2.1: +component-bind@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/component-bind/-/component-bind-1.0.0.tgz#00c608ab7dcd93897c0009651b1d3a8e1e73bbd1" + +component-emitter@1.2.1, component-emitter@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" +component-inherit@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/component-inherit/-/component-inherit-0.0.3.tgz#645fc4adf58b72b649d5cae65135619db26ff143" + compressible@~2.0.14: version "2.0.14" resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.14.tgz#326c5f507fbb055f54116782b969a81b67a29da7" @@ -1604,7 +1646,7 @@ debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.6, debug@^2.6. dependencies: ms "2.0.0" -debug@^3.1.0: +debug@^3.1.0, debug@~3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: @@ -1786,6 +1828,32 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" +engine.io-client@~3.2.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-3.2.1.tgz#6f54c0475de487158a1a7c77d10178708b6add36" + dependencies: + component-emitter "1.2.1" + component-inherit "0.0.3" + debug "~3.1.0" + engine.io-parser "~2.1.1" + has-cors "1.1.0" + indexof "0.0.1" + parseqs "0.0.5" + parseuri "0.0.5" + ws "~3.3.1" + xmlhttprequest-ssl "~1.5.4" + yeast "0.1.2" + +engine.io-parser@~2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-2.1.2.tgz#4c0f4cff79aaeecbbdcfdea66a823c6085409196" + dependencies: + after "0.8.2" + arraybuffer.slice "~0.0.7" + base64-arraybuffer "0.1.5" + blob "0.0.4" + has-binary2 "~1.0.2" + enhanced-resolve@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-3.4.1.tgz#0421e339fd71419b3da13d129b3979040230476e" @@ -2409,6 +2477,16 @@ has-ansi@^2.0.0: dependencies: ansi-regex "^2.0.0" +has-binary2@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-binary2/-/has-binary2-1.0.3.tgz#7776ac627f3ea77250cfc332dab7ddf5e4f5d11d" + dependencies: + isarray "2.0.1" + +has-cors@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-cors/-/has-cors-1.1.0.tgz#5e474793f7ea9843d1bb99c23eef49ff126fff39" + has-flag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-1.0.0.tgz#9d9e793165ce017a00f00418c43f942a7b1d11fa" @@ -2917,6 +2995,10 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" +isarray@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.1.tgz#a37d94ed9cda2d59865c9f76fe596ee1f338741e" + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -3645,6 +3727,10 @@ object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" +object-component@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291" + object-copy@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/object-copy/-/object-copy-0.1.0.tgz#7e7d858b781bd7c991a41ba975ed3812754e998c" @@ -3820,6 +3906,18 @@ parse-json@^4.0.0: error-ex "^1.3.1" json-parse-better-errors "^1.0.1" +parseqs@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d" + dependencies: + better-assert "~1.0.0" + +parseuri@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/parseuri/-/parseuri-0.0.5.tgz#80204a50d4dbb779bfdc6ebe2778d90e4bce320a" + dependencies: + better-assert "~1.0.0" + parseurl@~1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" @@ -5112,6 +5210,33 @@ snapdragon@^0.8.1: source-map-resolve "^0.5.0" use "^3.1.0" +socket.io-client@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-2.1.1.tgz#dcb38103436ab4578ddb026638ae2f21b623671f" + dependencies: + backo2 "1.0.2" + base64-arraybuffer "0.1.5" + component-bind "1.0.0" + component-emitter "1.2.1" + debug "~3.1.0" + engine.io-client "~3.2.0" + has-binary2 "~1.0.2" + has-cors "1.1.0" + indexof "0.0.1" + object-component "0.0.3" + parseqs "0.0.5" + parseuri "0.0.5" + socket.io-parser "~3.2.0" + to-array "0.1.4" + +socket.io-parser@~3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-3.2.0.tgz#e7c6228b6aa1f814e6148aea325b51aa9499e077" + dependencies: + component-emitter "1.2.1" + debug "~3.1.0" + isarray "2.0.1" + sockjs-client@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.1.4.tgz#5babe386b775e4cf14e7520911452654016c8b12" @@ -5446,6 +5571,10 @@ timers-browserify@^2.0.4: dependencies: setimmediate "^1.0.4" +to-array@0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/to-array/-/to-array-0.1.4.tgz#17e6c11f73dd4f3d74cda7a4ff3238e9ad9bf890" + to-arraybuffer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43" @@ -5569,6 +5698,10 @@ uglifyjs-webpack-plugin@^1.2.7: webpack-sources "^1.1.0" worker-farm "^1.5.2" +ultron@~1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ultron/-/ultron-1.1.1.tgz#9fe1536a10a664a65266a1e3ccf85fd36302bc9c" + union-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.0.tgz#5c71c34cb5bad5dcebe3ea0cd08207ba5aa1aea4" @@ -5862,6 +5995,18 @@ wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" +ws@~3.3.1: + version "3.3.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-3.3.3.tgz#f1cf84fe2d5e901ebce94efaece785f187a228f2" + dependencies: + async-limiter "~1.0.0" + safe-buffer "~5.1.0" + ultron "~1.1.0" + +xmlhttprequest-ssl@~1.5.4: + version "1.5.5" + resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz#c2876b06168aadc40e57d97e81191ac8f4398b3e" + xtend@^4.0.0, xtend@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" @@ -5962,3 +6107,7 @@ yargs@~3.10.0: cliui "^2.1.0" decamelize "^1.0.0" window-size "0.1.0" + +yeast@0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/yeast/-/yeast-0.1.2.tgz#008e06d8094320c372dbc2f8ed76a0ca6c8ac419"