// Broadcast Types class Signaling {} const JOIN_ROOM = "JOIN_ROOM"; const EXCHANGE = "EXCHANGE"; const REMOVE_USER = "REMOVE_USER"; // DOM Elements let currentUser; let localVideo; let remoteVideoContainer; // Objects let pcPeers = {}; // peer connection let localstream; window.onload = () => { currentUser = document.getElementById("current-user").innerHTML; localVideo = document.getElementById("local-video"); remoteVideoContainer = document.getElementById("remote-video-container"); }; // Ice Credentials const ice = { iceServers: [{urls: ['stun:stun.l.google.com:19302', 'stun:stun.1.google.com:19302']}]}; // Initialize user's own video document.onreadystatechange = async () => { if (document.readyState === "interactive") { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: true }) localstream = stream; localVideo.srcObject = stream localVideo.muted = true } catch (e) { console.error(e); } } }; const handleJoinSession = async () => { App.session = await App.cable.subscriptions.create("VideoSessionChannel", { connected: () => { broadcastData({ type: JOIN_ROOM, from: currentUser }); }, received: data => { console.log("received", data); if (data.from === currentUser) return; switch (data.type) { case JOIN_ROOM: return joinRoom(data); case EXCHANGE: if (data.to !== currentUser) return; return exchange(data); case REMOVE_USER: return removeUser(data); default: return; } } }); }; const handleLeaveSession = () => { for (user in pcPeers) { pcPeers[user].close(); } pcPeers = {}; App.session.unsubscribe(); remoteVideoContainer.innerHTML = ""; broadcastData({ type: REMOVE_USER, from: currentUser }); }; const joinRoom = data => { createPC(data.from, true); }; const removeUser = data => { console.log("removing user", data.from); let video = document.getElementById(`remoteVideoContainer+${data.from}`); video && video.remove(); delete pcPeers[data.from]; }; const broadcastData = data => { fetch("sessions", { method: "POST", body: JSON.stringify(data), headers: { "content-type": "application/json" } }); }; const createPC = (userId, isOffer) => { let pc = new RTCPeerConnection(ice); pcPeers[userId] = pc; pc.addStream(localstream); if (isOffer) { pc .createOffer() .then(offer => { pc.setLocalDescription(offer); broadcastData({ type: EXCHANGE, from: currentUser, to: userId, sdp: JSON.stringify(pc.localDescription) }); }) .catch(logError); } pc.onicecandidate = event => { if (event.candidate) { broadcastData({ type: EXCHANGE, from: currentUser, to: userId, candidate: JSON.stringify(event.candidate) }); } }; pc.onaddstream = event => { const element = document.createElement("video"); element.id = `remoteVideoContainer+${userId}`; // why is the userId being interpolated? element.autoplay = "autoplay"; element.srcObject = event.stream; remoteVideoContainer.appendChild(element); }; pc.oniceconnectionstatechange = event => { if (pc.iceConnectionState == "disconnected") { console.log("Disconnected:", userId); broadcastData({ type: REMOVE_USER, from: userId }); } }; return pc; }; const exchange = data => { let pc; if (!pcPeers[data.from]) { pc = createPC(data.from, false); } else { pc = pcPeers[data.from]; } if (data.candidate) { pc .addIceCandidate(new RTCIceCandidate(JSON.parse(data.candidate))) .then(() => console.log("Ice candidate added")) .catch(logError); } if (data.sdp) { sdp = JSON.parse(data.sdp); pc .setRemoteDescription(new RTCSessionDescription(sdp)) .then(() => { if (sdp.type === "offer") { pc.createAnswer().then(answer => { pc.setLocalDescription(answer); broadcastData({ type: EXCHANGE, from: currentUser, to: data.from, sdp: JSON.stringify(pc.localDescription) }); }); } }) .catch(logError); } }; const logError = error => console.warn("Whoops! Error:", error); // Google Cloud Speech Playground with node.js and socket.io // Created by Vinzenz Aubry for sansho 24.01.17 // Feel esee to improve! // Contact: vinzenz@sansho.studio const express = require('express'); // const bodyParser = require('body-parser'); // const path = require('path'); const fs = require('fs'); const environmentVars = require('dotenv').config(); // Google Cloud const speech = require('@google-cloud/speech'); const speechClient = new speech.SpeechClient(); // Creates a client const Translate = require('@google-cloud/translate'); const projectId = 'booming-banner-212315'; const translate = new Translate({ projectId: projectId, }); const target = 'en'; const app = express(); const port = process.env.PORT || 1337; const server = require('http').createServer(app); const io = require('socket.io')(server); app.use('/assets', express.static(__dirname + '/public')); app.use('/session/assets', express.static(__dirname + '/public')); app.set('view engine', 'ejs'); // =========================== ROUTERS ================================ // app.get('/', function (req, res) { res.render('index', {}); }); app.use('/', function (req, res, next) { next(); // console.log(`Request Url: ${req.url}`); }); // =========================== SOCKET.IO ================================ // io.on('connection', function (client) { console.log('Client Connected to server'); let recognizeStream = null; client.on('join', function (data) { client.emit('messages', 'Socket Connected to Server'); }); client.on('messages', function (data) { client.emit('broad', data); }); client.on('startGoogleCloudStream', function (data) { startRecognitionStream(this, data); }); client.on('endGoogleCloudStream', function (data) { stopRecognitionStream(); }); client.on('binaryData', function (data) { // console.log(data); //log binary data if (recognizeStream !== null) { recognizeStream.write(data); } }); function startRecognitionStream(client, data) { recognizeStream = speechClient.streamingRecognize(request) .on('error', console.error) .on('data', (data) => { process.stdout.write( (data.results[0] && data.results[0].alternatives[0]) ? `Transcription: ${data.results[0].alternatives[0].transcript}\n` : `\n\nReached transcription time limit, press Ctrl+C\n`); client.emit('speechData', data); if (data.results[0].alternatives[0] !== undefined) { let text = data.results[0].alternatives[0].transcript translate .translate(text, target) .then(results => { const translation = results[0]; client.emit('translateData', translation) console.log(`Text: ${text}`); console.log(`Translation: ${translation}`); }) .catch(err => { console.error('ERROR:', err); }); } // if end of utterance, let's restart stream // this is a small hack. After 65 seconds of silence, the stream will still throw an error for speech length limit if (data.results[0] && data.results[0].isFinal) { stopRecognitionStream(); startRecognitionStream(client); // console.log('restarted stream serverside'); } }); } function stopRecognitionStream() { if (recognizeStream) { recognizeStream.end(); } recognizeStream = null; } }); // =========================== GOOGLE CLOUD SETTINGS ================================ // // The encoding of the audio file, e.g. 'LINEAR16' // The sample rate of the audio file in hertz, e.g. 16000 // The BCP-47 language code to use, e.g. 'en-US' const encoding = 'LINEAR16'; const sampleRateHertz = 16000; const languageCode = 'fr-FR'; //en-US const request = { config: { encoding: encoding, sampleRateHertz: sampleRateHertz, languageCode: languageCode, profanityFilter: false, enableWordTimeOffsets: true }, interimResults: true // If you want interim results, set this to true }; // =========================== START SERVER ================================ // server.listen(port, "127.0.0.1", function () { //http listen, to make socket work // app.address = "127.0.0.1"; console.log('Server started on port:' + port) });