ruby implementation

app/controllers/chat_rooms_controller.rb

@@ -1,3 +1,13 @@
+SPEECH = Speech.new(
+  creds: JSON.parse(File.read(ENV["STREAMING_CREDENTIALS"])),
+  host_lang: "en",
+  recieve_lang: "fr"
+  )
+
+while true
+  SPEECH.stream
+end
+
 class ChatRoomsController < ApplicationController
 
   def show
@@ -6,12 +16,12 @@ class ChatRoomsController < ApplicationController
 
   def create
     #  HTTP status code 200 with an empty body
-    head :no_content
-    puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213"
-    puts params
-    puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213"
+
+    SPEECH.write_to_stream(params[:audio]) unless params[:audio].nil?
 
     ActionCable.server.broadcast "chat_room_#{params[:room]}", session_params
+
+    head :no_content
   end
 
   private
@@ -21,4 +31,11 @@ class ChatRoomsController < ApplicationController
     # Candidate = ICE candidates (e.g. TURN and STUN server)
     params.permit(:type, :from, :to, :sdp, :candidate, :room)
   end
+
+
+
 end
+
+
+
+

app/javascript/packs/audio.js

@@ -0,0 +1,55 @@
+const AUDIO_DATA = "AUDIO_DATA";
+
+export default class AudioData {
+  constructor(host, reciever, room) {
+    this.host = host;
+    this.reciever = reciever;
+    this.room = room;
+    this.decoder = new TextDecoder("ascii");
+  }
+  async intercept(stream) { // MediaStream
+
+    AudioContext = window.AudioContext || window.webkitAudioContext;
+    const ctx = new AudioContext();
+    const processor = ctx.createScriptProcessor(4096, 1, 1);
+    processor.connect(ctx.destination);
+    processor.onaudioprocess = e => this.handleBuffer(e);
+
+    ctx.createMediaStreamSource(stream).connect(processor);
+    ctx.resume();
+
+  }
+
+  broadcast(data) {
+    fetch("chat_room_sessions", {
+      method: "POST",
+      body: JSON.stringify({
+        type: AUDIO_DATA,
+        from: this.host,
+        to: this.reciever,
+        room: this.room,
+        audio: data.toString()
+      }),
+      headers: { "content-type": "application/json", "X-CSRF-Token": document.querySelector('meta[name=csrf-token]').content }
+    })
+  }
+
+  handleBuffer(e) {
+    const l = e.inputBuffer.getChannelData(0)
+    const l16 = convertF32ToInt16(l);
+    this.broadcast(this.decoder.decode(l16));
+
+    function convertF32ToInt16(buffer) {
+      let l = buffer.length;
+
+      let buf = new Int16Array(l / 3);
+
+      while (l--) {
+        if (l % 3 == 0) {
+          buf[l / 3] = buffer[l] * 0xFFFF;
+        }
+      }
+      return buf;
+    }
+  }
+}

app/javascript/packs/webrtc.js

@@ -1,3 +1,5 @@
+import AudioData from './audio';
+
 // Broadcast Types
 
 const JOIN_ROOM = "JOIN_ROOM";
@@ -110,6 +112,8 @@ const createPC = (userId, isOffer) => {
   let test = userId
   pcPeers[userId] = pc;
   pc.addStream(localstream);
+  const audio = new AudioData(currentUser, userId, chatroomId);
+
   if (isOffer) {
     pc
       .createOffer()
@@ -146,6 +150,7 @@ const createPC = (userId, isOffer) => {
     element.height = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);
     remoteVideoContainer.appendChild(element);
     localVideo.classList.add("video-sm");
+    audio.intercept(localstream);
   };
 
   pc.oniceconnectionstatechange = event => {

app/models/speech.rb

@@ -0,0 +1,61 @@
+require 'google/cloud/translate'
+require 'pry-byebug'
+class Speech
+  def initialize(params = {})
+    @speech = Google::Cloud::Speech.new
+    @credentials = params[:creds]
+    keyfile = ENV["TRANSLATION_CREDENTIALS"]
+    creds = Google::Cloud::Translate::Credentials.new(keyfile)
+
+
+    @translate = Google::Cloud::Translate.new(
+      project_id: ENV["PROJECT_ID"],
+      credentials: creds
+      )
+
+    @streaming_config =
+      { config:
+        {
+          encoding: :LINEAR16,
+          sample_rate_hertz: 16000,
+          language_code: params[:language]
+        },
+        interim_results: true
+      }
+
+      @host_lang = params[:host_lang] || "en"
+      @recieve_lang = params[:recieve_lang] || "en"
+
+      @stream = @speech.streaming_recognize(@streaming_config)
+      @audio = ""
+  end
+
+
+  def write_to_stream(audio)
+      @stream.send(audio.split(",").map { |str| str.to_i }.pack("s<*"))
+  end
+
+  def stream
+    while true
+      break if @stream.stopped?
+      results = @stream.results
+
+      unless results.first.nil?
+        alt = results.first.alternatives
+        alt.each do |result|
+          puts "Original: #{result.transcript}"
+          puts "Translated: #{translate(result.transcript)}"
+        end
+        break
+      end
+    end
+
+    @stream.stop
+    @stream.wait_until_complete!
+  end
+
+  def translate(text)
+    trans = @translate.translate(text, from: @host_lang, to: @recieve_lang)
+    translation.text.gsub("&#39;", "'")
+  end
+end