ruby implementation

Gemfile

@@ -1,6 +1,11 @@
 source 'https://rubygems.org'
 ruby '2.4.4'
 
+
+#Google Stuffs
+gem 'google-cloud-speech'
+gem 'google-cloud-translate'
+
 gem 'bootsnap', require: false
 gem 'devise'
 gem 'jbuilder', '~> 2.0'

Gemfile.lock

@@ -42,6 +42,8 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
     arel (9.0.0)
     autoprefixer-rails (9.1.3)
       execjs
@@ -79,11 +81,46 @@ GEM
       railties (>= 3.2, < 6.0)
     erubi (1.7.1)
     execjs (2.7.0)
+    faraday (0.15.2)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.25)
     font-awesome-sass (5.0.13)
       sassc (>= 1.11)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
+    google-cloud-core (1.2.3)
+      google-cloud-env (~> 1.0)
+    google-cloud-env (1.0.2)
+      faraday (~> 0.11)
+    google-cloud-speech (0.30.1)
+      google-gax (~> 1.3)
+    google-cloud-translate (1.2.1)
+      faraday (~> 0.13)
+      google-cloud-core (~> 1.2)
+      googleauth (~> 0.6.2)
+    google-gax (1.3.0)
+      google-protobuf (~> 3.2)
+      googleapis-common-protos (>= 1.3.5, < 2.0)
+      googleauth (~> 0.6.2)
+      grpc (>= 1.7.2, < 2.0)
+      rly (~> 0.2.3)
+    google-protobuf (3.6.1)
+    googleapis-common-protos (1.3.7)
+      google-protobuf (~> 3.0)
+      googleapis-common-protos-types (~> 1.0)
+      grpc (~> 1.0)
+    googleapis-common-protos-types (1.0.2)
+      google-protobuf (~> 3.0)
+    googleauth (0.6.6)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.12)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.7)
+    grpc (1.14.1)
+      google-protobuf (~> 3.1)
+      googleapis-common-protos-types (~> 1.0.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     i18n (1.1.0)
@@ -91,6 +128,7 @@ GEM
     jbuilder (2.7.0)
       activesupport (>= 4.2.0)
       multi_json (>= 1.2)
+    jwt (2.1.0)
     listen (3.0.8)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
@@ -101,6 +139,7 @@ GEM
       mini_mime (>= 0.1.1)
     marcel (0.3.2)
       mimemagic (~> 0.3.2)
+    memoist (0.16.0)
     method_source (0.9.0)
     mime-types (3.2.2)
       mime-types-data (~> 3.2015)
@@ -111,11 +150,13 @@ GEM
     minitest (5.11.3)
     msgpack (1.2.4)
     multi_json (1.13.1)
+    multipart-post (2.0.0)
     netrc (0.11.0)
     nio4r (2.3.1)
     nokogiri (1.8.4)
       mini_portile2 (~> 2.3.0)
     orm_adapter (0.5.0)
+    os (1.0.0)
     pg (0.21.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
@@ -125,6 +166,7 @@ GEM
       pry (~> 0.10)
     pry-rails (0.3.6)
       pry (>= 0.10.4)
+    public_suffix (3.0.3)
     puma (3.12.0)
     rack (2.0.5)
     rack-proxy (0.6.4)
@@ -167,6 +209,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    rly (0.2.3)
     sass (3.5.7)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
@@ -181,6 +224,11 @@ GEM
     sassc (1.12.1)
       ffi (~> 1.9.6)
       sass (>= 3.3.0)
+    signet (0.9.1)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simple_form (4.0.1)
       actionpack (>= 5.0)
       activemodel (>= 5.0)
@@ -233,6 +281,8 @@ DEPENDENCIES
   devise
   dotenv-rails
   font-awesome-sass (~> 5.0.9)
+  google-cloud-speech
+  google-cloud-translate
   jbuilder (~> 2.0)
   listen (~> 3.0.5)
   pg (~> 0.21)

app/controllers/chat_rooms_controller.rb

@@ -1,3 +1,13 @@
+SPEECH = Speech.new(
+  creds: JSON.parse(File.read(ENV["STREAMING_CREDENTIALS"])),
+  host_lang: "en",
+  recieve_lang: "fr"
+  )
+
+while true
+  SPEECH.stream
+end
+
 class ChatRoomsController < ApplicationController
 
   def show
@@ -6,12 +16,12 @@ class ChatRoomsController < ApplicationController
 
   def create
     #  HTTP status code 200 with an empty body
-    head :no_content
+
-    puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213"
+    SPEECH.write_to_stream(params[:audio]) unless params[:audio].nil?
-    puts params
-    puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213"
 
     ActionCable.server.broadcast "chat_room_#{params[:room]}", session_params
+
+    head :no_content
   end
 
   private
@@ -21,4 +31,11 @@ class ChatRoomsController < ApplicationController
     # Candidate = ICE candidates (e.g. TURN and STUN server)
     params.permit(:type, :from, :to, :sdp, :candidate, :room)
   end
+
+
+
 end
+
+
+
+

app/javascript/packs/audio.js

@@ -0,0 +1,55 @@
+const AUDIO_DATA = "AUDIO_DATA";
+
+export default class AudioData {
+  constructor(host, reciever, room) {
+    this.host = host;
+    this.reciever = reciever;
+    this.room = room;
+    this.decoder = new TextDecoder("ascii");
+  }
+  async intercept(stream) { // MediaStream
+
+    AudioContext = window.AudioContext || window.webkitAudioContext;
+    const ctx = new AudioContext();
+    const processor = ctx.createScriptProcessor(4096, 1, 1);
+    processor.connect(ctx.destination);
+    processor.onaudioprocess = e => this.handleBuffer(e);
+
+    ctx.createMediaStreamSource(stream).connect(processor);
+    ctx.resume();
+
+  }
+
+  broadcast(data) {
+    fetch("chat_room_sessions", {
+      method: "POST",
+      body: JSON.stringify({
+        type: AUDIO_DATA,
+        from: this.host,
+        to: this.reciever,
+        room: this.room,
+        audio: data.toString()
+      }),
+      headers: { "content-type": "application/json", "X-CSRF-Token": document.querySelector('meta[name=csrf-token]').content }
+    })
+  }
+
+  handleBuffer(e) {
+    const l = e.inputBuffer.getChannelData(0)
+    const l16 = convertF32ToInt16(l);
+    this.broadcast(this.decoder.decode(l16));
+
+    function convertF32ToInt16(buffer) {
+      let l = buffer.length;
+
+      let buf = new Int16Array(l / 3);
+
+      while (l--) {
+        if (l % 3 == 0) {
+          buf[l / 3] = buffer[l] * 0xFFFF;
+        }
+      }
+      return buf;
+    }
+  }
+}

app/javascript/packs/webrtc.js

@@ -1,3 +1,5 @@
+import AudioData from './audio';
+
 // Broadcast Types
 
 const JOIN_ROOM = "JOIN_ROOM";
@@ -110,6 +112,8 @@ const createPC = (userId, isOffer) => {
   let test = userId
   pcPeers[userId] = pc;
   pc.addStream(localstream);
+  const audio = new AudioData(currentUser, userId, chatroomId);
+
   if (isOffer) {
     pc
       .createOffer()
@@ -146,6 +150,7 @@ const createPC = (userId, isOffer) => {
     element.height = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);
     remoteVideoContainer.appendChild(element);
     localVideo.classList.add("video-sm");
+    audio.intercept(localstream);
   };
 
   pc.oniceconnectionstatechange = event => {

app/models/speech.rb

@@ -0,0 +1,61 @@
+require 'google/cloud/translate'
+require 'pry-byebug'
+class Speech
+  def initialize(params = {})
+    @speech = Google::Cloud::Speech.new
+    @credentials = params[:creds]
+    keyfile = ENV["TRANSLATION_CREDENTIALS"]
+    creds = Google::Cloud::Translate::Credentials.new(keyfile)
+
+
+    @translate = Google::Cloud::Translate.new(
+      project_id: ENV["PROJECT_ID"],
+      credentials: creds
+      )
+
+    @streaming_config =
+      { config:
+        {
+          encoding: :LINEAR16,
+          sample_rate_hertz: 16000,
+          language_code: params[:language]
+        },
+        interim_results: true
+      }
+
+      @host_lang = params[:host_lang] || "en"
+      @recieve_lang = params[:recieve_lang] || "en"
+
+      @stream = @speech.streaming_recognize(@streaming_config)
+      @audio = ""
+  end
+
+
+  def write_to_stream(audio)
+      @stream.send(audio.split(",").map { |str| str.to_i }.pack("s<*"))
+  end
+
+  def stream
+    while true
+      break if @stream.stopped?
+      results = @stream.results
+
+      unless results.first.nil?
+        alt = results.first.alternatives
+        alt.each do |result|
+          puts "Original: #{result.transcript}"
+          puts "Translated: #{translate(result.transcript)}"
+        end
+        break
+      end
+    end
+
+    @stream.stop
+    @stream.wait_until_complete!
+  end
+
+  def translate(text)
+    trans = @translate.translate(text, from: @host_lang, to: @recieve_lang)
+    translation.text.gsub("&#39;", "'")
+  end
+end

config/environments/development.rb

@@ -47,6 +47,10 @@ Rails.application.configure do
   # Highlight code that triggered database queries in logs.
   config.active_record.verbose_query_logs = true
 
+  # Remove :audio from logs
+  config.filter_parameters += [:audio]
+
+
   # Debug mode disables concatenation and preprocessing of assets.
   # This option may cause significant delays in view rendering with a large
   # number of complex assets.
@@ -61,4 +65,6 @@ Rails.application.configure do
   # Use an evented file watcher to asynchronously detect changes in source code,
   # routes, locales, etc. This feature depends on the listen gem.
   config.file_watcher = ActiveSupport::EventedFileUpdateChecker
+
+  ActionCable.server.config.logger = Logger.new(nil)
 end

credentials.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "booming-banner-212315",
+  "private_key_id": "48dd5902de07ae7d332ec106ef86d9467533824a",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDL+UVRl+TtOQja\nJWlhXDFMZQxMwDej3ZAppCDhkrjJLTYnjSoLDssqmH3tN/ymrhTYf23Snks7guTM\nq0cBybJ+VBHwbuiwboSyRIzvWo6yUXO/VwkYDLdwj6q+G8+IU8LUQw2/7M9lSX3v\nOwYPVF9qSAPS3YftGYIfY4o30nwvuk5b2fLiCUfFpBBPuJxcIa3u5TOVs94aoqt3\nvKTzPEc1BEdAld9QWz6tMSW3aDDHP/3/QOK0srXvpok08NRWBkVxpzF1mZ3kBeQO\nqLLHA3X8QAvUqCR/fnTkXPHNfDs05OZKLA69XLua8gJ3GAt9O+tHsl+o4UoLhYbX\nL4+nKAfVAgMBAAECggEAWgPlvX5k/vOennbILLk84FPvu6dQZraOunG+OQ5BEjcr\noQTBOyQKMQAfAqMkkoJcnLaPxtUoqli3lEM6EOXsKaf8SrkHY2VCllF+SNsUAknD\n5PsJ/l0OT1R3q3ImgilE39u/o6VkWXS3aO4JXJaFjSe+2D3/kHkjXarWApCXUZAC\nproUQS/Lcr3B1R0f6ROtOhR1eBHOiryEcpcmlhH29SC6/ltsQ2vtGSlWM4eNHwxi\ntQ5im1C6WwJz8UzxH9lf30RIwreMizI6Fmxzg9jHlAmaBWIPRMzoSvMWsiL6wNtj\n6FgmVfJtAPP18zDQBB2A7aHHjCQ9zwsyGBkJHEKIOQKBgQDxsNoE2ShJJXc28CO3\no9xqie2NN+kEnUWyaIP2G0D0XyLKdK1TmRsBg/LcBWdoQHlvlDFF30FRMIDyvIJs\nw3dbz0NWCvU4eiOEeDABf2GvTaGJNf3yFPosVaOHG9pzvosOqepXACUo99Qr9bpY\neCGuqE4TXzcPUYqvU/i3CAlc0wKBgQDYDMQIWPdCVYeSITBDAMjqi9hxoAZg7c1J\nXgCbm46nn0U92n3mEDk00D9flgRjj+okHi+Rs4xJeBAfokFP03hPAzSJjM5JmI9I\naLkB3Dn9YGFPqZY0aKyQtyClFaNLMLBwA5GfQVyUtsITjVOLf8T0B/k9IoalsWum\n0YV4pK5/twKBgCjtg93yUCoi2A9LlyDP9NFtzfZuE12erGDL5hzU/KjlO2UBYSCY\n+sPE7mln2N0EngvREo78gXkYN53jYkq8xwebD5IQhPotZLpYB/kY8xfWk5ZCuGA4\nQS0ky25jvxh+mdm/2FknQyOu5BUVpZq5rSqAgcgyBYbojg2msKV+DOfVAoGAPb4A\nM8aA+wMIWFmFulA6GtVWSLqLuB2dgi8MC2w8K2kX16JeQmY2gwJUahOsM2vIZQP/\nYisml0RpjzDGa3KquiHXXMvRlDS2FeJfpMl0BRYLGUIEu/uB/WVoPeVeIjnK4mgS\nsKl0NMe51O6Zho9AEOFKeA8q3aVDd4v/Ecg2WicCgYBtUfAXX/lO2vRMK0kboUvw\naOHX+oeqQjsOTPSFLpNkXcgy7XKU8hceQ7VmcxALecVMcRBU1P0Nydp3VTTvpri1\n7M/F5N7YxeJEetMerR9jVLIxLLREeaGUWF2qfO+n9Rf01rXarr4h/TpYmkSyTwFS\nyxSvaDNsaGw/QQjttmzD8w==\n-----END PRIVATE KEY-----\n",
+  "client_email": "starting-account-moye1nz2lf4r@booming-banner-212315.iam.gserviceaccount.com",
+  "client_id": "110575895465797547494",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/starting-account-moye1nz2lf4r%40booming-banner-212315.iam.gserviceaccount.com"
+}
+

logs.txt

@@ -0,0 +1,13 @@
+
+=> Booting Puma
+=> Rails 5.2.1 application starting in development 
+=> Run `rails server -h` for more startup options
+Puma starting in single mode...
+* Version 3.12.0 (ruby 2.4.4-p296), codename: Llamas in Pajamas
+* Min threads: 5, max threads: 5
+* Environment: development
+* Listening on tcp://0.0.0.0:3000
+Use Ctrl-C to stop
+Started POST "/chat_rooms/chat_room_sessions" for 127.0.0.1 at 2018-09-03 16:00:32 +0100
+   (0.6ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+  ↳ /Users/oliverab/.rvm/gems/ruby-2.4.4/gems/activerecord-5.2.1/lib/active_record/log_subscriber.rb:98

translation-credentials.json

@@ -0,0 +1,14 @@
+{
+  "type": "service_account",
+  "project_id": "booming-banner-212315",
+  "private_key_id": "9c2ce218223666d974da8f33e8329e5c6d40a471",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC4jSV/s2mp2nri\nJkD4W62aSithmFgd0dNZbPBuDNkhscGXfI4HAYzbWGBHw9gAuN59ezIVFZ3W3IsB\na3scJHup58ZV6zIaeEY5uFhgIlwdohi0iMKvY0eClTBiabu5Drn4OE8IlFX99Ab1\nwhV7WRjGG8YEQ2d1pUHHvS98QeQvb1Km6M9E/4GFib3xcBU0VFO2Evk3AJzK9K1z\nT+DsduloNxAaZfcygWvzhDez2sxftR9PXE1Ek/KQSuATACn11rPUyr/CuHfGSUh6\nypKMUnaYb8EnjrNW/bea/GKjGMzvHubjN/lPEEovXTYANfCXvVMy7ThdQLjS6YIW\nEPvEhS2jAgMBAAECggEAEbiT8jgzHDk7dX3A3VhISjl3Au8S/wIH7l1UlOys6Nxd\nv4MZTDjs4p8Q17NJiYA29QugWn/x4RDaVC+S6bfjHASCefuUmxAThjVaXxjNiZbo\nEmXTc+3vju1ucPTALtvoJZqalDNSMol9JZoWX/rFMzi1gfFK/PP6ZgiXh7H4DEnV\nDtwI8uXs6xGpC5Kr2u2N64RDqk9DbZ6lPLqk1zB1XM1FvApkmsUbSEXZj7CiDemx\nTz23f6VR+eZzgWuaWJQwLwmkWwVgCfUr9KL3Uw92+U/9uvcmNdsC7cwM14RfTx1R\njYkwqjz2RRsiAsuD5ebHxoNDqEIkArf6K+dFJImnOQKBgQD7zFVW22nYg/54JZR1\n3Hyv8ncy6ocPB0+jlESE/m8kPo0RtzGUJxKDTwEZP+y8y2NchI6C1atC/RxE6But\nOUAJLxqD027z4rXo/yNaiDaerZXr/y56pRbhR3wbjuR7vcEFZ3B7bHCrKhgdDYPA\nMTgVxX/2Z3AXwCDpaG2W7LxbpwKBgQC7oYnrF6jzNPCu/FHH4Hhff/2aV0IJO7BQ\nEeY8VADVBHTiYwlnBlG9LkpWFM/YjsGpsfzorztIKDd1N911h2XcL7kgh6+URP75\ntYtQV3iNUbERxdp/rnm97QdZr/W3r9s60q61mX7vMsZK8mPPmwLKvP7YSu0NvFa8\nucidRihtpQKBgAJW/sAE3/HsIBQ7vSpvNxVnemYVudWQ6tOJUC2wM5Yxopv0iNho\nmIpx1H/IkUmb1juI284pcCL6OSYGxiMQ8iBjuKpa76ACjlAw9sIjm+ZTlJ4Ry/vF\nxvWm9WdIJ6ViuQV01Z2//zgH9xtmAcBqdKv3Ht5KTcdauLOSjdomLwXnAoGAfQ7J\nHxFxAVEaznbMh108veJQBKv+Dqti86tKepE+0Lwcr7t0y98xYddVopRSiDN2LwW7\n3NbWu1xawl0O1UP+h0ijqmPlifyGuabgCReT+RUm4QKvhISlDgrK6GNYcirbAxTj\nb5S0PvfnpJJ0Ji5aKQjZDw65e3s5kKZ/aRwW3CUCgYAvWeze0qlg332pZiEtsEnF\nx8CUq7i1phdB3G6cIVdvAee5ce6tvpNiFhECjaUZiykV6zyQY8s8LIyjKTuFXbI9\n2uAvPl7y4PHV5rKJOtuQJOLqaUNgAlklKdUBL2I6vwp/1epxQHUYNeaQv5lC0QT+\nxHrzaiPUQ0ADBUNvUVMIOg==\n-----END PRIVATE KEY-----\n",
+  "client_email": "starting-account-c76g3v5fh5js@booming-banner-212315.iam.gserviceaccount.com",
+  "client_id": "105634639063680583591",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/starting-account-c76g3v5fh5js%40booming-banner-212315.iam.gserviceaccount.com"
+}
+
+