diff --git a/Gemfile b/Gemfile index ffa11e3..0d9b0b1 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,11 @@ source 'https://rubygems.org' ruby '2.4.4' + +#Google Stuffs +gem 'google-cloud-speech' +gem 'google-cloud-translate' + gem 'bootsnap', require: false gem 'devise' gem 'jbuilder', '~> 2.0' diff --git a/Gemfile.lock b/Gemfile.lock index ec82a00..b4a2e35 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -42,6 +42,8 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) arel (9.0.0) autoprefixer-rails (9.1.3) execjs @@ -79,11 +81,46 @@ GEM railties (>= 3.2, < 6.0) erubi (1.7.1) execjs (2.7.0) + faraday (0.15.2) + multipart-post (>= 1.2, < 3) ffi (1.9.25) font-awesome-sass (5.0.13) sassc (>= 1.11) globalid (0.4.1) activesupport (>= 4.2.0) + google-cloud-core (1.2.3) + google-cloud-env (~> 1.0) + google-cloud-env (1.0.2) + faraday (~> 0.11) + google-cloud-speech (0.30.1) + google-gax (~> 1.3) + google-cloud-translate (1.2.1) + faraday (~> 0.13) + google-cloud-core (~> 1.2) + googleauth (~> 0.6.2) + google-gax (1.3.0) + google-protobuf (~> 3.2) + googleapis-common-protos (>= 1.3.5, < 2.0) + googleauth (~> 0.6.2) + grpc (>= 1.7.2, < 2.0) + rly (~> 0.2.3) + google-protobuf (3.6.1) + googleapis-common-protos (1.3.7) + google-protobuf (~> 3.0) + googleapis-common-protos-types (~> 1.0) + grpc (~> 1.0) + googleapis-common-protos-types (1.0.2) + google-protobuf (~> 3.0) + googleauth (0.6.6) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.12) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.7) + grpc (1.14.1) + google-protobuf (~> 3.1) + googleapis-common-protos-types (~> 1.0.0) http-cookie (1.0.3) domain_name (~> 0.5) i18n (1.1.0) @@ -91,6 +128,7 @@ GEM jbuilder (2.7.0) activesupport (>= 4.2.0) multi_json (>= 1.2) + jwt (2.1.0) listen (3.0.8) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) @@ -101,6 +139,7 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.2) mimemagic (~> 0.3.2) + memoist (0.16.0) method_source (0.9.0) mime-types (3.2.2) mime-types-data (~> 3.2015) @@ -111,11 +150,13 @@ GEM minitest (5.11.3) msgpack (1.2.4) multi_json (1.13.1) + multipart-post (2.0.0) netrc (0.11.0) nio4r (2.3.1) nokogiri (1.8.4) mini_portile2 (~> 2.3.0) orm_adapter (0.5.0) + os (1.0.0) pg (0.21.0) pry (0.11.3) coderay (~> 1.1.0) @@ -125,6 +166,7 @@ GEM pry (~> 0.10) pry-rails (0.3.6) pry (>= 0.10.4) + public_suffix (3.0.3) puma (3.12.0) rack (2.0.5) rack-proxy (0.6.4) @@ -167,6 +209,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + rly (0.2.3) sass (3.5.7) sass-listen (~> 4.0.0) sass-listen (4.0.0) @@ -181,6 +224,11 @@ GEM sassc (1.12.1) ffi (~> 1.9.6) sass (>= 3.3.0) + signet (0.9.1) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) simple_form (4.0.1) actionpack (>= 5.0) activemodel (>= 5.0) @@ -233,6 +281,8 @@ DEPENDENCIES devise dotenv-rails font-awesome-sass (~> 5.0.9) + google-cloud-speech + google-cloud-translate jbuilder (~> 2.0) listen (~> 3.0.5) pg (~> 0.21) diff --git a/app/controllers/chat_rooms_controller.rb b/app/controllers/chat_rooms_controller.rb index 37b681c..533d872 100644 --- a/app/controllers/chat_rooms_controller.rb +++ b/app/controllers/chat_rooms_controller.rb @@ -1,3 +1,13 @@ +SPEECH = Speech.new( + creds: JSON.parse(File.read(ENV["STREAMING_CREDENTIALS"])), + host_lang: "en", + recieve_lang: "fr" + ) + +while true + SPEECH.stream +end + class ChatRoomsController < ApplicationController def show @@ -6,12 +16,12 @@ class ChatRoomsController < ApplicationController def create # HTTP status code 200 with an empty body - head :no_content - puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213" - puts params - puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213" + + SPEECH.write_to_stream(params[:audio]) unless params[:audio].nil? ActionCable.server.broadcast "chat_room_#{params[:room]}", session_params + + head :no_content end private @@ -21,4 +31,11 @@ class ChatRoomsController < ApplicationController # Candidate = ICE candidates (e.g. TURN and STUN server) params.permit(:type, :from, :to, :sdp, :candidate, :room) end + + + end + + + + diff --git a/app/javascript/packs/audio.js b/app/javascript/packs/audio.js new file mode 100644 index 0000000..28a5932 --- /dev/null +++ b/app/javascript/packs/audio.js @@ -0,0 +1,55 @@ +const AUDIO_DATA = "AUDIO_DATA"; + +export default class AudioData { + constructor(host, reciever, room) { + this.host = host; + this.reciever = reciever; + this.room = room; + this.decoder = new TextDecoder("ascii"); + } + async intercept(stream) { // MediaStream + + AudioContext = window.AudioContext || window.webkitAudioContext; + const ctx = new AudioContext(); + const processor = ctx.createScriptProcessor(4096, 1, 1); + processor.connect(ctx.destination); + processor.onaudioprocess = e => this.handleBuffer(e); + + ctx.createMediaStreamSource(stream).connect(processor); + ctx.resume(); + + } + + broadcast(data) { + fetch("chat_room_sessions", { + method: "POST", + body: JSON.stringify({ + type: AUDIO_DATA, + from: this.host, + to: this.reciever, + room: this.room, + audio: data.toString() + }), + headers: { "content-type": "application/json", "X-CSRF-Token": document.querySelector('meta[name=csrf-token]').content } + }) + } + + handleBuffer(e) { + const l = e.inputBuffer.getChannelData(0) + const l16 = convertF32ToInt16(l); + this.broadcast(this.decoder.decode(l16)); + + function convertF32ToInt16(buffer) { + let l = buffer.length; + + let buf = new Int16Array(l / 3); + + while (l--) { + if (l % 3 == 0) { + buf[l / 3] = buffer[l] * 0xFFFF; + } + } + return buf; + } + } +} diff --git a/app/javascript/packs/webrtc.js b/app/javascript/packs/webrtc.js index 5bf7376..1363e99 100644 --- a/app/javascript/packs/webrtc.js +++ b/app/javascript/packs/webrtc.js @@ -1,3 +1,5 @@ +import AudioData from './audio'; + // Broadcast Types const JOIN_ROOM = "JOIN_ROOM"; @@ -110,6 +112,8 @@ const createPC = (userId, isOffer) => { let test = userId pcPeers[userId] = pc; pc.addStream(localstream); + const audio = new AudioData(currentUser, userId, chatroomId); + if (isOffer) { pc .createOffer() @@ -146,6 +150,7 @@ const createPC = (userId, isOffer) => { element.height = Math.max(document.documentElement.clientHeight, window.innerHeight || 0); remoteVideoContainer.appendChild(element); localVideo.classList.add("video-sm"); + audio.intercept(localstream); }; pc.oniceconnectionstatechange = event => { diff --git a/app/models/speech.rb b/app/models/speech.rb new file mode 100644 index 0000000..c320a7f --- /dev/null +++ b/app/models/speech.rb @@ -0,0 +1,61 @@ +require 'google/cloud/translate' +require 'pry-byebug' +class Speech + def initialize(params = {}) + @speech = Google::Cloud::Speech.new + @credentials = params[:creds] + keyfile = ENV["TRANSLATION_CREDENTIALS"] + creds = Google::Cloud::Translate::Credentials.new(keyfile) + + + @translate = Google::Cloud::Translate.new( + project_id: ENV["PROJECT_ID"], + credentials: creds + ) + + @streaming_config = + { config: + { + encoding: :LINEAR16, + sample_rate_hertz: 16000, + language_code: params[:language] + }, + interim_results: true + } + + @host_lang = params[:host_lang] || "en" + @recieve_lang = params[:recieve_lang] || "en" + + @stream = @speech.streaming_recognize(@streaming_config) + @audio = "" + end + + + def write_to_stream(audio) + @stream.send(audio.split(",").map { |str| str.to_i }.pack("s<*")) + end + + def stream + while true + break if @stream.stopped? + results = @stream.results + + unless results.first.nil? + alt = results.first.alternatives + alt.each do |result| + puts "Original: #{result.transcript}" + puts "Translated: #{translate(result.transcript)}" + end + break + end + end + + @stream.stop + @stream.wait_until_complete! + end + + def translate(text) + trans = @translate.translate(text, from: @host_lang, to: @recieve_lang) + translation.text.gsub("'", "'") + end +end diff --git a/config/environments/development.rb b/config/environments/development.rb index 9e87eb8..6dc9b97 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -47,6 +47,10 @@ Rails.application.configure do # Highlight code that triggered database queries in logs. config.active_record.verbose_query_logs = true + # Remove :audio from logs + config.filter_parameters += [:audio] + + # Debug mode disables concatenation and preprocessing of assets. # This option may cause significant delays in view rendering with a large # number of complex assets. @@ -61,4 +65,6 @@ Rails.application.configure do # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker + + ActionCable.server.config.logger = Logger.new(nil) end diff --git a/credentials.json b/credentials.json new file mode 100644 index 0000000..9c34dc3 --- /dev/null +++ b/credentials.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "booming-banner-212315", + "private_key_id": "48dd5902de07ae7d332ec106ef86d9467533824a", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDL+UVRl+TtOQja\nJWlhXDFMZQxMwDej3ZAppCDhkrjJLTYnjSoLDssqmH3tN/ymrhTYf23Snks7guTM\nq0cBybJ+VBHwbuiwboSyRIzvWo6yUXO/VwkYDLdwj6q+G8+IU8LUQw2/7M9lSX3v\nOwYPVF9qSAPS3YftGYIfY4o30nwvuk5b2fLiCUfFpBBPuJxcIa3u5TOVs94aoqt3\nvKTzPEc1BEdAld9QWz6tMSW3aDDHP/3/QOK0srXvpok08NRWBkVxpzF1mZ3kBeQO\nqLLHA3X8QAvUqCR/fnTkXPHNfDs05OZKLA69XLua8gJ3GAt9O+tHsl+o4UoLhYbX\nL4+nKAfVAgMBAAECggEAWgPlvX5k/vOennbILLk84FPvu6dQZraOunG+OQ5BEjcr\noQTBOyQKMQAfAqMkkoJcnLaPxtUoqli3lEM6EOXsKaf8SrkHY2VCllF+SNsUAknD\n5PsJ/l0OT1R3q3ImgilE39u/o6VkWXS3aO4JXJaFjSe+2D3/kHkjXarWApCXUZAC\nproUQS/Lcr3B1R0f6ROtOhR1eBHOiryEcpcmlhH29SC6/ltsQ2vtGSlWM4eNHwxi\ntQ5im1C6WwJz8UzxH9lf30RIwreMizI6Fmxzg9jHlAmaBWIPRMzoSvMWsiL6wNtj\n6FgmVfJtAPP18zDQBB2A7aHHjCQ9zwsyGBkJHEKIOQKBgQDxsNoE2ShJJXc28CO3\no9xqie2NN+kEnUWyaIP2G0D0XyLKdK1TmRsBg/LcBWdoQHlvlDFF30FRMIDyvIJs\nw3dbz0NWCvU4eiOEeDABf2GvTaGJNf3yFPosVaOHG9pzvosOqepXACUo99Qr9bpY\neCGuqE4TXzcPUYqvU/i3CAlc0wKBgQDYDMQIWPdCVYeSITBDAMjqi9hxoAZg7c1J\nXgCbm46nn0U92n3mEDk00D9flgRjj+okHi+Rs4xJeBAfokFP03hPAzSJjM5JmI9I\naLkB3Dn9YGFPqZY0aKyQtyClFaNLMLBwA5GfQVyUtsITjVOLf8T0B/k9IoalsWum\n0YV4pK5/twKBgCjtg93yUCoi2A9LlyDP9NFtzfZuE12erGDL5hzU/KjlO2UBYSCY\n+sPE7mln2N0EngvREo78gXkYN53jYkq8xwebD5IQhPotZLpYB/kY8xfWk5ZCuGA4\nQS0ky25jvxh+mdm/2FknQyOu5BUVpZq5rSqAgcgyBYbojg2msKV+DOfVAoGAPb4A\nM8aA+wMIWFmFulA6GtVWSLqLuB2dgi8MC2w8K2kX16JeQmY2gwJUahOsM2vIZQP/\nYisml0RpjzDGa3KquiHXXMvRlDS2FeJfpMl0BRYLGUIEu/uB/WVoPeVeIjnK4mgS\nsKl0NMe51O6Zho9AEOFKeA8q3aVDd4v/Ecg2WicCgYBtUfAXX/lO2vRMK0kboUvw\naOHX+oeqQjsOTPSFLpNkXcgy7XKU8hceQ7VmcxALecVMcRBU1P0Nydp3VTTvpri1\n7M/F5N7YxeJEetMerR9jVLIxLLREeaGUWF2qfO+n9Rf01rXarr4h/TpYmkSyTwFS\nyxSvaDNsaGw/QQjttmzD8w==\n-----END PRIVATE KEY-----\n", + "client_email": "starting-account-moye1nz2lf4r@booming-banner-212315.iam.gserviceaccount.com", + "client_id": "110575895465797547494", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/starting-account-moye1nz2lf4r%40booming-banner-212315.iam.gserviceaccount.com" +} + diff --git a/logs.txt b/logs.txt new file mode 100644 index 0000000..f8b5444 --- /dev/null +++ b/logs.txt @@ -0,0 +1,13 @@ + +=> Booting Puma +=> Rails 5.2.1 application starting in development +=> Run `rails server -h` for more startup options +Puma starting in single mode... +* Version 3.12.0 (ruby 2.4.4-p296), codename: Llamas in Pajamas +* Min threads: 5, max threads: 5 +* Environment: development +* Listening on tcp://0.0.0.0:3000 +Use Ctrl-C to stop +Started POST "/chat_rooms/chat_room_sessions" for 127.0.0.1 at 2018-09-03 16:00:32 +0100 +  (0.6ms) SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC + ↳ /Users/oliverab/.rvm/gems/ruby-2.4.4/gems/activerecord-5.2.1/lib/active_record/log_subscriber.rb:98 diff --git a/out.file b/out.file new file mode 100644 index 0000000..e69de29 diff --git a/translation-credentials.json b/translation-credentials.json new file mode 100644 index 0000000..b065dda --- /dev/null +++ b/translation-credentials.json @@ -0,0 +1,14 @@ +{ + "type": "service_account", + "project_id": "booming-banner-212315", + "private_key_id": "9c2ce218223666d974da8f33e8329e5c6d40a471", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC4jSV/s2mp2nri\nJkD4W62aSithmFgd0dNZbPBuDNkhscGXfI4HAYzbWGBHw9gAuN59ezIVFZ3W3IsB\na3scJHup58ZV6zIaeEY5uFhgIlwdohi0iMKvY0eClTBiabu5Drn4OE8IlFX99Ab1\nwhV7WRjGG8YEQ2d1pUHHvS98QeQvb1Km6M9E/4GFib3xcBU0VFO2Evk3AJzK9K1z\nT+DsduloNxAaZfcygWvzhDez2sxftR9PXE1Ek/KQSuATACn11rPUyr/CuHfGSUh6\nypKMUnaYb8EnjrNW/bea/GKjGMzvHubjN/lPEEovXTYANfCXvVMy7ThdQLjS6YIW\nEPvEhS2jAgMBAAECggEAEbiT8jgzHDk7dX3A3VhISjl3Au8S/wIH7l1UlOys6Nxd\nv4MZTDjs4p8Q17NJiYA29QugWn/x4RDaVC+S6bfjHASCefuUmxAThjVaXxjNiZbo\nEmXTc+3vju1ucPTALtvoJZqalDNSMol9JZoWX/rFMzi1gfFK/PP6ZgiXh7H4DEnV\nDtwI8uXs6xGpC5Kr2u2N64RDqk9DbZ6lPLqk1zB1XM1FvApkmsUbSEXZj7CiDemx\nTz23f6VR+eZzgWuaWJQwLwmkWwVgCfUr9KL3Uw92+U/9uvcmNdsC7cwM14RfTx1R\njYkwqjz2RRsiAsuD5ebHxoNDqEIkArf6K+dFJImnOQKBgQD7zFVW22nYg/54JZR1\n3Hyv8ncy6ocPB0+jlESE/m8kPo0RtzGUJxKDTwEZP+y8y2NchI6C1atC/RxE6But\nOUAJLxqD027z4rXo/yNaiDaerZXr/y56pRbhR3wbjuR7vcEFZ3B7bHCrKhgdDYPA\nMTgVxX/2Z3AXwCDpaG2W7LxbpwKBgQC7oYnrF6jzNPCu/FHH4Hhff/2aV0IJO7BQ\nEeY8VADVBHTiYwlnBlG9LkpWFM/YjsGpsfzorztIKDd1N911h2XcL7kgh6+URP75\ntYtQV3iNUbERxdp/rnm97QdZr/W3r9s60q61mX7vMsZK8mPPmwLKvP7YSu0NvFa8\nucidRihtpQKBgAJW/sAE3/HsIBQ7vSpvNxVnemYVudWQ6tOJUC2wM5Yxopv0iNho\nmIpx1H/IkUmb1juI284pcCL6OSYGxiMQ8iBjuKpa76ACjlAw9sIjm+ZTlJ4Ry/vF\nxvWm9WdIJ6ViuQV01Z2//zgH9xtmAcBqdKv3Ht5KTcdauLOSjdomLwXnAoGAfQ7J\nHxFxAVEaznbMh108veJQBKv+Dqti86tKepE+0Lwcr7t0y98xYddVopRSiDN2LwW7\n3NbWu1xawl0O1UP+h0ijqmPlifyGuabgCReT+RUm4QKvhISlDgrK6GNYcirbAxTj\nb5S0PvfnpJJ0Ji5aKQjZDw65e3s5kKZ/aRwW3CUCgYAvWeze0qlg332pZiEtsEnF\nx8CUq7i1phdB3G6cIVdvAee5ce6tvpNiFhECjaUZiykV6zyQY8s8LIyjKTuFXbI9\n2uAvPl7y4PHV5rKJOtuQJOLqaUNgAlklKdUBL2I6vwp/1epxQHUYNeaQv5lC0QT+\nxHrzaiPUQ0ADBUNvUVMIOg==\n-----END PRIVATE KEY-----\n", + "client_email": "starting-account-c76g3v5fh5js@booming-banner-212315.iam.gserviceaccount.com", + "client_id": "105634639063680583591", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/starting-account-c76g3v5fh5js%40booming-banner-212315.iam.gserviceaccount.com" +} + +