ruby implementation

This commit is contained in:
Beatrice Olivera 2018-09-03 16:24:43 +01:00
parent 598d59319e
commit 523125bea0
11 changed files with 243 additions and 4 deletions

View File

@ -1,6 +1,11 @@
source 'https://rubygems.org'
ruby '2.4.4'
#Google Stuffs
gem 'google-cloud-speech'
gem 'google-cloud-translate'
gem 'bootsnap', require: false
gem 'devise'
gem 'jbuilder', '~> 2.0'

View File

@ -42,6 +42,8 @@ GEM
i18n (>= 0.7, < 2)
minitest (~> 5.1)
tzinfo (~> 1.1)
addressable (2.5.2)
public_suffix (>= 2.0.2, < 4.0)
arel (9.0.0)
autoprefixer-rails (9.1.3)
execjs
@ -79,11 +81,46 @@ GEM
railties (>= 3.2, < 6.0)
erubi (1.7.1)
execjs (2.7.0)
faraday (0.15.2)
multipart-post (>= 1.2, < 3)
ffi (1.9.25)
font-awesome-sass (5.0.13)
sassc (>= 1.11)
globalid (0.4.1)
activesupport (>= 4.2.0)
google-cloud-core (1.2.3)
google-cloud-env (~> 1.0)
google-cloud-env (1.0.2)
faraday (~> 0.11)
google-cloud-speech (0.30.1)
google-gax (~> 1.3)
google-cloud-translate (1.2.1)
faraday (~> 0.13)
google-cloud-core (~> 1.2)
googleauth (~> 0.6.2)
google-gax (1.3.0)
google-protobuf (~> 3.2)
googleapis-common-protos (>= 1.3.5, < 2.0)
googleauth (~> 0.6.2)
grpc (>= 1.7.2, < 2.0)
rly (~> 0.2.3)
google-protobuf (3.6.1)
googleapis-common-protos (1.3.7)
google-protobuf (~> 3.0)
googleapis-common-protos-types (~> 1.0)
grpc (~> 1.0)
googleapis-common-protos-types (1.0.2)
google-protobuf (~> 3.0)
googleauth (0.6.6)
faraday (~> 0.12)
jwt (>= 1.4, < 3.0)
memoist (~> 0.12)
multi_json (~> 1.11)
os (>= 0.9, < 2.0)
signet (~> 0.7)
grpc (1.14.1)
google-protobuf (~> 3.1)
googleapis-common-protos-types (~> 1.0.0)
http-cookie (1.0.3)
domain_name (~> 0.5)
i18n (1.1.0)
@ -91,6 +128,7 @@ GEM
jbuilder (2.7.0)
activesupport (>= 4.2.0)
multi_json (>= 1.2)
jwt (2.1.0)
listen (3.0.8)
rb-fsevent (~> 0.9, >= 0.9.4)
rb-inotify (~> 0.9, >= 0.9.7)
@ -101,6 +139,7 @@ GEM
mini_mime (>= 0.1.1)
marcel (0.3.2)
mimemagic (~> 0.3.2)
memoist (0.16.0)
method_source (0.9.0)
mime-types (3.2.2)
mime-types-data (~> 3.2015)
@ -111,11 +150,13 @@ GEM
minitest (5.11.3)
msgpack (1.2.4)
multi_json (1.13.1)
multipart-post (2.0.0)
netrc (0.11.0)
nio4r (2.3.1)
nokogiri (1.8.4)
mini_portile2 (~> 2.3.0)
orm_adapter (0.5.0)
os (1.0.0)
pg (0.21.0)
pry (0.11.3)
coderay (~> 1.1.0)
@ -125,6 +166,7 @@ GEM
pry (~> 0.10)
pry-rails (0.3.6)
pry (>= 0.10.4)
public_suffix (3.0.3)
puma (3.12.0)
rack (2.0.5)
rack-proxy (0.6.4)
@ -167,6 +209,7 @@ GEM
http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 4.0)
netrc (~> 0.8)
rly (0.2.3)
sass (3.5.7)
sass-listen (~> 4.0.0)
sass-listen (4.0.0)
@ -181,6 +224,11 @@ GEM
sassc (1.12.1)
ffi (~> 1.9.6)
sass (>= 3.3.0)
signet (0.9.1)
addressable (~> 2.3)
faraday (~> 0.9)
jwt (>= 1.5, < 3.0)
multi_json (~> 1.10)
simple_form (4.0.1)
actionpack (>= 5.0)
activemodel (>= 5.0)
@ -233,6 +281,8 @@ DEPENDENCIES
devise
dotenv-rails
font-awesome-sass (~> 5.0.9)
google-cloud-speech
google-cloud-translate
jbuilder (~> 2.0)
listen (~> 3.0.5)
pg (~> 0.21)

View File

@ -1,3 +1,13 @@
SPEECH = Speech.new(
creds: JSON.parse(File.read(ENV["STREAMING_CREDENTIALS"])),
host_lang: "en",
recieve_lang: "fr"
)
while true
SPEECH.stream
end
class ChatRoomsController < ApplicationController
def show
@ -6,12 +16,12 @@ class ChatRoomsController < ApplicationController
def create
# HTTP status code 200 with an empty body
head :no_content
puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213"
puts params
puts ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>11123213213213123213"
SPEECH.write_to_stream(params[:audio]) unless params[:audio].nil?
ActionCable.server.broadcast "chat_room_#{params[:room]}", session_params
head :no_content
end
private
@ -21,4 +31,11 @@ class ChatRoomsController < ApplicationController
# Candidate = ICE candidates (e.g. TURN and STUN server)
params.permit(:type, :from, :to, :sdp, :candidate, :room)
end
end

View File

@ -0,0 +1,55 @@
const AUDIO_DATA = "AUDIO_DATA";
export default class AudioData {
constructor(host, reciever, room) {
this.host = host;
this.reciever = reciever;
this.room = room;
this.decoder = new TextDecoder("ascii");
}
async intercept(stream) { // MediaStream
AudioContext = window.AudioContext || window.webkitAudioContext;
const ctx = new AudioContext();
const processor = ctx.createScriptProcessor(4096, 1, 1);
processor.connect(ctx.destination);
processor.onaudioprocess = e => this.handleBuffer(e);
ctx.createMediaStreamSource(stream).connect(processor);
ctx.resume();
}
broadcast(data) {
fetch("chat_room_sessions", {
method: "POST",
body: JSON.stringify({
type: AUDIO_DATA,
from: this.host,
to: this.reciever,
room: this.room,
audio: data.toString()
}),
headers: { "content-type": "application/json", "X-CSRF-Token": document.querySelector('meta[name=csrf-token]').content }
})
}
handleBuffer(e) {
const l = e.inputBuffer.getChannelData(0)
const l16 = convertF32ToInt16(l);
this.broadcast(this.decoder.decode(l16));
function convertF32ToInt16(buffer) {
let l = buffer.length;
let buf = new Int16Array(l / 3);
while (l--) {
if (l % 3 == 0) {
buf[l / 3] = buffer[l] * 0xFFFF;
}
}
return buf;
}
}
}

View File

@ -1,3 +1,5 @@
import AudioData from './audio';
// Broadcast Types
const JOIN_ROOM = "JOIN_ROOM";
@ -110,6 +112,8 @@ const createPC = (userId, isOffer) => {
let test = userId
pcPeers[userId] = pc;
pc.addStream(localstream);
const audio = new AudioData(currentUser, userId, chatroomId);
if (isOffer) {
pc
.createOffer()
@ -146,6 +150,7 @@ const createPC = (userId, isOffer) => {
element.height = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);
remoteVideoContainer.appendChild(element);
localVideo.classList.add("video-sm");
audio.intercept(localstream);
};
pc.oniceconnectionstatechange = event => {

61
app/models/speech.rb Normal file
View File

@ -0,0 +1,61 @@
require 'google/cloud/translate'
require 'pry-byebug'
class Speech
def initialize(params = {})
@speech = Google::Cloud::Speech.new
@credentials = params[:creds]
keyfile = ENV["TRANSLATION_CREDENTIALS"]
creds = Google::Cloud::Translate::Credentials.new(keyfile)
@translate = Google::Cloud::Translate.new(
project_id: ENV["PROJECT_ID"],
credentials: creds
)
@streaming_config =
{ config:
{
encoding: :LINEAR16,
sample_rate_hertz: 16000,
language_code: params[:language]
},
interim_results: true
}
@host_lang = params[:host_lang] || "en"
@recieve_lang = params[:recieve_lang] || "en"
@stream = @speech.streaming_recognize(@streaming_config)
@audio = ""
end
def write_to_stream(audio)
@stream.send(audio.split(",").map { |str| str.to_i }.pack("s<*"))
end
def stream
while true
break if @stream.stopped?
results = @stream.results
unless results.first.nil?
alt = results.first.alternatives
alt.each do |result|
puts "Original: #{result.transcript}"
puts "Translated: #{translate(result.transcript)}"
end
break
end
end
@stream.stop
@stream.wait_until_complete!
end
def translate(text)
trans = @translate.translate(text, from: @host_lang, to: @recieve_lang)
translation.text.gsub("&#39;", "'")
end
end

View File

@ -47,6 +47,10 @@ Rails.application.configure do
# Highlight code that triggered database queries in logs.
config.active_record.verbose_query_logs = true
# Remove :audio from logs
config.filter_parameters += [:audio]
# Debug mode disables concatenation and preprocessing of assets.
# This option may cause significant delays in view rendering with a large
# number of complex assets.
@ -61,4 +65,6 @@ Rails.application.configure do
# Use an evented file watcher to asynchronously detect changes in source code,
# routes, locales, etc. This feature depends on the listen gem.
config.file_watcher = ActiveSupport::EventedFileUpdateChecker
ActionCable.server.config.logger = Logger.new(nil)
end

13
credentials.json Normal file
View File

@ -0,0 +1,13 @@
{
"type": "service_account",
"project_id": "booming-banner-212315",
"private_key_id": "48dd5902de07ae7d332ec106ef86d9467533824a",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDL+UVRl+TtOQja\nJWlhXDFMZQxMwDej3ZAppCDhkrjJLTYnjSoLDssqmH3tN/ymrhTYf23Snks7guTM\nq0cBybJ+VBHwbuiwboSyRIzvWo6yUXO/VwkYDLdwj6q+G8+IU8LUQw2/7M9lSX3v\nOwYPVF9qSAPS3YftGYIfY4o30nwvuk5b2fLiCUfFpBBPuJxcIa3u5TOVs94aoqt3\nvKTzPEc1BEdAld9QWz6tMSW3aDDHP/3/QOK0srXvpok08NRWBkVxpzF1mZ3kBeQO\nqLLHA3X8QAvUqCR/fnTkXPHNfDs05OZKLA69XLua8gJ3GAt9O+tHsl+o4UoLhYbX\nL4+nKAfVAgMBAAECggEAWgPlvX5k/vOennbILLk84FPvu6dQZraOunG+OQ5BEjcr\noQTBOyQKMQAfAqMkkoJcnLaPxtUoqli3lEM6EOXsKaf8SrkHY2VCllF+SNsUAknD\n5PsJ/l0OT1R3q3ImgilE39u/o6VkWXS3aO4JXJaFjSe+2D3/kHkjXarWApCXUZAC\nproUQS/Lcr3B1R0f6ROtOhR1eBHOiryEcpcmlhH29SC6/ltsQ2vtGSlWM4eNHwxi\ntQ5im1C6WwJz8UzxH9lf30RIwreMizI6Fmxzg9jHlAmaBWIPRMzoSvMWsiL6wNtj\n6FgmVfJtAPP18zDQBB2A7aHHjCQ9zwsyGBkJHEKIOQKBgQDxsNoE2ShJJXc28CO3\no9xqie2NN+kEnUWyaIP2G0D0XyLKdK1TmRsBg/LcBWdoQHlvlDFF30FRMIDyvIJs\nw3dbz0NWCvU4eiOEeDABf2GvTaGJNf3yFPosVaOHG9pzvosOqepXACUo99Qr9bpY\neCGuqE4TXzcPUYqvU/i3CAlc0wKBgQDYDMQIWPdCVYeSITBDAMjqi9hxoAZg7c1J\nXgCbm46nn0U92n3mEDk00D9flgRjj+okHi+Rs4xJeBAfokFP03hPAzSJjM5JmI9I\naLkB3Dn9YGFPqZY0aKyQtyClFaNLMLBwA5GfQVyUtsITjVOLf8T0B/k9IoalsWum\n0YV4pK5/twKBgCjtg93yUCoi2A9LlyDP9NFtzfZuE12erGDL5hzU/KjlO2UBYSCY\n+sPE7mln2N0EngvREo78gXkYN53jYkq8xwebD5IQhPotZLpYB/kY8xfWk5ZCuGA4\nQS0ky25jvxh+mdm/2FknQyOu5BUVpZq5rSqAgcgyBYbojg2msKV+DOfVAoGAPb4A\nM8aA+wMIWFmFulA6GtVWSLqLuB2dgi8MC2w8K2kX16JeQmY2gwJUahOsM2vIZQP/\nYisml0RpjzDGa3KquiHXXMvRlDS2FeJfpMl0BRYLGUIEu/uB/WVoPeVeIjnK4mgS\nsKl0NMe51O6Zho9AEOFKeA8q3aVDd4v/Ecg2WicCgYBtUfAXX/lO2vRMK0kboUvw\naOHX+oeqQjsOTPSFLpNkXcgy7XKU8hceQ7VmcxALecVMcRBU1P0Nydp3VTTvpri1\n7M/F5N7YxeJEetMerR9jVLIxLLREeaGUWF2qfO+n9Rf01rXarr4h/TpYmkSyTwFS\nyxSvaDNsaGw/QQjttmzD8w==\n-----END PRIVATE KEY-----\n",
"client_email": "starting-account-moye1nz2lf4r@booming-banner-212315.iam.gserviceaccount.com",
"client_id": "110575895465797547494",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/starting-account-moye1nz2lf4r%40booming-banner-212315.iam.gserviceaccount.com"
}

13
logs.txt Normal file
View File

@ -0,0 +1,13 @@
=> Booting Puma
=> Rails 5.2.1 application starting in development
=> Run `rails server -h` for more startup options
Puma starting in single mode...
* Version 3.12.0 (ruby 2.4.4-p296), codename: Llamas in Pajamas
* Min threads: 5, max threads: 5
* Environment: development
* Listening on tcp://0.0.0.0:3000
Use Ctrl-C to stop
Started POST "/chat_rooms/chat_room_sessions" for 127.0.0.1 at 2018-09-03 16:00:32 +0100
 (0.6ms) SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
↳ /Users/oliverab/.rvm/gems/ruby-2.4.4/gems/activerecord-5.2.1/lib/active_record/log_subscriber.rb:98

0
out.file Normal file
View File

View File

@ -0,0 +1,14 @@
{
"type": "service_account",
"project_id": "booming-banner-212315",
"private_key_id": "9c2ce218223666d974da8f33e8329e5c6d40a471",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC4jSV/s2mp2nri\nJkD4W62aSithmFgd0dNZbPBuDNkhscGXfI4HAYzbWGBHw9gAuN59ezIVFZ3W3IsB\na3scJHup58ZV6zIaeEY5uFhgIlwdohi0iMKvY0eClTBiabu5Drn4OE8IlFX99Ab1\nwhV7WRjGG8YEQ2d1pUHHvS98QeQvb1Km6M9E/4GFib3xcBU0VFO2Evk3AJzK9K1z\nT+DsduloNxAaZfcygWvzhDez2sxftR9PXE1Ek/KQSuATACn11rPUyr/CuHfGSUh6\nypKMUnaYb8EnjrNW/bea/GKjGMzvHubjN/lPEEovXTYANfCXvVMy7ThdQLjS6YIW\nEPvEhS2jAgMBAAECggEAEbiT8jgzHDk7dX3A3VhISjl3Au8S/wIH7l1UlOys6Nxd\nv4MZTDjs4p8Q17NJiYA29QugWn/x4RDaVC+S6bfjHASCefuUmxAThjVaXxjNiZbo\nEmXTc+3vju1ucPTALtvoJZqalDNSMol9JZoWX/rFMzi1gfFK/PP6ZgiXh7H4DEnV\nDtwI8uXs6xGpC5Kr2u2N64RDqk9DbZ6lPLqk1zB1XM1FvApkmsUbSEXZj7CiDemx\nTz23f6VR+eZzgWuaWJQwLwmkWwVgCfUr9KL3Uw92+U/9uvcmNdsC7cwM14RfTx1R\njYkwqjz2RRsiAsuD5ebHxoNDqEIkArf6K+dFJImnOQKBgQD7zFVW22nYg/54JZR1\n3Hyv8ncy6ocPB0+jlESE/m8kPo0RtzGUJxKDTwEZP+y8y2NchI6C1atC/RxE6But\nOUAJLxqD027z4rXo/yNaiDaerZXr/y56pRbhR3wbjuR7vcEFZ3B7bHCrKhgdDYPA\nMTgVxX/2Z3AXwCDpaG2W7LxbpwKBgQC7oYnrF6jzNPCu/FHH4Hhff/2aV0IJO7BQ\nEeY8VADVBHTiYwlnBlG9LkpWFM/YjsGpsfzorztIKDd1N911h2XcL7kgh6+URP75\ntYtQV3iNUbERxdp/rnm97QdZr/W3r9s60q61mX7vMsZK8mPPmwLKvP7YSu0NvFa8\nucidRihtpQKBgAJW/sAE3/HsIBQ7vSpvNxVnemYVudWQ6tOJUC2wM5Yxopv0iNho\nmIpx1H/IkUmb1juI284pcCL6OSYGxiMQ8iBjuKpa76ACjlAw9sIjm+ZTlJ4Ry/vF\nxvWm9WdIJ6ViuQV01Z2//zgH9xtmAcBqdKv3Ht5KTcdauLOSjdomLwXnAoGAfQ7J\nHxFxAVEaznbMh108veJQBKv+Dqti86tKepE+0Lwcr7t0y98xYddVopRSiDN2LwW7\n3NbWu1xawl0O1UP+h0ijqmPlifyGuabgCReT+RUm4QKvhISlDgrK6GNYcirbAxTj\nb5S0PvfnpJJ0Ji5aKQjZDw65e3s5kKZ/aRwW3CUCgYAvWeze0qlg332pZiEtsEnF\nx8CUq7i1phdB3G6cIVdvAee5ce6tvpNiFhECjaUZiykV6zyQY8s8LIyjKTuFXbI9\n2uAvPl7y4PHV5rKJOtuQJOLqaUNgAlklKdUBL2I6vwp/1epxQHUYNeaQv5lC0QT+\nxHrzaiPUQ0ADBUNvUVMIOg==\n-----END PRIVATE KEY-----\n",
"client_email": "starting-account-c76g3v5fh5js@booming-banner-212315.iam.gserviceaccount.com",
"client_id": "105634639063680583591",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/starting-account-c76g3v5fh5js%40booming-banner-212315.iam.gserviceaccount.com"
}