diff --git a/package.json b/package.json index f8b4287197..a4b425d0cc 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop", "matrix-widget-api": "^0.1.0-beta.13", "minimist": "^1.2.5", + "opus-recorder": "^8.0.3", "pako": "^2.0.3", "parse5": "^6.0.1", "png-chunks-extract": "^1.0.0", diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index 4aa6df5488..051e5cc429 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -39,6 +39,7 @@ import {ModalWidgetStore} from "../stores/ModalWidgetStore"; import { WidgetLayoutStore } from "../stores/widgets/WidgetLayoutStore"; import VoipUserMapper from "../VoipUserMapper"; import {SpaceStoreClass} from "../stores/SpaceStore"; +import {VoiceRecorder} from "../voice/VoiceRecorder"; declare global { interface Window { @@ -70,6 +71,7 @@ declare global { mxModalWidgetStore: ModalWidgetStore; mxVoipUserMapper: VoipUserMapper; mxSpaceStore: SpaceStoreClass; + mxVoiceRecorder: typeof VoiceRecorder; } interface Document { diff --git a/src/index.js b/src/index.js index 008e15ad90..1ef760dab9 100644 --- a/src/index.js +++ b/src/index.js @@ -28,3 +28,5 @@ export function resetSkin() { export function getComponent(componentName) { return Skinner.getComponent(componentName); } + +import "./voice/VoiceRecorder"; // TODO: @@ REMOVE diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts new file mode 100644 index 0000000000..2764d94174 --- /dev/null +++ b/src/voice/VoiceRecorder.ts @@ -0,0 +1,116 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import * as Recorder from 'opus-recorder'; +import encoderPath from 'opus-recorder/dist/encoderWorker.min.js'; +import {MatrixClient} from "matrix-js-sdk/src/client"; +import CallMediaHandler from "../CallMediaHandler"; +import {sleep} from "../utils/promise"; + +export class VoiceRecorder { + private recorder = new Recorder({ + encoderPath, // magic from webpack + mediaTrackConstraints: { + deviceId: CallMediaHandler.getAudioInput(), + }, + encoderSampleRate: 16000, // we could go down to 12khz, but we lose quality + encoderApplication: 2048, // voice (default is "audio") + streamPages: true, // so we can have a live EQ for the user + encoderFrameSize: 10, // we want updates fairly regularly for the UI + }); + private buffer = new Uint8Array(0); + private mxc: string; + private recording = false; + + public constructor(private client: MatrixClient) { + this.recorder.ondataavailable = (a: ArrayBuffer) => { + // TODO: @@ We'll have to decode each frame and convert it to an EQ to observe + console.log(a); + const buf = new Uint8Array(a); + const newBuf = new Uint8Array(this.buffer.length + buf.length); + newBuf.set(this.buffer, 0); + newBuf.set(buf, this.buffer.length); + this.buffer = newBuf; + }; + } + + public get isSupported(): boolean { + return !!Recorder.isRecordingSupported(); + } + + public get hasRecording(): boolean { + return this.buffer.length > 0; + } + + public get mxcUri(): string { + if (!this.mxc) { + throw new Error("Recording has not been uploaded yet"); + } + return this.mxc; + } + + public async start(): Promise { + if (this.mxc || this.hasRecording) { + throw new Error("Recording already prepared"); + } + if (this.recording) { + throw new Error("Recording already in progress"); + } + return this.recorder.start().then(() => this.recording = true); + } + + public async stop(): Promise { + if (!this.recording) { + throw new Error("No recording to stop"); + } + return new Promise(resolve => { + this.recorder.stop().then(() => { + this.recording = false; + return this.recorder.close(); + }).then(() => resolve(this.buffer)); + }); + } + + public async upload(): Promise { + if (!this.hasRecording) { + throw new Error("No recording available to upload"); + } + + if (this.mxc) return this.mxc; + + this.mxc = await this.client.uploadContent(new Blob([this.buffer], { + type: "audio/ogg", + }), { + onlyContentUri: false, // to stop the warnings in the console + }).then(r => r['content_uri']); + return this.mxc; + } + + // TODO: @@ REMOVE + public async test() { + this.start() + .then(() => sleep(5000)) + .then(() => this.stop()) + .then(() => this.upload()) + .then(() => this.client.sendMessage("!HKjSnKDluFnCCnjayl:localhost", { + body: "Voice message", + msgtype: "m.audio", // TODO + url: this.mxc, + })); + } +} + +window.mxVoiceRecorder = VoiceRecorder; diff --git a/yarn.lock b/yarn.lock index 58686248f7..1763a42e75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6096,6 +6096,11 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" +opus-recorder@^8.0.3: + version "8.0.3" + resolved "https://registry.yarnpkg.com/opus-recorder/-/opus-recorder-8.0.3.tgz#f7b44f8f68500c9b96a15042a69f915fd9c1716d" + integrity sha512-8vXGiRwlJAavT9D3yYzukNVXQ8vEcKHcsQL/zXO24DQtJ0PLXvoPHNQPJrbMCdB4ypJgWDExvHF4JitQDL7dng== + os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"