From 097c2d8be0ca1dabca2d024347c72f9002901769 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 11 Mar 2021 21:28:40 -0700 Subject: [PATCH 01/18] Add labs flag for voice messages --- src/i18n/strings/en_EN.json | 1 + src/settings/Settings.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 07d292a0e7..1a66e0cfee 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -783,6 +783,7 @@ "%(senderName)s: %(stickerName)s": "%(senderName)s: %(stickerName)s", "Change notification settings": "Change notification settings", "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.": "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.", + "Send and receive voice messages": "Send and receive voice messages", "Render LaTeX maths in messages": "Render LaTeX maths in messages", "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.": "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.", "New spinner design": "New spinner design", diff --git a/src/settings/Settings.ts b/src/settings/Settings.ts index 4f589ba49a..b718ddd8d2 100644 --- a/src/settings/Settings.ts +++ b/src/settings/Settings.ts @@ -128,6 +128,12 @@ export const SETTINGS: {[setting: string]: ISetting} = { default: false, controller: new ReloadOnChangeController(), }, + "feature_voice_messages": { + isFeature: true, + displayName: _td("Send and receive voice messages"), + supportedLevels: LEVELS_FEATURE, + default: false, + }, "feature_latex_maths": { isFeature: true, displayName: _td("Render LaTeX maths in messages"), From be2e30df0d891b4df7fb78778c723a3b45b3fb1a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 11 Mar 2021 22:05:47 -0700 Subject: [PATCH 02/18] Add an early voice recorder utility class --- package.json | 1 + src/@types/global.d.ts | 2 + src/index.js | 2 + src/voice/VoiceRecorder.ts | 116 +++++++++++++++++++++++++++++++++++++ yarn.lock | 5 ++ 5 files changed, 126 insertions(+) create mode 100644 src/voice/VoiceRecorder.ts diff --git a/package.json b/package.json index f8b4287197..a4b425d0cc 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop", "matrix-widget-api": "^0.1.0-beta.13", "minimist": "^1.2.5", + "opus-recorder": "^8.0.3", "pako": "^2.0.3", "parse5": "^6.0.1", "png-chunks-extract": "^1.0.0", diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index 4aa6df5488..051e5cc429 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -39,6 +39,7 @@ import {ModalWidgetStore} from "../stores/ModalWidgetStore"; import { WidgetLayoutStore } from "../stores/widgets/WidgetLayoutStore"; import VoipUserMapper from "../VoipUserMapper"; import {SpaceStoreClass} from "../stores/SpaceStore"; +import {VoiceRecorder} from "../voice/VoiceRecorder"; declare global { interface Window { @@ -70,6 +71,7 @@ declare global { mxModalWidgetStore: ModalWidgetStore; mxVoipUserMapper: VoipUserMapper; mxSpaceStore: SpaceStoreClass; + mxVoiceRecorder: typeof VoiceRecorder; } interface Document { diff --git a/src/index.js b/src/index.js index 008e15ad90..1ef760dab9 100644 --- a/src/index.js +++ b/src/index.js @@ -28,3 +28,5 @@ export function resetSkin() { export function getComponent(componentName) { return Skinner.getComponent(componentName); } + +import "./voice/VoiceRecorder"; // TODO: @@ REMOVE diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts new file mode 100644 index 0000000000..2764d94174 --- /dev/null +++ b/src/voice/VoiceRecorder.ts @@ -0,0 +1,116 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import * as Recorder from 'opus-recorder'; +import encoderPath from 'opus-recorder/dist/encoderWorker.min.js'; +import {MatrixClient} from "matrix-js-sdk/src/client"; +import CallMediaHandler from "../CallMediaHandler"; +import {sleep} from "../utils/promise"; + +export class VoiceRecorder { + private recorder = new Recorder({ + encoderPath, // magic from webpack + mediaTrackConstraints: { + deviceId: CallMediaHandler.getAudioInput(), + }, + encoderSampleRate: 16000, // we could go down to 12khz, but we lose quality + encoderApplication: 2048, // voice (default is "audio") + streamPages: true, // so we can have a live EQ for the user + encoderFrameSize: 10, // we want updates fairly regularly for the UI + }); + private buffer = new Uint8Array(0); + private mxc: string; + private recording = false; + + public constructor(private client: MatrixClient) { + this.recorder.ondataavailable = (a: ArrayBuffer) => { + // TODO: @@ We'll have to decode each frame and convert it to an EQ to observe + console.log(a); + const buf = new Uint8Array(a); + const newBuf = new Uint8Array(this.buffer.length + buf.length); + newBuf.set(this.buffer, 0); + newBuf.set(buf, this.buffer.length); + this.buffer = newBuf; + }; + } + + public get isSupported(): boolean { + return !!Recorder.isRecordingSupported(); + } + + public get hasRecording(): boolean { + return this.buffer.length > 0; + } + + public get mxcUri(): string { + if (!this.mxc) { + throw new Error("Recording has not been uploaded yet"); + } + return this.mxc; + } + + public async start(): Promise { + if (this.mxc || this.hasRecording) { + throw new Error("Recording already prepared"); + } + if (this.recording) { + throw new Error("Recording already in progress"); + } + return this.recorder.start().then(() => this.recording = true); + } + + public async stop(): Promise { + if (!this.recording) { + throw new Error("No recording to stop"); + } + return new Promise(resolve => { + this.recorder.stop().then(() => { + this.recording = false; + return this.recorder.close(); + }).then(() => resolve(this.buffer)); + }); + } + + public async upload(): Promise { + if (!this.hasRecording) { + throw new Error("No recording available to upload"); + } + + if (this.mxc) return this.mxc; + + this.mxc = await this.client.uploadContent(new Blob([this.buffer], { + type: "audio/ogg", + }), { + onlyContentUri: false, // to stop the warnings in the console + }).then(r => r['content_uri']); + return this.mxc; + } + + // TODO: @@ REMOVE + public async test() { + this.start() + .then(() => sleep(5000)) + .then(() => this.stop()) + .then(() => this.upload()) + .then(() => this.client.sendMessage("!HKjSnKDluFnCCnjayl:localhost", { + body: "Voice message", + msgtype: "m.audio", // TODO + url: this.mxc, + })); + } +} + +window.mxVoiceRecorder = VoiceRecorder; diff --git a/yarn.lock b/yarn.lock index 58686248f7..1763a42e75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6096,6 +6096,11 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" +opus-recorder@^8.0.3: + version "8.0.3" + resolved "https://registry.yarnpkg.com/opus-recorder/-/opus-recorder-8.0.3.tgz#f7b44f8f68500c9b96a15042a69f915fd9c1716d" + integrity sha512-8vXGiRwlJAavT9D3yYzukNVXQ8vEcKHcsQL/zXO24DQtJ0PLXvoPHNQPJrbMCdB4ypJgWDExvHF4JitQDL7dng== + os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" From b5d32d92f31a9d65f02ae048c55360aef9e58296 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 15 Mar 2021 22:16:58 -0600 Subject: [PATCH 03/18] Wire up a simple record button --- .../views/rooms/_BasicMessageComposer.scss | 5 ++ res/css/views/rooms/_MessageComposer.scss | 4 + res/img/voip/mic-on-mask.svg | 3 + .../views/rooms/BasicMessageComposer.tsx | 3 + src/components/views/rooms/MessageComposer.js | 31 ++++++-- .../views/rooms/SendMessageComposer.js | 2 + .../views/rooms/VoiceRecordComposerTile.tsx | 74 +++++++++++++++++++ src/i18n/strings/en_EN.json | 1 + src/voice/VoiceRecorder.ts | 12 +++ 9 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 res/img/voip/mic-on-mask.svg create mode 100644 src/components/views/rooms/VoiceRecordComposerTile.tsx diff --git a/res/css/views/rooms/_BasicMessageComposer.scss b/res/css/views/rooms/_BasicMessageComposer.scss index e126e523a6..4f58c08617 100644 --- a/res/css/views/rooms/_BasicMessageComposer.scss +++ b/res/css/views/rooms/_BasicMessageComposer.scss @@ -66,6 +66,11 @@ limitations under the License. } } } + + &.mx_BasicMessageComposer_input_disabled { + pointer-events: none; + cursor: not-allowed; + } } .mx_BasicMessageComposer_AutoCompleteWrapper { diff --git a/res/css/views/rooms/_MessageComposer.scss b/res/css/views/rooms/_MessageComposer.scss index dea1b58741..e6c0cc3f46 100644 --- a/res/css/views/rooms/_MessageComposer.scss +++ b/res/css/views/rooms/_MessageComposer.scss @@ -227,6 +227,10 @@ limitations under the License. mask-image: url('$(res)/img/element-icons/room/composer/attach.svg'); } +.mx_MessageComposer_voiceMessage::before { + mask-image: url('$(res)/img/voip/mic-on-mask.svg'); +} + .mx_MessageComposer_emoji::before { mask-image: url('$(res)/img/element-icons/room/composer/emoji.svg'); } diff --git a/res/img/voip/mic-on-mask.svg b/res/img/voip/mic-on-mask.svg new file mode 100644 index 0000000000..418316b164 --- /dev/null +++ b/res/img/voip/mic-on-mask.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/components/views/rooms/BasicMessageComposer.tsx b/src/components/views/rooms/BasicMessageComposer.tsx index 5ab2b82a32..223a7a5395 100644 --- a/src/components/views/rooms/BasicMessageComposer.tsx +++ b/src/components/views/rooms/BasicMessageComposer.tsx @@ -93,6 +93,7 @@ interface IProps { placeholder?: string; label?: string; initialCaret?: DocumentOffset; + disabled?: boolean; onChange?(); onPaste?(event: ClipboardEvent, model: EditorModel): boolean; @@ -672,6 +673,7 @@ export default class BasicMessageEditor extends React.Component }); const classes = classNames("mx_BasicMessageComposer_input", { "mx_BasicMessageComposer_input_shouldShowPillAvatar": this.state.showPillAvatar, + "mx_BasicMessageComposer_input_disabled": this.props.disabled, }); const shortcuts = { @@ -704,6 +706,7 @@ export default class BasicMessageEditor extends React.Component aria-expanded={Boolean(this.state.autoComplete)} aria-activedescendant={completionIndex >= 0 ? generateCompletionDomId(completionIndex) : undefined} dir="auto" + aria-disabled={this.props.disabled} /> ); } diff --git a/src/components/views/rooms/MessageComposer.js b/src/components/views/rooms/MessageComposer.js index ccf097c4fd..2efda53bc2 100644 --- a/src/components/views/rooms/MessageComposer.js +++ b/src/components/views/rooms/MessageComposer.js @@ -33,6 +33,7 @@ import WidgetStore from "../../../stores/WidgetStore"; import {UPDATE_EVENT} from "../../../stores/AsyncStore"; import ActiveWidgetStore from "../../../stores/ActiveWidgetStore"; import {replaceableComponent} from "../../../utils/replaceableComponent"; +import VoiceRecordComposerTile from "./VoiceRecordComposerTile"; function ComposerAvatar(props) { const MemberStatusMessageAvatar = sdk.getComponent('avatars.MemberStatusMessageAvatar'); @@ -187,6 +188,7 @@ export default class MessageComposer extends React.Component { hasConference: WidgetStore.instance.doesRoomHaveConference(this.props.room), joinedConference: WidgetStore.instance.isJoinedToConferenceIn(this.props.room), isComposerEmpty: true, + haveRecording: false, }; } @@ -325,6 +327,10 @@ export default class MessageComposer extends React.Component { }); } + onVoiceUpdate = (haveRecording: boolean) => { + this.setState({haveRecording}); + }; + render() { const controls = [ this.state.me ? : null, @@ -346,17 +352,32 @@ export default class MessageComposer extends React.Component { permalinkCreator={this.props.permalinkCreator} replyToEvent={this.props.replyToEvent} onChange={this.onChange} - />, - , - , + // TODO: TravisR - Disabling the composer doesn't work + disabled={this.state.haveRecording} + /> ); + if (!this.state.haveRecording) { + controls.push( + , + , + ); + } + if (SettingsStore.getValue(UIFeature.Widgets) && - SettingsStore.getValue("MessageComposerInput.showStickersButton")) { + SettingsStore.getValue("MessageComposerInput.showStickersButton") && + !this.state.haveRecording) { controls.push(); } - if (!this.state.isComposerEmpty) { + if (SettingsStore.getValue("feature_voice_messages")) { + controls.push(); + } + + if (!this.state.isComposerEmpty || this.state.haveRecording) { controls.push( , ); diff --git a/src/components/views/rooms/SendMessageComposer.js b/src/components/views/rooms/SendMessageComposer.js index ba3076c07d..aed1bb36fe 100644 --- a/src/components/views/rooms/SendMessageComposer.js +++ b/src/components/views/rooms/SendMessageComposer.js @@ -120,6 +120,7 @@ export default class SendMessageComposer extends React.Component { permalinkCreator: PropTypes.object.isRequired, replyToEvent: PropTypes.object, onChange: PropTypes.func, + disabled: PropTypes.bool, }; static contextType = MatrixClientContext; @@ -556,6 +557,7 @@ export default class SendMessageComposer extends React.Component { label={this.props.placeholder} placeholder={this.props.placeholder} onPaste={this._onPaste} + disabled={this.props.disabled} /> ); diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx new file mode 100644 index 0000000000..9ba3764524 --- /dev/null +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -0,0 +1,74 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import AccessibleTooltipButton from "../elements/AccessibleTooltipButton"; +import {_t} from "../../../languageHandler"; +import React from "react"; +import {VoiceRecorder} from "../../../voice/VoiceRecorder"; +import {Room} from "matrix-js-sdk/src/models/room"; +import {MatrixClientPeg} from "../../../MatrixClientPeg"; + +interface IProps { + room: Room; + onRecording: (haveRecording: boolean) => void; +} + +interface IState { + recorder?: VoiceRecorder; +} + +export default class VoiceRecordComposerTile extends React.PureComponent { + public constructor(props) { + super(props); + + this.state = { + recorder: null, // not recording by default + }; + } + + private onStartVoiceMessage = async () => { + if (this.state.recorder) { + await this.state.recorder.stop(); + const mxc = await this.state.recorder.upload(); + MatrixClientPeg.get().sendMessage(this.props.room.roomId, { + body: "Voice message", + msgtype: "m.audio", // TODO + url: mxc, + }); + this.setState({recorder: null}); + this.props.onRecording(false); + return; + } + const recorder = new VoiceRecorder(MatrixClientPeg.get()); + await recorder.start(); + this.props.onRecording(true); + // TODO: Run through EQ component + recorder.rawData.onUpdate((frame) => { + console.log('@@ FRAME', frame); + }); + this.setState({recorder}); + }; + + public render() { + return ( + + ); + } +} diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 1a66e0cfee..4654cb1f99 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -1636,6 +1636,7 @@ "Invited by %(sender)s": "Invited by %(sender)s", "Jump to first unread message.": "Jump to first unread message.", "Mark all as read": "Mark all as read", + "Record a voice message": "Record a voice message", "Error updating main address": "Error updating main address", "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.", "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.", diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 2764d94174..df6621d00b 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -19,6 +19,7 @@ import encoderPath from 'opus-recorder/dist/encoderWorker.min.js'; import {MatrixClient} from "matrix-js-sdk/src/client"; import CallMediaHandler from "../CallMediaHandler"; import {sleep} from "../utils/promise"; +import {SimpleObservable} from "matrix-widget-api"; export class VoiceRecorder { private recorder = new Recorder({ @@ -34,6 +35,7 @@ export class VoiceRecorder { private buffer = new Uint8Array(0); private mxc: string; private recording = false; + private observable: SimpleObservable; public constructor(private client: MatrixClient) { this.recorder.ondataavailable = (a: ArrayBuffer) => { @@ -44,9 +46,15 @@ export class VoiceRecorder { newBuf.set(this.buffer, 0); newBuf.set(buf, this.buffer.length); this.buffer = newBuf; + this.observable.update(buf); // send the frame over the observable }; } + public get rawData(): SimpleObservable { + if (!this.recording) throw new Error("No observable when not recording"); + return this.observable; + } + public get isSupported(): boolean { return !!Recorder.isRecordingSupported(); } @@ -69,6 +77,10 @@ export class VoiceRecorder { if (this.recording) { throw new Error("Recording already in progress"); } + if (this.observable) { + this.observable.close(); + } + this.observable = new SimpleObservable(); return this.recorder.start().then(() => this.recording = true); } From dafa8786a1e17b7238a07c058b5e2be10131b32d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:01:37 -0600 Subject: [PATCH 04/18] Fix comments --- src/components/views/rooms/VoiceRecordComposerTile.tsx | 8 ++++---- src/index.js | 2 -- src/voice/VoiceRecorder.ts | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 9ba3764524..d15328eb5a 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -55,10 +55,10 @@ export default class VoiceRecordComposerTile extends React.PureComponent { - console.log('@@ FRAME', frame); - }); + // TODO: @@ TravisR: Run through EQ component + // recorder.rawData.onUpdate((frame) => { + // console.log('@@ FRAME', frame); + // }); this.setState({recorder}); }; diff --git a/src/index.js b/src/index.js index 1ef760dab9..008e15ad90 100644 --- a/src/index.js +++ b/src/index.js @@ -28,5 +28,3 @@ export function resetSkin() { export function getComponent(componentName) { return Skinner.getComponent(componentName); } - -import "./voice/VoiceRecorder"; // TODO: @@ REMOVE diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index df6621d00b..e968627fe3 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -39,7 +39,7 @@ export class VoiceRecorder { public constructor(private client: MatrixClient) { this.recorder.ondataavailable = (a: ArrayBuffer) => { - // TODO: @@ We'll have to decode each frame and convert it to an EQ to observe + // TODO: @@ TravisR: We'll have to decode each frame and convert it to an EQ to observe console.log(a); const buf = new Uint8Array(a); const newBuf = new Uint8Array(this.buffer.length + buf.length); @@ -111,7 +111,7 @@ export class VoiceRecorder { return this.mxc; } - // TODO: @@ REMOVE + // TODO: @@ TravisR: REMOVE public async test() { this.start() .then(() => sleep(5000)) From 9aa5348c7fd059ffc1e32f3821e04b5b0ea7345e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:43:59 -0600 Subject: [PATCH 05/18] Show a stop button to stop the recording --- res/css/_components.scss | 5 +-- .../views/rooms/_VoiceRecordComposerTile.scss | 36 +++++++++++++++++++ .../legacy-light/css/_legacy-light.scss | 3 ++ res/themes/light/css/_light.scss | 3 ++ .../views/rooms/VoiceRecordComposerTile.tsx | 20 +++++++++-- src/i18n/strings/en_EN.json | 1 + src/voice/VoiceRecorder.ts | 2 +- 7 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 res/css/views/rooms/_VoiceRecordComposerTile.scss diff --git a/res/css/_components.scss b/res/css/_components.scss index d894688cac..9c895490b3 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -111,8 +111,8 @@ @import "./views/elements/_AddressSelector.scss"; @import "./views/elements/_AddressTile.scss"; @import "./views/elements/_DesktopBuildsNotice.scss"; -@import "./views/elements/_DirectorySearchBox.scss"; @import "./views/elements/_DesktopCapturerSourcePicker.scss"; +@import "./views/elements/_DirectorySearchBox.scss"; @import "./views/elements/_Dropdown.scss"; @import "./views/elements/_EditableItemList.scss"; @import "./views/elements/_ErrorBoundary.scss"; @@ -211,13 +211,13 @@ @import "./views/rooms/_SendMessageComposer.scss"; @import "./views/rooms/_Stickers.scss"; @import "./views/rooms/_TopUnreadMessagesBar.scss"; +@import "./views/rooms/_VoiceRecordComposerTile.scss"; @import "./views/rooms/_WhoIsTypingTile.scss"; @import "./views/settings/_AvatarSetting.scss"; @import "./views/settings/_CrossSigningPanel.scss"; @import "./views/settings/_DevicesPanel.scss"; @import "./views/settings/_E2eAdvancedPanel.scss"; @import "./views/settings/_EmailAddresses.scss"; -@import "./views/settings/_SpellCheckLanguages.scss"; @import "./views/settings/_IntegrationManager.scss"; @import "./views/settings/_Notifications.scss"; @import "./views/settings/_PhoneNumbers.scss"; @@ -225,6 +225,7 @@ @import "./views/settings/_SecureBackupPanel.scss"; @import "./views/settings/_SetIdServer.scss"; @import "./views/settings/_SetIntegrationManager.scss"; +@import "./views/settings/_SpellCheckLanguages.scss"; @import "./views/settings/_UpdateCheckButton.scss"; @import "./views/settings/tabs/_SettingsTab.scss"; @import "./views/settings/tabs/room/_GeneralRoomSettingsTab.scss"; diff --git a/res/css/views/rooms/_VoiceRecordComposerTile.scss b/res/css/views/rooms/_VoiceRecordComposerTile.scss new file mode 100644 index 0000000000..469a3de66a --- /dev/null +++ b/res/css/views/rooms/_VoiceRecordComposerTile.scss @@ -0,0 +1,36 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_VoiceRecordComposerTile_stop { + // 28px plus a 2px border makes this a 32px square (as intended) + width: 28px; + height: 28px; + border: 2px solid $voice-record-stop-border-color; + border-radius: 32px; + margin-right: 16px; // between us and the send button + position: relative; + + &::after { + content: ''; + width: 14px; + height: 14px; + position: absolute; + top: 6px; + left: 6px; + border-radius: 2px; + background-color: $voice-record-stop-symbol-color; + } +} diff --git a/res/themes/legacy-light/css/_legacy-light.scss b/res/themes/legacy-light/css/_legacy-light.scss index 9ad154dd93..d7ee496d80 100644 --- a/res/themes/legacy-light/css/_legacy-light.scss +++ b/res/themes/legacy-light/css/_legacy-light.scss @@ -189,6 +189,9 @@ $roomsublist-skeleton-ui-bg: linear-gradient(180deg, #ffffff 0%, #ffffff00 100%) $groupFilterPanel-divider-color: $roomlist-header-color; $space-button-outline-color: #E3E8F0; +$voice-record-stop-border-color: #E3E8F0; +$voice-record-stop-symbol-color: $warning-color; + $roomtile-preview-color: #9e9e9e; $roomtile-default-badge-bg-color: #61708b; $roomtile-selected-bg-color: #fff; diff --git a/res/themes/light/css/_light.scss b/res/themes/light/css/_light.scss index 25fbd0201b..577204ef0c 100644 --- a/res/themes/light/css/_light.scss +++ b/res/themes/light/css/_light.scss @@ -180,6 +180,9 @@ $roomsublist-skeleton-ui-bg: linear-gradient(180deg, #ffffff 0%, #ffffff00 100%) $groupFilterPanel-divider-color: $roomlist-header-color; $space-button-outline-color: #E3E8F0; +$voice-record-stop-border-color: #E3E8F0; +$voice-record-stop-symbol-color: $warning-color; + $roomtile-preview-color: $secondary-fg-color; $roomtile-default-badge-bg-color: #61708b; $roomtile-selected-bg-color: #FFF; diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index d15328eb5a..3b31075de2 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -20,6 +20,7 @@ import React from "react"; import {VoiceRecorder} from "../../../voice/VoiceRecorder"; import {Room} from "matrix-js-sdk/src/models/room"; import {MatrixClientPeg} from "../../../MatrixClientPeg"; +import classNames from "classnames"; interface IProps { room: Room; @@ -40,12 +41,13 @@ export default class VoiceRecordComposerTile extends React.PureComponent { + // TODO: @@ TravisR: We do not want to auto-send on stop. if (this.state.recorder) { await this.state.recorder.stop(); const mxc = await this.state.recorder.upload(); MatrixClientPeg.get().sendMessage(this.props.room.roomId, { body: "Voice message", - msgtype: "m.audio", // TODO + msgtype: "m.audio", // TODO @@ url: mxc, }); this.setState({recorder: null}); @@ -63,11 +65,23 @@ export default class VoiceRecordComposerTile extends React.PureComponent ); } diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 4654cb1f99..74100ed4b0 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -1637,6 +1637,7 @@ "Jump to first unread message.": "Jump to first unread message.", "Mark all as read": "Mark all as read", "Record a voice message": "Record a voice message", + "Stop & send recording": "Stop & send recording", "Error updating main address": "Error updating main address", "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.", "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.", diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index e968627fe3..1e59e451bd 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -119,7 +119,7 @@ export class VoiceRecorder { .then(() => this.upload()) .then(() => this.client.sendMessage("!HKjSnKDluFnCCnjayl:localhost", { body: "Voice message", - msgtype: "m.audio", // TODO + msgtype: "m.audio", // TODO @@ url: this.mxc, })); } From 1dc2427128910a9e6cd5355758153490b62b778a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:44:39 -0600 Subject: [PATCH 06/18] Remove test function --- src/voice/VoiceRecorder.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 1e59e451bd..9080c35b91 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -18,7 +18,6 @@ import * as Recorder from 'opus-recorder'; import encoderPath from 'opus-recorder/dist/encoderWorker.min.js'; import {MatrixClient} from "matrix-js-sdk/src/client"; import CallMediaHandler from "../CallMediaHandler"; -import {sleep} from "../utils/promise"; import {SimpleObservable} from "matrix-widget-api"; export class VoiceRecorder { @@ -110,19 +109,6 @@ export class VoiceRecorder { }).then(r => r['content_uri']); return this.mxc; } - - // TODO: @@ TravisR: REMOVE - public async test() { - this.start() - .then(() => sleep(5000)) - .then(() => this.stop()) - .then(() => this.upload()) - .then(() => this.client.sendMessage("!HKjSnKDluFnCCnjayl:localhost", { - body: "Voice message", - msgtype: "m.audio", // TODO @@ - url: this.mxc, - })); - } } window.mxVoiceRecorder = VoiceRecorder; From f0d5edbc37b20e1421c4471617e0946a03b7fbbf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:48:47 -0600 Subject: [PATCH 07/18] Render voice messages as audio messages for now --- src/components/views/messages/MessageEvent.js | 4 ++++ src/components/views/rooms/VoiceRecordComposerTile.tsx | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/components/views/messages/MessageEvent.js b/src/components/views/messages/MessageEvent.js index 866e0f521d..28c2f8f9b9 100644 --- a/src/components/views/messages/MessageEvent.js +++ b/src/components/views/messages/MessageEvent.js @@ -71,6 +71,10 @@ export default class MessageEvent extends React.Component { 'm.file': sdk.getComponent('messages.MFileBody'), 'm.audio': sdk.getComponent('messages.MAudioBody'), 'm.video': sdk.getComponent('messages.MVideoBody'), + + // TODO: @@ TravisR: Use labs flag determination. + // MSC: https://github.com/matrix-org/matrix-doc/pull/2516 + 'org.matrix.msc2516.voice': sdk.getComponent('messages.MAudioBody'), }; const evTypes = { 'm.sticker': sdk.getComponent('messages.MStickerBody'), diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 3b31075de2..a1d0e8c12f 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -47,7 +47,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent Date: Tue, 16 Mar 2021 23:50:03 -0600 Subject: [PATCH 08/18] Remove debugging --- src/voice/VoiceRecorder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 9080c35b91..66eb64b424 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -39,7 +39,6 @@ export class VoiceRecorder { public constructor(private client: MatrixClient) { this.recorder.ondataavailable = (a: ArrayBuffer) => { // TODO: @@ TravisR: We'll have to decode each frame and convert it to an EQ to observe - console.log(a); const buf = new Uint8Array(a); const newBuf = new Uint8Array(this.buffer.length + buf.length); newBuf.set(this.buffer, 0); From 51dca8d13d670c8fe9bfe5676f8726d0210bc5ac Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:54:33 -0600 Subject: [PATCH 09/18] Fix positioning of stop square --- res/css/views/rooms/_VoiceRecordComposerTile.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/res/css/views/rooms/_VoiceRecordComposerTile.scss b/res/css/views/rooms/_VoiceRecordComposerTile.scss index 469a3de66a..bb36991b4f 100644 --- a/res/css/views/rooms/_VoiceRecordComposerTile.scss +++ b/res/css/views/rooms/_VoiceRecordComposerTile.scss @@ -28,8 +28,8 @@ limitations under the License. width: 14px; height: 14px; position: absolute; - top: 6px; - left: 6px; + top: 7px; + left: 7px; border-radius: 2px; background-color: $voice-record-stop-symbol-color; } From 69f90ee97eef64af7b23a8b91d1c53536b810012 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:55:07 -0600 Subject: [PATCH 10/18] Label labs flag as in development --- src/i18n/strings/en_EN.json | 2 +- src/settings/Settings.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 74100ed4b0..625a2e7334 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -783,7 +783,7 @@ "%(senderName)s: %(stickerName)s": "%(senderName)s: %(stickerName)s", "Change notification settings": "Change notification settings", "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.": "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.", - "Send and receive voice messages": "Send and receive voice messages", + "Send and receive voice messages (in development)": "Send and receive voice messages (in development)", "Render LaTeX maths in messages": "Render LaTeX maths in messages", "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.": "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.", "New spinner design": "New spinner design", diff --git a/src/settings/Settings.ts b/src/settings/Settings.ts index b718ddd8d2..b38dee6e1a 100644 --- a/src/settings/Settings.ts +++ b/src/settings/Settings.ts @@ -130,7 +130,7 @@ export const SETTINGS: {[setting: string]: ISetting} = { }, "feature_voice_messages": { isFeature: true, - displayName: _td("Send and receive voice messages"), + displayName: _td("Send and receive voice messages (in development)"), supportedLevels: LEVELS_FEATURE, default: false, }, From c7b72bc4c43c26b438f4a5de07a8ecaea2a0a390 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 16 Mar 2021 23:57:27 -0600 Subject: [PATCH 11/18] Appease the linter --- src/components/views/rooms/MessageComposer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/rooms/MessageComposer.js b/src/components/views/rooms/MessageComposer.js index 2efda53bc2..13f934c626 100644 --- a/src/components/views/rooms/MessageComposer.js +++ b/src/components/views/rooms/MessageComposer.js @@ -354,7 +354,7 @@ export default class MessageComposer extends React.Component { onChange={this.onChange} // TODO: TravisR - Disabling the composer doesn't work disabled={this.state.haveRecording} - /> + />, ); if (!this.state.haveRecording) { From 0f09eb3214984ceb256c30e3c222033edd8854c6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 17 Mar 2021 00:09:56 -0600 Subject: [PATCH 12/18] Add more notes --- src/components/views/rooms/BasicMessageComposer.tsx | 2 ++ src/components/views/rooms/MessageComposer.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/views/rooms/BasicMessageComposer.tsx b/src/components/views/rooms/BasicMessageComposer.tsx index 223a7a5395..1a95b4366a 100644 --- a/src/components/views/rooms/BasicMessageComposer.tsx +++ b/src/components/views/rooms/BasicMessageComposer.tsx @@ -673,6 +673,8 @@ export default class BasicMessageEditor extends React.Component }); const classes = classNames("mx_BasicMessageComposer_input", { "mx_BasicMessageComposer_input_shouldShowPillAvatar": this.state.showPillAvatar, + + // TODO: @@ TravisR: This doesn't work properly. The composer resets in a strange way. "mx_BasicMessageComposer_input_disabled": this.props.disabled, }); diff --git a/src/components/views/rooms/MessageComposer.js b/src/components/views/rooms/MessageComposer.js index 13f934c626..b7078766fb 100644 --- a/src/components/views/rooms/MessageComposer.js +++ b/src/components/views/rooms/MessageComposer.js @@ -352,7 +352,7 @@ export default class MessageComposer extends React.Component { permalinkCreator={this.props.permalinkCreator} replyToEvent={this.props.replyToEvent} onChange={this.onChange} - // TODO: TravisR - Disabling the composer doesn't work + // TODO: @@ TravisR - Disabling the composer doesn't work disabled={this.state.haveRecording} />, ); From 207ba11da12f09e8bd84f057c585f4bb93a85f53 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 19 Mar 2021 17:08:01 -0600 Subject: [PATCH 13/18] Tweak a bunch of settings --- src/voice/VoiceRecorder.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 66eb64b424..3d1008d45e 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -26,10 +26,15 @@ export class VoiceRecorder { mediaTrackConstraints: { deviceId: CallMediaHandler.getAudioInput(), }, - encoderSampleRate: 16000, // we could go down to 12khz, but we lose quality + encoderSampleRate: 48000, // we could go down to 12khz, but we lose quality. 48khz is a webrtc default encoderApplication: 2048, // voice (default is "audio") streamPages: true, // so we can have a live EQ for the user - encoderFrameSize: 10, // we want updates fairly regularly for the UI + encoderFrameSize: 20, // ms, we want updates fairly regularly for the UI + numberOfChannels: 1, // stereo isn't important for us + //sourceNode: instanceof MediaStreamAudioSourceNode, // TODO: @@ Travis: Use this for EQ stuff. + encoderBitRate: 64000, // 64kbps is average for webrtc + encoderComplexity: 3, // 0-10, 0 is fast and low complexity + resampleQuality: 3, // 0-10, 10 is slow and high quality }); private buffer = new Uint8Array(0); private mxc: string; From e352ed19085c76522a5963e89594cd09bfb18c9a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Mar 2021 19:32:24 -0600 Subject: [PATCH 14/18] Run audio through the Web Audio API instead This leads to more reliable frequency/timing information, and involves a whole lot less decoding. We still maintain ongoing encoded frames to avoid having to do one giant encode at the end, as that could take long enough to be disruptive. --- .../views/rooms/VoiceRecordComposerTile.tsx | 4 +- src/voice/VoiceRecorder.ts | 101 +++++++++++++----- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index a1d0e8c12f..7327bf2380 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -58,8 +58,8 @@ export default class VoiceRecordComposerTile extends React.PureComponent { - // console.log('@@ FRAME', frame); + // recorder.frequencyData.onUpdate((freq) => { + // console.log('@@ UPDATE', freq); // }); this.setState({recorder}); }; diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 3d1008d45e..5646b53cbe 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -20,40 +20,74 @@ import {MatrixClient} from "matrix-js-sdk/src/client"; import CallMediaHandler from "../CallMediaHandler"; import {SimpleObservable} from "matrix-widget-api"; +const CHANNELS = 1; // stereo isn't important +const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. +const BITRATE = 64000; // 64kbps is average for WebRTC, so we might as well use it too. +const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data. We don't need this super often. + +export interface IFrequencyPackage { + dbBars: Float32Array; + dbMin: number; + dbMax: number; + + // TODO: @@ TravisR: Generalize this for a timing package? +} + export class VoiceRecorder { - private recorder = new Recorder({ - encoderPath, // magic from webpack - mediaTrackConstraints: { - deviceId: CallMediaHandler.getAudioInput(), - }, - encoderSampleRate: 48000, // we could go down to 12khz, but we lose quality. 48khz is a webrtc default - encoderApplication: 2048, // voice (default is "audio") - streamPages: true, // so we can have a live EQ for the user - encoderFrameSize: 20, // ms, we want updates fairly regularly for the UI - numberOfChannels: 1, // stereo isn't important for us - //sourceNode: instanceof MediaStreamAudioSourceNode, // TODO: @@ Travis: Use this for EQ stuff. - encoderBitRate: 64000, // 64kbps is average for webrtc - encoderComplexity: 3, // 0-10, 0 is fast and low complexity - resampleQuality: 3, // 0-10, 10 is slow and high quality - }); + private recorder: Recorder; + private recorderContext: AudioContext; + private recorderSource: MediaStreamAudioSourceNode; + private recorderStream: MediaStream; + private recorderFreqNode: AnalyserNode; private buffer = new Uint8Array(0); private mxc: string; private recording = false; - private observable: SimpleObservable; + private observable: SimpleObservable; + private freqTimerId: number; public constructor(private client: MatrixClient) { + } + + private async makeRecorder() { + this.recorderStream = await navigator.mediaDevices.getUserMedia({ + audio: { + // specify some audio settings so we're feeding the recorder with the + // best possible values. The browser will handle resampling for us. + sampleRate: SAMPLE_RATE, + channelCount: CHANNELS, + noiseSuppression: true, // browsers ignore constraints they can't honour + deviceId: CallMediaHandler.getAudioInput(), + }, + }); + this.recorderContext = new AudioContext({ + latencyHint: "interactive", + sampleRate: SAMPLE_RATE, // once again, the browser will resample for us + }); + this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream); + this.recorderFreqNode = this.recorderContext.createAnalyser(); + this.recorderSource.connect(this.recorderFreqNode); + this.recorder = new Recorder({ + encoderPath, // magic from webpack + encoderSampleRate: SAMPLE_RATE, + encoderApplication: 2048, // voice (default is "audio") + streamPages: true, // this speeds up the encoding process by using CPU over time + encoderFrameSize: 20, // ms, arbitrary frame size we send to the encoder + numberOfChannels: CHANNELS, + sourceNode: this.recorderSource, + encoderBitRate: BITRATE, + encoderComplexity: 3, // 0-10, 0 is fast and low complexity + resampleQuality: 3, // 0-10, 10 is slow and high quality + }); this.recorder.ondataavailable = (a: ArrayBuffer) => { - // TODO: @@ TravisR: We'll have to decode each frame and convert it to an EQ to observe const buf = new Uint8Array(a); const newBuf = new Uint8Array(this.buffer.length + buf.length); newBuf.set(this.buffer, 0); newBuf.set(buf, this.buffer.length); this.buffer = newBuf; - this.observable.update(buf); // send the frame over the observable }; } - public get rawData(): SimpleObservable { + public get frequencyData(): SimpleObservable { if (!this.recording) throw new Error("No observable when not recording"); return this.observable; } @@ -83,7 +117,18 @@ export class VoiceRecorder { if (this.observable) { this.observable.close(); } - this.observable = new SimpleObservable(); + this.observable = new SimpleObservable(); + await this.makeRecorder(); + this.freqTimerId = setInterval(() => { + if (!this.recording) return; + const data = new Float32Array(this.recorderFreqNode.frequencyBinCount); + this.recorderFreqNode.getFloatFrequencyData(data); + this.observable.update({ + dbBars: data, + dbMin: this.recorderFreqNode.minDecibels, + dbMax: this.recorderFreqNode.maxDecibels, + }); + }, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment return this.recorder.start().then(() => this.recording = true); } @@ -91,12 +136,20 @@ export class VoiceRecorder { if (!this.recording) { throw new Error("No recording to stop"); } - return new Promise(resolve => { - this.recorder.stop().then(() => { + // Disconnect the source early to start shutting down resources + this.recorderSource.disconnect(); + return this.recorder.stop() + // close the context after the recorder so the recorder doesn't try to + // connect anything to the context (this would generate a warning) + .then(() => this.recorderContext.close()) + // Now stop all the media tracks so we can release them back to the user/OS + .then(() => this.recorderStream.getTracks().forEach(t => t.stop())) + // Finally do our post-processing and clean up + .then(() => { + clearInterval(this.freqTimerId); this.recording = false; return this.recorder.close(); - }).then(() => resolve(this.buffer)); - }); + }).then(() => this.buffer); } public async upload(): Promise { From 090cf28af497678ce7ebaa132b01112228faa5a2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Mar 2021 19:36:58 -0600 Subject: [PATCH 15/18] Appease the linter --- src/voice/VoiceRecorder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 5646b53cbe..ec1a745272 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -146,7 +146,7 @@ export class VoiceRecorder { .then(() => this.recorderStream.getTracks().forEach(t => t.stop())) // Finally do our post-processing and clean up .then(() => { - clearInterval(this.freqTimerId); + clearInterval(this.freqTimerId); this.recording = false; return this.recorder.close(); }).then(() => this.buffer); From b7e653268b6c8e41fd22db365813b586beba1fbb Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 23 Mar 2021 18:19:14 -0600 Subject: [PATCH 16/18] Rename function --- src/components/views/rooms/VoiceRecordComposerTile.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 7327bf2380..0d381001a1 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -40,7 +40,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent { + private onStartStopVoiceMessage = async () => { // TODO: @@ TravisR: We do not want to auto-send on stop. if (this.state.recorder) { await this.state.recorder.stop(); @@ -80,7 +80,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent ); From c9938ff704765414ad119e0ea42822d96eefc23f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 23 Mar 2021 18:24:40 -0600 Subject: [PATCH 17/18] Adjust settings/docs for encoder --- src/voice/VoiceRecorder.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index ec1a745272..9fa2faad1e 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -22,8 +22,8 @@ import {SimpleObservable} from "matrix-widget-api"; const CHANNELS = 1; // stereo isn't important const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. -const BITRATE = 64000; // 64kbps is average for WebRTC, so we might as well use it too. -const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data. We don't need this super often. +const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus. +const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often. export interface IFrequencyPackage { dbBars: Float32Array; @@ -75,7 +75,11 @@ export class VoiceRecorder { numberOfChannels: CHANNELS, sourceNode: this.recorderSource, encoderBitRate: BITRATE, - encoderComplexity: 3, // 0-10, 0 is fast and low complexity + + // We use low values for the following to ease CPU usage - the resulting waveform + // is indistinguishable for a voice message. Note that the underlying library will + // pick defaults which prefer the highest possible quality, CPU be damned. + encoderComplexity: 3, // 0-10, 10 is slow and high quality. resampleQuality: 3, // 0-10, 10 is slow and high quality }); this.recorder.ondataavailable = (a: ArrayBuffer) => { From d929d4839154927c415110ea89db90160fe934dc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 23 Mar 2021 18:26:43 -0600 Subject: [PATCH 18/18] Clean up promises --- src/voice/VoiceRecorder.ts | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 9fa2faad1e..06c0d939fc 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -133,27 +133,32 @@ export class VoiceRecorder { dbMax: this.recorderFreqNode.maxDecibels, }); }, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment - return this.recorder.start().then(() => this.recording = true); + await this.recorder.start(); + this.recording = true; } public async stop(): Promise { if (!this.recording) { throw new Error("No recording to stop"); } + // Disconnect the source early to start shutting down resources this.recorderSource.disconnect(); - return this.recorder.stop() - // close the context after the recorder so the recorder doesn't try to - // connect anything to the context (this would generate a warning) - .then(() => this.recorderContext.close()) - // Now stop all the media tracks so we can release them back to the user/OS - .then(() => this.recorderStream.getTracks().forEach(t => t.stop())) - // Finally do our post-processing and clean up - .then(() => { - clearInterval(this.freqTimerId); - this.recording = false; - return this.recorder.close(); - }).then(() => this.buffer); + await this.recorder.stop(); + + // close the context after the recorder so the recorder doesn't try to + // connect anything to the context (this would generate a warning) + await this.recorderContext.close(); + + // Now stop all the media tracks so we can release them back to the user/OS + this.recorderStream.getTracks().forEach(t => t.stop()); + + // Finally do our post-processing and clean up + clearInterval(this.freqTimerId); + this.recording = false; + await this.recorder.close(); + + return this.buffer; } public async upload(): Promise {