PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Merge pull request #5769 from matrix-org/travis/voice-messages/exp

Browse source 
Labs feature: Early implementation of voice messages
This commit is contained in:
Travis Ralston 2021-03-24 09:56:34 -06:00 committed by GitHub
commit 8587ec888b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 378 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -83,6 +83,7 @@
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
"matrix-widget-api": "^0.1.0-beta.13",
"minimist": "^1.2.5",
"opus-recorder": "^8.0.3",
"pako": "^2.0.3",
"parse5": "^6.0.1",
"png-chunks-extract": "^1.0.0",

5
res/css/_components.scss
View file

@ -111,8 +111,8 @@
@import "./views/elements/_AddressSelector.scss";
@import "./views/elements/_AddressTile.scss";
@import "./views/elements/_DesktopBuildsNotice.scss";
@import "./views/elements/_DirectorySearchBox.scss";
@import "./views/elements/_DesktopCapturerSourcePicker.scss";
@import "./views/elements/_DirectorySearchBox.scss";
@import "./views/elements/_Dropdown.scss";
@import "./views/elements/_EditableItemList.scss";
@import "./views/elements/_ErrorBoundary.scss";
@ -211,13 +211,13 @@
@import "./views/rooms/_SendMessageComposer.scss";
@import "./views/rooms/_Stickers.scss";
@import "./views/rooms/_TopUnreadMessagesBar.scss";
@import "./views/rooms/_VoiceRecordComposerTile.scss";
@import "./views/rooms/_WhoIsTypingTile.scss";
@import "./views/settings/_AvatarSetting.scss";
@import "./views/settings/_CrossSigningPanel.scss";
@import "./views/settings/_DevicesPanel.scss";
@import "./views/settings/_E2eAdvancedPanel.scss";
@import "./views/settings/_EmailAddresses.scss";
@import "./views/settings/_SpellCheckLanguages.scss";
@import "./views/settings/_IntegrationManager.scss";
@import "./views/settings/_Notifications.scss";
@import "./views/settings/_PhoneNumbers.scss";
@ -225,6 +225,7 @@
@import "./views/settings/_SecureBackupPanel.scss";
@import "./views/settings/_SetIdServer.scss";
@import "./views/settings/_SetIntegrationManager.scss";
@import "./views/settings/_SpellCheckLanguages.scss";
@import "./views/settings/_UpdateCheckButton.scss";
@import "./views/settings/tabs/_SettingsTab.scss";
@import "./views/settings/tabs/room/_GeneralRoomSettingsTab.scss";

5
res/css/views/rooms/_BasicMessageComposer.scss
View file

@ -66,6 +66,11 @@ limitations under the License.
}
}
}
&.mx_BasicMessageComposer_input_disabled {
pointer-events: none;
cursor: not-allowed;
}
}
.mx_BasicMessageComposer_AutoCompleteWrapper {

4
res/css/views/rooms/_MessageComposer.scss
View file

@ -227,6 +227,10 @@ limitations under the License.
mask-image: url('$(res)/img/element-icons/room/composer/attach.svg');
}
.mx_MessageComposer_voiceMessage::before {
mask-image: url('$(res)/img/voip/mic-on-mask.svg');
}
.mx_MessageComposer_emoji::before {
mask-image: url('$(res)/img/element-icons/room/composer/emoji.svg');
}

36
res/css/views/rooms/_VoiceRecordComposerTile.scss Normal file
View file

@ -0,0 +1,36 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.mx_VoiceRecordComposerTile_stop {
// 28px plus a 2px border makes this a 32px square (as intended)
width: 28px;
height: 28px;
border: 2px solid $voice-record-stop-border-color;
border-radius: 32px;
margin-right: 16px; // between us and the send button
position: relative;
&::after {
content: '';
width: 14px;
height: 14px;
position: absolute;
top: 7px;
left: 7px;
border-radius: 2px;
background-color: $voice-record-stop-symbol-color;
}
}

3
res/img/voip/mic-on-mask.svg Normal file
View file

@ -0,0 +1,3 @@
<svg width="20" height="21" viewBox="0 0 20 21" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M6.09999 4.15C6.09999 1.99609 7.84608 0.25 9.99999 0.25C12.1539 0.25 13.9 1.99609 13.9 4.15V9.98254C13.9 12.1365 12.1539 13.8825 9.99999 13.8825C7.84608 13.8825 6.09999 12.1365 6.09999 9.98254V4.15ZM3.1748 8.73755C3.86516 8.73755 4.4248 9.29719 4.4248 9.98755C4.4248 13.0574 6.91483 15.5493 9.9915 15.5538C9.99433 15.5538 9.99717 15.5538 10 15.5538C10.0028 15.5538 10.0056 15.5538 10.0084 15.5538C13.085 15.5492 15.5748 13.0573 15.5748 9.98755C15.5748 9.29719 16.1344 8.73755 16.8248 8.73755C17.5152 8.73755 18.0748 9.29719 18.0748 9.98755C18.0748 14.0189 15.115 17.3576 11.25 17.9577V18.7513C11.25 19.4416 10.6904 20.0013 10 20.0013C9.30964 20.0013 8.75 19.4416 8.75 18.7513V17.9578C4.88483 17.3578 1.9248 14.0191 1.9248 9.98755C1.9248 9.29719 2.48445 8.73755 3.1748 8.73755Z" fill="#C1C6CD"/>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 951 B

3
res/themes/legacy-light/css/_legacy-light.scss
View file

@ -189,6 +189,9 @@ $roomsublist-skeleton-ui-bg: linear-gradient(180deg, #ffffff 0%, #ffffff00 100%)
$groupFilterPanel-divider-color: $roomlist-header-color;
$space-button-outline-color: #E3E8F0;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: $warning-color;
$roomtile-preview-color: #9e9e9e;
$roomtile-default-badge-bg-color: #61708b;
$roomtile-selected-bg-color: #fff;

3
res/themes/light/css/_light.scss
View file

@ -180,6 +180,9 @@ $roomsublist-skeleton-ui-bg: linear-gradient(180deg, #ffffff 0%, #ffffff00 100%)
$groupFilterPanel-divider-color: $roomlist-header-color;
$space-button-outline-color: #E3E8F0;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: $warning-color;
$roomtile-preview-color: $secondary-fg-color;
$roomtile-default-badge-bg-color: #61708b;
$roomtile-selected-bg-color: #FFF;

2
src/@types/global.d.ts vendored
View file

@ -39,6 +39,7 @@ import {ModalWidgetStore} from "../stores/ModalWidgetStore";
import { WidgetLayoutStore } from "../stores/widgets/WidgetLayoutStore";
import VoipUserMapper from "../VoipUserMapper";
import {SpaceStoreClass} from "../stores/SpaceStore";
import {VoiceRecorder} from "../voice/VoiceRecorder";
declare global {
interface Window {
@ -70,6 +71,7 @@ declare global {
mxModalWidgetStore: ModalWidgetStore;
mxVoipUserMapper: VoipUserMapper;
mxSpaceStore: SpaceStoreClass;
mxVoiceRecorder: typeof VoiceRecorder;
}
interface Document {

4
src/components/views/messages/MessageEvent.js
View file

@ -71,6 +71,10 @@ export default class MessageEvent extends React.Component {
'm.file': sdk.getComponent('messages.MFileBody'),
'm.audio': sdk.getComponent('messages.MAudioBody'),
'm.video': sdk.getComponent('messages.MVideoBody'),
// TODO: @@ TravisR: Use labs flag determination.
// MSC: https://github.com/matrix-org/matrix-doc/pull/2516
'org.matrix.msc2516.voice': sdk.getComponent('messages.MAudioBody'),
};
const evTypes = {
'm.sticker': sdk.getComponent('messages.MStickerBody'),

5
src/components/views/rooms/BasicMessageComposer.tsx
View file

@ -93,6 +93,7 @@ interface IProps {
placeholder?: string;
label?: string;
initialCaret?: DocumentOffset;
disabled?: boolean;
onChange?();
onPaste?(event: ClipboardEvent<HTMLDivElement>, model: EditorModel): boolean;
@ -672,6 +673,9 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
});
const classes = classNames("mx_BasicMessageComposer_input", {
"mx_BasicMessageComposer_input_shouldShowPillAvatar": this.state.showPillAvatar,
// TODO: @@ TravisR: This doesn't work properly. The composer resets in a strange way.
"mx_BasicMessageComposer_input_disabled": this.props.disabled,
});
const shortcuts = {
@ -704,6 +708,7 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
aria-expanded={Boolean(this.state.autoComplete)}
aria-activedescendant={completionIndex >= 0 ? generateCompletionDomId(completionIndex) : undefined}
dir="auto"
aria-disabled={this.props.disabled}
/>
</div>);
}

29
src/components/views/rooms/MessageComposer.js
View file

@ -33,6 +33,7 @@ import WidgetStore from "../../../stores/WidgetStore";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import ActiveWidgetStore from "../../../stores/ActiveWidgetStore";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import VoiceRecordComposerTile from "./VoiceRecordComposerTile";
function ComposerAvatar(props) {
const MemberStatusMessageAvatar = sdk.getComponent('avatars.MemberStatusMessageAvatar');
@ -187,6 +188,7 @@ export default class MessageComposer extends React.Component {
hasConference: WidgetStore.instance.doesRoomHaveConference(this.props.room),
joinedConference: WidgetStore.instance.isJoinedToConferenceIn(this.props.room),
isComposerEmpty: true,
haveRecording: false,
};
}
@ -325,6 +327,10 @@ export default class MessageComposer extends React.Component {
});
}
onVoiceUpdate = (haveRecording: boolean) => {
this.setState({haveRecording});
};
render() {
const controls = [
this.state.me ? <ComposerAvatar key="controls_avatar" me={this.state.me} /> : null,
@ -346,17 +352,32 @@ export default class MessageComposer extends React.Component {
permalinkCreator={this.props.permalinkCreator}
replyToEvent={this.props.replyToEvent}
onChange={this.onChange}
// TODO: @@ TravisR - Disabling the composer doesn't work
disabled={this.state.haveRecording}
/>,
<UploadButton key="controls_upload" roomId={this.props.room.roomId} />,
<EmojiButton key="emoji_button" addEmoji={this.addEmoji} />,
);
if (!this.state.haveRecording) {
controls.push(
<UploadButton key="controls_upload" roomId={this.props.room.roomId} />,
<EmojiButton key="emoji_button" addEmoji={this.addEmoji} />,
);
}
if (SettingsStore.getValue(UIFeature.Widgets) &&
SettingsStore.getValue("MessageComposerInput.showStickersButton")) {
SettingsStore.getValue("MessageComposerInput.showStickersButton") &&
!this.state.haveRecording) {
controls.push(<Stickerpicker key="stickerpicker_controls_button" room={this.props.room} />);
}
if (!this.state.isComposerEmpty) {
if (SettingsStore.getValue("feature_voice_messages")) {
controls.push(<VoiceRecordComposerTile
key="controls_voice_record"
room={this.props.room}
onRecording={this.onVoiceUpdate} />);
}
if (!this.state.isComposerEmpty || this.state.haveRecording) {
controls.push(
<SendButton key="controls_send" onClick={this.sendMessage} />,
);

2
src/components/views/rooms/SendMessageComposer.js
View file

@ -120,6 +120,7 @@ export default class SendMessageComposer extends React.Component {
permalinkCreator: PropTypes.object.isRequired,
replyToEvent: PropTypes.object,
onChange: PropTypes.func,
disabled: PropTypes.bool,
};
static contextType = MatrixClientContext;
@ -556,6 +557,7 @@ export default class SendMessageComposer extends React.Component {
label={this.props.placeholder}
placeholder={this.props.placeholder}
onPaste={this._onPaste}
disabled={this.props.disabled}
/>
</div>
);

88
src/components/views/rooms/VoiceRecordComposerTile.tsx Normal file
View file

@ -0,0 +1,88 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import React from "react";
import {VoiceRecorder} from "../../../voice/VoiceRecorder";
import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
interface IProps {
room: Room;
onRecording: (haveRecording: boolean) => void;
}
interface IState {
recorder?: VoiceRecorder;
}
export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
recorder: null, // not recording by default
};
}
private onStartStopVoiceMessage = async () => {
// TODO: @@ TravisR: We do not want to auto-send on stop.
if (this.state.recorder) {
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
body: "Voice message",
msgtype: "org.matrix.msc2516.voice",
url: mxc,
});
this.setState({recorder: null});
this.props.onRecording(false);
return;
}
const recorder = new VoiceRecorder(MatrixClientPeg.get());
await recorder.start();
this.props.onRecording(true);
// TODO: @@ TravisR: Run through EQ component
// recorder.frequencyData.onUpdate((freq) => {
// console.log('@@ UPDATE', freq);
// });
this.setState({recorder});
};
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
});
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
// TODO: @@ TravisR: Change to match behaviour
tooltip = _t("Stop & send recording");
}
return (
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}
title={tooltip}
/>
);
}
}

3
src/i18n/strings/en_EN.json
View file

@ -783,6 +783,7 @@
"%(senderName)s: %(stickerName)s": "%(senderName)s: %(stickerName)s",
"Change notification settings": "Change notification settings",
"Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.": "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.",
"Send and receive voice messages (in development)": "Send and receive voice messages (in development)",
"Render LaTeX maths in messages": "Render LaTeX maths in messages",
"Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.": "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.",
"New spinner design": "New spinner design",
@ -1636,6 +1637,8 @@
"Invited by %(sender)s": "Invited by %(sender)s",
"Jump to first unread message.": "Jump to first unread message.",
"Mark all as read": "Mark all as read",
"Record a voice message": "Record a voice message",
"Stop & send recording": "Stop & send recording",
"Error updating main address": "Error updating main address",
"There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
"There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",

6
src/settings/Settings.ts
View file

@ -128,6 +128,12 @@ export const SETTINGS: {[setting: string]: ISetting} = {
default: false,
controller: new ReloadOnChangeController(),
},
"feature_voice_messages": {
isFeature: true,
displayName: _td("Send and receive voice messages (in development)"),
supportedLevels: LEVELS_FEATURE,
default: false,
},
"feature_latex_maths": {
isFeature: true,
displayName: _td("Render LaTeX maths in messages"),

180
src/voice/VoiceRecorder.ts Normal file
View file

@ -0,0 +1,180 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import * as Recorder from 'opus-recorder';
import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
import {MatrixClient} from "matrix-js-sdk/src/client";
import CallMediaHandler from "../CallMediaHandler";
import {SimpleObservable} from "matrix-widget-api";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus.
const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often.
export interface IFrequencyPackage {
dbBars: Float32Array;
dbMin: number;
dbMax: number;
// TODO: @@ TravisR: Generalize this for a timing package?
}
export class VoiceRecorder {
private recorder: Recorder;
private recorderContext: AudioContext;
private recorderSource: MediaStreamAudioSourceNode;
private recorderStream: MediaStream;
private recorderFreqNode: AnalyserNode;
private buffer = new Uint8Array(0);
private mxc: string;
private recording = false;
private observable: SimpleObservable<IFrequencyPackage>;
private freqTimerId: number;
public constructor(private client: MatrixClient) {
}
private async makeRecorder() {
this.recorderStream = await navigator.mediaDevices.getUserMedia({
audio: {
// specify some audio settings so we're feeding the recorder with the
// best possible values. The browser will handle resampling for us.
sampleRate: SAMPLE_RATE,
channelCount: CHANNELS,
noiseSuppression: true, // browsers ignore constraints they can't honour
deviceId: CallMediaHandler.getAudioInput(),
},
});
this.recorderContext = new AudioContext({
latencyHint: "interactive",
sampleRate: SAMPLE_RATE, // once again, the browser will resample for us
});
this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);
this.recorderFreqNode = this.recorderContext.createAnalyser();
this.recorderSource.connect(this.recorderFreqNode);
this.recorder = new Recorder({
encoderPath, // magic from webpack
encoderSampleRate: SAMPLE_RATE,
encoderApplication: 2048, // voice (default is "audio")
streamPages: true, // this speeds up the encoding process by using CPU over time
encoderFrameSize: 20, // ms, arbitrary frame size we send to the encoder
numberOfChannels: CHANNELS,
sourceNode: this.recorderSource,
encoderBitRate: BITRATE,
// We use low values for the following to ease CPU usage - the resulting waveform
// is indistinguishable for a voice message. Note that the underlying library will
// pick defaults which prefer the highest possible quality, CPU be damned.
encoderComplexity: 3, // 0-10, 10 is slow and high quality.
resampleQuality: 3, // 0-10, 10 is slow and high quality
});
this.recorder.ondataavailable = (a: ArrayBuffer) => {
const buf = new Uint8Array(a);
const newBuf = new Uint8Array(this.buffer.length + buf.length);
newBuf.set(this.buffer, 0);
newBuf.set(buf, this.buffer.length);
this.buffer = newBuf;
};
}
public get frequencyData(): SimpleObservable<IFrequencyPackage> {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
}
public get isSupported(): boolean {
return !!Recorder.isRecordingSupported();
}
public get hasRecording(): boolean {
return this.buffer.length > 0;
}
public get mxcUri(): string {
if (!this.mxc) {
throw new Error("Recording has not been uploaded yet");
}
return this.mxc;
}
public async start(): Promise<void> {
if (this.mxc || this.hasRecording) {
throw new Error("Recording already prepared");
}
if (this.recording) {
throw new Error("Recording already in progress");
}
if (this.observable) {
this.observable.close();
}
this.observable = new SimpleObservable<IFrequencyPackage>();
await this.makeRecorder();
this.freqTimerId = setInterval(() => {
if (!this.recording) return;
const data = new Float32Array(this.recorderFreqNode.frequencyBinCount);
this.recorderFreqNode.getFloatFrequencyData(data);
this.observable.update({
dbBars: data,
dbMin: this.recorderFreqNode.minDecibels,
dbMax: this.recorderFreqNode.maxDecibels,
});
}, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment
await this.recorder.start();
this.recording = true;
}
public async stop(): Promise<Uint8Array> {
if (!this.recording) {
throw new Error("No recording to stop");
}
// Disconnect the source early to start shutting down resources
this.recorderSource.disconnect();
await this.recorder.stop();
// close the context after the recorder so the recorder doesn't try to
// connect anything to the context (this would generate a warning)
await this.recorderContext.close();
// Now stop all the media tracks so we can release them back to the user/OS
this.recorderStream.getTracks().forEach(t => t.stop());
// Finally do our post-processing and clean up
clearInterval(this.freqTimerId);
this.recording = false;
await this.recorder.close();
return this.buffer;
}
public async upload(): Promise<string> {
if (!this.hasRecording) {
throw new Error("No recording available to upload");
}
if (this.mxc) return this.mxc;
this.mxc = await this.client.uploadContent(new Blob([this.buffer], {
type: "audio/ogg",
}), {
onlyContentUri: false, // to stop the warnings in the console
}).then(r => r['content_uri']);
return this.mxc;
}
}
window.mxVoiceRecorder = VoiceRecorder;

5
yarn.lock
View file

@ -6096,6 +6096,11 @@ optionator@^0.9.1:
type-check "^0.4.0"
word-wrap "^1.2.3"
opus-recorder@^8.0.3:
version "8.0.3"
resolved "https://registry.yarnpkg.com/opus-recorder/-/opus-recorder-8.0.3.tgz#f7b44f8f68500c9b96a15042a69f915fd9c1716d"
integrity sha512-8vXGiRwlJAavT9D3yYzukNVXQ8vEcKHcsQL/zXO24DQtJ0PLXvoPHNQPJrbMCdB4ypJgWDExvHF4JitQDL7dng==
os-tmpdir@~1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"