PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Handle basic state machine of recordings

Browse source
This commit is contained in:
Travis Ralston 2021-04-20 21:30:21 -06:00
parent afd53d8b53
commit 32e3ce3dea
9 changed files with 241 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
res/css/views/rooms/_VoiceRecordComposerTile.scss
View file

@ -36,11 +36,12 @@ limitations under the License.
}
.mx_VoiceRecordComposerTile_waveformContainer {
padding: 5px;
padding: 8px; // makes us 4px taller than the send/stop button
padding-right: 4px; // there's 1px from the waveform itself, so account for that
padding-left: 15px; // +10px for the live circle, +5px for regular padding
background-color: $voice-record-waveform-bg-color;
border-radius: 12px;
margin: 6px; // force the composer area to put a gutter around us
margin-right: 12px; // isolate from stop button
// Cheat at alignment a bit
@ -52,7 +53,7 @@ limitations under the License.
color: $voice-record-waveform-fg-color;
font-size: $font-14px;
&::before {
&.mx_VoiceRecordComposerTile_recording::before {
animation: recording-pulse 2s infinite;
content: '';
@ -61,7 +62,7 @@ limitations under the License.
height: 10px;
position: absolute;
left: 8px;
top: 16px; // vertically center
top: 18px; // vertically center
border-radius: 10px;
}

160
src/components/views/rooms/VoiceRecordComposerTile.tsx
View file

@ -17,7 +17,7 @@ limitations under the License.
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import React from "react";
import {VoiceRecording} from "../../../voice/VoiceRecording";
import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording";
import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
@ -25,6 +25,8 @@ import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import PlaybackWaveform from "../voice_messages/PlaybackWaveform";
interface IProps {
room: Room;
@ -32,6 +34,7 @@ interface IProps {
interface IState {
recorder?: VoiceRecording;
recordingPhase?: RecordingState;
}
/**
@ -43,87 +46,126 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
super(props);
this.state = {
recorder: null, // not recording by default
recorder: null, // no recording started by default
};
}
private onStartStopVoiceMessage = async () => {
// TODO: @@ TravisR: We do not want to auto-send on stop.
public async componentWillUnmount() {
await VoiceRecordingStore.instance.disposeRecording();
}
// called by composer
public async send() {
if (!this.state.recorder) {
throw new Error("No recording started - cannot send anything");
}
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
"msgtype": "org.matrix.msc2516.voice",
//"msgtype": MsgType.Audio,
"url": mxc,
"info": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
// MSC1767 experiment
"org.matrix.msc1767.text": "Voice message",
"org.matrix.msc1767.file": {
url: mxc,
name: "Voice message.ogg",
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
"org.matrix.msc1767.audio": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
},
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// Events can't have floats, so we try to maintain resolution by using 1024
// as a maximum value. The waveform contains values between zero and 1, so this
// should come out largely sane.
//
// We're expecting about one data point per second of audio.
waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
},
});
await VoiceRecordingStore.instance.disposeRecording();
this.setState({recorder: null});
}
private onRecordStartEndClick = async () => {
if (this.state.recorder) {
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
"msgtype": "org.matrix.msc2516.voice",
//"msgtype": MsgType.Audio,
"url": mxc,
"info": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
// MSC1767 experiment
"org.matrix.msc1767.text": "Voice message",
"org.matrix.msc1767.file": {
url: mxc,
name: "Voice message.ogg",
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
"org.matrix.msc1767.audio": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
},
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// Events can't have floats, so we try to maintain resolution by using 1024
// as a maximum value. The waveform contains values between zero and 1, so this
// should come out largely sane.
//
// We're expecting about one data point per second of audio.
waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
},
});
await VoiceRecordingStore.instance.disposeRecording();
this.setState({recorder: null});
return;
}
const recorder = VoiceRecordingStore.instance.startRecording();
await recorder.start();
this.setState({recorder});
// We don't need to remove the listener: the recorder will clean that up for us.
recorder.on(UPDATE_EVENT, (ev: RecordingState) => {
if (ev === RecordingState.EndingSoon) return; // ignore this state: it has no UI purpose here
this.setState({recordingPhase: ev});
});
this.setState({recorder, recordingPhase: RecordingState.Started});
};
private renderWaveformArea() {
if (!this.state.recorder) return null;
return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
<LiveRecordingClock recorder={this.state.recorder} />
<LiveRecordingWaveform recorder={this.state.recorder} />
const classes = classNames({
'mx_VoiceRecordComposerTile_waveformContainer': true,
'mx_VoiceRecordComposerTile_recording': this.state.recordingPhase === RecordingState.Started,
});
const clock = <LiveRecordingClock recorder={this.state.recorder} />;
let waveform = <LiveRecordingWaveform recorder={this.state.recorder} />;
if (this.state.recordingPhase !== RecordingState.Started) {
waveform = <PlaybackWaveform recorder={this.state.recorder} />;
}
return <div className={classes}>
{clock}
{waveform}
</div>;
}
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
});
let recordingInfo;
if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording,
});
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
// TODO: @@ TravisR: Change to match behaviour
tooltip = _t("Stop & send recording");
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
tooltip = _t("Stop the recording");
}
let stopOrRecordBtn = <AccessibleTooltipButton
className={classes}
onClick={this.onRecordStartEndClick}
title={tooltip}
/>;
if (this.state.recorder && !this.state.recorder?.isRecording) {
stopOrRecordBtn = null;
}
recordingInfo = stopOrRecordBtn;
}
return (<>
{this.renderWaveformArea()}
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}
title={tooltip}
/>
{recordingInfo}
</>);
}
}

26
src/components/views/voice_messages/IRecordingWaveformStateProps.ts Normal file
View file

@ -0,0 +1,26 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import {VoiceRecording} from "../../../voice/VoiceRecording";
export interface IRecordingWaveformProps {
recorder: VoiceRecording;
}
export interface IRecordingWaveformState {
heights: number[];
}
export const DOWNSAMPLE_TARGET = 35; // number of bars we want

13
src/components/views/voice_messages/LiveRecordingWaveform.tsx
View file

@ -20,22 +20,13 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arrayFastResample, arraySeed} from "../../../utils/arrays";
import {percentageOf} from "../../../utils/numbers";
import Waveform from "./Waveform";
interface IProps {
recorder: VoiceRecording;
}
interface IState {
heights: number[];
}
const DOWNSAMPLE_TARGET = 35; // number of bars we want
import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps";
/**
* A waveform which shows the waveform of a live recording
*/
@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
export default class LiveRecordingWaveform extends React.PureComponent<IRecordingWaveformProps, IRecordingWaveformState> {
public constructor(props) {
super(props);

43
src/components/views/voice_messages/PlaybackWaveform.tsx Normal file
View file

@ -0,0 +1,43 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {IRecordingUpdate, VoiceRecording} from "../../../voice/VoiceRecording";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arrayFastResample, arraySeed, arrayTrimFill} from "../../../utils/arrays";
import {percentageOf} from "../../../utils/numbers";
import Waveform from "./Waveform";
import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps";
/**
* A waveform which shows the waveform of a previously recorded recording
*/
@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
export default class PlaybackWaveform extends React.PureComponent<IRecordingWaveformProps, IRecordingWaveformState> {
public constructor(props) {
super(props);
// Like the live recording waveform
const bars = arrayFastResample(this.props.recorder.finalWaveform, DOWNSAMPLE_TARGET);
const seed = arraySeed(0, DOWNSAMPLE_TARGET);
const heights = arrayTrimFill(bars, DOWNSAMPLE_TARGET, seed).map(b => percentageOf(b, 0, 0.5));
this.state = {heights};
}
public render() {
return <Waveform relHeights={this.state.heights} />;
}
}

2
src/i18n/strings/en_EN.json
View file

@ -1645,7 +1645,7 @@
"Jump to first unread message.": "Jump to first unread message.",
"Mark all as read": "Mark all as read",
"Record a voice message": "Record a voice message",
"Stop & send recording": "Stop & send recording",
"Stop the recording": "Stop the recording",
"Error updating main address": "Error updating main address",
"There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
"There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",

20
src/utils/arrays.ts
View file

@ -73,6 +73,26 @@ export function arraySeed<T>(val: T, length: number): T[] {
return a;
}
/**
* Trims or fills the array to ensure it meets the desired length. The seed array
* given is pulled from to fill any missing slots - it is recommended that this be
* at least `len` long. The resulting array will be exactly `len` long, either
* trimmed from the source or filled with the some/all of the seed array.
* @param {T[]} a The array to trim/fill.
* @param {number} len The length to trim or fill to, as needed.
* @param {T[]} seed Values to pull from if the array needs filling.
* @returns {T[]} The resulting array of `len` length.
*/
export function arrayTrimFill<T>(a: T[], len: number, seed: T[]): T[] {
// Dev note: we do length checks because the spread operator can result in some
// performance penalties in more critical code paths. As a utility, it should be
// as fast as possible to not cause a problem for the call stack, no matter how
// critical that stack is.
if (a.length === len) return a;
if (a.length > len) return a.slice(0, len);
return a.concat(seed.slice(0, len - a.length));
}
/**
* Clones an array as fast as possible, retaining references of the array's values.
* @param a The array to clone. Must be defined.

11
src/voice/VoiceRecording.ts
View file

@ -25,6 +25,7 @@ import {IDestroyable} from "../utils/IDestroyable";
import {Singleflight} from "../utils/Singleflight";
import {PayloadEvent, WORKLET_NAME} from "./consts";
import {arrayFastClone} from "../utils/arrays";
import {UPDATE_EVENT} from "../stores/AsyncStore";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
@ -79,6 +80,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return this.recorderContext.currentTime;
}
public get isRecording(): boolean {
return this.recording;
}
public emit(event: string, ...args: any[]): boolean {
super.emit(event, ...args);
super.emit(UPDATE_EVENT, event, ...args);
return true; // we don't ever care if the event had listeners, so just return "yes"
}
private async makeRecorder() {
this.recorderStream = await navigator.mediaDevices.getUserMedia({
audio: {

33
test/utils/arrays-test.ts
View file

@ -22,6 +22,7 @@ import {
arrayHasOrderChange,
arrayMerge,
arraySeed,
arrayTrimFill,
arrayUnion,
ArrayUtil,
GroupedArray,
@ -64,6 +65,38 @@ describe('arrays', () => {
});
});
describe('arrayTrimFill', () => {
it('should shrink arrays', () => {
const input = [1, 2, 3];
const output = [1, 2];
const seed = [4, 5, 6];
const result = arrayTrimFill(input, output.length, seed);
expect(result).toBeDefined();
expect(result).toHaveLength(output.length);
expect(result).toEqual(output);
});
it('should expand arrays', () => {
const input = [1, 2, 3];
const output = [1, 2, 3, 4, 5];
const seed = [4, 5, 6];
const result = arrayTrimFill(input, output.length, seed);
expect(result).toBeDefined();
expect(result).toHaveLength(output.length);
expect(result).toEqual(output);
});
it('should keep arrays the same', () => {
const input = [1, 2, 3];
const output = [1, 2, 3];
const seed = [4, 5, 6];
const result = arrayTrimFill(input, output.length, seed);
expect(result).toBeDefined();
expect(result).toHaveLength(output.length);
expect(result).toEqual(output);
});
});
describe('arraySeed', () => {
it('should create an array of given length', () => {
const val = 1;