Handle basic state machine of recordings

res/css/views/rooms/_VoiceRecordComposerTile.scss

@@ -36,11 +36,12 @@ limitations under the License.
 }
 
 .mx_VoiceRecordComposerTile_waveformContainer {
-    padding: 5px;
+    padding: 8px; // makes us 4px taller than the send/stop button
     padding-right: 4px; // there's 1px from the waveform itself, so account for that
     padding-left: 15px; // +10px for the live circle, +5px for regular padding
     background-color: $voice-record-waveform-bg-color;
     border-radius: 12px;
+    margin: 6px; // force the composer area to put a gutter around us
     margin-right: 12px; // isolate from stop button
 
     // Cheat at alignment a bit
@@ -52,7 +53,7 @@ limitations under the License.
     color: $voice-record-waveform-fg-color;
     font-size: $font-14px;
 
-    &::before {
+    &.mx_VoiceRecordComposerTile_recording::before {
         animation: recording-pulse 2s infinite;
 
         content: '';
@@ -61,7 +62,7 @@ limitations under the License.
         height: 10px;
         position: absolute;
         left: 8px;
-        top: 16px; // vertically center
+        top: 18px; // vertically center
         border-radius: 10px;
     }
 

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -17,7 +17,7 @@ limitations under the License.
 import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
 import {_t} from "../../../languageHandler";
 import React from "react";
-import {VoiceRecording} from "../../../voice/VoiceRecording";
+import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording";
 import {Room} from "matrix-js-sdk/src/models/room";
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
 import classNames from "classnames";
@@ -25,6 +25,8 @@ import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
 import {replaceableComponent} from "../../../utils/replaceableComponent";
 import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
 import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
+import {UPDATE_EVENT} from "../../../stores/AsyncStore";
+import PlaybackWaveform from "../voice_messages/PlaybackWaveform";
 
 interface IProps {
     room: Room;
@@ -32,6 +34,7 @@ interface IProps {
 
 interface IState {
     recorder?: VoiceRecording;
+    recordingPhase?: RecordingState;
 }
 
 /**
@@ -43,87 +46,126 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
         super(props);
 
         this.state = {
-            recorder: null, // not recording by default
+            recorder: null, // no recording started by default
         };
     }
 
-    private onStartStopVoiceMessage = async () => {
+    public async componentWillUnmount() {
-        // TODO: @@ TravisR: We do not want to auto-send on stop.
+        await VoiceRecordingStore.instance.disposeRecording();
+    }
+
+    // called by composer
+    public async send() {
+        if (!this.state.recorder) {
+            throw new Error("No recording started - cannot send anything");
+        }
+
+        await this.state.recorder.stop();
+        const mxc = await this.state.recorder.upload();
+        MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
+            "body": "Voice message",
+            "msgtype": "org.matrix.msc2516.voice",
+            //"msgtype": MsgType.Audio,
+            "url": mxc,
+            "info": {
+                duration: Math.round(this.state.recorder.durationSeconds * 1000),
+                mimetype: this.state.recorder.contentType,
+                size: this.state.recorder.contentLength,
+            },
+
+            // MSC1767 experiment
+            "org.matrix.msc1767.text": "Voice message",
+            "org.matrix.msc1767.file": {
+                url: mxc,
+                name: "Voice message.ogg",
+                mimetype: this.state.recorder.contentType,
+                size: this.state.recorder.contentLength,
+            },
+            "org.matrix.msc1767.audio": {
+                duration: Math.round(this.state.recorder.durationSeconds * 1000),
+                // TODO: @@ TravisR: Waveform? (MSC1767 decision)
+            },
+            "org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
+                duration: Math.round(this.state.recorder.durationSeconds * 1000),
+
+                // Events can't have floats, so we try to maintain resolution by using 1024
+                // as a maximum value. The waveform contains values between zero and 1, so this
+                // should come out largely sane.
+                //
+                // We're expecting about one data point per second of audio.
+                waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
+            },
+        });
+        await VoiceRecordingStore.instance.disposeRecording();
+        this.setState({recorder: null});
+    }
+
+    private onRecordStartEndClick = async () => {
         if (this.state.recorder) {
             await this.state.recorder.stop();
-            const mxc = await this.state.recorder.upload();
-            MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
-                "body": "Voice message",
-                "msgtype": "org.matrix.msc2516.voice",
-                //"msgtype": MsgType.Audio,
-                "url": mxc,
-                "info": {
-                    duration: Math.round(this.state.recorder.durationSeconds * 1000),
-                    mimetype: this.state.recorder.contentType,
-                    size: this.state.recorder.contentLength,
-                },
-
-                // MSC1767 experiment
-                "org.matrix.msc1767.text": "Voice message",
-                "org.matrix.msc1767.file": {
-                    url: mxc,
-                    name: "Voice message.ogg",
-                    mimetype: this.state.recorder.contentType,
-                    size: this.state.recorder.contentLength,
-                },
-                "org.matrix.msc1767.audio": {
-                    duration: Math.round(this.state.recorder.durationSeconds * 1000),
-                    // TODO: @@ TravisR: Waveform? (MSC1767 decision)
-                },
-                "org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
-                    duration: Math.round(this.state.recorder.durationSeconds * 1000),
-
-                    // Events can't have floats, so we try to maintain resolution by using 1024
-                    // as a maximum value. The waveform contains values between zero and 1, so this
-                    // should come out largely sane.
-                    //
-                    // We're expecting about one data point per second of audio.
-                    waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
-                },
-            });
-            await VoiceRecordingStore.instance.disposeRecording();
-            this.setState({recorder: null});
             return;
         }
         const recorder = VoiceRecordingStore.instance.startRecording();
         await recorder.start();
-        this.setState({recorder});
+
+        // We don't need to remove the listener: the recorder will clean that up for us.
+        recorder.on(UPDATE_EVENT, (ev: RecordingState) => {
+            if (ev === RecordingState.EndingSoon) return; // ignore this state: it has no UI purpose here
+            this.setState({recordingPhase: ev});
+        });
+
+        this.setState({recorder, recordingPhase: RecordingState.Started});
     };
 
     private renderWaveformArea() {
         if (!this.state.recorder) return null;
 
-        return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
+        const classes = classNames({
-            <LiveRecordingClock recorder={this.state.recorder} />
+            'mx_VoiceRecordComposerTile_waveformContainer': true,
-            <LiveRecordingWaveform recorder={this.state.recorder} />
+            'mx_VoiceRecordComposerTile_recording': this.state.recordingPhase === RecordingState.Started,
+        });
+
+        const clock = <LiveRecordingClock recorder={this.state.recorder} />;
+        let waveform = <LiveRecordingWaveform recorder={this.state.recorder} />;
+        if (this.state.recordingPhase !== RecordingState.Started) {
+            waveform = <PlaybackWaveform recorder={this.state.recorder} />;
+        }
+
+        return <div className={classes}>
+            {clock}
+            {waveform}
         </div>;
     }
 
     public render() {
-        const classes = classNames({
+        let recordingInfo;
-            'mx_MessageComposer_button': !this.state.recorder,
+        if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) {
-            'mx_MessageComposer_voiceMessage': !this.state.recorder,
+            const classes = classNames({
-            'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
+                'mx_MessageComposer_button': !this.state.recorder,
-        });
+                'mx_MessageComposer_voiceMessage': !this.state.recorder,
+                'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording,
+            });
 
-        let tooltip = _t("Record a voice message");
+            let tooltip = _t("Record a voice message");
-        if (!!this.state.recorder) {
+            if (!!this.state.recorder) {
-            // TODO: @@ TravisR: Change to match behaviour
+                tooltip = _t("Stop the recording");
-            tooltip = _t("Stop & send recording");
+            }
+
+            let stopOrRecordBtn = <AccessibleTooltipButton
+                className={classes}
+                onClick={this.onRecordStartEndClick}
+                title={tooltip}
+            />;
+            if (this.state.recorder && !this.state.recorder?.isRecording) {
+                stopOrRecordBtn = null;
+            }
+
+            recordingInfo = stopOrRecordBtn;
         }
 
         return (<>
             {this.renderWaveformArea()}
-            <AccessibleTooltipButton
+            {recordingInfo}
-                className={classes}
-                onClick={this.onStartStopVoiceMessage}
-                title={tooltip}
-            />
         </>);
     }
 }

src/components/views/voice_messages/IRecordingWaveformStateProps.ts

@@ -0,0 +1,26 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {VoiceRecording} from "../../../voice/VoiceRecording";
+
+export interface IRecordingWaveformProps {
+    recorder: VoiceRecording;
+}
+
+export interface IRecordingWaveformState {
+    heights: number[];
+}
+
+export const DOWNSAMPLE_TARGET = 35; // number of bars we want

src/components/views/voice_messages/LiveRecordingWaveform.tsx

@@ -20,22 +20,13 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
 import {arrayFastResample, arraySeed} from "../../../utils/arrays";
 import {percentageOf} from "../../../utils/numbers";
 import Waveform from "./Waveform";
-
+import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps";
-interface IProps {
-    recorder: VoiceRecording;
-}
-
-interface IState {
-    heights: number[];
-}
-
-const DOWNSAMPLE_TARGET = 35; // number of bars we want
 
 /**
  * A waveform which shows the waveform of a live recording
  */
 @replaceableComponent("views.voice_messages.LiveRecordingWaveform")
-export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
+export default class LiveRecordingWaveform extends React.PureComponent<IRecordingWaveformProps, IRecordingWaveformState> {
     public constructor(props) {
         super(props);
 

src/components/views/voice_messages/PlaybackWaveform.tsx

@@ -0,0 +1,43 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {IRecordingUpdate, VoiceRecording} from "../../../voice/VoiceRecording";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import {arrayFastResample, arraySeed, arrayTrimFill} from "../../../utils/arrays";
+import {percentageOf} from "../../../utils/numbers";
+import Waveform from "./Waveform";
+import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps";
+
+/**
+ * A waveform which shows the waveform of a previously recorded recording
+ */
+@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
+export default class PlaybackWaveform extends React.PureComponent<IRecordingWaveformProps, IRecordingWaveformState> {
+    public constructor(props) {
+        super(props);
+
+        // Like the live recording waveform
+        const bars = arrayFastResample(this.props.recorder.finalWaveform, DOWNSAMPLE_TARGET);
+        const seed = arraySeed(0, DOWNSAMPLE_TARGET);
+        const heights = arrayTrimFill(bars, DOWNSAMPLE_TARGET, seed).map(b => percentageOf(b, 0, 0.5));
+        this.state = {heights};
+    }
+
+    public render() {
+        return <Waveform relHeights={this.state.heights} />;
+    }
+}

src/i18n/strings/en_EN.json

@@ -1645,7 +1645,7 @@
     "Jump to first unread message.": "Jump to first unread message.",
     "Mark all as read": "Mark all as read",
     "Record a voice message": "Record a voice message",
-    "Stop & send recording": "Stop & send recording",
+    "Stop the recording": "Stop the recording",
     "Error updating main address": "Error updating main address",
     "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
     "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",

src/utils/arrays.ts

@@ -73,6 +73,26 @@ export function arraySeed<T>(val: T, length: number): T[] {
     return a;
 }
 
+/**
+ * Trims or fills the array to ensure it meets the desired length. The seed array
+ * given is pulled from to fill any missing slots - it is recommended that this be
+ * at least `len` long. The resulting array will be exactly `len` long, either
+ * trimmed from the source or filled with the some/all of the seed array.
+ * @param {T[]} a The array to trim/fill.
+ * @param {number} len The length to trim or fill to, as needed.
+ * @param {T[]} seed Values to pull from if the array needs filling.
+ * @returns {T[]} The resulting array of `len` length.
+ */
+export function arrayTrimFill<T>(a: T[], len: number, seed: T[]): T[] {
+    // Dev note: we do length checks because the spread operator can result in some
+    // performance penalties in more critical code paths. As a utility, it should be
+    // as fast as possible to not cause a problem for the call stack, no matter how
+    // critical that stack is.
+    if (a.length === len) return a;
+    if (a.length > len) return a.slice(0, len);
+    return a.concat(seed.slice(0, len - a.length));
+}
+
 /**
  * Clones an array as fast as possible, retaining references of the array's values.
  * @param a The array to clone. Must be defined.

src/voice/VoiceRecording.ts

@@ -25,6 +25,7 @@ import {IDestroyable} from "../utils/IDestroyable";
 import {Singleflight} from "../utils/Singleflight";
 import {PayloadEvent, WORKLET_NAME} from "./consts";
 import {arrayFastClone} from "../utils/arrays";
+import {UPDATE_EVENT} from "../stores/AsyncStore";
 
 const CHANNELS = 1; // stereo isn't important
 const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
@@ -79,6 +80,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         return this.recorderContext.currentTime;
     }
 
+    public get isRecording(): boolean {
+        return this.recording;
+    }
+
+    public emit(event: string, ...args: any[]): boolean {
+        super.emit(event, ...args);
+        super.emit(UPDATE_EVENT, event, ...args);
+        return true; // we don't ever care if the event had listeners, so just return "yes"
+    }
+
     private async makeRecorder() {
         this.recorderStream = await navigator.mediaDevices.getUserMedia({
             audio: {

test/utils/arrays-test.ts

@@ -22,6 +22,7 @@ import {
     arrayHasOrderChange,
     arrayMerge,
     arraySeed,
+    arrayTrimFill,
     arrayUnion,
     ArrayUtil,
     GroupedArray,
@@ -64,6 +65,38 @@ describe('arrays', () => {
         });
     });
 
+    describe('arrayTrimFill', () => {
+        it('should shrink arrays', () => {
+            const input = [1, 2, 3];
+            const output = [1, 2];
+            const seed = [4, 5, 6];
+            const result = arrayTrimFill(input, output.length, seed);
+            expect(result).toBeDefined();
+            expect(result).toHaveLength(output.length);
+            expect(result).toEqual(output);
+        });
+
+        it('should expand arrays', () => {
+            const input = [1, 2, 3];
+            const output = [1, 2, 3, 4, 5];
+            const seed = [4, 5, 6];
+            const result = arrayTrimFill(input, output.length, seed);
+            expect(result).toBeDefined();
+            expect(result).toHaveLength(output.length);
+            expect(result).toEqual(output);
+        });
+
+        it('should keep arrays the same', () => {
+            const input = [1, 2, 3];
+            const output = [1, 2, 3];
+            const seed = [4, 5, 6];
+            const result = arrayTrimFill(input, output.length, seed);
+            expect(result).toBeDefined();
+            expect(result).toHaveLength(output.length);
+            expect(result).toEqual(output);
+        });
+    });
+
     describe('arraySeed', () => {
         it('should create an array of given length', () => {
             const val = 1;