Move playback to its own set of classes

This all started with a bug where the clock wouldn't update appropriately, and ended with a whole refactoring to support later playback in the timeline.

Playback and recording instances are now independent, and this applies to the <Playback* /> components as well. Instead of those playback components taking a recording, they take a playback instance which has all the information the components need.

The clock was incredibly difficult to do because of the audio context's time tracking and the source's inability to say where it is at in the buffer/in time. This means we have to track when we started playing the clip so we can capture the audio context's current time, which may be a few seconds by the first time the user hits play. We also track stops so we know when to reset that flag.

Waveform calculations have also been moved into the base component, deduplicating the math a bit.

res/css/_components.scss

@@ -249,6 +249,7 @@
 @import "./views/toasts/_NonUrgentEchoFailureToast.scss";
 @import "./views/verification/_VerificationShowSas.scss";
 @import "./views/voice_messages/_PlayPauseButton.scss";
+@import "./views/voice_messages/_PlaybackContainer.scss";
 @import "./views/voice_messages/_Waveform.scss";
 @import "./views/voip/_CallContainer.scss";
 @import "./views/voip/_CallView.scss";

res/css/views/rooms/_VoiceRecordComposerTile.scss

@@ -46,23 +46,14 @@ limitations under the License.
     mask-image: url('$(res)/img/element-icons/trashcan.svg');
 }
 
-.mx_VoiceRecordComposerTile_waveformContainer {
-    padding: 6px; // makes us 4px taller than the send/stop button
-    padding-right: 5px; // there's 1px from the waveform itself, so account for that
-    background-color: $voice-record-waveform-bg-color;
-    border-radius: 12px;
+.mx_VoiceMessagePrimaryContainer {
+    // Note: remaining class properties are in the PlayerContainer CSS.
+
     margin: 6px; // force the composer area to put a gutter around us
     margin-right: 12px; // isolate from stop button
 
-    // Cheat at alignment a bit
-    display: flex;
-    align-items: center;
-
     position: relative; // important for the live circle
 
-    color: $voice-record-waveform-fg-color;
-    font-size: $font-14px;
-
     &.mx_VoiceRecordComposerTile_recording {
         padding-left: 16px; // +10px for the live circle, +6px for regular padding
 
@@ -79,21 +70,6 @@ limitations under the License.
             border-radius: 10px;
         }
     }
-
-    .mx_Waveform {
-        // We want the bars to be 2px shorter than the play/pause button in the waveform control
-        height: 28px; // default is 30px, so we're subtracting the 2px border off the bars
-
-        .mx_Waveform_bar {
-            background-color: $voice-record-waveform-fg-color;
-        }
-    }
-
-    .mx_Clock {
-        padding-right: 4px; // isolate from waveform
-        padding-left: 8px; // isolate from live circle
-        width: 40px; // we're not using a monospace font, so fake it
-    }
 }
 
 // The keyframes are slightly weird here to help make a ramping/punch effect

res/css/views/voice_messages/_PlaybackContainer.scss

@@ -0,0 +1,48 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Dev note: there's no actual component called <PlaybackContainer />. These classes
+// are shared amongst multiple voice message components.
+
+// Container for live recording and playback controls
+.mx_VoiceMessagePrimaryContainer {
+    padding: 6px; // makes us 4px taller than the send/stop button
+    padding-right: 5px; // there's 1px from the waveform itself, so account for that
+    background-color: $voice-record-waveform-bg-color;
+    border-radius: 12px;
+
+    // Cheat at alignment a bit
+    display: flex;
+    align-items: center;
+
+    color: $voice-record-waveform-fg-color;
+    font-size: $font-14px;
+
+    .mx_Waveform {
+        // We want the bars to be 2px shorter than the play/pause button in the waveform control
+        height: 28px; // default is 30px, so we're subtracting the 2px border off the bars
+
+        .mx_Waveform_bar {
+            background-color: $voice-record-waveform-fg-color;
+        }
+    }
+
+    .mx_Clock {
+        padding-right: 4px; // isolate from waveform
+        padding-left: 8px; // isolate from live circle
+        width: 40px; // we're not using a monospace font, so fake it
+    }
+}

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -16,7 +16,7 @@ limitations under the License.
 
 import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
 import {_t} from "../../../languageHandler";
-import React from "react";
+import React, {ReactNode} from "react";
 import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording";
 import {Room} from "matrix-js-sdk/src/models/room";
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
@@ -26,8 +26,7 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
 import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
 import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
 import {UPDATE_EVENT} from "../../../stores/AsyncStore";
-import PlaybackWaveform from "../voice_messages/PlaybackWaveform";
-import PlayPauseButton from "../voice_messages/PlayPauseButton";
+import RecordingPlayback from "../voice_messages/RecordingPlayback";
 
 interface IProps {
     room: Room;
@@ -94,7 +93,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
                 // should come out largely sane.
                 //
                 // We're expecting about one data point per second of audio.
-                waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
+                waveform: this.state.recorder.getPlayback().waveform.map(v => Math.round(v * 1024)),
             },
         });
         await this.disposeRecording();
@@ -128,34 +127,22 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
         this.setState({recorder, recordingPhase: RecordingState.Started});
     };
 
-    private renderWaveformArea() {
-        if (!this.state.recorder) return null;
+    private renderWaveformArea(): ReactNode {
+        if (!this.state.recorder) return null; // no recorder means we're not recording: no waveform
 
-        const classes = classNames({
-            'mx_VoiceRecordComposerTile_waveformContainer': true,
-            'mx_VoiceRecordComposerTile_recording': this.state.recordingPhase === RecordingState.Started,
-        });
-
-        const clock = <LiveRecordingClock recorder={this.state.recorder} />;
-        let waveform = <LiveRecordingWaveform recorder={this.state.recorder} />;
         if (this.state.recordingPhase !== RecordingState.Started) {
-            waveform = <PlaybackWaveform recorder={this.state.recorder} />;
-        }
-
-        let playPause = null;
-        if (this.state.recordingPhase === RecordingState.Ended) {
             // TODO: @@ TR: Should we disable this during upload? What does a failed upload look like?
-            playPause = <PlayPauseButton recorder={this.state.recorder} />;
+            return <RecordingPlayback playback={this.state.recorder.getPlayback()} />;
         }
 
-        return <div className={classes}>
-            {playPause}
-            {clock}
-            {waveform}
+        // only other UI is the recording-in-progress UI
+        return <div className="mx_VoiceMessagePrimaryContainer mx_VoiceRecordComposerTile_recording">
+            <LiveRecordingClock recorder={this.state.recorder} />
+            <LiveRecordingWaveform recorder={this.state.recorder} />
         </div>;
     }
 
-    public render() {
+    public render(): ReactNode {
         let recordingInfo;
         let deleteButton;
         if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) {

src/components/views/voice_messages/Clock.tsx

@@ -29,11 +29,17 @@ interface IState {
  * displayed, making it possible to see "82:29".
  */
 @replaceableComponent("views.voice_messages.Clock")
-export default class Clock extends React.PureComponent<IProps, IState> {
+export default class Clock extends React.Component<IProps, IState> {
     public constructor(props) {
         super(props);
     }
 
+    shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
+        const currentFloor = Math.floor(this.props.seconds);
+        const nextFloor = Math.floor(nextProps.seconds);
+        return currentFloor !== nextFloor;
+    }
+
     public render() {
         const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
         const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis

src/components/views/voice_messages/IRecordingWaveformStateProps.ts

@@ -1,26 +0,0 @@
-/*
-Copyright 2021 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-import {VoiceRecording} from "../../../voice/VoiceRecording";
-
-export interface IRecordingWaveformProps {
-    recorder: VoiceRecording;
-}
-
-export interface IRecordingWaveformState {
-    heights: number[];
-}
-
-export const DOWNSAMPLE_TARGET = 35; // number of bars we want

src/components/views/voice_messages/LiveRecordingClock.tsx

@@ -31,7 +31,7 @@ interface IState {
  * A clock for a live recording.
  */
 @replaceableComponent("views.voice_messages.LiveRecordingClock")
-export default class LiveRecordingClock extends React.Component<IProps, IState> {
+export default class LiveRecordingClock extends React.PureComponent<IProps, IState> {
     public constructor(props) {
         super(props);
 
@@ -39,12 +39,6 @@ export default class LiveRecordingClock extends React.Component<IProps, IState>
         this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
     }
 
-    shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
-        const currentFloor = Math.floor(this.state.seconds);
-        const nextFloor = Math.floor(nextState.seconds);
-        return currentFloor !== nextFloor;
-    }
-
     private onRecordingUpdate = (update: IRecordingUpdate) => {
         this.setState({seconds: update.timeSeconds});
     };

src/components/views/voice_messages/LiveRecordingWaveform.tsx

@@ -20,24 +20,32 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
 import {arrayFastResample, arraySeed} from "../../../utils/arrays";
 import {percentageOf} from "../../../utils/numbers";
 import Waveform from "./Waveform";
-import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps";
+import {PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
+
+interface IProps {
+    recorder: VoiceRecording;
+}
+
+interface IState {
+    heights: number[];
+}
 
 /**
  * A waveform which shows the waveform of a live recording
  */
 @replaceableComponent("views.voice_messages.LiveRecordingWaveform")
-export default class LiveRecordingWaveform extends React.PureComponent<IRecordingWaveformProps, IRecordingWaveformState> {
+export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
     public constructor(props) {
         super(props);
 
-        this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)};
+        this.state = {heights: arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES)};
         this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
     }
 
     private onRecordingUpdate = (update: IRecordingUpdate) => {
         // The waveform and the downsample target are pretty close, so we should be fine to
         // do this, despite the docs on arrayFastResample.
-        const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
+        const bars = arrayFastResample(Array.from(update.waveform), PLAYBACK_WAVEFORM_SAMPLES);
         this.setState({
             // The incoming data is between zero and one, but typically even screaming into a
             // microphone won't send you over 0.6, so we artificially adjust the gain for the

src/components/views/voice_messages/PlayPauseButton.tsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React from "react";
+import React, {ReactNode} from "react";
 import {replaceableComponent} from "../../../utils/replaceableComponent";
 import {VoiceRecording} from "../../../voice/VoiceRecording";
 import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
@@ -24,12 +24,14 @@ import classNames from "classnames";
 import {UPDATE_EVENT} from "../../../stores/AsyncStore";
 
 interface IProps {
-    recorder: VoiceRecording;
+    // Playback instance to manipulate. Cannot change during the component lifecycle.
+    playback: Playback;
+
+    // The playback phase to render. Able to change during the component lifecycle.
+    playbackPhase: PlaybackState;
 }
 
 interface IState {
-    playback: Playback;
-    playbackPhase: PlaybackState;
 }
 
 /**
@@ -40,40 +42,16 @@ interface IState {
 export default class PlayPauseButton extends React.PureComponent<IProps, IState> {
     public constructor(props) {
         super(props);
-        this.state = {
-            playback: null, // not ready yet
-            playbackPhase: PlaybackState.Decoding,
-        };
+        this.state = {};
     }
 
-    public async componentDidMount() {
-        const playback = await this.props.recorder.getPlayback();
-        playback.on(UPDATE_EVENT, this.onPlaybackState);
-        this.setState({
-            playback: playback,
-
-            // We know the playback is no longer decoding when we get here. It'll emit an update
-            // before we've bound a listener, so we just update the state here.
-            playbackPhase: PlaybackState.Stopped,
-        });
-    }
-
-    public componentWillUnmount() {
-        if (this.state.playback) this.state.playback.off(UPDATE_EVENT, this.onPlaybackState);
-    }
-
-    private onPlaybackState = (newState: PlaybackState) => {
-        this.setState({playbackPhase: newState});
-    };
-
     private onClick = async () => {
-        if (!this.state.playback) return; // ignore for now
-        await this.state.playback.toggle();
+        await this.props.playback.toggle();
     };
 
-    public render() {
-        const isPlaying = this.state.playback?.isPlaying;
-        const isDisabled = this.state.playbackPhase === PlaybackState.Decoding;
+    public render(): ReactNode {
+        const isPlaying = this.props.playback.isPlaying;
+        const isDisabled = this.props.playbackPhase === PlaybackState.Decoding;
         const classes = classNames('mx_PlayPauseButton', {
             'mx_PlayPauseButton_play': !isPlaying,
             'mx_PlayPauseButton_pause': isPlaying,

src/components/views/voice_messages/PlaybackClock.tsx

@@ -0,0 +1,71 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import Clock from "./Clock";
+import {Playback, PlaybackState} from "../../../voice/Playback";
+import {UPDATE_EVENT} from "../../../stores/AsyncStore";
+
+interface IProps {
+    playback: Playback;
+}
+
+interface IState {
+    seconds: number;
+    durationSeconds: number;
+    playbackPhase: PlaybackState;
+}
+
+/**
+ * A clock for a playback of a recording.
+ */
+@replaceableComponent("views.voice_messages.PlaybackClock")
+export default class PlaybackClock extends React.PureComponent<IProps, IState> {
+    public constructor(props) {
+        super(props);
+
+        this.state = {
+            seconds: this.props.playback.clockInfo.timeSeconds,
+            // we track the duration on state because we won't really know what the clip duration
+            // is until the first time update, and as a PureComponent we are trying to dedupe state
+            // updates as much as possible. This is just the easiest way to avoid a forceUpdate() or
+            // member property to track "did we get a duration".
+            durationSeconds: this.props.playback.clockInfo.durationSeconds,
+            playbackPhase: PlaybackState.Stopped, // assume not started, so full clock
+        };
+        this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
+        this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
+    }
+
+    private onPlaybackUpdate = (ev: PlaybackState) => {
+        // Convert Decoding -> Stopped because we don't care about the distinction here
+        if (ev === PlaybackState.Decoding) ev = PlaybackState.Stopped;
+        this.setState({playbackPhase: ev});
+    };
+
+    private onTimeUpdate = (time: number[]) => {
+        this.setState({seconds: time[0], durationSeconds: time[1]});
+    };
+
+    public render() {
+        let seconds = this.state.seconds;
+        if (this.state.playbackPhase === PlaybackState.Stopped) {
+            seconds = this.state.durationSeconds;
+        }
+        return <Clock seconds={seconds} />;
+    }
+}

src/components/views/voice_messages/PlaybackWaveform.tsx

@@ -15,28 +15,41 @@ limitations under the License.
 */
 
 import React from "react";
-import {IRecordingUpdate, VoiceRecording} from "../../../voice/VoiceRecording";
 import {replaceableComponent} from "../../../utils/replaceableComponent";
-import {arrayFastResample, arraySeed, arrayTrimFill} from "../../../utils/arrays";
-import {percentageOf} from "../../../utils/numbers";
+import {arraySeed, arrayTrimFill} from "../../../utils/arrays";
 import Waveform from "./Waveform";
-import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps";
+import {Playback, PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
+
+interface IProps {
+    playback: Playback;
+}
+
+interface IState {
+    heights: number[];
+}
 
 /**
  * A waveform which shows the waveform of a previously recorded recording
  */
-@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
-export default class PlaybackWaveform extends React.PureComponent<IRecordingWaveformProps, IRecordingWaveformState> {
+@replaceableComponent("views.voice_messages.PlaybackWaveform")
+export default class PlaybackWaveform extends React.PureComponent<IProps, IState> {
     public constructor(props) {
         super(props);
 
-        // Like the live recording waveform
-        const bars = arrayFastResample(this.props.recorder.finalWaveform, DOWNSAMPLE_TARGET);
-        const seed = arraySeed(0, DOWNSAMPLE_TARGET);
-        const heights = arrayTrimFill(bars, DOWNSAMPLE_TARGET, seed).map(b => percentageOf(b, 0, 0.5));
-        this.state = {heights};
+        this.state = {heights: this.toHeights(this.props.playback.waveform)};
+
+        this.props.playback.waveformData.onUpdate(this.onWaveformUpdate);
     }
 
+    private toHeights(waveform: number[]) {
+        const seed = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
+        return arrayTrimFill(waveform, PLAYBACK_WAVEFORM_SAMPLES, seed);
+    }
+
+    private onWaveformUpdate = (waveform: number[]) => {
+        this.setState({heights: this.toHeights(waveform)});
+    };
+
     public render() {
         return <Waveform relHeights={this.state.heights} />;
     }

src/components/views/voice_messages/RecordingPlayback.tsx

@@ -0,0 +1,62 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {Playback, PlaybackState} from "../../../voice/Playback";
+import React, {ReactNode} from "react";
+import {UPDATE_EVENT} from "../../../stores/AsyncStore";
+import PlaybackWaveform from "./PlaybackWaveform";
+import PlayPauseButton from "./PlayPauseButton";
+import PlaybackClock from "./PlaybackClock";
+
+interface IProps {
+    // Playback instance to render. Cannot change during component lifecycle: create
+    // an all-new component instead.
+    playback: Playback;
+}
+
+interface IState {
+    playbackPhase: PlaybackState;
+}
+
+export default class RecordingPlayback extends React.PureComponent<IProps, IState> {
+    constructor(props: IProps) {
+        super(props);
+
+        this.state = {
+            playbackPhase: PlaybackState.Decoding, // default assumption
+        };
+
+        // We don't need to de-register: the class handles this for us internally
+        this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
+
+        // Don't wait for the promise to complete - it will emit a progress update when it
+        // is done, and it's not meant to take long anyhow.
+        // noinspection JSIgnoredPromiseFromCall
+        this.props.playback.prepare();
+    }
+
+    private onPlaybackUpdate = (ev: PlaybackState) => {
+        this.setState({playbackPhase: ev});
+    };
+
+    public render(): ReactNode {
+        return <div className='mx_VoiceMessagePrimaryContainer'>
+            <PlayPauseButton playback={this.props.playback} playbackPhase={this.state.playbackPhase} />
+            <PlaybackClock playback={this.props.playback} />
+            <PlaybackWaveform playback={this.props.playback} />
+        </div>
+    }
+}

src/voice/Playback.ts

@@ -16,6 +16,10 @@ limitations under the License.
 
 import EventEmitter from "events";
 import {UPDATE_EVENT} from "../stores/AsyncStore";
+import {arrayFastResample, arraySeed} from "../utils/arrays";
+import {SimpleObservable} from "matrix-widget-api";
+import {IDestroyable} from "../utils/IDestroyable";
+import {PlaybackClock} from "./PlaybackClock";
 
 export enum PlaybackState {
     Decoding = "decoding",
@@ -24,15 +28,52 @@ export enum PlaybackState {
     Playing = "playing", // active progress through timeline
 }
 
-export class Playback extends EventEmitter {
-    private context: AudioContext;
+export const PLAYBACK_WAVEFORM_SAMPLES = 35;
+const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
+
+export class Playback extends EventEmitter implements IDestroyable {
+    private readonly context: AudioContext;
     private source: AudioBufferSourceNode;
     private state = PlaybackState.Decoding;
     private audioBuf: AudioBuffer;
+    private resampledWaveform: number[];
+    private waveformObservable = new SimpleObservable<number[]>();
+    private readonly clock: PlaybackClock;
 
-    constructor(private buf: ArrayBuffer) {
+    /**
+     * Creates a new playback instance from a buffer.
+     * @param {ArrayBuffer} buf The buffer containing the sound sample.
+     * @param {number[]} seedWaveform Optional seed waveform to present until the proper waveform
+     * can be calculated. Contains values between zero and one, inclusive.
+     */
+    constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
         super();
         this.context = new AudioContext();
+        this.resampledWaveform = arrayFastResample(seedWaveform, PLAYBACK_WAVEFORM_SAMPLES);
+        this.waveformObservable.update(this.resampledWaveform);
+        this.clock = new PlaybackClock(this.context);
+
+        // TODO: @@ TR: Calculate real waveform
+    }
+
+    public get waveform(): number[] {
+        return this.resampledWaveform;
+    }
+
+    public get waveformData(): SimpleObservable<number[]> {
+        return this.waveformObservable;
+    }
+
+    public get clockInfo(): PlaybackClock {
+        return this.clock;
+    }
+
+    public get currentState(): PlaybackState {
+        return this.state;
+    }
+
+    public get isPlaying(): boolean {
+        return this.currentState === PlaybackState.Playing;
     }
 
     public emit(event: PlaybackState, ...args: any[]): boolean {
@@ -42,17 +83,18 @@ export class Playback extends EventEmitter {
         return true; // we don't ever care if the event had listeners, so just return "yes"
     }
 
+    public destroy() {
+        // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
+        this.stop();
+        this.removeAllListeners();
+        this.clock.destroy();
+        this.waveformObservable.close();
+    }
+
     public async prepare() {
         this.audioBuf = await this.context.decodeAudioData(this.buf);
         this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
-    }
-
-    public get currentState(): PlaybackState {
-        return this.state;
-    }
-
-    public get isPlaying(): boolean {
-        return this.currentState === PlaybackState.Playing;
+        this.clock.durationSeconds = this.audioBuf.duration;
     }
 
     private onPlaybackEnd = async () => {
@@ -78,6 +120,7 @@ export class Playback extends EventEmitter {
         // We use the context suspend/resume functions because it allows us to pause a source
         // node, but that still doesn't help us when the source node runs out (see above).
         await this.context.resume();
+        this.clock.flagStart();
         this.emit(PlaybackState.Playing);
     }
 
@@ -88,6 +131,7 @@ export class Playback extends EventEmitter {
 
     public async stop() {
         await this.onPlaybackEnd();
+        this.clock.flagStop();
     }
 
     public async toggle() {

src/voice/PlaybackClock.ts

@@ -0,0 +1,78 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {SimpleObservable} from "matrix-widget-api";
+import {IDestroyable} from "../utils/IDestroyable";
+
+// Because keeping track of time is sufficiently complicated...
+export class PlaybackClock implements IDestroyable {
+    private clipStart = 0;
+    private stopped = true;
+    private lastCheck = 0;
+    private observable = new SimpleObservable<number[]>();
+    private timerId: number;
+    private clipDuration = 0;
+
+    public constructor(private context: AudioContext) {
+    }
+
+    public get durationSeconds(): number {
+        return this.clipDuration;
+    }
+
+    public set durationSeconds(val: number) {
+        this.clipDuration = val;
+        this.observable.update([this.timeSeconds, this.clipDuration]);
+    }
+
+    public get timeSeconds(): number {
+        return (this.context.currentTime - this.clipStart) % this.clipDuration;
+    }
+
+    public get liveData(): SimpleObservable<number[]> {
+        return this.observable;
+    }
+
+    private checkTime = () => {
+        const now = this.timeSeconds;
+        if (this.lastCheck !== now) {
+            this.observable.update([now, this.durationSeconds]);
+            this.lastCheck = now;
+        }
+    };
+
+    public flagStart() {
+        if (this.stopped) {
+            this.clipStart = this.context.currentTime;
+            this.stopped = false;
+        }
+
+        if (!this.timerId) {
+            // case to number because the types are wrong
+            // 100ms interval to make sure the time is as accurate as possible
+            this.timerId = <number><any>setInterval(this.checkTime, 100);
+        }
+    }
+
+    public flagStop() {
+        this.stopped = true;
+    }
+
+    public destroy() {
+        this.observable.close();
+        if (this.timerId) clearInterval(this.timerId);
+    }
+}

src/voice/VoiceRecording.ts

@@ -24,7 +24,6 @@ import EventEmitter from "events";
 import {IDestroyable} from "../utils/IDestroyable";
 import {Singleflight} from "../utils/Singleflight";
 import {PayloadEvent, WORKLET_NAME} from "./consts";
-import {arrayFastClone} from "../utils/arrays";
 import {UPDATE_EVENT} from "../stores/AsyncStore";
 import {Playback} from "./Playback";
 
@@ -59,15 +58,12 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
     private recording = false;
     private observable: SimpleObservable<IRecordingUpdate>;
     private amplitudes: number[] = []; // at each second mark, generated
+    private playback: Playback;
 
     public constructor(private client: MatrixClient) {
         super();
     }
 
-    public get finalWaveform(): number[] {
-        return arrayFastClone(this.amplitudes);
-    }
-
     public get contentType(): string {
         return "audio/ogg";
     }
@@ -277,12 +273,19 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         });
     }
 
-    public getPlayback(): Promise<Playback> {
-        return Singleflight.for(this, "playback").do(async () => {
-            const playback = new Playback(this.audioBuffer.buffer); // cast to ArrayBuffer proper
-            await playback.prepare();
-            return playback;
+    /**
+     * Gets a playback instance for this voice recording. Note that the playback will not
+     * have been prepared fully, meaning the `prepare()` function needs to be called on it.
+     *
+     * The same playback instance is returned each time.
+     *
+     * @returns {Playback} The playback instance.
+     */
+    public getPlayback(): Playback {
+        this.playback = Singleflight.for(this, "playback").do(() => {
+            return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
         });
+        return this.playback;
     }
 
     public destroy() {
@@ -290,6 +293,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         this.stop();
         this.removeAllListeners();
         Singleflight.forgetAllFor(this);
+        // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
+        this.playback?.destroy();
+        this.observable.close();
     }
 
     public async upload(): Promise<string> {