Merge pull request #6710 from matrix-org/travis/voice-messages/audio-queue

Add autoplay for voice messages

src/audio/ManagedPlayback.ts

@@ -26,7 +26,7 @@ export class ManagedPlayback extends Playback {
     }
 
     public async play(): Promise<void> {
-        this.manager.playOnly(this);
+        this.manager.pauseAllExcept(this);
         return super.play();
     }
 

src/audio/Playback.ts

@@ -117,6 +117,8 @@ export class Playback extends EventEmitter implements IDestroyable {
     }
 
     public destroy() {
+        // Dev note: It's critical that we call stop() during cleanup to ensure that downstream callers
+        // are aware of the final clock position before the user triggered an unload.
         // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
         this.stop();
         this.removeAllListeners();
@@ -177,9 +179,12 @@ export class Playback extends EventEmitter implements IDestroyable {
 
         this.waveformObservable.update(this.resampledWaveform);
 
-        this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
         this.clock.flagLoadTime(); // must happen first because setting the duration fires a clock update
         this.clock.durationSeconds = this.element ? this.element.duration : this.audioBuf.duration;
+
+        // Signal that we're not decoding anymore. This is done last to ensure the clock is updated for
+        // when the downstream callers try to use it.
+        this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
     }
 
     private onPlaybackEnd = async () => {

src/audio/PlaybackClock.ts

@@ -89,9 +89,9 @@ export class PlaybackClock implements IDestroyable {
         return this.observable;
     }
 
-    private checkTime = () => {
+    private checkTime = (force = false) => {
         const now = this.timeSeconds; // calculated dynamically
-        if (this.lastCheck !== now) {
+        if (this.lastCheck !== now || force) {
             this.observable.update([now, this.durationSeconds]);
             this.lastCheck = now;
         }
@@ -141,7 +141,7 @@ export class PlaybackClock implements IDestroyable {
     public syncTo(contextTime: number, clipTime: number) {
         this.clipStart = contextTime - clipTime;
         this.stopped = false; // count as a mid-stream pause (if we were stopped)
-        this.checkTime();
+        this.checkTime(true);
     }
 
     public destroy() {

src/audio/PlaybackManager.ts

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { DEFAULT_WAVEFORM, Playback } from "./Playback";
+import { DEFAULT_WAVEFORM, Playback, PlaybackState } from "./Playback";
 import { ManagedPlayback } from "./ManagedPlayback";
 
 /**
@@ -34,12 +34,14 @@ export class PlaybackManager {
     }
 
     /**
-     * Stops all other playback instances. If no playback is provided, all instances
-     * are stopped.
+     * Pauses all other playback instances. If no playback is provided, all playing
+     * instances are paused.
      * @param playback Optional. The playback to leave untouched.
      */
-    public playOnly(playback?: Playback) {
-        this.instances.filter(p => p !== playback).forEach(p => p.stop());
+    public pauseAllExcept(playback?: Playback) {
+        this.instances
+            .filter(p => p !== playback && p.currentState === PlaybackState.Playing)
+            .forEach(p => p.pause());
     }
 
     public destroyPlaybackInstance(playback: ManagedPlayback) {

src/audio/PlaybackQueue.ts

@@ -0,0 +1,212 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk";
+import { Playback, PlaybackState } from "./Playback";
+import { UPDATE_EVENT } from "../stores/AsyncStore";
+import { MatrixClientPeg } from "../MatrixClientPeg";
+import { arrayFastClone } from "../utils/arrays";
+import { PlaybackManager } from "./PlaybackManager";
+import { isVoiceMessage } from "../utils/EventUtils";
+import RoomViewStore from "../stores/RoomViewStore";
+
+/**
+ * Audio playback queue management for a given room. This keeps track of where the user
+ * was at for each playback, what order the playbacks were played in, and triggers subsequent
+ * playbacks.
+ *
+ * Currently this is only intended to be used by voice messages.
+ *
+ * The primary mechanics are:
+ * * Persisted clock state for each playback instance (tied to Event ID).
+ * * Limited memory of playback order (see code; not persisted).
+ * * Autoplay of next eligible playback instance.
+ */
+export class PlaybackQueue {
+    private static queues = new Map<string, PlaybackQueue>(); // keyed by room ID
+
+    private playbacks = new Map<string, Playback>(); // keyed by event ID
+    private clockStates = new Map<string, number>(); // keyed by event ID
+    private playbackIdOrder: string[] = []; // event IDs, last == current
+    private currentPlaybackId: string; // event ID, broken out from above for ease of use
+    private recentFullPlays = new Set<string>(); // event IDs
+
+    constructor(private client: MatrixClient, private room: Room) {
+        this.loadClocks();
+
+        RoomViewStore.addListener(() => {
+            if (RoomViewStore.getRoomId() === this.room.roomId) {
+                // Reset the state of the playbacks before they start mounting and enqueuing updates.
+                // We reset the entirety of the queue, including order, to ensure the user isn't left
+                // confused with what order the messages are playing in.
+                this.currentPlaybackId = null; // this in particular stops autoplay when the room is switched to
+                this.recentFullPlays = new Set<string>();
+                this.playbackIdOrder = [];
+            }
+        });
+    }
+
+    public static forRoom(roomId: string): PlaybackQueue {
+        const cli = MatrixClientPeg.get();
+        const room = cli.getRoom(roomId);
+        if (!room) throw new Error("Unknown room");
+        if (PlaybackQueue.queues.has(room.roomId)) {
+            return PlaybackQueue.queues.get(room.roomId);
+        }
+        const queue = new PlaybackQueue(cli, room);
+        PlaybackQueue.queues.set(room.roomId, queue);
+        return queue;
+    }
+
+    private persistClocks() {
+        localStorage.setItem(
+            `mx_voice_message_clocks_${this.room.roomId}`,
+            JSON.stringify(Array.from(this.clockStates.entries())),
+        );
+    }
+
+    private loadClocks() {
+        const val = localStorage.getItem(`mx_voice_message_clocks_${this.room.roomId}`);
+        if (!!val) {
+            this.clockStates = new Map<string, number>(JSON.parse(val));
+        }
+    }
+
+    public unsortedEnqueue(mxEvent: MatrixEvent, playback: Playback) {
+        // We don't ever detach our listeners: we expect the Playback to clean up for us
+        this.playbacks.set(mxEvent.getId(), playback);
+        playback.on(UPDATE_EVENT, (state) => this.onPlaybackStateChange(playback, mxEvent, state));
+        playback.clockInfo.liveData.onUpdate((clock) => this.onPlaybackClock(playback, mxEvent, clock));
+    }
+
+    private onPlaybackStateChange(playback: Playback, mxEvent: MatrixEvent, newState: PlaybackState) {
+        // Remember where the user got to in playback
+        const wasLastPlaying = this.currentPlaybackId === mxEvent.getId();
+        if (newState === PlaybackState.Stopped && this.clockStates.has(mxEvent.getId()) && !wasLastPlaying) {
+            // noinspection JSIgnoredPromiseFromCall
+            playback.skipTo(this.clockStates.get(mxEvent.getId()));
+        } else if (newState === PlaybackState.Stopped) {
+            // Remove the now-useless clock for some space savings
+            this.clockStates.delete(mxEvent.getId());
+
+            if (wasLastPlaying) {
+                this.recentFullPlays.add(this.currentPlaybackId);
+                const orderClone = arrayFastClone(this.playbackIdOrder);
+                const last = orderClone.pop();
+                if (last === this.currentPlaybackId) {
+                    const next = orderClone.pop();
+                    if (next) {
+                        const instance = this.playbacks.get(next);
+                        if (!instance) {
+                            console.warn(
+                                "Voice message queue desync: Missing playback for next message: "
+                                + `Current=${this.currentPlaybackId} Last=${last} Next=${next}`,
+                            );
+                        } else {
+                            this.playbackIdOrder = orderClone;
+                            PlaybackManager.instance.pauseAllExcept(instance);
+
+                            // This should cause a Play event, which will re-populate our playback order
+                            // and update our current playback ID.
+                            // noinspection JSIgnoredPromiseFromCall
+                            instance.play();
+                        }
+                    } else {
+                        // else no explicit next event, so find an event we haven't played that comes next. The live
+                        // timeline is already most recent last, so we can iterate down that.
+                        const timeline = arrayFastClone(this.room.getLiveTimeline().getEvents());
+                        let scanForVoiceMessage = false;
+                        let nextEv: MatrixEvent;
+                        for (const event of timeline) {
+                            if (event.getId() === mxEvent.getId()) {
+                                scanForVoiceMessage = true;
+                                continue;
+                            }
+                            if (!scanForVoiceMessage) continue;
+
+                            // Dev note: This is where we'd break to cause text/non-voice messages to
+                            // interrupt automatic playback.
+
+                            const isRightType = isVoiceMessage(event);
+                            const havePlayback = this.playbacks.has(event.getId());
+                            const isRecentlyCompleted = this.recentFullPlays.has(event.getId());
+                            if (isRightType && havePlayback && !isRecentlyCompleted) {
+                                nextEv = event;
+                                break;
+                            }
+                        }
+                        if (!nextEv) {
+                            // if we don't have anywhere to go, reset the recent playback queue so the user
+                            // can start a new chain of playbacks.
+                            this.recentFullPlays = new Set<string>();
+                            this.playbackIdOrder = [];
+                        } else {
+                            this.playbackIdOrder = orderClone;
+
+                            const instance = this.playbacks.get(nextEv.getId());
+                            PlaybackManager.instance.pauseAllExcept(instance);
+
+                            // This should cause a Play event, which will re-populate our playback order
+                            // and update our current playback ID.
+                            // noinspection JSIgnoredPromiseFromCall
+                            instance.play();
+                        }
+                    }
+                } else {
+                    console.warn(
+                        "Voice message queue desync: Expected playback stop to be last in order. "
+                        + `Current=${this.currentPlaybackId} Last=${last} EventID=${mxEvent.getId()}`,
+                    );
+                }
+            }
+        }
+
+        if (newState === PlaybackState.Playing) {
+            const order = this.playbackIdOrder;
+            if (this.currentPlaybackId !== mxEvent.getId() && !!this.currentPlaybackId) {
+                if (order.length === 0 || order[order.length - 1] !== this.currentPlaybackId) {
+                    const lastInstance = this.playbacks.get(this.currentPlaybackId);
+                    if (
+                        lastInstance.currentState === PlaybackState.Playing
+                        || lastInstance.currentState === PlaybackState.Paused
+                    ) {
+                        order.push(this.currentPlaybackId);
+                    }
+                }
+            }
+
+            this.currentPlaybackId = mxEvent.getId();
+            if (order.length === 0 || order[order.length - 1] !== this.currentPlaybackId) {
+                order.push(this.currentPlaybackId);
+            }
+        }
+
+        // Only persist clock information on pause/stop (end) to avoid overwhelming the storage.
+        // This should get triggered from normal voice message component unmount due to the playback
+        // stopping itself for cleanup.
+        if (newState === PlaybackState.Paused || newState === PlaybackState.Stopped) {
+            this.persistClocks();
+        }
+    }
+
+    private onPlaybackClock(playback: Playback, mxEvent: MatrixEvent, clocks: number[]) {
+        if (playback.currentState === PlaybackState.Decoding) return; // ignore pre-ready values
+
+        if (playback.currentState !== PlaybackState.Stopped) {
+            this.clockStates.set(mxEvent.getId(), clocks[0]); // [0] is the current seek position
+        }
+    }
+}

src/components/views/messages/MAudioBody.tsx

@@ -24,6 +24,8 @@ import { IMediaEventContent } from "../../../customisations/models/IMediaEventCo
 import MFileBody from "./MFileBody";
 import { IBodyProps } from "./IBodyProps";
 import { PlaybackManager } from "../../../audio/PlaybackManager";
+import { isVoiceMessage } from "../../../utils/EventUtils";
+import { PlaybackQueue } from "../../../audio/PlaybackQueue";
 
 interface IState {
     error?: Error;
@@ -67,6 +69,10 @@ export default class MAudioBody extends React.PureComponent<IBodyProps, IState>
         playback.clockInfo.populatePlaceholdersFrom(this.props.mxEvent);
         this.setState({ playback });
 
+        if (isVoiceMessage(this.props.mxEvent)) {
+            PlaybackQueue.forRoom(this.props.mxEvent.getRoomId()).unsortedEnqueue(this.props.mxEvent, playback);
+        }
+
         // Note: the components later on will handle preparing the Playback class for us.
     }
 

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -179,7 +179,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
 
         try {
             // stop any noises which might be happening
-            await PlaybackManager.instance.playOnly(null);
+            await PlaybackManager.instance.pauseAllExcept(null);
 
             const recorder = VoiceRecordingStore.instance.startRecording();
             await recorder.start();