Implement broadcast UTD handling (#10021)

src/events/EventTileFactory.tsx

@@ -48,7 +48,11 @@ import ViewSourceEvent from "../components/views/messages/ViewSourceEvent";
 import { shouldDisplayAsBeaconTile } from "../utils/beacon/timeline";
 import { shouldDisplayAsVoiceBroadcastTile } from "../voice-broadcast/utils/shouldDisplayAsVoiceBroadcastTile";
 import { ElementCall } from "../models/Call";
-import { shouldDisplayAsVoiceBroadcastStoppedText, VoiceBroadcastChunkEventType } from "../voice-broadcast";
+import {
+    isRelatedToVoiceBroadcast,
+    shouldDisplayAsVoiceBroadcastStoppedText,
+    VoiceBroadcastChunkEventType,
+} from "../voice-broadcast";
 
 // Subset of EventTile's IProps plus some mixins
 export interface EventTileTypeProps {
@@ -74,13 +78,13 @@ export interface EventTileTypeProps {
 type FactoryProps = Omit<EventTileTypeProps, "ref">;
 type Factory<X = FactoryProps> = (ref: Optional<React.RefObject<any>>, props: X) => JSX.Element;
 
-const MessageEventFactory: Factory = (ref, props) => <MessageEvent ref={ref} {...props} />;
+export const MessageEventFactory: Factory = (ref, props) => <MessageEvent ref={ref} {...props} />;
 const KeyVerificationConclFactory: Factory = (ref, props) => <MKeyVerificationConclusion ref={ref} {...props} />;
 const LegacyCallEventFactory: Factory<FactoryProps & { callEventGrouper: LegacyCallEventGrouper }> = (ref, props) => (
     <LegacyCallEvent ref={ref} {...props} />
 );
 const CallEventFactory: Factory = (ref, props) => <CallEvent ref={ref} {...props} />;
-const TextualEventFactory: Factory = (ref, props) => <TextualEvent ref={ref} {...props} />;
+export const TextualEventFactory: Factory = (ref, props) => <TextualEvent ref={ref} {...props} />;
 const VerificationReqFactory: Factory = (ref, props) => <MKeyVerificationRequest ref={ref} {...props} />;
 const HiddenEventFactory: Factory = (ref, props) => <HiddenBody ref={ref} {...props} />;
 
@@ -260,6 +264,11 @@ export function pickFactory(
         return noEventFactoryFactory();
     }
 
+    if (!showHiddenEvents && mxEvent.isDecryptionFailure() && isRelatedToVoiceBroadcast(mxEvent, cli)) {
+        // hide utd events related to a broadcast
+        return noEventFactoryFactory();
+    }
+
     return EVENT_TILE_TYPES.get(evType) ?? noEventFactoryFactory();
 }
 

src/i18n/strings/en_EN.json

@@ -659,6 +659,7 @@
     "%(senderName)s ended a <a>voice broadcast</a>": "%(senderName)s ended a <a>voice broadcast</a>",
     "You ended a voice broadcast": "You ended a voice broadcast",
     "%(senderName)s ended a voice broadcast": "%(senderName)s ended a voice broadcast",
+    "Unable to decrypt voice broadcast": "Unable to decrypt voice broadcast",
     "Unable to play this voice broadcast": "Unable to play this voice broadcast",
     "Stop live broadcasting?": "Stop live broadcasting?",
     "Are you sure you want to stop your live broadcast?This will end the broadcast and the full recording will be available in the room.": "Are you sure you want to stop your live broadcast?This will end the broadcast and the full recording will be available in the room.",

src/voice-broadcast/index.ts

@@ -52,6 +52,7 @@ export * from "./utils/doMaybeSetCurrentVoiceBroadcastPlayback";
 export * from "./utils/getChunkLength";
 export * from "./utils/getMaxBroadcastLength";
 export * from "./utils/hasRoomLiveVoiceBroadcast";
+export * from "./utils/isRelatedToVoiceBroadcast";
 export * from "./utils/isVoiceBroadcastStartedEvent";
 export * from "./utils/findRoomLiveVoiceBroadcastFromUserAndDevice";
 export * from "./utils/retrieveStartedInfoEvent";

src/voice-broadcast/models/VoiceBroadcastPlayback.ts

@@ -82,6 +82,8 @@ export class VoiceBroadcastPlayback
 {
     private state = VoiceBroadcastPlaybackState.Stopped;
     private chunkEvents = new VoiceBroadcastChunkEvents();
+    /** @var Map: event Id → undecryptable event */
+    private utdChunkEvents: Map<string, MatrixEvent> = new Map();
     private playbacks = new Map<string, Playback>();
     private currentlyPlaying: MatrixEvent | null = null;
     /** @var total duration of all chunks in milliseconds */
@@ -154,13 +156,18 @@ export class VoiceBroadcastPlayback
     }
 
     private addChunkEvent = async (event: MatrixEvent): Promise<boolean> => {
-        if (event.getContent()?.msgtype !== MsgType.Audio) {
+        if (!event.getId() && !event.getTxnId()) {
-            // skip non-audio event
+            // skip events without id and txn id
             return false;
         }
 
-        if (!event.getId() && !event.getTxnId()) {
+        if (event.isDecryptionFailure()) {
-            // skip events without id and txn id
+            this.onChunkEventDecryptionFailure(event);
+            return false;
+        }
+
+        if (event.getContent()?.msgtype !== MsgType.Audio) {
+            // skip non-audio event
             return false;
         }
 
@@ -174,6 +181,45 @@ export class VoiceBroadcastPlayback
         return true;
     };
 
+    private onChunkEventDecryptionFailure = (event: MatrixEvent): void => {
+        const eventId = event.getId();
+
+        if (!eventId) {
+            // This should not happen, as the existence of the Id is checked before the call.
+            // Log anyway and return.
+            logger.warn("Broadcast chunk decryption failure for event without Id", {
+                broadcast: this.infoEvent.getId(),
+            });
+            return;
+        }
+
+        if (!this.utdChunkEvents.has(eventId)) {
+            event.once(MatrixEventEvent.Decrypted, this.onChunkEventDecrypted);
+        }
+
+        this.utdChunkEvents.set(eventId, event);
+        this.setError();
+    };
+
+    private onChunkEventDecrypted = async (event: MatrixEvent): Promise<void> => {
+        const eventId = event.getId();
+
+        if (!eventId) {
+            // This should not happen, as the existence of the Id is checked before the call.
+            // Log anyway and return.
+            logger.warn("Broadcast chunk decrypted for event without Id", { broadcast: this.infoEvent.getId() });
+            return;
+        }
+
+        this.utdChunkEvents.delete(eventId);
+        await this.addChunkEvent(event);
+
+        if (this.utdChunkEvents.size === 0) {
+            // no more UTD events, recover from error to paused
+            this.setState(VoiceBroadcastPlaybackState.Paused);
+        }
+    };
+
     private startOrPlayNext = async (): Promise<void> => {
         if (this.currentlyPlaying) {
             return this.playNext();
@@ -210,7 +256,7 @@ export class VoiceBroadcastPlayback
     private async tryLoadPlayback(chunkEvent: MatrixEvent): Promise<void> {
         try {
             return await this.loadPlayback(chunkEvent);
-        } catch (err) {
+        } catch (err: any) {
             logger.warn("Unable to load broadcast playback", {
                 message: err.message,
                 broadcastId: this.infoEvent.getId(),
@@ -332,7 +378,7 @@ export class VoiceBroadcastPlayback
     private async tryGetOrLoadPlaybackForEvent(event: MatrixEvent): Promise<Playback | undefined> {
         try {
             return await this.getOrLoadPlaybackForEvent(event);
-        } catch (err) {
+        } catch (err: any) {
             logger.warn("Unable to load broadcast playback", {
                 message: err.message,
                 broadcastId: this.infoEvent.getId(),
@@ -551,9 +597,6 @@ export class VoiceBroadcastPlayback
     }
 
     private setState(state: VoiceBroadcastPlaybackState): void {
-        // error is a final state
-        if (this.getState() === VoiceBroadcastPlaybackState.Error) return;
-
         if (this.state === state) {
             return;
         }
@@ -587,10 +630,18 @@ export class VoiceBroadcastPlayback
     }
 
     public get errorMessage(): string {
-        return this.getState() === VoiceBroadcastPlaybackState.Error ? _t("Unable to play this voice broadcast") : "";
+        if (this.getState() !== VoiceBroadcastPlaybackState.Error) return "";
+        if (this.utdChunkEvents.size) return _t("Unable to decrypt voice broadcast");
+        return _t("Unable to play this voice broadcast");
     }
 
     public destroy(): void {
+        for (const [, utdEvent] of this.utdChunkEvents) {
+            utdEvent.off(MatrixEventEvent.Decrypted, this.onChunkEventDecrypted);
+        }
+
+        this.utdChunkEvents.clear();
+
         this.chunkRelationHelper.destroy();
         this.infoRelationHelper.destroy();
         this.removeAllListeners();

src/voice-broadcast/utils/isRelatedToVoiceBroadcast.ts

@@ -0,0 +1,29 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MatrixClient, MatrixEvent, RelationType } from "matrix-js-sdk/src/matrix";
+
+import { VoiceBroadcastInfoEventType } from "../types";
+
+export const isRelatedToVoiceBroadcast = (event: MatrixEvent, client: MatrixClient): boolean => {
+    const relation = event.getRelation();
+
+    return (
+        relation?.rel_type === RelationType.Reference &&
+        !!relation.event_id &&
+        client.getRoom(event.getRoomId())?.findEventById(relation.event_id)?.getType() === VoiceBroadcastInfoEventType
+    );
+};

test/events/EventTileFactory-test.ts

@@ -11,9 +11,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { EventType, MatrixClient, MatrixEvent, MsgType } from "matrix-js-sdk/src/matrix";
+import { mocked } from "jest-mock";
+import { EventType, MatrixClient, MatrixEvent, MsgType, RelationType, Room } from "matrix-js-sdk/src/matrix";
 
-import { JSONEventFactory, pickFactory } from "../../src/events/EventTileFactory";
+import {
+    JSONEventFactory,
+    MessageEventFactory,
+    pickFactory,
+    TextualEventFactory,
+} from "../../src/events/EventTileFactory";
 import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoState } from "../../src/voice-broadcast";
 import { createTestClient, mkEvent } from "../test-utils";
 import { mkVoiceBroadcastInfoStateEvent } from "../voice-broadcast/utils/test-utils";
@@ -21,15 +27,32 @@ import { mkVoiceBroadcastInfoStateEvent } from "../voice-broadcast/utils/test-ut
 const roomId = "!room:example.com";
 
 describe("pickFactory", () => {
+    let voiceBroadcastStartedEvent: MatrixEvent;
     let voiceBroadcastStoppedEvent: MatrixEvent;
     let voiceBroadcastChunkEvent: MatrixEvent;
+    let utdEvent: MatrixEvent;
+    let utdBroadcastChunkEvent: MatrixEvent;
     let audioMessageEvent: MatrixEvent;
     let client: MatrixClient;
 
     beforeAll(() => {
         client = createTestClient();
+
+        const room = new Room(roomId, client, client.getSafeUserId());
+        mocked(client.getRoom).mockImplementation((getRoomId: string): Room | null => {
+            if (getRoomId === room.roomId) return room;
+            return null;
+        });
+
+        voiceBroadcastStartedEvent = mkVoiceBroadcastInfoStateEvent(
+            roomId,
+            VoiceBroadcastInfoState.Started,
+            client.getUserId()!,
+            client.deviceId!,
+        );
+        room.addLiveEvents([voiceBroadcastStartedEvent]);
         voiceBroadcastStoppedEvent = mkVoiceBroadcastInfoStateEvent(
-            "!room:example.com",
+            roomId,
             VoiceBroadcastInfoState.Stopped,
             client.getUserId()!,
             client.deviceId!,
@@ -53,6 +76,29 @@ describe("pickFactory", () => {
                 msgtype: MsgType.Audio,
             },
         });
+        utdEvent = mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            user: client.getUserId()!,
+            room: roomId,
+            content: {
+                msgtype: "m.bad.encrypted",
+            },
+        });
+        utdBroadcastChunkEvent = mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            user: client.getUserId()!,
+            room: roomId,
+            content: {
+                "msgtype": "m.bad.encrypted",
+                "m.relates_to": {
+                    rel_type: RelationType.Reference,
+                    event_id: voiceBroadcastStartedEvent.getId(),
+                },
+            },
+        });
+        jest.spyOn(utdBroadcastChunkEvent, "isDecryptionFailure").mockReturnValue(true);
     });
 
     it("should return JSONEventFactory for a no-op m.room.power_levels event", () => {
@@ -67,16 +113,20 @@ describe("pickFactory", () => {
     });
 
     describe("when showing hidden events", () => {
-        it("should return a function for a voice broadcast event", () => {
+        it("should return a JSONEventFactory for a voice broadcast event", () => {
-            expect(pickFactory(voiceBroadcastChunkEvent, client, true)).toBeInstanceOf(Function);
+            expect(pickFactory(voiceBroadcastChunkEvent, client, true)).toBe(JSONEventFactory);
         });
 
-        it("should return a Function for a voice broadcast stopped event", () => {
+        it("should return a TextualEventFactory for a voice broadcast stopped event", () => {
-            expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBeInstanceOf(Function);
+            expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBe(TextualEventFactory);
         });
 
-        it("should return a function for an audio message event", () => {
+        it("should return a MessageEventFactory for an audio message event", () => {
-            expect(pickFactory(audioMessageEvent, client, true)).toBeInstanceOf(Function);
+            expect(pickFactory(audioMessageEvent, client, true)).toBe(MessageEventFactory);
+        });
+
+        it("should return a MessageEventFactory for a UTD broadcast chunk event", () => {
+            expect(pickFactory(utdBroadcastChunkEvent, client, true)).toBe(MessageEventFactory);
         });
     });
 
@@ -85,12 +135,20 @@ describe("pickFactory", () => {
             expect(pickFactory(voiceBroadcastChunkEvent, client, false)).toBeUndefined();
         });
 
-        it("should return a Function for a voice broadcast stopped event", () => {
+        it("should return a TextualEventFactory for a voice broadcast stopped event", () => {
-            expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBeInstanceOf(Function);
+            expect(pickFactory(voiceBroadcastStoppedEvent, client, false)).toBe(TextualEventFactory);
         });
 
-        it("should return a function for an audio message event", () => {
+        it("should return a MessageEventFactory for an audio message event", () => {
-            expect(pickFactory(audioMessageEvent, client, false)).toBeInstanceOf(Function);
+            expect(pickFactory(audioMessageEvent, client, false)).toBe(MessageEventFactory);
+        });
+
+        it("should return a MessageEventFactory for a UTD event", () => {
+            expect(pickFactory(utdEvent, client, false)).toBe(MessageEventFactory);
+        });
+
+        it("should return undefined for a UTD broadcast chunk event", () => {
+            expect(pickFactory(utdBroadcastChunkEvent, client, false)).toBeUndefined();
         });
     });
 });

test/voice-broadcast/models/VoiceBroadcastPlayback-test.tsx

@@ -17,7 +17,7 @@ limitations under the License.
 import { mocked } from "jest-mock";
 import { screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
-import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk/src/matrix";
+import { MatrixClient, MatrixEvent, MatrixEventEvent, Room } from "matrix-js-sdk/src/matrix";
 
 import { Playback, PlaybackState } from "../../../src/audio/Playback";
 import { PlaybackManager } from "../../../src/audio/PlaybackManager";
@@ -268,6 +268,32 @@ describe("VoiceBroadcastPlayback", () => {
                     expect(chunk1Playback.play).toHaveBeenCalled();
                 });
             });
+
+            describe("and receiving the first undecryptable chunk", () => {
+                beforeEach(() => {
+                    jest.spyOn(chunk1Event, "isDecryptionFailure").mockReturnValue(true);
+                    room.relations.aggregateChildEvent(chunk1Event);
+                });
+
+                itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Error);
+
+                it("should not update the duration", () => {
+                    expect(playback.durationSeconds).toBe(0);
+                });
+
+                describe("and the chunk is decrypted", () => {
+                    beforeEach(() => {
+                        mocked(chunk1Event.isDecryptionFailure).mockReturnValue(false);
+                        chunk1Event.emit(MatrixEventEvent.Decrypted, chunk1Event);
+                    });
+
+                    itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused);
+
+                    it("should not update the duration", () => {
+                        expect(playback.durationSeconds).toBe(2.3);
+                    });
+                });
+            });
         });
     });
 

test/voice-broadcast/utils/isRelatedToVoiceBroadcast-test.ts

@@ -0,0 +1,118 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventType, MatrixEvent, RelationType, Room } from "matrix-js-sdk/src/matrix";
+import { MatrixClient } from "matrix-js-sdk/src/client";
+import { mocked } from "jest-mock";
+
+import { isRelatedToVoiceBroadcast, VoiceBroadcastInfoState } from "../../../src/voice-broadcast";
+import { mkEvent, stubClient } from "../../test-utils";
+import { mkVoiceBroadcastInfoStateEvent } from "./test-utils";
+
+const mkRelatedEvent = (
+    room: Room,
+    relationType: RelationType,
+    relatesTo: MatrixEvent | undefined,
+    client: MatrixClient,
+): MatrixEvent => {
+    const event = mkEvent({
+        event: true,
+        type: EventType.RoomMessage,
+        room: room.roomId,
+        content: {
+            "m.relates_to": {
+                rel_type: relationType,
+                event_id: relatesTo?.getId(),
+            },
+        },
+        user: client.getSafeUserId(),
+    });
+    room.addLiveEvents([event]);
+    return event;
+};
+
+describe("isRelatedToVoiceBroadcast", () => {
+    const roomId = "!room:example.com";
+    let client: MatrixClient;
+    let room: Room;
+    let broadcastEvent: MatrixEvent;
+    let nonBroadcastEvent: MatrixEvent;
+
+    beforeAll(() => {
+        client = stubClient();
+        room = new Room(roomId, client, client.getSafeUserId());
+
+        mocked(client.getRoom).mockImplementation((getRoomId: string): Room | null => {
+            if (getRoomId === roomId) return room;
+            return null;
+        });
+
+        broadcastEvent = mkVoiceBroadcastInfoStateEvent(
+            roomId,
+            VoiceBroadcastInfoState.Started,
+            client.getSafeUserId(),
+            "ABC123",
+        );
+        nonBroadcastEvent = mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            room: roomId,
+            content: {},
+            user: client.getSafeUserId(),
+        });
+
+        room.addLiveEvents([broadcastEvent, nonBroadcastEvent]);
+    });
+
+    it("should return true if related (reference) to a broadcast event", () => {
+        expect(
+            isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Reference, broadcastEvent, client), client),
+        ).toBe(true);
+    });
+
+    it("should return false if related (reference) is undefeind", () => {
+        expect(isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Reference, undefined, client), client)).toBe(
+            false,
+        );
+    });
+
+    it("should return false if related (referenireplace) to a broadcast event", () => {
+        expect(
+            isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Replace, broadcastEvent, client), client),
+        ).toBe(false);
+    });
+
+    it("should return false if the event has no relation", () => {
+        const noRelationEvent = mkEvent({
+            event: true,
+            type: EventType.RoomMessage,
+            room: room.roomId,
+            content: {},
+            user: client.getSafeUserId(),
+        });
+        expect(isRelatedToVoiceBroadcast(noRelationEvent, client)).toBe(false);
+    });
+
+    it("should return false for an unknown room", () => {
+        const otherRoom = new Room("!other:example.com", client, client.getSafeUserId());
+        expect(
+            isRelatedToVoiceBroadcast(
+                mkRelatedEvent(otherRoom, RelationType.Reference, broadcastEvent, client),
+                client,
+            ),
+        ).toBe(false);
+    });
+});