From c6f6fa62f787d34460bcf4bba386814a62f3f9f7 Mon Sep 17 00:00:00 2001 From: Michael Weimann Date: Wed, 1 Feb 2023 13:32:49 +0100 Subject: [PATCH] Implement broadcast UTD handling (#10021) --- src/events/EventTileFactory.tsx | 15 ++- src/i18n/strings/en_EN.json | 1 + src/voice-broadcast/index.ts | 1 + .../models/VoiceBroadcastPlayback.ts | 71 +++++++++-- .../utils/isRelatedToVoiceBroadcast.ts | 29 +++++ test/events/EventTileFactory-test.ts | 84 +++++++++++-- .../models/VoiceBroadcastPlayback-test.tsx | 28 ++++- .../utils/isRelatedToVoiceBroadcast-test.ts | 118 ++++++++++++++++++ 8 files changed, 320 insertions(+), 27 deletions(-) create mode 100644 src/voice-broadcast/utils/isRelatedToVoiceBroadcast.ts create mode 100644 test/voice-broadcast/utils/isRelatedToVoiceBroadcast-test.ts diff --git a/src/events/EventTileFactory.tsx b/src/events/EventTileFactory.tsx index d6b5e18ad5..9c28678e7d 100644 --- a/src/events/EventTileFactory.tsx +++ b/src/events/EventTileFactory.tsx @@ -48,7 +48,11 @@ import ViewSourceEvent from "../components/views/messages/ViewSourceEvent"; import { shouldDisplayAsBeaconTile } from "../utils/beacon/timeline"; import { shouldDisplayAsVoiceBroadcastTile } from "../voice-broadcast/utils/shouldDisplayAsVoiceBroadcastTile"; import { ElementCall } from "../models/Call"; -import { shouldDisplayAsVoiceBroadcastStoppedText, VoiceBroadcastChunkEventType } from "../voice-broadcast"; +import { + isRelatedToVoiceBroadcast, + shouldDisplayAsVoiceBroadcastStoppedText, + VoiceBroadcastChunkEventType, +} from "../voice-broadcast"; // Subset of EventTile's IProps plus some mixins export interface EventTileTypeProps { @@ -74,13 +78,13 @@ export interface EventTileTypeProps { type FactoryProps = Omit; type Factory = (ref: Optional>, props: X) => JSX.Element; -const MessageEventFactory: Factory = (ref, props) => ; +export const MessageEventFactory: Factory = (ref, props) => ; const KeyVerificationConclFactory: Factory = (ref, props) => ; const LegacyCallEventFactory: Factory = (ref, props) => ( ); const CallEventFactory: Factory = (ref, props) => ; -const TextualEventFactory: Factory = (ref, props) => ; +export const TextualEventFactory: Factory = (ref, props) => ; const VerificationReqFactory: Factory = (ref, props) => ; const HiddenEventFactory: Factory = (ref, props) => ; @@ -260,6 +264,11 @@ export function pickFactory( return noEventFactoryFactory(); } + if (!showHiddenEvents && mxEvent.isDecryptionFailure() && isRelatedToVoiceBroadcast(mxEvent, cli)) { + // hide utd events related to a broadcast + return noEventFactoryFactory(); + } + return EVENT_TILE_TYPES.get(evType) ?? noEventFactoryFactory(); } diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 168d371905..ab21c2fd59 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -659,6 +659,7 @@ "%(senderName)s ended a voice broadcast": "%(senderName)s ended a voice broadcast", "You ended a voice broadcast": "You ended a voice broadcast", "%(senderName)s ended a voice broadcast": "%(senderName)s ended a voice broadcast", + "Unable to decrypt voice broadcast": "Unable to decrypt voice broadcast", "Unable to play this voice broadcast": "Unable to play this voice broadcast", "Stop live broadcasting?": "Stop live broadcasting?", "Are you sure you want to stop your live broadcast?This will end the broadcast and the full recording will be available in the room.": "Are you sure you want to stop your live broadcast?This will end the broadcast and the full recording will be available in the room.", diff --git a/src/voice-broadcast/index.ts b/src/voice-broadcast/index.ts index 4fb03bc485..e397c342b5 100644 --- a/src/voice-broadcast/index.ts +++ b/src/voice-broadcast/index.ts @@ -52,6 +52,7 @@ export * from "./utils/doMaybeSetCurrentVoiceBroadcastPlayback"; export * from "./utils/getChunkLength"; export * from "./utils/getMaxBroadcastLength"; export * from "./utils/hasRoomLiveVoiceBroadcast"; +export * from "./utils/isRelatedToVoiceBroadcast"; export * from "./utils/isVoiceBroadcastStartedEvent"; export * from "./utils/findRoomLiveVoiceBroadcastFromUserAndDevice"; export * from "./utils/retrieveStartedInfoEvent"; diff --git a/src/voice-broadcast/models/VoiceBroadcastPlayback.ts b/src/voice-broadcast/models/VoiceBroadcastPlayback.ts index 80ec38bd75..94ccaac1fb 100644 --- a/src/voice-broadcast/models/VoiceBroadcastPlayback.ts +++ b/src/voice-broadcast/models/VoiceBroadcastPlayback.ts @@ -82,6 +82,8 @@ export class VoiceBroadcastPlayback { private state = VoiceBroadcastPlaybackState.Stopped; private chunkEvents = new VoiceBroadcastChunkEvents(); + /** @var Map: event Id → undecryptable event */ + private utdChunkEvents: Map = new Map(); private playbacks = new Map(); private currentlyPlaying: MatrixEvent | null = null; /** @var total duration of all chunks in milliseconds */ @@ -154,13 +156,18 @@ export class VoiceBroadcastPlayback } private addChunkEvent = async (event: MatrixEvent): Promise => { - if (event.getContent()?.msgtype !== MsgType.Audio) { - // skip non-audio event + if (!event.getId() && !event.getTxnId()) { + // skip events without id and txn id return false; } - if (!event.getId() && !event.getTxnId()) { - // skip events without id and txn id + if (event.isDecryptionFailure()) { + this.onChunkEventDecryptionFailure(event); + return false; + } + + if (event.getContent()?.msgtype !== MsgType.Audio) { + // skip non-audio event return false; } @@ -174,6 +181,45 @@ export class VoiceBroadcastPlayback return true; }; + private onChunkEventDecryptionFailure = (event: MatrixEvent): void => { + const eventId = event.getId(); + + if (!eventId) { + // This should not happen, as the existence of the Id is checked before the call. + // Log anyway and return. + logger.warn("Broadcast chunk decryption failure for event without Id", { + broadcast: this.infoEvent.getId(), + }); + return; + } + + if (!this.utdChunkEvents.has(eventId)) { + event.once(MatrixEventEvent.Decrypted, this.onChunkEventDecrypted); + } + + this.utdChunkEvents.set(eventId, event); + this.setError(); + }; + + private onChunkEventDecrypted = async (event: MatrixEvent): Promise => { + const eventId = event.getId(); + + if (!eventId) { + // This should not happen, as the existence of the Id is checked before the call. + // Log anyway and return. + logger.warn("Broadcast chunk decrypted for event without Id", { broadcast: this.infoEvent.getId() }); + return; + } + + this.utdChunkEvents.delete(eventId); + await this.addChunkEvent(event); + + if (this.utdChunkEvents.size === 0) { + // no more UTD events, recover from error to paused + this.setState(VoiceBroadcastPlaybackState.Paused); + } + }; + private startOrPlayNext = async (): Promise => { if (this.currentlyPlaying) { return this.playNext(); @@ -210,7 +256,7 @@ export class VoiceBroadcastPlayback private async tryLoadPlayback(chunkEvent: MatrixEvent): Promise { try { return await this.loadPlayback(chunkEvent); - } catch (err) { + } catch (err: any) { logger.warn("Unable to load broadcast playback", { message: err.message, broadcastId: this.infoEvent.getId(), @@ -332,7 +378,7 @@ export class VoiceBroadcastPlayback private async tryGetOrLoadPlaybackForEvent(event: MatrixEvent): Promise { try { return await this.getOrLoadPlaybackForEvent(event); - } catch (err) { + } catch (err: any) { logger.warn("Unable to load broadcast playback", { message: err.message, broadcastId: this.infoEvent.getId(), @@ -551,9 +597,6 @@ export class VoiceBroadcastPlayback } private setState(state: VoiceBroadcastPlaybackState): void { - // error is a final state - if (this.getState() === VoiceBroadcastPlaybackState.Error) return; - if (this.state === state) { return; } @@ -587,10 +630,18 @@ export class VoiceBroadcastPlayback } public get errorMessage(): string { - return this.getState() === VoiceBroadcastPlaybackState.Error ? _t("Unable to play this voice broadcast") : ""; + if (this.getState() !== VoiceBroadcastPlaybackState.Error) return ""; + if (this.utdChunkEvents.size) return _t("Unable to decrypt voice broadcast"); + return _t("Unable to play this voice broadcast"); } public destroy(): void { + for (const [, utdEvent] of this.utdChunkEvents) { + utdEvent.off(MatrixEventEvent.Decrypted, this.onChunkEventDecrypted); + } + + this.utdChunkEvents.clear(); + this.chunkRelationHelper.destroy(); this.infoRelationHelper.destroy(); this.removeAllListeners(); diff --git a/src/voice-broadcast/utils/isRelatedToVoiceBroadcast.ts b/src/voice-broadcast/utils/isRelatedToVoiceBroadcast.ts new file mode 100644 index 0000000000..c27c270337 --- /dev/null +++ b/src/voice-broadcast/utils/isRelatedToVoiceBroadcast.ts @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixClient, MatrixEvent, RelationType } from "matrix-js-sdk/src/matrix"; + +import { VoiceBroadcastInfoEventType } from "../types"; + +export const isRelatedToVoiceBroadcast = (event: MatrixEvent, client: MatrixClient): boolean => { + const relation = event.getRelation(); + + return ( + relation?.rel_type === RelationType.Reference && + !!relation.event_id && + client.getRoom(event.getRoomId())?.findEventById(relation.event_id)?.getType() === VoiceBroadcastInfoEventType + ); +}; diff --git a/test/events/EventTileFactory-test.ts b/test/events/EventTileFactory-test.ts index 4a57e4d5ff..911abb71b4 100644 --- a/test/events/EventTileFactory-test.ts +++ b/test/events/EventTileFactory-test.ts @@ -11,9 +11,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { EventType, MatrixClient, MatrixEvent, MsgType } from "matrix-js-sdk/src/matrix"; +import { mocked } from "jest-mock"; +import { EventType, MatrixClient, MatrixEvent, MsgType, RelationType, Room } from "matrix-js-sdk/src/matrix"; -import { JSONEventFactory, pickFactory } from "../../src/events/EventTileFactory"; +import { + JSONEventFactory, + MessageEventFactory, + pickFactory, + TextualEventFactory, +} from "../../src/events/EventTileFactory"; import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoState } from "../../src/voice-broadcast"; import { createTestClient, mkEvent } from "../test-utils"; import { mkVoiceBroadcastInfoStateEvent } from "../voice-broadcast/utils/test-utils"; @@ -21,15 +27,32 @@ import { mkVoiceBroadcastInfoStateEvent } from "../voice-broadcast/utils/test-ut const roomId = "!room:example.com"; describe("pickFactory", () => { + let voiceBroadcastStartedEvent: MatrixEvent; let voiceBroadcastStoppedEvent: MatrixEvent; let voiceBroadcastChunkEvent: MatrixEvent; + let utdEvent: MatrixEvent; + let utdBroadcastChunkEvent: MatrixEvent; let audioMessageEvent: MatrixEvent; let client: MatrixClient; beforeAll(() => { client = createTestClient(); + + const room = new Room(roomId, client, client.getSafeUserId()); + mocked(client.getRoom).mockImplementation((getRoomId: string): Room | null => { + if (getRoomId === room.roomId) return room; + return null; + }); + + voiceBroadcastStartedEvent = mkVoiceBroadcastInfoStateEvent( + roomId, + VoiceBroadcastInfoState.Started, + client.getUserId()!, + client.deviceId!, + ); + room.addLiveEvents([voiceBroadcastStartedEvent]); voiceBroadcastStoppedEvent = mkVoiceBroadcastInfoStateEvent( - "!room:example.com", + roomId, VoiceBroadcastInfoState.Stopped, client.getUserId()!, client.deviceId!, @@ -53,6 +76,29 @@ describe("pickFactory", () => { msgtype: MsgType.Audio, }, }); + utdEvent = mkEvent({ + event: true, + type: EventType.RoomMessage, + user: client.getUserId()!, + room: roomId, + content: { + msgtype: "m.bad.encrypted", + }, + }); + utdBroadcastChunkEvent = mkEvent({ + event: true, + type: EventType.RoomMessage, + user: client.getUserId()!, + room: roomId, + content: { + "msgtype": "m.bad.encrypted", + "m.relates_to": { + rel_type: RelationType.Reference, + event_id: voiceBroadcastStartedEvent.getId(), + }, + }, + }); + jest.spyOn(utdBroadcastChunkEvent, "isDecryptionFailure").mockReturnValue(true); }); it("should return JSONEventFactory for a no-op m.room.power_levels event", () => { @@ -67,16 +113,20 @@ describe("pickFactory", () => { }); describe("when showing hidden events", () => { - it("should return a function for a voice broadcast event", () => { - expect(pickFactory(voiceBroadcastChunkEvent, client, true)).toBeInstanceOf(Function); + it("should return a JSONEventFactory for a voice broadcast event", () => { + expect(pickFactory(voiceBroadcastChunkEvent, client, true)).toBe(JSONEventFactory); }); - it("should return a Function for a voice broadcast stopped event", () => { - expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBeInstanceOf(Function); + it("should return a TextualEventFactory for a voice broadcast stopped event", () => { + expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBe(TextualEventFactory); }); - it("should return a function for an audio message event", () => { - expect(pickFactory(audioMessageEvent, client, true)).toBeInstanceOf(Function); + it("should return a MessageEventFactory for an audio message event", () => { + expect(pickFactory(audioMessageEvent, client, true)).toBe(MessageEventFactory); + }); + + it("should return a MessageEventFactory for a UTD broadcast chunk event", () => { + expect(pickFactory(utdBroadcastChunkEvent, client, true)).toBe(MessageEventFactory); }); }); @@ -85,12 +135,20 @@ describe("pickFactory", () => { expect(pickFactory(voiceBroadcastChunkEvent, client, false)).toBeUndefined(); }); - it("should return a Function for a voice broadcast stopped event", () => { - expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBeInstanceOf(Function); + it("should return a TextualEventFactory for a voice broadcast stopped event", () => { + expect(pickFactory(voiceBroadcastStoppedEvent, client, false)).toBe(TextualEventFactory); }); - it("should return a function for an audio message event", () => { - expect(pickFactory(audioMessageEvent, client, false)).toBeInstanceOf(Function); + it("should return a MessageEventFactory for an audio message event", () => { + expect(pickFactory(audioMessageEvent, client, false)).toBe(MessageEventFactory); + }); + + it("should return a MessageEventFactory for a UTD event", () => { + expect(pickFactory(utdEvent, client, false)).toBe(MessageEventFactory); + }); + + it("should return undefined for a UTD broadcast chunk event", () => { + expect(pickFactory(utdBroadcastChunkEvent, client, false)).toBeUndefined(); }); }); }); diff --git a/test/voice-broadcast/models/VoiceBroadcastPlayback-test.tsx b/test/voice-broadcast/models/VoiceBroadcastPlayback-test.tsx index 81eda2c986..ac4217e048 100644 --- a/test/voice-broadcast/models/VoiceBroadcastPlayback-test.tsx +++ b/test/voice-broadcast/models/VoiceBroadcastPlayback-test.tsx @@ -17,7 +17,7 @@ limitations under the License. import { mocked } from "jest-mock"; import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk/src/matrix"; +import { MatrixClient, MatrixEvent, MatrixEventEvent, Room } from "matrix-js-sdk/src/matrix"; import { Playback, PlaybackState } from "../../../src/audio/Playback"; import { PlaybackManager } from "../../../src/audio/PlaybackManager"; @@ -268,6 +268,32 @@ describe("VoiceBroadcastPlayback", () => { expect(chunk1Playback.play).toHaveBeenCalled(); }); }); + + describe("and receiving the first undecryptable chunk", () => { + beforeEach(() => { + jest.spyOn(chunk1Event, "isDecryptionFailure").mockReturnValue(true); + room.relations.aggregateChildEvent(chunk1Event); + }); + + itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Error); + + it("should not update the duration", () => { + expect(playback.durationSeconds).toBe(0); + }); + + describe("and the chunk is decrypted", () => { + beforeEach(() => { + mocked(chunk1Event.isDecryptionFailure).mockReturnValue(false); + chunk1Event.emit(MatrixEventEvent.Decrypted, chunk1Event); + }); + + itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused); + + it("should not update the duration", () => { + expect(playback.durationSeconds).toBe(2.3); + }); + }); + }); }); }); diff --git a/test/voice-broadcast/utils/isRelatedToVoiceBroadcast-test.ts b/test/voice-broadcast/utils/isRelatedToVoiceBroadcast-test.ts new file mode 100644 index 0000000000..ce5c61e5b8 --- /dev/null +++ b/test/voice-broadcast/utils/isRelatedToVoiceBroadcast-test.ts @@ -0,0 +1,118 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { EventType, MatrixEvent, RelationType, Room } from "matrix-js-sdk/src/matrix"; +import { MatrixClient } from "matrix-js-sdk/src/client"; +import { mocked } from "jest-mock"; + +import { isRelatedToVoiceBroadcast, VoiceBroadcastInfoState } from "../../../src/voice-broadcast"; +import { mkEvent, stubClient } from "../../test-utils"; +import { mkVoiceBroadcastInfoStateEvent } from "./test-utils"; + +const mkRelatedEvent = ( + room: Room, + relationType: RelationType, + relatesTo: MatrixEvent | undefined, + client: MatrixClient, +): MatrixEvent => { + const event = mkEvent({ + event: true, + type: EventType.RoomMessage, + room: room.roomId, + content: { + "m.relates_to": { + rel_type: relationType, + event_id: relatesTo?.getId(), + }, + }, + user: client.getSafeUserId(), + }); + room.addLiveEvents([event]); + return event; +}; + +describe("isRelatedToVoiceBroadcast", () => { + const roomId = "!room:example.com"; + let client: MatrixClient; + let room: Room; + let broadcastEvent: MatrixEvent; + let nonBroadcastEvent: MatrixEvent; + + beforeAll(() => { + client = stubClient(); + room = new Room(roomId, client, client.getSafeUserId()); + + mocked(client.getRoom).mockImplementation((getRoomId: string): Room | null => { + if (getRoomId === roomId) return room; + return null; + }); + + broadcastEvent = mkVoiceBroadcastInfoStateEvent( + roomId, + VoiceBroadcastInfoState.Started, + client.getSafeUserId(), + "ABC123", + ); + nonBroadcastEvent = mkEvent({ + event: true, + type: EventType.RoomMessage, + room: roomId, + content: {}, + user: client.getSafeUserId(), + }); + + room.addLiveEvents([broadcastEvent, nonBroadcastEvent]); + }); + + it("should return true if related (reference) to a broadcast event", () => { + expect( + isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Reference, broadcastEvent, client), client), + ).toBe(true); + }); + + it("should return false if related (reference) is undefeind", () => { + expect(isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Reference, undefined, client), client)).toBe( + false, + ); + }); + + it("should return false if related (referenireplace) to a broadcast event", () => { + expect( + isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Replace, broadcastEvent, client), client), + ).toBe(false); + }); + + it("should return false if the event has no relation", () => { + const noRelationEvent = mkEvent({ + event: true, + type: EventType.RoomMessage, + room: room.roomId, + content: {}, + user: client.getSafeUserId(), + }); + expect(isRelatedToVoiceBroadcast(noRelationEvent, client)).toBe(false); + }); + + it("should return false for an unknown room", () => { + const otherRoom = new Room("!other:example.com", client, client.getSafeUserId()); + expect( + isRelatedToVoiceBroadcast( + mkRelatedEvent(otherRoom, RelationType.Reference, broadcastEvent, client), + client, + ), + ).toBe(false); + }); +});