PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Implement broadcast UTD handling (#10021)

Browse source
This commit is contained in:
Michael Weimann 2023-02-01 13:32:49 +01:00 committed by GitHub
parent afda774471
commit c6f6fa62f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 320 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/events/EventTileFactory.tsx
View file

@ -48,7 +48,11 @@ import ViewSourceEvent from "../components/views/messages/ViewSourceEvent";
import { shouldDisplayAsBeaconTile } from "../utils/beacon/timeline";
import { shouldDisplayAsVoiceBroadcastTile } from "../voice-broadcast/utils/shouldDisplayAsVoiceBroadcastTile";
import { ElementCall } from "../models/Call";
import { shouldDisplayAsVoiceBroadcastStoppedText, VoiceBroadcastChunkEventType } from "../voice-broadcast";
import {
isRelatedToVoiceBroadcast,
shouldDisplayAsVoiceBroadcastStoppedText,
VoiceBroadcastChunkEventType,
} from "../voice-broadcast";
// Subset of EventTile's IProps plus some mixins
export interface EventTileTypeProps {
@ -74,13 +78,13 @@ export interface EventTileTypeProps {
type FactoryProps = Omit<EventTileTypeProps, "ref">;
type Factory<X = FactoryProps> = (ref: Optional<React.RefObject<any>>, props: X) => JSX.Element;
const MessageEventFactory: Factory = (ref, props) => <MessageEvent ref={ref} {...props} />;
export const MessageEventFactory: Factory = (ref, props) => <MessageEvent ref={ref} {...props} />;
const KeyVerificationConclFactory: Factory = (ref, props) => <MKeyVerificationConclusion ref={ref} {...props} />;
const LegacyCallEventFactory: Factory<FactoryProps & { callEventGrouper: LegacyCallEventGrouper }> = (ref, props) => (
<LegacyCallEvent ref={ref} {...props} />
);
const CallEventFactory: Factory = (ref, props) => <CallEvent ref={ref} {...props} />;
const TextualEventFactory: Factory = (ref, props) => <TextualEvent ref={ref} {...props} />;
export const TextualEventFactory: Factory = (ref, props) => <TextualEvent ref={ref} {...props} />;
const VerificationReqFactory: Factory = (ref, props) => <MKeyVerificationRequest ref={ref} {...props} />;
const HiddenEventFactory: Factory = (ref, props) => <HiddenBody ref={ref} {...props} />;
@ -260,6 +264,11 @@ export function pickFactory(
return noEventFactoryFactory();
}
if (!showHiddenEvents && mxEvent.isDecryptionFailure() && isRelatedToVoiceBroadcast(mxEvent, cli)) {
// hide utd events related to a broadcast
return noEventFactoryFactory();
}
return EVENT_TILE_TYPES.get(evType) ?? noEventFactoryFactory();
}

1
src/i18n/strings/en_EN.json
View file

@ -659,6 +659,7 @@
"%(senderName)s ended a <a>voice broadcast</a>": "%(senderName)s ended a <a>voice broadcast</a>",
"You ended a voice broadcast": "You ended a voice broadcast",
"%(senderName)s ended a voice broadcast": "%(senderName)s ended a voice broadcast",
"Unable to decrypt voice broadcast": "Unable to decrypt voice broadcast",
"Unable to play this voice broadcast": "Unable to play this voice broadcast",
"Stop live broadcasting?": "Stop live broadcasting?",
"Are you sure you want to stop your live broadcast?This will end the broadcast and the full recording will be available in the room.": "Are you sure you want to stop your live broadcast?This will end the broadcast and the full recording will be available in the room.",

1
src/voice-broadcast/index.ts
View file

@ -52,6 +52,7 @@ export * from "./utils/doMaybeSetCurrentVoiceBroadcastPlayback";
export * from "./utils/getChunkLength";
export * from "./utils/getMaxBroadcastLength";
export * from "./utils/hasRoomLiveVoiceBroadcast";
export * from "./utils/isRelatedToVoiceBroadcast";
export * from "./utils/isVoiceBroadcastStartedEvent";
export * from "./utils/findRoomLiveVoiceBroadcastFromUserAndDevice";
export * from "./utils/retrieveStartedInfoEvent";

71
src/voice-broadcast/models/VoiceBroadcastPlayback.ts
View file

@ -82,6 +82,8 @@ export class VoiceBroadcastPlayback
{
private state = VoiceBroadcastPlaybackState.Stopped;
private chunkEvents = new VoiceBroadcastChunkEvents();
/** @var Map: event Id → undecryptable event */
private utdChunkEvents: Map<string, MatrixEvent> = new Map();
private playbacks = new Map<string, Playback>();
private currentlyPlaying: MatrixEvent | null = null;
/** @var total duration of all chunks in milliseconds */
@ -154,13 +156,18 @@ export class VoiceBroadcastPlayback
}
private addChunkEvent = async (event: MatrixEvent): Promise<boolean> => {
if (event.getContent()?.msgtype !== MsgType.Audio) {
// skip non-audio event
if (!event.getId() && !event.getTxnId()) {
// skip events without id and txn id
return false;
}
if (!event.getId() && !event.getTxnId()) {
// skip events without id and txn id
if (event.isDecryptionFailure()) {
this.onChunkEventDecryptionFailure(event);
return false;
}
if (event.getContent()?.msgtype !== MsgType.Audio) {
// skip non-audio event
return false;
}
@ -174,6 +181,45 @@ export class VoiceBroadcastPlayback
return true;
};
private onChunkEventDecryptionFailure = (event: MatrixEvent): void => {
const eventId = event.getId();
if (!eventId) {
// This should not happen, as the existence of the Id is checked before the call.
// Log anyway and return.
logger.warn("Broadcast chunk decryption failure for event without Id", {
broadcast: this.infoEvent.getId(),
});
return;
}
if (!this.utdChunkEvents.has(eventId)) {
event.once(MatrixEventEvent.Decrypted, this.onChunkEventDecrypted);
}
this.utdChunkEvents.set(eventId, event);
this.setError();
};
private onChunkEventDecrypted = async (event: MatrixEvent): Promise<void> => {
const eventId = event.getId();
if (!eventId) {
// This should not happen, as the existence of the Id is checked before the call.
// Log anyway and return.
logger.warn("Broadcast chunk decrypted for event without Id", { broadcast: this.infoEvent.getId() });
return;
}
this.utdChunkEvents.delete(eventId);
await this.addChunkEvent(event);
if (this.utdChunkEvents.size === 0) {
// no more UTD events, recover from error to paused
this.setState(VoiceBroadcastPlaybackState.Paused);
}
};
private startOrPlayNext = async (): Promise<void> => {
if (this.currentlyPlaying) {
return this.playNext();
@ -210,7 +256,7 @@ export class VoiceBroadcastPlayback
private async tryLoadPlayback(chunkEvent: MatrixEvent): Promise<void> {
try {
return await this.loadPlayback(chunkEvent);
} catch (err) {
} catch (err: any) {
logger.warn("Unable to load broadcast playback", {
message: err.message,
broadcastId: this.infoEvent.getId(),
@ -332,7 +378,7 @@ export class VoiceBroadcastPlayback
private async tryGetOrLoadPlaybackForEvent(event: MatrixEvent): Promise<Playback | undefined> {
try {
return await this.getOrLoadPlaybackForEvent(event);
} catch (err) {
} catch (err: any) {
logger.warn("Unable to load broadcast playback", {
message: err.message,
broadcastId: this.infoEvent.getId(),
@ -551,9 +597,6 @@ export class VoiceBroadcastPlayback
}
private setState(state: VoiceBroadcastPlaybackState): void {
// error is a final state
if (this.getState() === VoiceBroadcastPlaybackState.Error) return;
if (this.state === state) {
return;
}
@ -587,10 +630,18 @@ export class VoiceBroadcastPlayback
}
public get errorMessage(): string {
return this.getState() === VoiceBroadcastPlaybackState.Error ? _t("Unable to play this voice broadcast") : "";
if (this.getState() !== VoiceBroadcastPlaybackState.Error) return "";
if (this.utdChunkEvents.size) return _t("Unable to decrypt voice broadcast");
return _t("Unable to play this voice broadcast");
}
public destroy(): void {
for (const [, utdEvent] of this.utdChunkEvents) {
utdEvent.off(MatrixEventEvent.Decrypted, this.onChunkEventDecrypted);
}
this.utdChunkEvents.clear();
this.chunkRelationHelper.destroy();
this.infoRelationHelper.destroy();
this.removeAllListeners();

29
src/voice-broadcast/utils/isRelatedToVoiceBroadcast.ts Normal file
View file

@ -0,0 +1,29 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { MatrixClient, MatrixEvent, RelationType } from "matrix-js-sdk/src/matrix";
import { VoiceBroadcastInfoEventType } from "../types";
export const isRelatedToVoiceBroadcast = (event: MatrixEvent, client: MatrixClient): boolean => {
const relation = event.getRelation();
return (
relation?.rel_type === RelationType.Reference &&
!!relation.event_id &&
client.getRoom(event.getRoomId())?.findEventById(relation.event_id)?.getType() === VoiceBroadcastInfoEventType
);
};

84
test/events/EventTileFactory-test.ts
View file

@ -11,9 +11,15 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import { EventType, MatrixClient, MatrixEvent, MsgType } from "matrix-js-sdk/src/matrix";
import { mocked } from "jest-mock";
import { EventType, MatrixClient, MatrixEvent, MsgType, RelationType, Room } from "matrix-js-sdk/src/matrix";
import { JSONEventFactory, pickFactory } from "../../src/events/EventTileFactory";
import {
JSONEventFactory,
MessageEventFactory,
pickFactory,
TextualEventFactory,
} from "../../src/events/EventTileFactory";
import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoState } from "../../src/voice-broadcast";
import { createTestClient, mkEvent } from "../test-utils";
import { mkVoiceBroadcastInfoStateEvent } from "../voice-broadcast/utils/test-utils";
@ -21,15 +27,32 @@ import { mkVoiceBroadcastInfoStateEvent } from "../voice-broadcast/utils/test-ut
const roomId = "!room:example.com";
describe("pickFactory", () => {
let voiceBroadcastStartedEvent: MatrixEvent;
let voiceBroadcastStoppedEvent: MatrixEvent;
let voiceBroadcastChunkEvent: MatrixEvent;
let utdEvent: MatrixEvent;
let utdBroadcastChunkEvent: MatrixEvent;
let audioMessageEvent: MatrixEvent;
let client: MatrixClient;
beforeAll(() => {
client = createTestClient();
const room = new Room(roomId, client, client.getSafeUserId());
mocked(client.getRoom).mockImplementation((getRoomId: string): Room | null => {
if (getRoomId === room.roomId) return room;
return null;
});
voiceBroadcastStartedEvent = mkVoiceBroadcastInfoStateEvent(
roomId,
VoiceBroadcastInfoState.Started,
client.getUserId()!,
client.deviceId!,
);
room.addLiveEvents([voiceBroadcastStartedEvent]);
voiceBroadcastStoppedEvent = mkVoiceBroadcastInfoStateEvent(
"!room:example.com",
roomId,
VoiceBroadcastInfoState.Stopped,
client.getUserId()!,
client.deviceId!,
@ -53,6 +76,29 @@ describe("pickFactory", () => {
msgtype: MsgType.Audio,
},
});
utdEvent = mkEvent({
event: true,
type: EventType.RoomMessage,
user: client.getUserId()!,
room: roomId,
content: {
msgtype: "m.bad.encrypted",
},
});
utdBroadcastChunkEvent = mkEvent({
event: true,
type: EventType.RoomMessage,
user: client.getUserId()!,
room: roomId,
content: {
"msgtype": "m.bad.encrypted",
"m.relates_to": {
rel_type: RelationType.Reference,
event_id: voiceBroadcastStartedEvent.getId(),
},
},
});
jest.spyOn(utdBroadcastChunkEvent, "isDecryptionFailure").mockReturnValue(true);
});
it("should return JSONEventFactory for a no-op m.room.power_levels event", () => {
@ -67,16 +113,20 @@ describe("pickFactory", () => {
});
describe("when showing hidden events", () => {
it("should return a function for a voice broadcast event", () => {
expect(pickFactory(voiceBroadcastChunkEvent, client, true)).toBeInstanceOf(Function);
it("should return a JSONEventFactory for a voice broadcast event", () => {
expect(pickFactory(voiceBroadcastChunkEvent, client, true)).toBe(JSONEventFactory);
});
it("should return a Function for a voice broadcast stopped event", () => {
expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBeInstanceOf(Function);
it("should return a TextualEventFactory for a voice broadcast stopped event", () => {
expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBe(TextualEventFactory);
});
it("should return a function for an audio message event", () => {
expect(pickFactory(audioMessageEvent, client, true)).toBeInstanceOf(Function);
it("should return a MessageEventFactory for an audio message event", () => {
expect(pickFactory(audioMessageEvent, client, true)).toBe(MessageEventFactory);
});
it("should return a MessageEventFactory for a UTD broadcast chunk event", () => {
expect(pickFactory(utdBroadcastChunkEvent, client, true)).toBe(MessageEventFactory);
});
});
@ -85,12 +135,20 @@ describe("pickFactory", () => {
expect(pickFactory(voiceBroadcastChunkEvent, client, false)).toBeUndefined();
});
it("should return a Function for a voice broadcast stopped event", () => {
expect(pickFactory(voiceBroadcastStoppedEvent, client, true)).toBeInstanceOf(Function);
it("should return a TextualEventFactory for a voice broadcast stopped event", () => {
expect(pickFactory(voiceBroadcastStoppedEvent, client, false)).toBe(TextualEventFactory);
});
it("should return a function for an audio message event", () => {
expect(pickFactory(audioMessageEvent, client, false)).toBeInstanceOf(Function);
it("should return a MessageEventFactory for an audio message event", () => {
expect(pickFactory(audioMessageEvent, client, false)).toBe(MessageEventFactory);
});
it("should return a MessageEventFactory for a UTD event", () => {
expect(pickFactory(utdEvent, client, false)).toBe(MessageEventFactory);
});
it("should return undefined for a UTD broadcast chunk event", () => {
expect(pickFactory(utdBroadcastChunkEvent, client, false)).toBeUndefined();
});
});
});

28
test/voice-broadcast/models/VoiceBroadcastPlayback-test.tsx
View file

@ -17,7 +17,7 @@ limitations under the License.
import { mocked } from "jest-mock";
import { screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk/src/matrix";
import { MatrixClient, MatrixEvent, MatrixEventEvent, Room } from "matrix-js-sdk/src/matrix";
import { Playback, PlaybackState } from "../../../src/audio/Playback";
import { PlaybackManager } from "../../../src/audio/PlaybackManager";
@ -268,6 +268,32 @@ describe("VoiceBroadcastPlayback", () => {
expect(chunk1Playback.play).toHaveBeenCalled();
});
});
describe("and receiving the first undecryptable chunk", () => {
beforeEach(() => {
jest.spyOn(chunk1Event, "isDecryptionFailure").mockReturnValue(true);
room.relations.aggregateChildEvent(chunk1Event);
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Error);
it("should not update the duration", () => {
expect(playback.durationSeconds).toBe(0);
});
describe("and the chunk is decrypted", () => {
beforeEach(() => {
mocked(chunk1Event.isDecryptionFailure).mockReturnValue(false);
chunk1Event.emit(MatrixEventEvent.Decrypted, chunk1Event);
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused);
it("should not update the duration", () => {
expect(playback.durationSeconds).toBe(2.3);
});
});
});
});
});

118
test/voice-broadcast/utils/isRelatedToVoiceBroadcast-test.ts Normal file
View file

@ -0,0 +1,118 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { EventType, MatrixEvent, RelationType, Room } from "matrix-js-sdk/src/matrix";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { mocked } from "jest-mock";
import { isRelatedToVoiceBroadcast, VoiceBroadcastInfoState } from "../../../src/voice-broadcast";
import { mkEvent, stubClient } from "../../test-utils";
import { mkVoiceBroadcastInfoStateEvent } from "./test-utils";
const mkRelatedEvent = (
room: Room,
relationType: RelationType,
relatesTo: MatrixEvent | undefined,
client: MatrixClient,
): MatrixEvent => {
const event = mkEvent({
event: true,
type: EventType.RoomMessage,
room: room.roomId,
content: {
"m.relates_to": {
rel_type: relationType,
event_id: relatesTo?.getId(),
},
},
user: client.getSafeUserId(),
});
room.addLiveEvents([event]);
return event;
};
describe("isRelatedToVoiceBroadcast", () => {
const roomId = "!room:example.com";
let client: MatrixClient;
let room: Room;
let broadcastEvent: MatrixEvent;
let nonBroadcastEvent: MatrixEvent;
beforeAll(() => {
client = stubClient();
room = new Room(roomId, client, client.getSafeUserId());
mocked(client.getRoom).mockImplementation((getRoomId: string): Room | null => {
if (getRoomId === roomId) return room;
return null;
});
broadcastEvent = mkVoiceBroadcastInfoStateEvent(
roomId,
VoiceBroadcastInfoState.Started,
client.getSafeUserId(),
"ABC123",
);
nonBroadcastEvent = mkEvent({
event: true,
type: EventType.RoomMessage,
room: roomId,
content: {},
user: client.getSafeUserId(),
});
room.addLiveEvents([broadcastEvent, nonBroadcastEvent]);
});
it("should return true if related (reference) to a broadcast event", () => {
expect(
isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Reference, broadcastEvent, client), client),
).toBe(true);
});
it("should return false if related (reference) is undefeind", () => {
expect(isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Reference, undefined, client), client)).toBe(
false,
);
});
it("should return false if related (referenireplace) to a broadcast event", () => {
expect(
isRelatedToVoiceBroadcast(mkRelatedEvent(room, RelationType.Replace, broadcastEvent, client), client),
).toBe(false);
});
it("should return false if the event has no relation", () => {
const noRelationEvent = mkEvent({
event: true,
type: EventType.RoomMessage,
room: room.roomId,
content: {},
user: client.getSafeUserId(),
});
expect(isRelatedToVoiceBroadcast(noRelationEvent, client)).toBe(false);
});
it("should return false for an unknown room", () => {
const otherRoom = new Room("!other:example.com", client, client.getSafeUserId());
expect(
isRelatedToVoiceBroadcast(
mkRelatedEvent(otherRoom, RelationType.Reference, broadcastEvent, client),
client,
),
).toBe(false);
});
});