PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Populate info.duration for audio & video file uploads (#11225)

Browse source 
* Improve m.file m.image m.audio m.video types

* Populate `info.duration` for audio & video file uploads

* Fix tests

* Iterate types

* Improve coverage

* Fix test

* Add small delay to stabilise cypress test

* Fix test idempotency

* Improve coverage

* Slow down

* iterate
This commit is contained in:
Michael Telatynski 2023-07-17 13:07:58 +01:00 committed by GitHub
parent 8b8ca425d7
commit f04a0e2860
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 556 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cypress/e2e/right-panel/file-panel.spec.ts
View file

@ -183,7 +183,7 @@ describe("FilePanel", () => {
});
});
it("should render the audio pleyer and play the audio file on the panel", () => {
it("should render the audio player and play the audio file on the panel", () => {
// Upload an image file
uploadFile("cypress/fixtures/1sec.ogg");
@ -202,10 +202,14 @@ describe("FilePanel", () => {
cy.contains(".mx_AudioPlayer_byline", "(3.56 KB)").should("exist"); // actual size
});
// Assert that the duration counter is 00:01 before clicking the play button
cy.contains(".mx_AudioPlayer_mediaInfo time", "00:01").should("exist");
// Assert that the counter is zero before clicking the play button
cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist");
// Click the play button
cy.wait(500);
cy.findByRole("button", { name: "Play" }).click();
// Assert that the pause button is rendered

87
src/ContentMessages.ts
View file

@ -33,7 +33,14 @@ import {
import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread";
import { removeElement } from "matrix-js-sdk/src/utils";
import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent";
import {
AudioInfo,
EncryptedFile,
ImageInfo,
IMediaEventContent,
IMediaEventInfo,
VideoInfo,
} from "./customisations/models/IMediaEventContent";
import dis from "./dispatcher/dispatcher";
import { _t } from "./languageHandler";
import Modal from "./Modal";
@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];
* @param {File} imageFile The image to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
async function infoForImageFile(
matrixClient: MatrixClient,
roomId: string,
imageFile: File,
): Promise<Partial<IMediaEventInfo>> {
async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
let thumbnailType = "image/png";
if (imageFile.type === "image/jpeg") {
thumbnailType = "image/jpeg";
@ -184,16 +187,59 @@ async function infoForImageFile(
return imageInfo;
}
/**
* Load a file into a newly created audio element and load the metadata
*
* @param {File} audioFile The file to load in an audio element.
* @return {Promise} A promise that resolves with the audio element.
*/
function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
return new Promise((resolve, reject) => {
// Load the file into a html element
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.muted = true;
const reader = new FileReader();
reader.onload = function (ev): void {
audio.onloadedmetadata = async function (): Promise<void> {
resolve(audio);
};
audio.onerror = function (e): void {
reject(e);
};
audio.src = ev.target?.result as string;
};
reader.onerror = function (e): void {
reject(e);
};
reader.readAsDataURL(audioFile);
});
}
/**
* Read the metadata for an audio file.
*
* @param {File} audioFile The audio to read.
* @return {Promise} A promise that resolves with the attachment info.
*/
async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
const audio = await loadAudioElement(audioFile);
return { duration: Math.ceil(audio.duration * 1000) };
}
/**
* Load a file into a newly created video element and pull some strings
* in an attempt to guarantee the first frame will be showing.
*
* @param {File} videoFile The file to load in an video element.
* @return {Promise} A promise that resolves with the video image element.
* @param {File} videoFile The file to load in a video element.
* @return {Promise} A promise that resolves with the video element.
*/
function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
return new Promise((resolve, reject) => {
// Load the file into an html element
// Load the file into a html element
const video = document.createElement("video");
video.preload = "metadata";
video.playsInline = true;
@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
* @param {File} videoFile The video to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
function infoForVideoFile(
matrixClient: MatrixClient,
roomId: string,
videoFile: File,
): Promise<Partial<IMediaEventInfo>> {
function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
const thumbnailType = "image/jpeg";
let videoInfo: Partial<IMediaEventInfo>;
const videoInfo: VideoInfo = {};
return loadVideoElement(videoFile)
.then((video) => {
videoInfo.duration = Math.ceil(video.duration * 1000);
return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
})
.then((result) => {
videoInfo = result.info;
Object.assign(videoInfo, result.info);
return uploadFile(matrixClient, roomId, result.thumbnail);
})
.then((result) => {
@ -299,7 +342,7 @@ export async function uploadFile(
file: File | Blob,
progressHandler?: UploadOpts["progressHandler"],
controller?: AbortController,
): Promise<{ url?: string; file?: IEncryptedFile }> {
): Promise<{ url?: string; file?: EncryptedFile }> {
const abortController = controller ?? new AbortController();
// If the room is encrypted then encrypt the file before uploading it.
@ -329,7 +372,7 @@ export async function uploadFile(
file: {
...encryptResult.info,
url,
} as IEncryptedFile,
} as EncryptedFile,
};
} else {
const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController });
@ -546,6 +589,14 @@ export default class ContentMessages {
}
} else if (file.type.indexOf("audio/") === 0) {
content.msgtype = MsgType.Audio;
try {
const audioInfo = await infoForAudioFile(file);
Object.assign(content.info, audioInfo);
} catch (e) {
// Failed to process audio file, fall back to uploading an m.file
logger.error(e);
content.msgtype = MsgType.File;
}
} else if (file.type.indexOf("video/") === 0) {
content.msgtype = MsgType.Video;
try {

2
src/components/views/messages/MFileBody.tsx
View file

@ -198,7 +198,7 @@ export default class MFileBody extends React.Component<IProps, IState> {
const isEncrypted = this.props.mediaEventHelper?.media.isEncrypted;
const contentUrl = this.getContentUrl();
const contentFileSize = this.content.info ? this.content.info.size : null;
const fileType = this.content.info ? this.content.info.mimetype : "application/octet-stream";
const fileType = this.content.info?.mimetype ?? "application/octet-stream";
let placeholder: React.ReactNode = null;
if (this.props.showGenericPlaceholder) {

22
src/components/views/messages/MImageBody.tsx
View file

@ -29,7 +29,7 @@ import SettingsStore from "../../../settings/SettingsStore";
import Spinner from "../elements/Spinner";
import { Media, mediaFromContent } from "../../../customisations/Media";
import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media";
import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
import { ImageContent } from "../../../customisations/models/IMediaEventContent";
import ImageView from "../elements/ImageView";
import { IBodyProps } from "./IBodyProps";
import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize";
@ -102,7 +102,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
return;
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
const httpUrl = this.state.contentUrl;
if (!httpUrl) return;
const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = {
@ -212,7 +212,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
const thumbWidth = 800;
const thumbHeight = 600;
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
const media = mediaFromContent(content);
const info = content.info;
@ -287,7 +287,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
contentUrl = this.getContentUrl();
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
let isAnimated = mayBeAnimated(content.info?.mimetype);
// If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
@ -317,7 +317,13 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
}
if (isAnimated) {
const thumb = await createThumbnail(img, img.width, img.height, content.info!.mimetype, false);
const thumb = await createThumbnail(
img,
img.width,
img.height,
content.info?.mimetype ?? "image/jpeg",
false,
);
thumbUrl = URL.createObjectURL(thumb.thumbnail);
}
} catch (error) {
@ -381,7 +387,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
}
}
protected getBanner(content: IMediaEventContent): ReactNode {
protected getBanner(content: ImageContent): ReactNode {
// Hide it for the threads list & the file panel where we show it as text anyway.
if (
[TimelineRenderingType.ThreadsList, TimelineRenderingType.File].includes(this.context.timelineRenderingType)
@ -395,7 +401,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
protected messageContent(
contentUrl: string | null,
thumbUrl: string | null,
content: IMediaEventContent,
content: ImageContent,
forcedHeight?: number,
): ReactNode {
if (!thumbUrl) thumbUrl = contentUrl; // fallback
@ -591,7 +597,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
}
public render(): React.ReactNode {
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
if (this.state.error) {
let errorText = _t("Unable to show image due to error");

4
src/components/views/messages/MImageReplyBody.tsx
View file

@ -17,7 +17,7 @@ limitations under the License.
import React from "react";
import MImageBody from "./MImageBody";
import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
import { ImageContent } from "../../../customisations/models/IMediaEventContent";
const FORCED_IMAGE_HEIGHT = 44;
@ -35,7 +35,7 @@ export default class MImageReplyBody extends MImageBody {
return super.render();
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
const thumbnail = this.state.contentUrl
? this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT)
: undefined;

223
src/customisations/models/IMediaEventContent.ts
View file

@ -16,8 +16,21 @@
// TODO: These types should be elsewhere.
export interface IEncryptedFile {
import { MsgType } from "matrix-js-sdk/src/matrix";
import { BLURHASH_FIELD } from "../../utils/image-media";
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#extensions-to-mroommessage-msgtypes
*/
export interface EncryptedFile {
/**
* The URL to the file.
*/
url: string;
/**
* A JSON Web Key object.
*/
key: {
alg: string;
key_ops: string[]; // eslint-disable-line camelcase
@ -25,43 +38,204 @@ export interface IEncryptedFile {
k: string;
ext: boolean;
};
/**
* The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
*/
iv: string;
/**
* A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
* Clients should support the SHA-256 hash, which uses the key sha256.
*/
hashes: { [alg: string]: string };
/**
* Version of the encrypted attachment's protocol. Must be v2.
*/
v: string;
}
export interface IMediaEventInfo {
thumbnail_url?: string; // eslint-disable-line camelcase
thumbnail_file?: IEncryptedFile; // eslint-disable-line camelcase
thumbnail_info?: {
// eslint-disable-line camelcase
mimetype: string;
w?: number;
h?: number;
size?: number;
};
mimetype: string;
interface ThumbnailInfo {
/**
* The mimetype of the image, e.g. image/jpeg.
*/
mimetype?: string;
/**
* The intended display width of the image in pixels.
* This may differ from the intrinsic dimensions of the image file.
*/
w?: number;
/**
* The intended display height of the image in pixels.
* This may differ from the intrinsic dimensions of the image file.
*/
h?: number;
/**
* Size of the image in bytes.
*/
size?: number;
}
export interface IMediaEventContent {
msgtype: string;
body?: string;
filename?: string; // `m.file` optional field
url?: string; // required on unencrypted media
file?: IEncryptedFile; // required for *encrypted* media
info?: IMediaEventInfo;
interface BaseInfo {
mimetype?: string;
size?: number;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mfile
*/
export interface FileInfo extends BaseInfo {
/**
* @see https://github.com/matrix-org/matrix-spec-proposals/pull/2448
*/
[BLURHASH_FIELD]?: string;
/**
* Information on the encrypted thumbnail file, as specified in End-to-end encryption.
* Only present if the thumbnail is encrypted.
* @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments
*/
thumbnail_file?: EncryptedFile;
/**
* Metadata about the image referred to in thumbnail_url.
*/
thumbnail_info?: ThumbnailInfo;
/**
* The URL to the thumbnail of the file. Only present if the thumbnail is unencrypted.
*/
thumbnail_url?: string;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mimage
*
*/
export interface ImageInfo extends FileInfo, ThumbnailInfo {}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mimage
*/
export interface AudioInfo extends BaseInfo {
/**
* The duration of the audio in milliseconds.
*/
duration?: number;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mvideo
*/
export interface VideoInfo extends AudioInfo, ImageInfo {
/**
* The duration of the video in milliseconds.
*/
duration?: number;
}
export type IMediaEventInfo = FileInfo | ImageInfo | AudioInfo | VideoInfo;
interface BaseContent {
/**
* Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
* @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments
*/
file?: EncryptedFile;
/**
* Required if the file is unencrypted. The URL (typically mxc:// URI) to the file.
*/
url?: string;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mfile
*/
export interface FileContent extends BaseContent {
/**
* A human-readable description of the file.
* This is recommended to be the filename of the original upload.
*/
body: string;
/**
* The original filename of the uploaded file.
*/
filename?: string;
/**
* Information about the file referred to in url.
*/
info?: FileInfo;
/**
* One of: [m.file].
*/
msgtype: MsgType.File;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mimage
*/
export interface ImageContent extends BaseContent {
/**
* A textual representation of the image.
* This could be the alt text of the image, the filename of the image,
* or some kind of content description for accessibility e.g. ‘image attachment’.
*/
body: string;
/**
* Metadata about the image referred to in url.
*/
info?: ImageInfo;
/**
* One of: [m.image].
*/
msgtype: MsgType.Image;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#maudio
*/
export interface AudioContent extends BaseContent {
/**
* A description of the audio e.g. ‘Bee Gees - Stayin’ Alive’,
* or some kind of content description for accessibility e.g. ‘audio attachment’.
*/
body: string;
/**
* Metadata for the audio clip referred to in url.
*/
info?: AudioInfo;
/**
* One of: [m.audio].
*/
msgtype: MsgType.Audio;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mvideo
*/
export interface VideoContent extends BaseContent {
/**
* A description of the video e.g. ‘Gangnam style’,
* or some kind of content description for accessibility e.g. ‘video attachment’.
*/
body: string;
/**
* Metadata about the video clip referred to in url.
*/
info?: VideoInfo;
/**
* One of: [m.video].
*/
msgtype: MsgType.Video;
}
/**
* Type representing media event contents for `m.room.message` events listed in the Matrix specification
*/
export type IMediaEventContent = FileContent | ImageContent | AudioContent | VideoContent;
export interface IPreparedMedia extends IMediaObject {
thumbnail?: IMediaObject;
}
export interface IMediaObject {
mxc: string;
file?: IEncryptedFile;
file?: EncryptedFile;
}
/**
@ -73,12 +247,17 @@ export interface IMediaObject {
*/
export function prepEventContentAsMedia(content: Partial<IMediaEventContent>): IPreparedMedia {
let thumbnail: IMediaObject | undefined;
if (content?.info?.thumbnail_url) {
if (typeof content?.info === "object" && "thumbnail_url" in content.info && content.info.thumbnail_url) {
thumbnail = {
mxc: content.info.thumbnail_url,
file: content.info.thumbnail_file,
};
} else if (content?.info?.thumbnail_file?.url) {
} else if (
typeof content?.info === "object" &&
"thumbnail_file" in content.info &&
typeof content?.info?.thumbnail_file === "object" &&
content?.info?.thumbnail_file?.url
) {
thumbnail = {
mxc: content.info.thumbnail_file.url,
file: content.info.thumbnail_file,

4
src/models/RoomUpload.ts
View file

@ -16,11 +16,11 @@ limitations under the License.
import { IEventRelation, UploadProgress } from "matrix-js-sdk/src/matrix";
import { IEncryptedFile } from "../customisations/models/IMediaEventContent";
import { EncryptedFile } from "../customisations/models/IMediaEventContent";
export class RoomUpload {
public readonly abortController = new AbortController();
public promise?: Promise<{ url?: string; file?: IEncryptedFile }>;
public promise?: Promise<{ url?: string; file?: EncryptedFile }>;
private uploaded = 0;
public constructor(

6
src/utils/DecryptFile.ts
View file

@ -19,7 +19,7 @@ import encrypt from "matrix-encrypt-attachment";
import { parseErrorResponse } from "matrix-js-sdk/src/http-api";
import { mediaFromContent } from "../customisations/Media";
import { IEncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent";
import { EncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent";
import { getBlobSafeMimeType } from "./blobs";
export class DownloadError extends Error {
@ -40,14 +40,14 @@ export class DecryptError extends Error {
/**
* Decrypt a file attached to a matrix event.
* @param {IEncryptedFile} file The encrypted file information taken from the matrix event.
* @param {EncryptedFile} file The encrypted file information taken from the matrix event.
* This passed to [link]{@link https://github.com/matrix-org/matrix-encrypt-attachment}
* as the encryption info object, so will also have the those keys in addition to
* the keys below.
* @param {IMediaEventInfo} info The info parameter taken from the matrix event.
* @returns {Promise<Blob>} Resolves to a Blob of the file.
*/
export async function decryptFile(file?: IEncryptedFile, info?: IMediaEventInfo): Promise<Blob> {
export async function decryptFile(file?: EncryptedFile, info?: IMediaEventInfo): Promise<Blob> {
// throws if file is falsy
const media = mediaFromContent({ file });

6
src/utils/MediaEventHelper.ts
View file

@ -21,7 +21,7 @@ import { logger } from "matrix-js-sdk/src/logger";
import { LazyValue } from "./LazyValue";
import { Media, mediaFromContent } from "../customisations/Media";
import { decryptFile } from "./DecryptFile";
import { IMediaEventContent } from "../customisations/models/IMediaEventContent";
import { FileContent, ImageContent, IMediaEventContent } from "../customisations/models/IMediaEventContent";
import { IDestroyable } from "./IDestroyable";
// TODO: We should consider caching the blobs. https://github.com/vector-im/element-web/issues/17192
@ -48,7 +48,7 @@ export class MediaEventHelper implements IDestroyable {
public get fileName(): string {
return (
this.event.getContent<IMediaEventContent>().filename ||
this.event.getContent<FileContent>().filename ||
this.event.getContent<IMediaEventContent>().body ||
"download"
);
@ -92,7 +92,7 @@ export class MediaEventHelper implements IDestroyable {
if (!this.media.hasThumbnail) return Promise.resolve(null);
if (this.media.isEncrypted) {
const content = this.event.getContent<IMediaEventContent>();
const content = this.event.getContent<ImageContent>();
if (content.info?.thumbnail_file) {
return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
} else {

4
src/utils/image-media.ts
View file

@ -15,7 +15,7 @@ limitations under the License.
*/
import { BlurhashEncoder } from "../BlurhashEncoder";
import { IEncryptedFile } from "../customisations/models/IMediaEventContent";
import { EncryptedFile } from "../customisations/models/IMediaEventContent";
type ThumbnailableElement = HTMLImageElement | HTMLVideoElement;
@ -33,7 +33,7 @@ interface IThumbnail {
h: number;
[BLURHASH_FIELD]?: string;
thumbnail_url?: string;
thumbnail_file?: IEncryptedFile;
thumbnail_file?: EncryptedFile;
};
thumbnail: Blob;
}

4
src/voice-broadcast/models/VoiceBroadcastRecording.ts
View file

@ -38,7 +38,7 @@ import {
VoiceBroadcastRecorderEvent,
} from "..";
import { uploadFile } from "../../ContentMessages";
import { IEncryptedFile } from "../../customisations/models/IMediaEventContent";
import { EncryptedFile } from "../../customisations/models/IMediaEventContent";
import { createVoiceMessageContent } from "../../utils/createVoiceMessageContent";
import { IDestroyable } from "../../utils/IDestroyable";
import dis from "../../dispatcher/dispatcher";
@ -367,7 +367,7 @@ export class VoiceBroadcastRecording
);
}
private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: IEncryptedFile): Promise<void> {
private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: EncryptedFile): Promise<void> {
/**
* Increment the last sequence number and use it for this message.
* Done outside of the sendMessageFn to get a scoped value.

53
test/ContentMessages-test.ts
View file

@ -163,6 +163,11 @@ describe("ContentMessages", () => {
return 800;
},
});
Object.defineProperty(element, "duration", {
get() {
return 123;
},
});
}
return element;
});
@ -176,11 +181,31 @@ describe("ContentMessages", () => {
expect.objectContaining({
url: "mxc://server/file",
msgtype: "m.video",
info: expect.objectContaining({
duration: 123000,
}),
}),
);
});
it("should use m.audio for audio files", async () => {
jest.spyOn(document, "createElement").mockImplementation((tagName) => {
const element = createElement(tagName);
if (tagName === "audio") {
Object.defineProperty(element, "duration", {
get() {
return 621;
},
});
Object.defineProperty(element, "src", {
set() {
element.onloadedmetadata!(new Event("loadedmetadata"));
},
});
}
return element;
});
mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" });
const file = new File([], "fileName", { type: "audio/mp3" });
await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined);
@ -190,6 +215,34 @@ describe("ContentMessages", () => {
expect.objectContaining({
url: "mxc://server/file",
msgtype: "m.audio",
info: expect.objectContaining({
duration: 621000,
}),
}),
);
});
it("should fall back to m.file for invalid audio files", async () => {
jest.spyOn(document, "createElement").mockImplementation((tagName) => {
const element = createElement(tagName);
if (tagName === "audio") {
Object.defineProperty(element, "src", {
set() {
element.onerror!("fail");
},
});
}
return element;
});
mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" });
const file = new File([], "fileName", { type: "audio/mp3" });
await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined);
expect(client.sendMessage).toHaveBeenCalledWith(
roomId,
null,
expect.objectContaining({
url: "mxc://server/file",
msgtype: "m.file",
}),
);
});

68
test/components/views/messages/MImageBody-test.tsx
View file

@ -15,11 +15,13 @@ limitations under the License.
*/
import React from "react";
import { fireEvent, render, screen } from "@testing-library/react";
import { fireEvent, render, screen, waitForElementToBeRemoved } from "@testing-library/react";
import { EventType, getHttpUriForMxc, MatrixEvent, Room } from "matrix-js-sdk/src/matrix";
import fetchMock from "fetch-mock-jest";
import encrypt from "matrix-encrypt-attachment";
import { mocked } from "jest-mock";
import fs from "fs";
import path from "path";
import MImageBody from "../../../../src/components/views/messages/MImageBody";
import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
@ -56,7 +58,11 @@ describe("<MImageBody/>", () => {
});
const url = "https://server/_matrix/media/r0/download/server/encrypted-image";
// eslint-disable-next-line no-restricted-properties
cli.mxcUrlToHttp.mockReturnValue(url);
cli.mxcUrlToHttp.mockImplementation(
(mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
},
);
const encryptedMediaEvent = new MatrixEvent({
room_id: "!room:server",
sender: userId,
@ -175,12 +181,6 @@ describe("<MImageBody/>", () => {
it("should fall back to /download/ if /thumbnail/ fails", async () => {
const thumbUrl = "https://server/_matrix/media/r0/thumbnail/server/image?width=800&height=600&method=scale";
const downloadUrl = "https://server/_matrix/media/r0/download/server/image";
// eslint-disable-next-line no-restricted-properties
cli.mxcUrlToHttp.mockImplementation(
(mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
},
);
const event = new MatrixEvent({
room_id: "!room:server",
@ -206,4 +206,56 @@ describe("<MImageBody/>", () => {
fireEvent.error(img);
expect(img).toHaveProperty("src", downloadUrl);
});
it("should generate a thumbnail if one isn't included for animated media", async () => {
Object.defineProperty(global.Image.prototype, "src", {
set(src) {
window.setTimeout(() => this.onload());
},
});
Object.defineProperty(global.Image.prototype, "height", {
get() {
return 600;
},
});
Object.defineProperty(global.Image.prototype, "width", {
get() {
return 800;
},
});
mocked(global.URL.createObjectURL).mockReturnValue("blob:generated-thumb");
fetchMock.getOnce(
"https://server/_matrix/media/r0/download/server/image",
{
body: fs.readFileSync(path.resolve(__dirname, "..", "..", "..", "images", "animated-logo.webp")),
},
{ sendAsJson: false },
);
const event = new MatrixEvent({
room_id: "!room:server",
sender: userId,
type: EventType.RoomMessage,
content: {
body: "alt for a test image",
info: {
w: 40,
h: 50,
mimetype: "image/webp",
},
url: "mxc://server/image",
},
});
const { container } = render(
<MImageBody {...props} mxEvent={event} mediaEventHelper={new MediaEventHelper(event)} />,
);
// Wait for spinners to go away
await waitForElementToBeRemoved(screen.getAllByRole("progressbar"));
// thumbnail with dimensions present
expect(container).toMatchSnapshot();
});
});

71
test/components/views/messages/MVideoBody-test.tsx
View file

@ -15,23 +15,22 @@ limitations under the License.
*/
import React, { ComponentProps } from "react";
import { IContent, MatrixEvent } from "matrix-js-sdk/src/matrix";
import { EventType, getHttpUriForMxc, IContent, MatrixEvent } from "matrix-js-sdk/src/matrix";
import { render, RenderResult } from "@testing-library/react";
import fetchMock from "fetch-mock-jest";
import MatrixClientContext from "../../../../src/contexts/MatrixClientContext";
import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
import { MediaEventHelper } from "../../../../src/utils/MediaEventHelper";
import { getMockClientWithEventEmitter } from "../../../test-utils";
import {
getMockClientWithEventEmitter,
mockClientMethodsCrypto,
mockClientMethodsDevice,
mockClientMethodsServer,
mockClientMethodsUser,
} from "../../../test-utils";
import MVideoBody from "../../../../src/components/views/messages/MVideoBody";
jest.mock("../../../../src/customisations/Media", () => {
return {
mediaFromContent: () => {
return { isEncrypted: false };
},
};
});
describe("MVideoBody", () => {
it("does not crash when given a portrait image", () => {
// Check for an unreliable crash caused by a fractional-sized
@ -40,6 +39,58 @@ describe("MVideoBody", () => {
expect(asFragment()).toMatchSnapshot();
// If we get here, we did not crash.
});
it("should show poster for encrypted media before downloading it", async () => {
const userId = "@user:server";
const deviceId = "DEADB33F";
const cli = getMockClientWithEventEmitter({
...mockClientMethodsUser(userId),
...mockClientMethodsServer(),
...mockClientMethodsDevice(deviceId),
...mockClientMethodsCrypto(),
getRooms: jest.fn().mockReturnValue([]),
getIgnoredUsers: jest.fn(),
getVersions: jest.fn().mockResolvedValue({
unstable_features: {
"org.matrix.msc3882": true,
"org.matrix.msc3886": true,
},
}),
});
const thumbUrl = "https://server/_matrix/media/r0/download/server/encrypted-poster";
fetchMock.getOnce(thumbUrl, { status: 200 });
// eslint-disable-next-line no-restricted-properties
cli.mxcUrlToHttp.mockImplementation(
(mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
},
);
const encryptedMediaEvent = new MatrixEvent({
room_id: "!room:server",
sender: userId,
type: EventType.RoomMessage,
content: {
body: "alt for a test video",
info: {
duration: 420,
w: 40,
h: 50,
thumbnail_file: {
url: "mxc://server/encrypted-poster",
},
},
file: {
url: "mxc://server/encrypted-image",
},
},
});
const { asFragment } = render(
<MVideoBody mxEvent={encryptedMediaEvent} mediaEventHelper={new MediaEventHelper(encryptedMediaEvent)} />,
);
expect(asFragment()).toMatchSnapshot();
});
});
function makeMVideoBody(w: number, h: number): RenderResult {

50
test/components/views/messages/__snapshots__/MImageBody-test.tsx.snap
View file

@ -1,5 +1,55 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`<MImageBody/> should generate a thumbnail if one isn't included for animated media 1`] = `
<div>
<div
class="mx_MImageBody"
>
<a
href="https://server/_matrix/media/r0/download/server/image"
>
<div
class="mx_MImageBody_thumbnail_container"
style="max-height: 50px; max-width: 40px;"
>
<div
class="mx_MImageBody_placeholder"
>
<div
class="mx_Spinner"
>
<div
aria-label="Loading…"
class="mx_Spinner_icon"
data-testid="spinner"
role="progressbar"
style="width: 32px; height: 32px;"
/>
</div>
</div>
<div
style="max-height: 50px; max-width: 40px;"
>
<img
alt="alt for a test image"
class="mx_MImageBody_thumbnail"
src="blob:generated-thumb"
/>
<p
class="mx_MImageBody_gifLabel"
>
GIF
</p>
</div>
<div
style="height: 50px; width: 40px;"
/>
</div>
</a>
</div>
</div>
`;
exports[`<MImageBody/> should show a thumbnail while image is being downloaded 1`] = `
<div>
<div

25
test/components/views/messages/__snapshots__/MVideoBody-test.tsx.snap
View file

@ -23,3 +23,28 @@ exports[`MVideoBody does not crash when given a portrait image 1`] = `
</span>
</DocumentFragment>
`;
exports[`MVideoBody should show poster for encrypted media before downloading it 1`] = `
<DocumentFragment>
<span
class="mx_MVideoBody"
>
<div
class="mx_MVideoBody_container"
style="max-width: 40px; max-height: 50px;"
>
<video
class="mx_MVideoBody"
controls=""
controlslist="nodownload"
poster="https://server/_matrix/media/r0/download/server/encrypted-poster"
preload="none"
title="alt for a test video"
/>
<div
style="width: 40px; height: 50px;"
/>
</div>
</span>
</DocumentFragment>
`;

6
test/voice-broadcast/models/VoiceBroadcastRecording-test.ts
View file

@ -31,7 +31,7 @@ import { Relations } from "matrix-js-sdk/src/models/relations";
import { SyncState } from "matrix-js-sdk/src/sync";
import { uploadFile } from "../../../src/ContentMessages";
import { IEncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
import { EncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
import { createVoiceMessageContent } from "../../../src/utils/createVoiceMessageContent";
import {
createVoiceBroadcastRecorder,
@ -82,7 +82,7 @@ jest.mock("../../../src/utils/createVoiceMessageContent", () => ({
describe("VoiceBroadcastRecording", () => {
const roomId = "!room:example.com";
const uploadedUrl = "mxc://example.com/vb";
const uploadedFile = { file: true } as unknown as IEncryptedFile;
const uploadedFile = { file: true } as unknown as EncryptedFile;
const maxLength = getMaxBroadcastLength();
let room: Room;
let client: MatrixClient;
@ -223,7 +223,7 @@ describe("VoiceBroadcastRecording", () => {
mimetype: string,
duration: number,
size: number,
file?: IEncryptedFile,
file?: EncryptedFile,
waveform?: number[],
) => {
return {