Populate info.duration for audio & video file uploads (#11225)

* Improve m.file m.image m.audio m.video types

* Populate `info.duration` for audio & video file uploads

* Fix tests

* Iterate types

* Improve coverage

* Fix test

* Add small delay to stabilise cypress test

* Fix test idempotency

* Improve coverage

* Slow down

* iterate

cypress/e2e/right-panel/file-panel.spec.ts

@@ -183,7 +183,7 @@ describe("FilePanel", () => {
             });
         });
 
-        it("should render the audio pleyer and play the audio file on the panel", () => {
+        it("should render the audio player and play the audio file on the panel", () => {
             // Upload an image file
             uploadFile("cypress/fixtures/1sec.ogg");
 
@@ -202,10 +202,14 @@ describe("FilePanel", () => {
                                 cy.contains(".mx_AudioPlayer_byline", "(3.56 KB)").should("exist"); // actual size
                             });
 
+                            // Assert that the duration counter is 00:01 before clicking the play button
+                            cy.contains(".mx_AudioPlayer_mediaInfo time", "00:01").should("exist");
+
                             // Assert that the counter is zero before clicking the play button
                             cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist");
 
                             // Click the play button
+                            cy.wait(500);
                             cy.findByRole("button", { name: "Play" }).click();
 
                             // Assert that the pause button is rendered

src/ContentMessages.ts

@@ -33,7 +33,14 @@ import {
 import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread";
 import { removeElement } from "matrix-js-sdk/src/utils";
 
-import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent";
+import {
+    AudioInfo,
+    EncryptedFile,
+    ImageInfo,
+    IMediaEventContent,
+    IMediaEventInfo,
+    VideoInfo,
+} from "./customisations/models/IMediaEventContent";
 import dis from "./dispatcher/dispatcher";
 import { _t } from "./languageHandler";
 import Modal from "./Modal";
@@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];
  * @param {File} imageFile The image to read and thumbnail.
  * @return {Promise} A promise that resolves with the attachment info.
  */
-async function infoForImageFile(
+async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
-    matrixClient: MatrixClient,
-    roomId: string,
-    imageFile: File,
-): Promise<Partial<IMediaEventInfo>> {
     let thumbnailType = "image/png";
     if (imageFile.type === "image/jpeg") {
         thumbnailType = "image/jpeg";
@@ -184,16 +187,59 @@ async function infoForImageFile(
     return imageInfo;
 }
 
+/**
+ * Load a file into a newly created audio element and load the metadata
+ *
+ * @param {File} audioFile The file to load in an audio element.
+ * @return {Promise} A promise that resolves with the audio element.
+ */
+function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
+    return new Promise((resolve, reject) => {
+        // Load the file into a html element
+        const audio = document.createElement("audio");
+        audio.preload = "metadata";
+        audio.muted = true;
+
+        const reader = new FileReader();
+
+        reader.onload = function (ev): void {
+            audio.onloadedmetadata = async function (): Promise<void> {
+                resolve(audio);
+            };
+            audio.onerror = function (e): void {
+                reject(e);
+            };
+
+            audio.src = ev.target?.result as string;
+        };
+        reader.onerror = function (e): void {
+            reject(e);
+        };
+        reader.readAsDataURL(audioFile);
+    });
+}
+
+/**
+ * Read the metadata for an audio file.
+ *
+ * @param {File} audioFile The audio to read.
+ * @return {Promise} A promise that resolves with the attachment info.
+ */
+async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
+    const audio = await loadAudioElement(audioFile);
+    return { duration: Math.ceil(audio.duration * 1000) };
+}
+
 /**
  * Load a file into a newly created video element and pull some strings
  * in an attempt to guarantee the first frame will be showing.
  *
- * @param {File} videoFile The file to load in an video element.
+ * @param {File} videoFile The file to load in a video element.
- * @return {Promise} A promise that resolves with the video image element.
+ * @return {Promise} A promise that resolves with the video element.
  */
 function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
     return new Promise((resolve, reject) => {
-        // Load the file into an html element
+        // Load the file into a html element
         const video = document.createElement("video");
         video.preload = "metadata";
         video.playsInline = true;
@@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
  * @param {File} videoFile The video to read and thumbnail.
  * @return {Promise} A promise that resolves with the attachment info.
  */
-function infoForVideoFile(
+function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
-    matrixClient: MatrixClient,
-    roomId: string,
-    videoFile: File,
-): Promise<Partial<IMediaEventInfo>> {
     const thumbnailType = "image/jpeg";
 
-    let videoInfo: Partial<IMediaEventInfo>;
+    const videoInfo: VideoInfo = {};
     return loadVideoElement(videoFile)
         .then((video) => {
+            videoInfo.duration = Math.ceil(video.duration * 1000);
             return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
         })
         .then((result) => {
-            videoInfo = result.info;
+            Object.assign(videoInfo, result.info);
             return uploadFile(matrixClient, roomId, result.thumbnail);
         })
         .then((result) => {
@@ -299,7 +342,7 @@ export async function uploadFile(
     file: File | Blob,
     progressHandler?: UploadOpts["progressHandler"],
     controller?: AbortController,
-): Promise<{ url?: string; file?: IEncryptedFile }> {
+): Promise<{ url?: string; file?: EncryptedFile }> {
     const abortController = controller ?? new AbortController();
 
     // If the room is encrypted then encrypt the file before uploading it.
@@ -329,7 +372,7 @@ export async function uploadFile(
             file: {
                 ...encryptResult.info,
                 url,
-            } as IEncryptedFile,
+            } as EncryptedFile,
         };
     } else {
         const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController });
@@ -546,6 +589,14 @@ export default class ContentMessages {
                 }
             } else if (file.type.indexOf("audio/") === 0) {
                 content.msgtype = MsgType.Audio;
+                try {
+                    const audioInfo = await infoForAudioFile(file);
+                    Object.assign(content.info, audioInfo);
+                } catch (e) {
+                    // Failed to process audio file, fall back to uploading an m.file
+                    logger.error(e);
+                    content.msgtype = MsgType.File;
+                }
             } else if (file.type.indexOf("video/") === 0) {
                 content.msgtype = MsgType.Video;
                 try {

src/components/views/messages/MFileBody.tsx

@@ -198,7 +198,7 @@ export default class MFileBody extends React.Component<IProps, IState> {
         const isEncrypted = this.props.mediaEventHelper?.media.isEncrypted;
         const contentUrl = this.getContentUrl();
         const contentFileSize = this.content.info ? this.content.info.size : null;
-        const fileType = this.content.info ? this.content.info.mimetype : "application/octet-stream";
+        const fileType = this.content.info?.mimetype ?? "application/octet-stream";
 
         let placeholder: React.ReactNode = null;
         if (this.props.showGenericPlaceholder) {

src/components/views/messages/MImageBody.tsx

@@ -29,7 +29,7 @@ import SettingsStore from "../../../settings/SettingsStore";
 import Spinner from "../elements/Spinner";
 import { Media, mediaFromContent } from "../../../customisations/Media";
 import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media";
-import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
+import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 import ImageView from "../elements/ImageView";
 import { IBodyProps } from "./IBodyProps";
 import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize";
@@ -102,7 +102,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                 return;
             }
 
-            const content = this.props.mxEvent.getContent<IMediaEventContent>();
+            const content = this.props.mxEvent.getContent<ImageContent>();
             const httpUrl = this.state.contentUrl;
             if (!httpUrl) return;
             const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = {
@@ -212,7 +212,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         const thumbWidth = 800;
         const thumbHeight = 600;
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         const media = mediaFromContent(content);
         const info = content.info;
 
@@ -287,7 +287,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
             contentUrl = this.getContentUrl();
         }
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         let isAnimated = mayBeAnimated(content.info?.mimetype);
 
         // If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
@@ -317,7 +317,13 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                     }
 
                     if (isAnimated) {
-                        const thumb = await createThumbnail(img, img.width, img.height, content.info!.mimetype, false);
+                        const thumb = await createThumbnail(
+                            img,
+                            img.width,
+                            img.height,
+                            content.info?.mimetype ?? "image/jpeg",
+                            false,
+                        );
                         thumbUrl = URL.createObjectURL(thumb.thumbnail);
                     }
                 } catch (error) {
@@ -381,7 +387,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         }
     }
 
-    protected getBanner(content: IMediaEventContent): ReactNode {
+    protected getBanner(content: ImageContent): ReactNode {
         // Hide it for the threads list & the file panel where we show it as text anyway.
         if (
             [TimelineRenderingType.ThreadsList, TimelineRenderingType.File].includes(this.context.timelineRenderingType)
@@ -395,7 +401,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
     protected messageContent(
         contentUrl: string | null,
         thumbUrl: string | null,
-        content: IMediaEventContent,
+        content: ImageContent,
         forcedHeight?: number,
     ): ReactNode {
         if (!thumbUrl) thumbUrl = contentUrl; // fallback
@@ -591,7 +597,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
     }
 
     public render(): React.ReactNode {
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
 
         if (this.state.error) {
             let errorText = _t("Unable to show image due to error");

src/components/views/messages/MImageReplyBody.tsx

@@ -17,7 +17,7 @@ limitations under the License.
 import React from "react";
 
 import MImageBody from "./MImageBody";
-import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
+import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 
 const FORCED_IMAGE_HEIGHT = 44;
 
@@ -35,7 +35,7 @@ export default class MImageReplyBody extends MImageBody {
             return super.render();
         }
 
-        const content = this.props.mxEvent.getContent<IMediaEventContent>();
+        const content = this.props.mxEvent.getContent<ImageContent>();
         const thumbnail = this.state.contentUrl
             ? this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT)
             : undefined;

src/customisations/models/IMediaEventContent.ts

@@ -16,8 +16,21 @@
 
 // TODO: These types should be elsewhere.
 
-export interface IEncryptedFile {
+import { MsgType } from "matrix-js-sdk/src/matrix";
+
+import { BLURHASH_FIELD } from "../../utils/image-media";
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#extensions-to-mroommessage-msgtypes
+ */
+export interface EncryptedFile {
+    /**
+     * The URL to the file.
+     */
     url: string;
+    /**
+     * A JSON Web Key object.
+     */
     key: {
         alg: string;
         key_ops: string[]; // eslint-disable-line camelcase
@@ -25,43 +38,204 @@ export interface IEncryptedFile {
         k: string;
         ext: boolean;
     };
+    /**
+     * The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
+     */
     iv: string;
+    /**
+     * A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
+     * Clients should support the SHA-256 hash, which uses the key sha256.
+     */
     hashes: { [alg: string]: string };
+    /**
+     * Version of the encrypted attachment's protocol. Must be v2.
+     */
     v: string;
 }
 
-export interface IMediaEventInfo {
+interface ThumbnailInfo {
-    thumbnail_url?: string; // eslint-disable-line camelcase
+    /**
-    thumbnail_file?: IEncryptedFile; // eslint-disable-line camelcase
+     * The mimetype of the image, e.g. image/jpeg.
-    thumbnail_info?: {
+     */
-        // eslint-disable-line camelcase
+    mimetype?: string;
-        mimetype: string;
+    /**
-        w?: number;
+     * The intended display width of the image in pixels.
-        h?: number;
+     * This may differ from the intrinsic dimensions of the image file.
-        size?: number;
+     */
-    };
-    mimetype: string;
     w?: number;
+    /**
+     * The intended display height of the image in pixels.
+     * This may differ from the intrinsic dimensions of the image file.
+     */
     h?: number;
+    /**
+     * Size of the image in bytes.
+     */
     size?: number;
 }
 
-export interface IMediaEventContent {
+interface BaseInfo {
-    msgtype: string;
+    mimetype?: string;
-    body?: string;
+    size?: number;
-    filename?: string; // `m.file` optional field
-    url?: string; // required on unencrypted media
-    file?: IEncryptedFile; // required for *encrypted* media
-    info?: IMediaEventInfo;
 }
 
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mfile
+ */
+export interface FileInfo extends BaseInfo {
+    /**
+     * @see https://github.com/matrix-org/matrix-spec-proposals/pull/2448
+     */
+    [BLURHASH_FIELD]?: string;
+    /**
+     * Information on the encrypted thumbnail file, as specified in End-to-end encryption.
+     * Only present if the thumbnail is encrypted.
+     * @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments
+     */
+    thumbnail_file?: EncryptedFile;
+    /**
+     * Metadata about the image referred to in thumbnail_url.
+     */
+    thumbnail_info?: ThumbnailInfo;
+    /**
+     * The URL to the thumbnail of the file. Only present if the thumbnail is unencrypted.
+     */
+    thumbnail_url?: string;
+}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mimage
+ *
+ */
+export interface ImageInfo extends FileInfo, ThumbnailInfo {}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mimage
+ */
+export interface AudioInfo extends BaseInfo {
+    /**
+     * The duration of the audio in milliseconds.
+     */
+    duration?: number;
+}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mvideo
+ */
+export interface VideoInfo extends AudioInfo, ImageInfo {
+    /**
+     * The duration of the video in milliseconds.
+     */
+    duration?: number;
+}
+
+export type IMediaEventInfo = FileInfo | ImageInfo | AudioInfo | VideoInfo;
+
+interface BaseContent {
+    /**
+     * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
+     * @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments
+     */
+    file?: EncryptedFile;
+    /**
+     * Required if the file is unencrypted. The URL (typically mxc:// URI) to the file.
+     */
+    url?: string;
+}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mfile
+ */
+export interface FileContent extends BaseContent {
+    /**
+     * A human-readable description of the file.
+     * This is recommended to be the filename of the original upload.
+     */
+    body: string;
+    /**
+     * The original filename of the uploaded file.
+     */
+    filename?: string;
+    /**
+     * Information about the file referred to in url.
+     */
+    info?: FileInfo;
+    /**
+     * One of: [m.file].
+     */
+    msgtype: MsgType.File;
+}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mimage
+ */
+export interface ImageContent extends BaseContent {
+    /**
+     * A textual representation of the image.
+     * This could be the alt text of the image, the filename of the image,
+     * or some kind of content description for accessibility e.g. ‘image attachment’.
+     */
+    body: string;
+    /**
+     * Metadata about the image referred to in url.
+     */
+    info?: ImageInfo;
+    /**
+     * One of: [m.image].
+     */
+    msgtype: MsgType.Image;
+}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#maudio
+ */
+export interface AudioContent extends BaseContent {
+    /**
+     * A description of the audio e.g. ‘Bee Gees - Stayin’ Alive’,
+     * or some kind of content description for accessibility e.g. ‘audio attachment’.
+     */
+    body: string;
+    /**
+     * Metadata for the audio clip referred to in url.
+     */
+    info?: AudioInfo;
+    /**
+     * One of: [m.audio].
+     */
+    msgtype: MsgType.Audio;
+}
+
+/**
+ * @see https://spec.matrix.org/v1.7/client-server-api/#mvideo
+ */
+export interface VideoContent extends BaseContent {
+    /**
+     * A description of the video e.g. ‘Gangnam style’,
+     * or some kind of content description for accessibility e.g. ‘video attachment’.
+     */
+    body: string;
+    /**
+     * Metadata about the video clip referred to in url.
+     */
+    info?: VideoInfo;
+    /**
+     * One of: [m.video].
+     */
+    msgtype: MsgType.Video;
+}
+
+/**
+ * Type representing media event contents for `m.room.message` events listed in the Matrix specification
+ */
+export type IMediaEventContent = FileContent | ImageContent | AudioContent | VideoContent;
+
 export interface IPreparedMedia extends IMediaObject {
     thumbnail?: IMediaObject;
 }
 
 export interface IMediaObject {
     mxc: string;
-    file?: IEncryptedFile;
+    file?: EncryptedFile;
 }
 
 /**
@@ -73,12 +247,17 @@ export interface IMediaObject {
  */
 export function prepEventContentAsMedia(content: Partial<IMediaEventContent>): IPreparedMedia {
     let thumbnail: IMediaObject | undefined;
-    if (content?.info?.thumbnail_url) {
+    if (typeof content?.info === "object" && "thumbnail_url" in content.info && content.info.thumbnail_url) {
         thumbnail = {
             mxc: content.info.thumbnail_url,
             file: content.info.thumbnail_file,
         };
-    } else if (content?.info?.thumbnail_file?.url) {
+    } else if (
+        typeof content?.info === "object" &&
+        "thumbnail_file" in content.info &&
+        typeof content?.info?.thumbnail_file === "object" &&
+        content?.info?.thumbnail_file?.url
+    ) {
         thumbnail = {
             mxc: content.info.thumbnail_file.url,
             file: content.info.thumbnail_file,

src/models/RoomUpload.ts

@@ -16,11 +16,11 @@ limitations under the License.
 
 import { IEventRelation, UploadProgress } from "matrix-js-sdk/src/matrix";
 
-import { IEncryptedFile } from "../customisations/models/IMediaEventContent";
+import { EncryptedFile } from "../customisations/models/IMediaEventContent";
 
 export class RoomUpload {
     public readonly abortController = new AbortController();
-    public promise?: Promise<{ url?: string; file?: IEncryptedFile }>;
+    public promise?: Promise<{ url?: string; file?: EncryptedFile }>;
     private uploaded = 0;
 
     public constructor(

src/utils/DecryptFile.ts

@@ -19,7 +19,7 @@ import encrypt from "matrix-encrypt-attachment";
 import { parseErrorResponse } from "matrix-js-sdk/src/http-api";
 
 import { mediaFromContent } from "../customisations/Media";
-import { IEncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent";
+import { EncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent";
 import { getBlobSafeMimeType } from "./blobs";
 
 export class DownloadError extends Error {
@@ -40,14 +40,14 @@ export class DecryptError extends Error {
 
 /**
  * Decrypt a file attached to a matrix event.
- * @param {IEncryptedFile} file The encrypted file information taken from the matrix event.
+ * @param {EncryptedFile} file The encrypted file information taken from the matrix event.
  *   This passed to [link]{@link https://github.com/matrix-org/matrix-encrypt-attachment}
  *   as the encryption info object, so will also have the those keys in addition to
  *   the keys below.
  * @param {IMediaEventInfo} info The info parameter taken from the matrix event.
  * @returns {Promise<Blob>} Resolves to a Blob of the file.
  */
-export async function decryptFile(file?: IEncryptedFile, info?: IMediaEventInfo): Promise<Blob> {
+export async function decryptFile(file?: EncryptedFile, info?: IMediaEventInfo): Promise<Blob> {
     // throws if file is falsy
     const media = mediaFromContent({ file });
 

src/utils/MediaEventHelper.ts

@@ -21,7 +21,7 @@ import { logger } from "matrix-js-sdk/src/logger";
 import { LazyValue } from "./LazyValue";
 import { Media, mediaFromContent } from "../customisations/Media";
 import { decryptFile } from "./DecryptFile";
-import { IMediaEventContent } from "../customisations/models/IMediaEventContent";
+import { FileContent, ImageContent, IMediaEventContent } from "../customisations/models/IMediaEventContent";
 import { IDestroyable } from "./IDestroyable";
 
 // TODO: We should consider caching the blobs. https://github.com/vector-im/element-web/issues/17192
@@ -48,7 +48,7 @@ export class MediaEventHelper implements IDestroyable {
 
     public get fileName(): string {
         return (
-            this.event.getContent<IMediaEventContent>().filename ||
+            this.event.getContent<FileContent>().filename ||
             this.event.getContent<IMediaEventContent>().body ||
             "download"
         );
@@ -92,7 +92,7 @@ export class MediaEventHelper implements IDestroyable {
         if (!this.media.hasThumbnail) return Promise.resolve(null);
 
         if (this.media.isEncrypted) {
-            const content = this.event.getContent<IMediaEventContent>();
+            const content = this.event.getContent<ImageContent>();
             if (content.info?.thumbnail_file) {
                 return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
             } else {

src/utils/image-media.ts

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 import { BlurhashEncoder } from "../BlurhashEncoder";
-import { IEncryptedFile } from "../customisations/models/IMediaEventContent";
+import { EncryptedFile } from "../customisations/models/IMediaEventContent";
 
 type ThumbnailableElement = HTMLImageElement | HTMLVideoElement;
 
@@ -33,7 +33,7 @@ interface IThumbnail {
         h: number;
         [BLURHASH_FIELD]?: string;
         thumbnail_url?: string;
-        thumbnail_file?: IEncryptedFile;
+        thumbnail_file?: EncryptedFile;
     };
     thumbnail: Blob;
 }

src/voice-broadcast/models/VoiceBroadcastRecording.ts

@@ -38,7 +38,7 @@ import {
     VoiceBroadcastRecorderEvent,
 } from "..";
 import { uploadFile } from "../../ContentMessages";
-import { IEncryptedFile } from "../../customisations/models/IMediaEventContent";
+import { EncryptedFile } from "../../customisations/models/IMediaEventContent";
 import { createVoiceMessageContent } from "../../utils/createVoiceMessageContent";
 import { IDestroyable } from "../../utils/IDestroyable";
 import dis from "../../dispatcher/dispatcher";
@@ -367,7 +367,7 @@ export class VoiceBroadcastRecording
         );
     }
 
-    private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: IEncryptedFile): Promise<void> {
+    private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: EncryptedFile): Promise<void> {
         /**
          * Increment the last sequence number and use it for this message.
          * Done outside of the sendMessageFn to get a scoped value.

test/ContentMessages-test.ts

@@ -163,6 +163,11 @@ describe("ContentMessages", () => {
                             return 800;
                         },
                     });
+                    Object.defineProperty(element, "duration", {
+                        get() {
+                            return 123;
+                        },
+                    });
                 }
                 return element;
             });
@@ -176,11 +181,31 @@ describe("ContentMessages", () => {
                 expect.objectContaining({
                     url: "mxc://server/file",
                     msgtype: "m.video",
+                    info: expect.objectContaining({
+                        duration: 123000,
+                    }),
                 }),
             );
         });
 
         it("should use m.audio for audio files", async () => {
+            jest.spyOn(document, "createElement").mockImplementation((tagName) => {
+                const element = createElement(tagName);
+                if (tagName === "audio") {
+                    Object.defineProperty(element, "duration", {
+                        get() {
+                            return 621;
+                        },
+                    });
+                    Object.defineProperty(element, "src", {
+                        set() {
+                            element.onloadedmetadata!(new Event("loadedmetadata"));
+                        },
+                    });
+                }
+                return element;
+            });
+
             mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" });
             const file = new File([], "fileName", { type: "audio/mp3" });
             await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined);
@@ -190,6 +215,34 @@ describe("ContentMessages", () => {
                 expect.objectContaining({
                     url: "mxc://server/file",
                     msgtype: "m.audio",
+                    info: expect.objectContaining({
+                        duration: 621000,
+                    }),
+                }),
+            );
+        });
+
+        it("should fall back to m.file for invalid audio files", async () => {
+            jest.spyOn(document, "createElement").mockImplementation((tagName) => {
+                const element = createElement(tagName);
+                if (tagName === "audio") {
+                    Object.defineProperty(element, "src", {
+                        set() {
+                            element.onerror!("fail");
+                        },
+                    });
+                }
+                return element;
+            });
+            mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" });
+            const file = new File([], "fileName", { type: "audio/mp3" });
+            await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined);
+            expect(client.sendMessage).toHaveBeenCalledWith(
+                roomId,
+                null,
+                expect.objectContaining({
+                    url: "mxc://server/file",
+                    msgtype: "m.file",
                 }),
             );
         });

test/components/views/messages/MImageBody-test.tsx

@@ -15,11 +15,13 @@ limitations under the License.
 */
 
 import React from "react";
-import { fireEvent, render, screen } from "@testing-library/react";
+import { fireEvent, render, screen, waitForElementToBeRemoved } from "@testing-library/react";
 import { EventType, getHttpUriForMxc, MatrixEvent, Room } from "matrix-js-sdk/src/matrix";
 import fetchMock from "fetch-mock-jest";
 import encrypt from "matrix-encrypt-attachment";
 import { mocked } from "jest-mock";
+import fs from "fs";
+import path from "path";
 
 import MImageBody from "../../../../src/components/views/messages/MImageBody";
 import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
@@ -56,7 +58,11 @@ describe("<MImageBody/>", () => {
     });
     const url = "https://server/_matrix/media/r0/download/server/encrypted-image";
     // eslint-disable-next-line no-restricted-properties
-    cli.mxcUrlToHttp.mockReturnValue(url);
+    cli.mxcUrlToHttp.mockImplementation(
+        (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
+            return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
+        },
+    );
     const encryptedMediaEvent = new MatrixEvent({
         room_id: "!room:server",
         sender: userId,
@@ -175,12 +181,6 @@ describe("<MImageBody/>", () => {
     it("should fall back to /download/ if /thumbnail/ fails", async () => {
         const thumbUrl = "https://server/_matrix/media/r0/thumbnail/server/image?width=800&height=600&method=scale";
         const downloadUrl = "https://server/_matrix/media/r0/download/server/image";
-        // eslint-disable-next-line no-restricted-properties
-        cli.mxcUrlToHttp.mockImplementation(
-            (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
-                return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
-            },
-        );
 
         const event = new MatrixEvent({
             room_id: "!room:server",
@@ -206,4 +206,56 @@ describe("<MImageBody/>", () => {
         fireEvent.error(img);
         expect(img).toHaveProperty("src", downloadUrl);
     });
+
+    it("should generate a thumbnail if one isn't included for animated media", async () => {
+        Object.defineProperty(global.Image.prototype, "src", {
+            set(src) {
+                window.setTimeout(() => this.onload());
+            },
+        });
+        Object.defineProperty(global.Image.prototype, "height", {
+            get() {
+                return 600;
+            },
+        });
+        Object.defineProperty(global.Image.prototype, "width", {
+            get() {
+                return 800;
+            },
+        });
+
+        mocked(global.URL.createObjectURL).mockReturnValue("blob:generated-thumb");
+
+        fetchMock.getOnce(
+            "https://server/_matrix/media/r0/download/server/image",
+            {
+                body: fs.readFileSync(path.resolve(__dirname, "..", "..", "..", "images", "animated-logo.webp")),
+            },
+            { sendAsJson: false },
+        );
+
+        const event = new MatrixEvent({
+            room_id: "!room:server",
+            sender: userId,
+            type: EventType.RoomMessage,
+            content: {
+                body: "alt for a test image",
+                info: {
+                    w: 40,
+                    h: 50,
+                    mimetype: "image/webp",
+                },
+                url: "mxc://server/image",
+            },
+        });
+
+        const { container } = render(
+            <MImageBody {...props} mxEvent={event} mediaEventHelper={new MediaEventHelper(event)} />,
+        );
+
+        // Wait for spinners to go away
+        await waitForElementToBeRemoved(screen.getAllByRole("progressbar"));
+        // thumbnail with dimensions present
+        expect(container).toMatchSnapshot();
+    });
 });

test/components/views/messages/MVideoBody-test.tsx

@@ -15,23 +15,22 @@ limitations under the License.
 */
 
 import React, { ComponentProps } from "react";
-import { IContent, MatrixEvent } from "matrix-js-sdk/src/matrix";
+import { EventType, getHttpUriForMxc, IContent, MatrixEvent } from "matrix-js-sdk/src/matrix";
 import { render, RenderResult } from "@testing-library/react";
+import fetchMock from "fetch-mock-jest";
 
 import MatrixClientContext from "../../../../src/contexts/MatrixClientContext";
 import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
 import { MediaEventHelper } from "../../../../src/utils/MediaEventHelper";
-import { getMockClientWithEventEmitter } from "../../../test-utils";
+import {
+    getMockClientWithEventEmitter,
+    mockClientMethodsCrypto,
+    mockClientMethodsDevice,
+    mockClientMethodsServer,
+    mockClientMethodsUser,
+} from "../../../test-utils";
 import MVideoBody from "../../../../src/components/views/messages/MVideoBody";
 
-jest.mock("../../../../src/customisations/Media", () => {
-    return {
-        mediaFromContent: () => {
-            return { isEncrypted: false };
-        },
-    };
-});
-
 describe("MVideoBody", () => {
     it("does not crash when given a portrait image", () => {
         // Check for an unreliable crash caused by a fractional-sized
@@ -40,6 +39,58 @@ describe("MVideoBody", () => {
         expect(asFragment()).toMatchSnapshot();
         // If we get here, we did not crash.
     });
+
+    it("should show poster for encrypted media before downloading it", async () => {
+        const userId = "@user:server";
+        const deviceId = "DEADB33F";
+        const cli = getMockClientWithEventEmitter({
+            ...mockClientMethodsUser(userId),
+            ...mockClientMethodsServer(),
+            ...mockClientMethodsDevice(deviceId),
+            ...mockClientMethodsCrypto(),
+            getRooms: jest.fn().mockReturnValue([]),
+            getIgnoredUsers: jest.fn(),
+            getVersions: jest.fn().mockResolvedValue({
+                unstable_features: {
+                    "org.matrix.msc3882": true,
+                    "org.matrix.msc3886": true,
+                },
+            }),
+        });
+        const thumbUrl = "https://server/_matrix/media/r0/download/server/encrypted-poster";
+        fetchMock.getOnce(thumbUrl, { status: 200 });
+
+        // eslint-disable-next-line no-restricted-properties
+        cli.mxcUrlToHttp.mockImplementation(
+            (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
+                return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
+            },
+        );
+        const encryptedMediaEvent = new MatrixEvent({
+            room_id: "!room:server",
+            sender: userId,
+            type: EventType.RoomMessage,
+            content: {
+                body: "alt for a test video",
+                info: {
+                    duration: 420,
+                    w: 40,
+                    h: 50,
+                    thumbnail_file: {
+                        url: "mxc://server/encrypted-poster",
+                    },
+                },
+                file: {
+                    url: "mxc://server/encrypted-image",
+                },
+            },
+        });
+
+        const { asFragment } = render(
+            <MVideoBody mxEvent={encryptedMediaEvent} mediaEventHelper={new MediaEventHelper(encryptedMediaEvent)} />,
+        );
+        expect(asFragment()).toMatchSnapshot();
+    });
 });
 
 function makeMVideoBody(w: number, h: number): RenderResult {

test/components/views/messages/__snapshots__/MImageBody-test.tsx.snap

@@ -1,5 +1,55 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`<MImageBody/> should generate a thumbnail if one isn't included for animated media 1`] = `
+<div>
+  <div
+    class="mx_MImageBody"
+  >
+    <a
+      href="https://server/_matrix/media/r0/download/server/image"
+    >
+      <div
+        class="mx_MImageBody_thumbnail_container"
+        style="max-height: 50px; max-width: 40px;"
+      >
+        <div
+          class="mx_MImageBody_placeholder"
+        >
+          <div
+            class="mx_Spinner"
+          >
+            <div
+              aria-label="Loading…"
+              class="mx_Spinner_icon"
+              data-testid="spinner"
+              role="progressbar"
+              style="width: 32px; height: 32px;"
+            />
+          </div>
+        </div>
+        <div
+          style="max-height: 50px; max-width: 40px;"
+        >
+          <img
+            alt="alt for a test image"
+            class="mx_MImageBody_thumbnail"
+            src="blob:generated-thumb"
+          />
+          <p
+            class="mx_MImageBody_gifLabel"
+          >
+            GIF
+          </p>
+        </div>
+        <div
+          style="height: 50px; width: 40px;"
+        />
+      </div>
+    </a>
+  </div>
+</div>
+`;
+
 exports[`<MImageBody/> should show a thumbnail while image is being downloaded 1`] = `
 <div>
   <div

test/components/views/messages/__snapshots__/MVideoBody-test.tsx.snap

@@ -23,3 +23,28 @@ exports[`MVideoBody does not crash when given a portrait image 1`] = `
   </span>
 </DocumentFragment>
 `;
+
+exports[`MVideoBody should show poster for encrypted media before downloading it 1`] = `
+<DocumentFragment>
+  <span
+    class="mx_MVideoBody"
+  >
+    <div
+      class="mx_MVideoBody_container"
+      style="max-width: 40px; max-height: 50px;"
+    >
+      <video
+        class="mx_MVideoBody"
+        controls=""
+        controlslist="nodownload"
+        poster="https://server/_matrix/media/r0/download/server/encrypted-poster"
+        preload="none"
+        title="alt for a test video"
+      />
+      <div
+        style="width: 40px; height: 50px;"
+      />
+    </div>
+  </span>
+</DocumentFragment>
+`;

test/voice-broadcast/models/VoiceBroadcastRecording-test.ts

@@ -31,7 +31,7 @@ import { Relations } from "matrix-js-sdk/src/models/relations";
 import { SyncState } from "matrix-js-sdk/src/sync";
 
 import { uploadFile } from "../../../src/ContentMessages";
-import { IEncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
+import { EncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
 import { createVoiceMessageContent } from "../../../src/utils/createVoiceMessageContent";
 import {
     createVoiceBroadcastRecorder,
@@ -82,7 +82,7 @@ jest.mock("../../../src/utils/createVoiceMessageContent", () => ({
 describe("VoiceBroadcastRecording", () => {
     const roomId = "!room:example.com";
     const uploadedUrl = "mxc://example.com/vb";
-    const uploadedFile = { file: true } as unknown as IEncryptedFile;
+    const uploadedFile = { file: true } as unknown as EncryptedFile;
     const maxLength = getMaxBroadcastLength();
     let room: Room;
     let client: MatrixClient;
@@ -223,7 +223,7 @@ describe("VoiceBroadcastRecording", () => {
                 mimetype: string,
                 duration: number,
                 size: number,
-                file?: IEncryptedFile,
+                file?: EncryptedFile,
                 waveform?: number[],
             ) => {
                 return {