Improve handling of animated GIF and WEBP images (#8153)

This commit is contained in:
Michael Telatynski 2022-03-25 16:31:40 +00:00 committed by GitHub
parent 50fd24581c
commit bc01efa124
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 297 additions and 126 deletions

View file

@ -166,6 +166,7 @@
"@wojtekmaj/enzyme-adapter-react-17": "^0.6.1", "@wojtekmaj/enzyme-adapter-react-17": "^0.6.1",
"allchange": "^1.0.6", "allchange": "^1.0.6",
"babel-jest": "^26.6.3", "babel-jest": "^26.6.3",
"blob-polyfill": "^6.0.20211015",
"chokidar": "^3.5.1", "chokidar": "^3.5.1",
"concurrently": "^5.3.0", "concurrently": "^5.3.0",
"enzyme": "^3.11.0", "enzyme": "^3.11.0",

View file

@ -106,15 +106,17 @@ interface IThumbnail {
* @param {HTMLElement} element The element to thumbnail. * @param {HTMLElement} element The element to thumbnail.
* @param {number} inputWidth The width of the image in the input element. * @param {number} inputWidth The width of the image in the input element.
* @param {number} inputHeight the width of the image in the input element. * @param {number} inputHeight the width of the image in the input element.
* @param {String} mimeType The mimeType to save the blob as. * @param {string} mimeType The mimeType to save the blob as.
* @param {boolean} calculateBlurhash Whether to calculate a blurhash of the given image too.
* @return {Promise} A promise that resolves with an object with an info key * @return {Promise} A promise that resolves with an object with an info key
* and a thumbnail key. * and a thumbnail key.
*/ */
async function createThumbnail( export async function createThumbnail(
element: ThumbnailableElement, element: ThumbnailableElement,
inputWidth: number, inputWidth: number,
inputHeight: number, inputHeight: number,
mimeType: string, mimeType: string,
calculateBlurhash = true,
): Promise<IThumbnail> { ): Promise<IThumbnail> {
let targetWidth = inputWidth; let targetWidth = inputWidth;
let targetHeight = inputHeight; let targetHeight = inputHeight;
@ -152,7 +154,7 @@ async function createThumbnail(
const imageData = context.getImageData(0, 0, targetWidth, targetHeight); const imageData = context.getImageData(0, 0, targetWidth, targetHeight);
// thumbnailPromise and blurhash promise are being awaited concurrently // thumbnailPromise and blurhash promise are being awaited concurrently
const blurhash = await BlurhashEncoder.instance.getBlurhash(imageData); const blurhash = calculateBlurhash ? await BlurhashEncoder.instance.getBlurhash(imageData) : undefined;
const thumbnail = await thumbnailPromise; const thumbnail = await thumbnailPromise;
return { return {

View file

@ -30,19 +30,25 @@ import SettingsStore from "../../../settings/SettingsStore";
import InlineSpinner from '../elements/InlineSpinner'; import InlineSpinner from '../elements/InlineSpinner';
import { replaceableComponent } from "../../../utils/replaceableComponent"; import { replaceableComponent } from "../../../utils/replaceableComponent";
import { Media, mediaFromContent } from "../../../customisations/Media"; import { Media, mediaFromContent } from "../../../customisations/Media";
import { BLURHASH_FIELD } from "../../../ContentMessages"; import { BLURHASH_FIELD, createThumbnail } from "../../../ContentMessages";
import { IMediaEventContent } from '../../../customisations/models/IMediaEventContent'; import { IMediaEventContent } from '../../../customisations/models/IMediaEventContent';
import ImageView from '../elements/ImageView'; import ImageView from '../elements/ImageView';
import { IBodyProps } from "./IBodyProps"; import { IBodyProps } from "./IBodyProps";
import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize"; import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize";
import { MatrixClientPeg } from '../../../MatrixClientPeg'; import { MatrixClientPeg } from '../../../MatrixClientPeg';
import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext"; import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext";
import { blobIsAnimated, mayBeAnimated } from '../../../utils/Image';
enum Placeholder {
NoImage,
Blurhash,
}
interface IState { interface IState {
decryptedUrl?: string; contentUrl?: string;
decryptedThumbnailUrl?: string; thumbUrl?: string;
decryptedBlob?: Blob; isAnimated?: boolean;
error; error?: Error;
imgError: boolean; imgError: boolean;
imgLoaded: boolean; imgLoaded: boolean;
loadedImageDimensions?: { loadedImageDimensions?: {
@ -51,7 +57,7 @@ interface IState {
}; };
hover: boolean; hover: boolean;
showImage: boolean; showImage: boolean;
placeholder: 'no-image' | 'blurhash'; placeholder: Placeholder;
} }
@replaceableComponent("views.messages.MImageBody") @replaceableComponent("views.messages.MImageBody")
@ -68,16 +74,11 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
super(props); super(props);
this.state = { this.state = {
decryptedUrl: null,
decryptedThumbnailUrl: null,
decryptedBlob: null,
error: null,
imgError: false, imgError: false,
imgLoaded: false, imgLoaded: false,
loadedImageDimensions: null,
hover: false, hover: false,
showImage: SettingsStore.getValue("showImages"), showImage: SettingsStore.getValue("showImages"),
placeholder: 'no-image', placeholder: Placeholder.NoImage,
}; };
} }
@ -86,7 +87,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
if (this.unmounted) return; if (this.unmounted) return;
// Consider the client reconnected if there is no error with syncing. // Consider the client reconnected if there is no error with syncing.
// This means the state could be RECONNECTING, SYNCING, PREPARED or CATCHUP. // This means the state could be RECONNECTING, SYNCING, PREPARED or CATCHUP.
const reconnected = syncState !== "ERROR" && prevState !== syncState; const reconnected = syncState !== SyncState.Error && prevState !== syncState;
if (reconnected && this.state.imgError) { if (reconnected && this.state.imgError) {
// Load the image again // Load the image again
this.setState({ this.setState({
@ -110,7 +111,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
} }
const content = this.props.mxEvent.getContent<IMediaEventContent>(); const content = this.props.mxEvent.getContent<IMediaEventContent>();
const httpUrl = this.getContentUrl(); const httpUrl = this.state.contentUrl;
const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = { const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = {
src: httpUrl, src: httpUrl,
name: content.body?.length > 0 ? content.body : _t('Attachment'), name: content.body?.length > 0 ? content.body : _t('Attachment'),
@ -139,29 +140,24 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
} }
}; };
private isGif = (): boolean => {
const content = this.props.mxEvent.getContent();
return content.info?.mimetype === "image/gif";
};
private onImageEnter = (e: React.MouseEvent<HTMLImageElement>): void => { private onImageEnter = (e: React.MouseEvent<HTMLImageElement>): void => {
this.setState({ hover: true }); this.setState({ hover: true });
if (!this.state.showImage || !this.isGif() || SettingsStore.getValue("autoplayGifs")) { if (!this.state.showImage || !this.state.isAnimated || SettingsStore.getValue("autoplayGifs")) {
return; return;
} }
const imgElement = e.currentTarget; const imgElement = e.currentTarget;
imgElement.src = this.getContentUrl(); imgElement.src = this.state.contentUrl;
}; };
private onImageLeave = (e: React.MouseEvent<HTMLImageElement>): void => { private onImageLeave = (e: React.MouseEvent<HTMLImageElement>): void => {
this.setState({ hover: false }); this.setState({ hover: false });
if (!this.state.showImage || !this.isGif() || SettingsStore.getValue("autoplayGifs")) { if (!this.state.showImage || !this.state.isAnimated || SettingsStore.getValue("autoplayGifs")) {
return; return;
} }
const imgElement = e.currentTarget; const imgElement = e.currentTarget;
imgElement.src = this.getThumbUrl(); imgElement.src = this.state.thumbUrl ?? this.state.contentUrl;
}; };
private onImageError = (): void => { private onImageError = (): void => {
@ -175,7 +171,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
this.clearBlurhashTimeout(); this.clearBlurhashTimeout();
this.props.onHeightChanged(); this.props.onHeightChanged();
let loadedImageDimensions; let loadedImageDimensions: IState["loadedImageDimensions"];
if (this.image.current) { if (this.image.current) {
const { naturalWidth, naturalHeight } = this.image.current; const { naturalWidth, naturalHeight } = this.image.current;
@ -185,22 +181,17 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
this.setState({ imgLoaded: true, loadedImageDimensions }); this.setState({ imgLoaded: true, loadedImageDimensions });
}; };
protected getContentUrl(): string { private getContentUrl(): string {
const content: IMediaEventContent = this.props.mxEvent.getContent();
// During export, the content url will point to the MSC, which will later point to a local url // During export, the content url will point to the MSC, which will later point to a local url
if (this.props.forExport) return content.url || content.file?.url; if (this.props.forExport) return this.media.srcMxc;
if (this.media.isEncrypted) {
return this.state.decryptedUrl;
} else {
return this.media.srcHttp; return this.media.srcHttp;
} }
}
private get media(): Media { private get media(): Media {
return mediaFromContent(this.props.mxEvent.getContent()); return mediaFromContent(this.props.mxEvent.getContent());
} }
protected getThumbUrl(): string { private getThumbUrl(): string {
// FIXME: we let images grow as wide as you like, rather than capped to 800x600. // FIXME: we let images grow as wide as you like, rather than capped to 800x600.
// So either we need to support custom timeline widths here, or reimpose the cap, otherwise the // So either we need to support custom timeline widths here, or reimpose the cap, otherwise the
// thumbnail resolution will be unnecessarily reduced. // thumbnail resolution will be unnecessarily reduced.
@ -210,46 +201,37 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
const content = this.props.mxEvent.getContent<IMediaEventContent>(); const content = this.props.mxEvent.getContent<IMediaEventContent>();
const media = mediaFromContent(content); const media = mediaFromContent(content);
const info = content.info;
if (media.isEncrypted) { if (info?.mimetype === "image/svg+xml" && media.hasThumbnail) {
// Don't use the thumbnail for clients wishing to autoplay gifs. // Special-case to return clientside sender-generated thumbnails for SVGs, if any,
if (this.state.decryptedThumbnailUrl) { // given we deliberately don't thumbnail them serverside to prevent billion lol attacks and similar.
return this.state.decryptedThumbnailUrl;
}
return this.state.decryptedUrl;
} else if (content.info && content.info.mimetype === "image/svg+xml" && media.hasThumbnail) {
// special case to return clientside sender-generated thumbnails for SVGs, if any,
// given we deliberately don't thumbnail them serverside to prevent
// billion lol attacks and similar
return media.getThumbnailHttp(thumbWidth, thumbHeight, 'scale'); return media.getThumbnailHttp(thumbWidth, thumbHeight, 'scale');
} else { }
// we try to download the correct resolution
// for hi-res images (like retina screenshots). // we try to download the correct resolution for hi-res images (like retina screenshots).
// synapse only supports 800x600 thumbnails for now though, // Synapse only supports 800x600 thumbnails for now though,
// so we'll need to download the original image for this to work // so we'll need to download the original image for this to work well for now.
// well for now. First, let's try a few cases that let us avoid // First, let's try a few cases that let us avoid downloading the original, including:
// downloading the original, including:
// - When displaying a GIF, we always want to thumbnail so that we can // - When displaying a GIF, we always want to thumbnail so that we can
// properly respect the user's GIF autoplay setting (which relies on // properly respect the user's GIF autoplay setting (which relies on
// thumbnailing to produce the static preview image) // thumbnailing to produce the static preview image)
// - On a low DPI device, always thumbnail to save bandwidth // - On a low DPI device, always thumbnail to save bandwidth
// - If there's no sizing info in the event, default to thumbnail // - If there's no sizing info in the event, default to thumbnail
const info = content.info;
if ( if (
this.isGif() || this.state.isAnimated ||
window.devicePixelRatio === 1.0 || window.devicePixelRatio === 1.0 ||
(!info || !info.w || !info.h || !info.size) (!info || !info.w || !info.h || !info.size)
) { ) {
return media.getThumbnailOfSourceHttp(thumbWidth, thumbHeight); return media.getThumbnailOfSourceHttp(thumbWidth, thumbHeight);
} else { }
// we should only request thumbnails if the image is bigger than 800x600
// (or 1600x1200 on retina) otherwise the image in the timeline will just // We should only request thumbnails if the image is bigger than 800x600 (or 1600x1200 on retina) otherwise
// end up resampled and de-retina'd for no good reason. // the image in the timeline will just end up resampled and de-retina'd for no good reason.
// Ideally the server would pregen 1600x1200 thumbnails in order to provide retina // Ideally the server would pre-gen 1600x1200 thumbnails in order to provide retina thumbnails,
// thumbnails, but we don't do this currently in synapse for fear of disk space. // but we don't do this currently in synapse for fear of disk space.
// As a compromise, let's switch to non-retina thumbnails only if the original // As a compromise, let's switch to non-retina thumbnails only if the original image is both
// image is both physically too large and going to be massive to load in the // physically too large and going to be massive to load in the timeline (e.g. >1MB).
// timeline (e.g. >1MB).
const isLargerThanThumbnail = ( const isLargerThanThumbnail = (
info.w > thumbWidth || info.w > thumbWidth ||
@ -258,38 +240,74 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
const isLargeFileSize = info.size > 1 * 1024 * 1024; // 1mb const isLargeFileSize = info.size > 1 * 1024 * 1024; // 1mb
if (isLargeFileSize && isLargerThanThumbnail) { if (isLargeFileSize && isLargerThanThumbnail) {
// image is too large physically and bytewise to clutter our timeline so // image is too large physically and byte-wise to clutter our timeline so,
// we ask for a thumbnail, despite knowing that it will be max 800x600 // we ask for a thumbnail, despite knowing that it will be max 800x600
// despite us being retina (as synapse doesn't do 1600x1200 thumbs yet). // despite us being retina (as synapse doesn't do 1600x1200 thumbs yet).
return media.getThumbnailOfSourceHttp(thumbWidth, thumbHeight); return media.getThumbnailOfSourceHttp(thumbWidth, thumbHeight);
} else { }
// download the original image otherwise, so we can scale it client side
// to take pixelRatio into account. // download the original image otherwise, so we can scale it client side to take pixelRatio into account.
return media.srcHttp; return media.srcHttp;
} }
private async downloadImage() {
if (this.state.contentUrl) return; // already downloaded
let thumbUrl: string;
let contentUrl: string;
if (this.props.mediaEventHelper.media.isEncrypted) {
try {
([contentUrl, thumbUrl] = await Promise.all([
this.props.mediaEventHelper.sourceUrl.value,
this.props.mediaEventHelper.thumbnailUrl.value,
]));
} catch (error) {
if (this.unmounted) return;
logger.warn("Unable to decrypt attachment: ", error);
// Set a placeholder image when we can't decrypt the image.
this.setState({ error });
}
} else {
thumbUrl = this.getThumbUrl();
contentUrl = this.getContentUrl();
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
let isAnimated = mayBeAnimated(content.info?.mimetype);
// If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
// because 1. encryption and 2. we can't ask the server specifically for a non-animated thumbnail.
if (isAnimated && !SettingsStore.getValue("autoplayGifs")) {
if (!thumbUrl || !content?.info.thumbnail_info || mayBeAnimated(content.info.thumbnail_info.mimetype)) {
const img = document.createElement("img");
const loadPromise = new Promise((resolve, reject) => {
img.onload = resolve;
img.onerror = reject;
});
img.crossOrigin = "Anonymous"; // CORS allow canvas access
img.src = contentUrl;
await loadPromise;
const blob = await this.props.mediaEventHelper.sourceBlob.value;
if (!await blobIsAnimated(content.info.mimetype, blob)) {
isAnimated = false;
}
if (isAnimated) {
const thumb = await createThumbnail(img, img.width, img.height, content.info.mimetype, false);
thumbUrl = URL.createObjectURL(thumb.thumbnail);
} }
} }
} }
private async downloadImage() {
if (this.props.mediaEventHelper.media.isEncrypted && this.state.decryptedUrl === null) {
try {
const thumbnailUrl = await this.props.mediaEventHelper.thumbnailUrl.value;
this.setState({
decryptedUrl: await this.props.mediaEventHelper.sourceUrl.value,
decryptedThumbnailUrl: thumbnailUrl,
decryptedBlob: await this.props.mediaEventHelper.sourceBlob.value,
});
} catch (err) {
if (this.unmounted) return; if (this.unmounted) return;
logger.warn("Unable to decrypt attachment: ", err);
// Set a placeholder image when we can't decrypt the image.
this.setState({ this.setState({
error: err, contentUrl,
thumbUrl,
isAnimated,
}); });
} }
}
}
private clearBlurhashTimeout() { private clearBlurhashTimeout() {
if (this.timeout) { if (this.timeout) {
@ -317,7 +335,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
this.timeout = setTimeout(() => { this.timeout = setTimeout(() => {
if (!this.state.imgLoaded || !this.state.imgError) { if (!this.state.imgLoaded || !this.state.imgError) {
this.setState({ this.setState({
placeholder: 'blurhash', placeholder: Placeholder.Blurhash,
}); });
} }
}, 150); }, 150);
@ -333,6 +351,9 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
MatrixClientPeg.get().removeListener(ClientEvent.Sync, this.onClientSync); MatrixClientPeg.get().removeListener(ClientEvent.Sync, this.onClientSync);
this.clearBlurhashTimeout(); this.clearBlurhashTimeout();
SettingsStore.unwatchSetting(this.sizeWatcher); SettingsStore.unwatchSetting(this.sizeWatcher);
if (this.state.isAnimated && this.state.thumbUrl) {
URL.revokeObjectURL(this.state.thumbUrl);
}
} }
protected messageContent( protected messageContent(
@ -341,10 +362,12 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
content: IMediaEventContent, content: IMediaEventContent,
forcedHeight?: number, forcedHeight?: number,
): JSX.Element { ): JSX.Element {
if (!thumbUrl) thumbUrl = contentUrl; // fallback
let infoWidth: number; let infoWidth: number;
let infoHeight: number; let infoHeight: number;
if (content && content.info && content.info.w && content.info.h) { if (content.info?.w && content.info?.h) {
infoWidth = content.info.w; infoWidth = content.info.w;
infoHeight = content.info.h; infoHeight = content.info.h;
} else { } else {
@ -385,9 +408,9 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
forcedHeight ?? this.props.maxImageHeight, forcedHeight ?? this.props.maxImageHeight,
); );
let img = null; let img: JSX.Element;
let placeholder = null; let placeholder: JSX.Element;
let gifLabel = null; let gifLabel: JSX.Element;
if (!this.props.forExport && !this.state.imgLoaded) { if (!this.props.forExport && !this.state.imgLoaded) {
placeholder = this.getPlaceholder(maxWidth, maxHeight); placeholder = this.getPlaceholder(maxWidth, maxHeight);
@ -427,7 +450,8 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
showPlaceholder = false; // because we're hiding the image, so don't show the placeholder. showPlaceholder = false; // because we're hiding the image, so don't show the placeholder.
} }
if (this.isGif() && !SettingsStore.getValue("autoplayGifs") && !this.state.hover) { if (this.state.isAnimated && !SettingsStore.getValue("autoplayGifs") && !this.state.hover) {
// XXX: Arguably we may want a different label when the animated image is WEBP and not GIF
gifLabel = <p className="mx_MImageBody_gifLabel">GIF</p>; gifLabel = <p className="mx_MImageBody_gifLabel">GIF</p>;
} }
@ -489,9 +513,9 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
const blurhash = this.props.mxEvent.getContent().info?.[BLURHASH_FIELD]; const blurhash = this.props.mxEvent.getContent().info?.[BLURHASH_FIELD];
if (blurhash) { if (blurhash) {
if (this.state.placeholder === 'no-image') { if (this.state.placeholder === Placeholder.NoImage) {
return <div className="mx_no-image-placeholder" style={{ width: width, height: height }} />; return <div className="mx_no-image-placeholder" style={{ width: width, height: height }} />;
} else if (this.state.placeholder === 'blurhash') { } else if (this.state.placeholder === Placeholder.Blurhash) {
return <Blurhash className="mx_Blurhash" hash={blurhash} width={width} height={height} />; return <Blurhash className="mx_Blurhash" hash={blurhash} width={width} height={height} />;
} }
} }
@ -510,13 +534,15 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
if (this.props.forExport) return null; if (this.props.forExport) return null;
/* /*
* In the room timeline or the thread context we don't need the download * In the room timeline or the thread context we don't need the download
* link as the message action bar will fullfil that * link as the message action bar will fulfill that
*/ */
const hasMessageActionBar = this.context.timelineRenderingType === TimelineRenderingType.Room const hasMessageActionBar = (
|| this.context.timelineRenderingType === TimelineRenderingType.Pinned this.context.timelineRenderingType === TimelineRenderingType.Room ||
|| this.context.timelineRenderingType === TimelineRenderingType.Search this.context.timelineRenderingType === TimelineRenderingType.Pinned ||
|| this.context.timelineRenderingType === TimelineRenderingType.Thread this.context.timelineRenderingType === TimelineRenderingType.Search ||
|| this.context.timelineRenderingType === TimelineRenderingType.ThreadsList; this.context.timelineRenderingType === TimelineRenderingType.Thread ||
this.context.timelineRenderingType === TimelineRenderingType.ThreadsList
);
if (!hasMessageActionBar) { if (!hasMessageActionBar) {
return <MFileBody {...this.props} showGenericPlaceholder={false} />; return <MFileBody {...this.props} showGenericPlaceholder={false} />;
} }
@ -525,7 +551,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
render() { render() {
const content = this.props.mxEvent.getContent<IMediaEventContent>(); const content = this.props.mxEvent.getContent<IMediaEventContent>();
if (this.state.error !== null) { if (this.state.error) {
return ( return (
<div className="mx_MImageBody"> <div className="mx_MImageBody">
<img src={require("../../../../res/img/warning.svg").default} width="16" height="16" /> <img src={require("../../../../res/img/warning.svg").default} width="16" height="16" />
@ -534,12 +560,12 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
); );
} }
const contentUrl = this.getContentUrl(); const contentUrl = this.state.contentUrl;
let thumbUrl; let thumbUrl: string;
if (this.props.forExport || (this.isGif() && SettingsStore.getValue("autoplayGifs"))) { if (this.props.forExport || (this.state.isAnimated && SettingsStore.getValue("autoplayGifs"))) {
thumbUrl = contentUrl; thumbUrl = contentUrl;
} else { } else {
thumbUrl = this.getThumbUrl(); thumbUrl = this.state.thumbUrl ?? this.state.contentUrl;
} }
const thumbnail = this.messageContent(contentUrl, thumbUrl, content); const thumbnail = this.messageContent(contentUrl, thumbUrl, content);

View file

@ -41,14 +41,12 @@ export default class MImageReplyBody extends MImageBody {
} }
render() { render() {
if (this.state.error !== null) { if (this.state.error) {
return super.render(); return super.render();
} }
const content = this.props.mxEvent.getContent<IMediaEventContent>(); const content = this.props.mxEvent.getContent<IMediaEventContent>();
const thumbnail = this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT);
const contentUrl = this.getContentUrl();
const thumbnail = this.messageContent(contentUrl, this.getThumbUrl(), content, FORCED_IMAGE_HEIGHT);
const fileBody = this.getFileBody(); const fileBody = this.getFileBody();
const sender = <SenderProfile const sender = <SenderProfile
mxEvent={this.props.mxEvent} mxEvent={this.props.mxEvent}

76
src/utils/Image.ts Normal file
View file

@ -0,0 +1,76 @@
/*
* Copyright 2022 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export function mayBeAnimated(mimeType: string): boolean {
return ["image/gif", "image/webp"].includes(mimeType);
}
function arrayBufferRead(arr: ArrayBuffer, start: number, len: number): Uint8Array {
return new Uint8Array(arr.slice(start, start + len));
}
function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): string {
return String.fromCharCode.apply(null, arrayBufferRead(arr, start, len));
}
export async function blobIsAnimated(mimeType: string, blob: Blob): Promise<boolean> {
if (mimeType === "image/webp") {
// Only extended file format WEBP images support animation, so grab the expected data range and verify header.
// Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format
const arr = await blob.slice(0, 17).arrayBuffer();
if (
arrayBufferReadStr(arr, 0, 4) === "RIFF" &&
arrayBufferReadStr(arr, 8, 4) === "WEBP" &&
arrayBufferReadStr(arr, 12, 4) === "VP8X"
) {
const [flags] = arrayBufferRead(arr, 16, 1);
// Flags: R R I L E X _A_ R (reversed)
const animationFlagMask = 1 << 1;
return (flags & animationFlagMask) != 0;
}
} else if (mimeType === "image/gif") {
// Based on https://gist.github.com/zakirt/faa4a58cec5a7505b10e3686a226f285
// More info at http://www.matthewflickinger.com/lab/whatsinagif/bits_and_bytes.asp
const dv = new DataView(await blob.arrayBuffer(), 10);
const globalColorTable = dv.getUint8(0);
let globalColorTableSize = 0;
// check first bit, if 0, then we don't have a Global Color Table
if (globalColorTable & 0x80) {
// grab the last 3 bits, to calculate the global color table size -> RGB * 2^(N+1)
// N is the value in the last 3 bits.
globalColorTableSize = 3 * Math.pow(2, (globalColorTable & 0x7) + 1);
}
// move on to the Graphics Control Extension
const offset = 3 + globalColorTableSize;
const extensionIntroducer = dv.getUint8(offset);
const graphicsControlLabel = dv.getUint8(offset + 1);
let delayTime = 0;
// Graphics Control Extension section is where GIF animation data is stored
// First 2 bytes must be 0x21 and 0xF9
if ((extensionIntroducer & 0x21) && (graphicsControlLabel & 0xF9)) {
// skip to the 2 bytes with the delay time
delayTime = dv.getUint16(offset + 4);
}
return !!delayTime;
}
return false;
}

View file

@ -52,6 +52,7 @@ const ALLOWED_BLOB_MIMETYPES = [
'image/jpeg', 'image/jpeg',
'image/gif', 'image/gif',
'image/png', 'image/png',
'image/webp',
'video/mp4', 'video/mp4',
'video/webm', 'video/webm',

61
test/Image-test.ts Normal file
View file

@ -0,0 +1,61 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import fs from "fs";
import path from "path";
import './skinned-sdk';
import { blobIsAnimated, mayBeAnimated } from "../src/utils/Image";
describe("Image", () => {
describe("mayBeAnimated", () => {
it("image/gif", async () => {
expect(mayBeAnimated("image/gif")).toBeTruthy();
});
it("image/webp", async () => {
expect(mayBeAnimated("image/webp")).toBeTruthy();
});
it("image/png", async () => {
expect(mayBeAnimated("image/png")).toBeFalsy();
});
it("image/jpeg", async () => {
expect(mayBeAnimated("image/jpeg")).toBeFalsy();
});
});
describe("blobIsAnimated", () => {
const animatedGif = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))]);
const animatedWebp = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))]);
const staticGif = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))]);
const staticWebp = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))]);
it("Animated GIF", async () => {
expect(await blobIsAnimated("image/gif", animatedGif)).toBeTruthy();
});
it("Static GIF", async () => {
expect(await blobIsAnimated("image/gif", staticGif)).toBeFalsy();
});
it("Animated WEBP", async () => {
expect(await blobIsAnimated("image/webp", animatedWebp)).toBeTruthy();
});
it("Static WEBP", async () => {
expect(await blobIsAnimated("image/webp", staticWebp)).toBeFalsy();
});
});
});

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

BIN
test/images/static-logo.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 999 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 146 B

View file

@ -1,6 +1,7 @@
import { TextEncoder, TextDecoder } from 'util'; import { TextEncoder, TextDecoder } from 'util';
import Adapter from "@wojtekmaj/enzyme-adapter-react-17"; import Adapter from "@wojtekmaj/enzyme-adapter-react-17";
import { configure } from "enzyme"; import { configure } from "enzyme";
import "blob-polyfill"; // https://github.com/jsdom/jsdom/issues/2555
import * as languageHandler from "../src/languageHandler"; import * as languageHandler from "../src/languageHandler";
import SdkConfig, { DEFAULTS } from '../src/SdkConfig'; import SdkConfig, { DEFAULTS } from '../src/SdkConfig';

View file

@ -2627,6 +2627,11 @@ binary-extensions@^2.0.0:
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
blob-polyfill@^6.0.20211015:
version "6.0.20211015"
resolved "https://registry.yarnpkg.com/blob-polyfill/-/blob-polyfill-6.0.20211015.tgz#7c47e62347e302e8d1d1ee5e140b881f74bdb23e"
integrity sha512-OGL4bm6ZNpdFAvQugRlQy5MNly8gk15aWi/ZhQHimQsrx9WKD05r+v+xNgHCChLER3MH+9KLAhzuFlwFKrH1Yw==
bluebird@^3.5.0: bluebird@^3.5.0:
version "3.7.2" version "3.7.2"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"