From a294fe08886ccf43f446094d8287f4aaa01349cd Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 1 Oct 2025 16:44:09 +0100 Subject: [PATCH 1/7] Only set MSC4230 is_animated flag if we are able to tell if the media is animated Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- src/ContentMessages.ts | 5 ++++- src/utils/Image.ts | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/ContentMessages.ts b/src/ContentMessages.ts index e608b3470a9..e2e01ac9591 100644 --- a/src/ContentMessages.ts +++ b/src/ContentMessages.ts @@ -165,7 +165,10 @@ async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imag const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType); const imageInfo = result.info; - imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise; + const isAnimated = await isAnimatedPromise; + if (isAnimated !== undefined) { + imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise; + } // For lesser supported image types, always include the thumbnail even if it is larger if (!ALWAYS_INCLUDE_THUMBNAIL.includes(imageFile.type)) { diff --git a/src/utils/Image.ts b/src/utils/Image.ts index ab7c67d4777..b7a89556c70 100644 --- a/src/utils/Image.ts +++ b/src/utils/Image.ts @@ -26,7 +26,12 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin return String.fromCharCode.apply(null, Array.from(arrayBufferRead(arr, start, len))); } -export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise { +/** + * Check if a Blob contains an animated image. + * @param blob The Blob to check. + * @returns True if the image is animated, false if not, or undefined if it could not be determined. + */ +export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise { switch (mimeType) { case "image/webp": { // Only extended file format WEBP images support animation, so grab the expected data range and verify header. @@ -42,7 +47,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): const animationFlagMask = 1 << 1; return (flags & animationFlagMask) != 0; } - break; + return false; } case "image/gif": { @@ -100,9 +105,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): } i += length + 4; } - break; + return false; } } - - return false; } From 1d203fdae3eb59e444be40445655d01e98b6391d Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 1 Oct 2025 16:44:55 +0100 Subject: [PATCH 2/7] Set blob type correctly to not need to weave the mimetype around Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- src/ContentMessages.ts | 2 +- src/components/views/messages/MImageBody.tsx | 5 +---- src/utils/Image.ts | 4 ++-- src/utils/MediaEventHelper.ts | 8 ++++++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ContentMessages.ts b/src/ContentMessages.ts index e2e01ac9591..c22a6bab33a 100644 --- a/src/ContentMessages.ts +++ b/src/ContentMessages.ts @@ -158,7 +158,7 @@ async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imag } // We don't await this immediately so it can happen in the background - const isAnimatedPromise = blobIsAnimated(imageFile.type, imageFile); + const isAnimatedPromise = blobIsAnimated(imageFile); const imageElement = await loadImageElement(imageFile); diff --git a/src/components/views/messages/MImageBody.tsx b/src/components/views/messages/MImageBody.tsx index c113c36c415..023322476ea 100644 --- a/src/components/views/messages/MImageBody.tsx +++ b/src/components/views/messages/MImageBody.tsx @@ -311,10 +311,7 @@ export class MImageBodyInner extends React.Component { // then we need to check if the image is animated by downloading it. if ( content.info?.["org.matrix.msc4230.is_animated"] === false || - !(await blobIsAnimated( - content.info?.mimetype, - await this.props.mediaEventHelper!.sourceBlob.value, - )) + (await blobIsAnimated(await this.props.mediaEventHelper!.sourceBlob.value)) === false ) { isAnimated = false; } diff --git a/src/utils/Image.ts b/src/utils/Image.ts index b7a89556c70..6899d64f964 100644 --- a/src/utils/Image.ts +++ b/src/utils/Image.ts @@ -31,8 +31,8 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin * @param blob The Blob to check. * @returns True if the image is animated, false if not, or undefined if it could not be determined. */ -export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise { - switch (mimeType) { +export async function blobIsAnimated(blob: Blob): Promise { + switch (blob.type) { case "image/webp": { // Only extended file format WEBP images support animation, so grab the expected data range and verify header. // Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format diff --git a/src/utils/MediaEventHelper.ts b/src/utils/MediaEventHelper.ts index 075ef11c4e2..e7e148b24c2 100644 --- a/src/utils/MediaEventHelper.ts +++ b/src/utils/MediaEventHelper.ts @@ -72,11 +72,15 @@ export class MediaEventHelper implements IDestroyable { }; private fetchSource = (): Promise => { + const content = this.event.getContent(); if (this.media.isEncrypted) { - const content = this.event.getContent(); return decryptFile(content.file!, content.info); } - return this.media.downloadSource().then((r) => r.blob()); + + return this.media + .downloadSource() + .then((r) => r.blob()) + .then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type)); }; private fetchThumbnail = (): Promise => { From d7aa927a25d02e287941ec27bf99054078c27345 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 1 Oct 2025 16:47:30 +0100 Subject: [PATCH 3/7] Use ImageDecoder to determine whether media is animated or not, adding support for AVIF and other formats Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- src/utils/Image.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/utils/Image.ts b/src/utils/Image.ts index 6899d64f964..99352e26a01 100644 --- a/src/utils/Image.ts +++ b/src/utils/Image.ts @@ -9,7 +9,6 @@ import { arrayHasDiff } from "./arrays"; export function mayBeAnimated(mimeType?: string): boolean { - // AVIF animation support at the time of writing is only available in Chrome hence not having `blobIsAnimated` check return ["image/gif", "image/webp", "image/png", "image/apng", "image/avif"].includes(mimeType!); } @@ -32,6 +31,21 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin * @returns True if the image is animated, false if not, or undefined if it could not be determined. */ export async function blobIsAnimated(blob: Blob): Promise { + try { + // Try parse the image using ImageDecoder as this is the most coherent way of asserting whether a piece of media + // is or is not animated. Limited availability at time of writing, notably Safari lacks support. + // https://developer.mozilla.org/en-US/docs/Web/API/ImageDecoder + const data = await blob.arrayBuffer(); + const decoder = new ImageDecoder({ data, type: blob.type }); + await decoder.tracks.ready; + if ([...decoder.tracks].some((track) => track.animated)) { + return true; + } + } catch (e) { + console.warn("ImageDecoder not supported or failed to decode image", e); + // Not supported by this browser, fall through to manual checks + } + switch (blob.type) { case "image/webp": { // Only extended file format WEBP images support animation, so grab the expected data range and verify header. From 82f3db82f31e43aa8dff347358da2b7b52856eaa Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 1 Oct 2025 17:32:04 +0100 Subject: [PATCH 4/7] Fix test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- test/unit-tests/Image-test.ts | 45 ++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/test/unit-tests/Image-test.ts b/test/unit-tests/Image-test.ts index 966236bc8a2..1104788e6f6 100644 --- a/test/unit-tests/Image-test.ts +++ b/test/unit-tests/Image-test.ts @@ -32,42 +32,55 @@ describe("Image", () => { describe("blobIsAnimated", () => { it("Animated GIF", async () => { - const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))]); - expect(await blobIsAnimated("image/gif", img)).toBeTruthy(); + const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))], { + type: "image/gif", + }); + expect(await blobIsAnimated(img)).toBeTruthy(); }); it("Static GIF", async () => { - const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))]); - expect(await blobIsAnimated("image/gif", img)).toBeFalsy(); + const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))], { + type: "image/gif", + }); + expect(await blobIsAnimated(img)).toBeFalsy(); }); it("Animated WEBP", async () => { - const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))]); - expect(await blobIsAnimated("image/webp", img)).toBeTruthy(); + const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))], { + type: "image/webp", + }); + expect(await blobIsAnimated(img)).toBeTruthy(); }); it("Static WEBP", async () => { - const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))]); - expect(await blobIsAnimated("image/webp", img)).toBeFalsy(); + const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))], { + type: "image/webp", + }); + expect(await blobIsAnimated(img)).toBeFalsy(); }); it("Static WEBP in extended file format", async () => { - const img = new Blob([ - fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp")), - ]); - expect(await blobIsAnimated("image/webp", img)).toBeFalsy(); + const img = new Blob( + [fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp"))], + { type: "image/webp" }, + ); + expect(await blobIsAnimated(img)).toBeFalsy(); }); it("Animated PNG", async () => { const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.apng"))]); - expect(await blobIsAnimated("image/png", img)).toBeTruthy(); - expect(await blobIsAnimated("image/apng", img)).toBeTruthy(); + const pngBlob = img.slice(0, img.size, "image/png"); + const apngBlob = img.slice(0, img.size, "image/apng"); + expect(await blobIsAnimated(pngBlob)).toBeTruthy(); + expect(await blobIsAnimated(apngBlob)).toBeTruthy(); }); it("Static PNG", async () => { const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.png"))]); - expect(await blobIsAnimated("image/png", img)).toBeFalsy(); - expect(await blobIsAnimated("image/apng", img)).toBeFalsy(); + const pngBlob = img.slice(0, img.size, "image/png"); + const apngBlob = img.slice(0, img.size, "image/apng"); + expect(await blobIsAnimated(pngBlob)).toBeFalsy(); + expect(await blobIsAnimated(apngBlob)).toBeFalsy(); }); }); }); From c5bc81dbe3ec4d10efed5e84edc942b67764e1f3 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 2 Oct 2025 12:34:24 +0100 Subject: [PATCH 5/7] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- src/utils/MediaEventHelper.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/utils/MediaEventHelper.ts b/src/utils/MediaEventHelper.ts index e7e148b24c2..308d85b2fbc 100644 --- a/src/utils/MediaEventHelper.ts +++ b/src/utils/MediaEventHelper.ts @@ -77,17 +77,20 @@ export class MediaEventHelper implements IDestroyable { return decryptFile(content.file!, content.info); } - return this.media - .downloadSource() - .then((r) => r.blob()) - .then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type)); + return ( + this.media + .downloadSource() + .then((r) => r.blob()) + // Set the mime type from the event info on the blob + .then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type)) + ); }; private fetchThumbnail = (): Promise => { if (!this.media.hasThumbnail) return Promise.resolve(null); + const content = this.event.getContent(); if (this.media.isEncrypted) { - const content = this.event.getContent(); if (content.info?.thumbnail_file) { return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info); } else { @@ -100,7 +103,12 @@ export class MediaEventHelper implements IDestroyable { const thumbnailHttp = this.media.thumbnailHttp; if (!thumbnailHttp) return Promise.resolve(null); - return fetch(thumbnailHttp).then((r) => r.blob()); + return ( + fetch(thumbnailHttp) + .then((r) => r.blob()) + // Set the mime type from the event info on the blob + .then((blob) => blob.slice(0, blob.size, content.info?.thumbnail_info?.mimetype ?? blob.type)) + ); }; public static isEligible(event: MatrixEvent): boolean { From 6f2f3d76228b37c529c1714924158908caa60428 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 3 Oct 2025 13:50:54 +0100 Subject: [PATCH 6/7] Add test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .../unit-tests/utils/MediaEventHelper-test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 test/unit-tests/utils/MediaEventHelper-test.ts diff --git a/test/unit-tests/utils/MediaEventHelper-test.ts b/test/unit-tests/utils/MediaEventHelper-test.ts new file mode 100644 index 00000000000..ca5813b7aed --- /dev/null +++ b/test/unit-tests/utils/MediaEventHelper-test.ts @@ -0,0 +1,36 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE files in the repository root for full details. +*/ + +import { MatrixEvent } from "matrix-js-sdk/src/matrix"; + +import { MediaEventHelper } from "../../../src/utils/MediaEventHelper.ts"; +import { stubClient } from "../../test-utils"; + +describe("MediaEventHelper", () => { + it("should set the mime type on the blob based on the event metadata", async () => { + stubClient(); + + const event = new MatrixEvent({ + type: "m.room.message", + content: { + msgtype: "m.image", + body: "image.png", + info: { + mimetype: "image/png", + size: 1234, + w: 100, + h: 100, + }, + url: "mxc://matrix.org/abcdef", + }, + }); + const helper = new MediaEventHelper(event); + + const blob = await helper.sourceBlob.value; + expect(blob.type).toBe(event.getContent().info?.mimetype); + }); +}); From 945cf9760a615fda07040c734a73a6fa23698911 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 3 Oct 2025 14:13:31 +0100 Subject: [PATCH 7/7] Fix test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- test/unit-tests/utils/MediaEventHelper-test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/unit-tests/utils/MediaEventHelper-test.ts b/test/unit-tests/utils/MediaEventHelper-test.ts index ca5813b7aed..803924f74cc 100644 --- a/test/unit-tests/utils/MediaEventHelper-test.ts +++ b/test/unit-tests/utils/MediaEventHelper-test.ts @@ -24,13 +24,17 @@ describe("MediaEventHelper", () => { size: 1234, w: 100, h: 100, + thumbnail_info: { + mimetype: "image/png", + }, + thumbnail_url: "mxc://matrix.org/thumbnail", }, url: "mxc://matrix.org/abcdef", }, }); const helper = new MediaEventHelper(event); - const blob = await helper.sourceBlob.value; - expect(blob.type).toBe(event.getContent().info?.mimetype); + const blob = await helper.thumbnailBlob.value; + expect(blob?.type).toBe(event.getContent().info.thumbnail_info?.mimetype); }); });