Allow image pasting in rich text mode in RTE (#11049)

* add comments to rough first solution

* allow eventRelation prop to pass to both composers

* use eventRelation in image paste

* add image pasting to rich text mode of rich text editor

* extract error handling to function

* type the error handler

* add tests

* make behaviour mimic SendMessage

* add sad path tests

* refactor to use catch throughout

* update comments

* tidy up tests

* add special case and change function signature

* add comment

* bump rte to 2.2.2

package.json

@@ -61,7 +61,7 @@
     "dependencies": {
         "@babel/runtime": "^7.12.5",
         "@matrix-org/analytics-events": "^0.5.0",
-        "@matrix-org/matrix-wysiwyg": "^2.0.0",
+        "@matrix-org/matrix-wysiwyg": "^2.2.2",
         "@matrix-org/react-sdk-module-api": "^0.0.5",
         "@sentry/browser": "^7.0.0",
         "@sentry/tracing": "^7.0.0",

src/components/views/rooms/wysiwyg_composer/SendWysiwygComposer.tsx

@@ -57,11 +57,10 @@ export default function SendWysiwygComposer({
     isRichTextEnabled,
     e2eStatus,
     menuPosition,
-    eventRelation,
     ...props
 }: SendWysiwygComposerProps): JSX.Element {
     const Composer = isRichTextEnabled ? WysiwygComposer : PlainTextComposer;
-    const defaultContextValue = useRef(getDefaultContextValue({ eventRelation }));
+    const defaultContextValue = useRef(getDefaultContextValue({ eventRelation: props.eventRelation }));
 
     return (
         <ComposerContext.Provider value={defaultContextValue.current}>

src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import classNames from "classnames";
+import { IEventRelation } from "matrix-js-sdk/src/matrix";
 import React, { MutableRefObject, ReactNode } from "react";
 
 import { useComposerFunctions } from "../hooks/useComposerFunctions";
@@ -36,6 +37,7 @@ interface PlainTextComposerProps {
     leftComponent?: ReactNode;
     rightComponent?: ReactNode;
     children?: (ref: MutableRefObject<HTMLDivElement | null>, composerFunctions: ComposerFunctions) => ReactNode;
+    eventRelation?: IEventRelation;
 }
 
 export function PlainTextComposer({

src/components/views/rooms/wysiwyg_composer/components/WysiwygComposer.tsx

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import React, { memo, MutableRefObject, ReactNode, useEffect, useRef } from "react";
+import { IEventRelation } from "matrix-js-sdk/src/matrix";
 import { useWysiwyg, FormattingFunctions } from "@matrix-org/matrix-wysiwyg";
 import classNames from "classnames";
 
@@ -40,6 +41,7 @@ interface WysiwygComposerProps {
     leftComponent?: ReactNode;
     rightComponent?: ReactNode;
     children?: (ref: MutableRefObject<HTMLDivElement | null>, wysiwyg: FormattingFunctions) => ReactNode;
+    eventRelation?: IEventRelation;
 }
 
 export const WysiwygComposer = memo(function WysiwygComposer({
@@ -52,11 +54,12 @@ export const WysiwygComposer = memo(function WysiwygComposer({
     leftComponent,
     rightComponent,
     children,
+    eventRelation,
 }: WysiwygComposerProps) {
     const { room } = useRoomContext();
     const autocompleteRef = useRef<Autocomplete | null>(null);
 
-    const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent);
+    const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent, eventRelation);
     const { ref, isWysiwygReady, content, actionStates, wysiwyg, suggestion } = useWysiwyg({
         initialContent,
         inputEventProcessor,

src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import { Wysiwyg, WysiwygEvent } from "@matrix-org/matrix-wysiwyg";
 import { useCallback } from "react";
-import { MatrixClient } from "matrix-js-sdk/src/matrix";
+import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix";
 
 import { useSettingValue } from "../../../../../hooks/useSettings";
 import { getKeyBindingsManager } from "../../../../../KeyBindingsManager";
@@ -34,11 +34,15 @@ import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/ev
 import { endEditing } from "../utils/editing";
 import Autocomplete from "../../Autocomplete";
 import { handleEventWithAutocomplete } from "./utils";
+import ContentMessages from "../../../../../ContentMessages";
+import { getBlobSafeMimeType } from "../../../../../utils/blobs";
+import { isNotNull } from "../../../../../Typeguards";
 
 export function useInputEventProcessor(
     onSend: () => void,
     autocompleteRef: React.RefObject<Autocomplete>,
     initialContent?: string,
+    eventRelation?: IEventRelation,
 ): (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => WysiwygEvent | null {
     const roomContext = useRoomContext();
     const composerContext = useComposerContext();
@@ -47,10 +51,6 @@ export function useInputEventProcessor(
 
     return useCallback(
         (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => {
-            if (event instanceof ClipboardEvent) {
-                return event;
-            }
-
             const send = (): void => {
                 event.stopPropagation?.();
                 event.preventDefault?.();
@@ -61,6 +61,21 @@ export function useInputEventProcessor(
                 onSend();
             };
 
+            // this is required to handle edge case image pasting in Safari, see
+            // https://github.com/vector-im/element-web/issues/25327 and it is caught by the
+            // `beforeinput` listener attached to the composer
+            const isInputEventForClipboard =
+                event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
+            const isClipboardEvent = event instanceof ClipboardEvent;
+
+            const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard;
+
+            if (shouldHandleAsClipboardEvent) {
+                const data = isClipboardEvent ? event.clipboardData : event.dataTransfer;
+                const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation);
+                return handled ? null : event;
+            }
+
             const isKeyboardEvent = event instanceof KeyboardEvent;
             if (isKeyboardEvent) {
                 return handleKeyboardEvent(
@@ -78,7 +93,16 @@ export function useInputEventProcessor(
                 return handleInputEvent(event, send, isCtrlEnterToSend);
             }
         },
-        [isCtrlEnterToSend, onSend, initialContent, roomContext, composerContext, mxClient, autocompleteRef],
+        [
+            isCtrlEnterToSend,
+            onSend,
+            initialContent,
+            roomContext,
+            composerContext,
+            mxClient,
+            autocompleteRef,
+            eventRelation,
+        ],
     );
 }
 
@@ -220,3 +244,88 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool
 
     return event;
 }
+
+/**
+ * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
+ * the event or not. Must accept either clipboard or input events in order to prevent issue:
+ * https://github.com/vector-im/element-web/issues/25327
+ *
+ * @param event - event to process
+ * @param roomContext - room in which the event occurs
+ * @param mxClient - current matrix client
+ * @param eventRelation - used to send the event to the correct place eg timeline vs thread
+ * @returns - boolean to show if the event was handled or not
+ */
+export function handleClipboardEvent(
+    event: ClipboardEvent | InputEvent,
+    data: DataTransfer | null,
+    roomContext: IRoomState,
+    mxClient: MatrixClient,
+    eventRelation?: IEventRelation,
+): boolean {
+    // Logic in this function follows that of `SendMessageComposer.onPaste`
+    const { room, timelineRenderingType, replyToEvent } = roomContext;
+
+    function handleError(error: unknown): void {
+        if (error instanceof Error) {
+            console.log(error.message);
+        } else if (typeof error === "string") {
+            console.log(error);
+        }
+    }
+
+    if (event.type !== "paste" || data === null || room === undefined) {
+        return false;
+    }
+
+    // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
+    // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
+    // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
+    // it puts the filename in as text/plain which we want to ignore.
+    if (data.files.length && !data.types.includes("text/rtf")) {
+        ContentMessages.sharedInstance()
+            .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
+            .catch(handleError);
+        return true;
+    }
+
+    // Safari `Insert from iPhone or iPad`
+    // data.getData("text/html") returns a string like: <img src="blob:https://...">
+    if (data.types.includes("text/html")) {
+        const imgElementStr = data.getData("text/html");
+        const parser = new DOMParser();
+        const imgDoc = parser.parseFromString(imgElementStr, "text/html");
+
+        if (
+            imgDoc.getElementsByTagName("img").length !== 1 ||
+            !imgDoc.querySelector("img")?.src.startsWith("blob:") ||
+            imgDoc.childNodes.length !== 1
+        ) {
+            handleError("Failed to handle pasted content as Safari inserted content");
+            return false;
+        }
+        const imgSrc = imgDoc.querySelector("img")!.src;
+
+        fetch(imgSrc)
+            .then((response) => {
+                response
+                    .blob()
+                    .then((imgBlob) => {
+                        const type = imgBlob.type;
+                        const safetype = getBlobSafeMimeType(type);
+                        const ext = type.split("/")[1];
+                        const parts = response.url.split("/");
+                        const filename = parts[parts.length - 1];
+                        const file = new File([imgBlob], filename + "." + ext, { type: safetype });
+                        ContentMessages.sharedInstance()
+                            .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
+                            .catch(handleError);
+                    })
+                    .catch(handleError);
+            })
+            .catch(handleError);
+        return true;
+    }
+
+    return false;
+}

test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx

@@ -0,0 +1,287 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix";
+import { waitFor } from "@testing-library/react";
+
+import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor";
+import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext";
+import { mkStubRoom, stubClient } from "../../../../../test-utils";
+import ContentMessages from "../../../../../../src/ContentMessages";
+import { IRoomState } from "../../../../../../src/components/structures/RoomView";
+
+const mockClient = stubClient();
+const mockRoom = mkStubRoom("mock room", "mock room", mockClient);
+const mockRoomState = {
+    room: mockRoom,
+    timelineRenderingType: TimelineRenderingType.Room,
+    replyToEvent: {} as unknown as MatrixEvent,
+} as unknown as IRoomState;
+
+const sendContentListToRoomSpy = jest.spyOn(ContentMessages.sharedInstance(), "sendContentListToRoom");
+const sendContentToRoomSpy = jest.spyOn(ContentMessages.sharedInstance(), "sendContentToRoom");
+const fetchSpy = jest.spyOn(window, "fetch");
+const logSpy = jest.spyOn(console, "log").mockImplementation(() => {});
+
+describe("handleClipboardEvent", () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+
+    afterAll(() => {
+        jest.restoreAllMocks();
+    });
+
+    function createMockClipboardEvent(props: any): ClipboardEvent {
+        return { clipboardData: { files: [], types: [] }, ...props } as ClipboardEvent;
+    }
+
+    it("returns false if it is not a paste event", () => {
+        const originalEvent = createMockClipboardEvent({ type: "copy" });
+        const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
+
+        expect(output).toBe(false);
+    });
+
+    it("returns false if clipboard data is null", () => {
+        const originalEvent = createMockClipboardEvent({ type: "paste", clipboardData: null });
+        const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
+
+        expect(output).toBe(false);
+    });
+
+    it("returns false if room is undefined", () => {
+        const originalEvent = createMockClipboardEvent({ type: "paste" });
+        const { room, ...roomStateWithoutRoom } = mockRoomState;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            roomStateWithoutRoom,
+            mockClient,
+        );
+
+        expect(output).toBe(false);
+    });
+
+    it("returns false if room clipboardData files and types are empty", () => {
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: { files: [], types: [] },
+        });
+        const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
+        expect(output).toBe(false);
+    });
+
+    it("handles event and calls sendContentListToRoom when data files are present", () => {
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: { files: ["something here"], types: [] },
+        });
+        const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
+
+        expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1);
+        expect(sendContentListToRoomSpy).toHaveBeenCalledWith(
+            originalEvent.clipboardData?.files,
+            mockRoom.roomId,
+            undefined, // this is the event relation, an optional arg
+            mockClient,
+            mockRoomState.timelineRenderingType,
+        );
+        expect(output).toBe(true);
+    });
+
+    it("calls sendContentListToRoom with eventRelation when present", () => {
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: { files: ["something here"], types: [] },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            mockRoomState,
+            mockClient,
+            mockEventRelation,
+        );
+
+        expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1);
+        expect(sendContentListToRoomSpy).toHaveBeenCalledWith(
+            originalEvent.clipboardData?.files,
+            mockRoom.roomId,
+            mockEventRelation, // this is the event relation, an optional arg
+            mockClient,
+            mockRoomState.timelineRenderingType,
+        );
+        expect(output).toBe(true);
+    });
+
+    it("calls the error handler when sentContentListToRoom errors", async () => {
+        const mockErrorMessage = "something went wrong";
+        sendContentListToRoomSpy.mockRejectedValueOnce(new Error(mockErrorMessage));
+
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: { files: ["something here"], types: [] },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            mockRoomState,
+            mockClient,
+            mockEventRelation,
+        );
+
+        expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1);
+        await waitFor(() => {
+            expect(logSpy).toHaveBeenCalledWith(mockErrorMessage);
+        });
+        expect(output).toBe(true);
+    });
+
+    it("calls the error handler when data types has text/html but data can not be parsed", () => {
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: {
+                files: [],
+                types: ["text/html"],
+                getData: jest.fn().mockReturnValue("<div>invalid html"),
+            },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            mockRoomState,
+            mockClient,
+            mockEventRelation,
+        );
+
+        expect(logSpy).toHaveBeenCalledWith("Failed to handle pasted content as Safari inserted content");
+        expect(output).toBe(false);
+    });
+
+    it("calls fetch when data types has text/html and data can parsed", () => {
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: {
+                files: [],
+                types: ["text/html"],
+                getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
+            },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient, mockEventRelation);
+
+        expect(fetchSpy).toHaveBeenCalledTimes(1);
+        expect(fetchSpy).toHaveBeenCalledWith("blob:");
+    });
+
+    it("calls error handler when fetch fails", async () => {
+        const mockErrorMessage = "fetch failed";
+        fetchSpy.mockRejectedValueOnce(mockErrorMessage);
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: {
+                files: [],
+                types: ["text/html"],
+                getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
+            },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            mockRoomState,
+            mockClient,
+            mockEventRelation,
+        );
+
+        await waitFor(() => {
+            expect(logSpy).toHaveBeenCalledWith(mockErrorMessage);
+        });
+        expect(output).toBe(true);
+    });
+
+    it("calls sendContentToRoom when parsing is successful", async () => {
+        fetchSpy.mockResolvedValueOnce({
+            url: "test/file",
+            blob: () => {
+                return Promise.resolve({ type: "image/jpeg" } as Blob);
+            },
+        } as Response);
+
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: {
+                files: [],
+                types: ["text/html"],
+                getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
+            },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            mockRoomState,
+            mockClient,
+            mockEventRelation,
+        );
+
+        await waitFor(() => {
+            expect(sendContentToRoomSpy).toHaveBeenCalledWith(
+                expect.any(File),
+                mockRoom.roomId,
+                mockEventRelation,
+                mockClient,
+                mockRoomState.replyToEvent,
+            );
+        });
+        expect(output).toBe(true);
+    });
+
+    it("calls error handler when parsing is not successful", async () => {
+        fetchSpy.mockResolvedValueOnce({
+            url: "test/file",
+            blob: () => {
+                return Promise.resolve({ type: "image/jpeg" } as Blob);
+            },
+        } as Response);
+        const mockErrorMessage = "sendContentToRoom failed";
+        sendContentToRoomSpy.mockRejectedValueOnce(mockErrorMessage);
+
+        const originalEvent = createMockClipboardEvent({
+            type: "paste",
+            clipboardData: {
+                files: [],
+                types: ["text/html"],
+                getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
+            },
+        });
+        const mockEventRelation = {} as unknown as IEventRelation;
+        const output = handleClipboardEvent(
+            originalEvent,
+            originalEvent.clipboardData,
+            mockRoomState,
+            mockClient,
+            mockEventRelation,
+        );
+
+        await waitFor(() => {
+            expect(logSpy).toHaveBeenCalledWith(mockErrorMessage);
+        });
+        expect(output).toBe(true);
+    });
+});

yarn.lock

@@ -1598,10 +1598,10 @@
   resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.10.tgz#b6a6395cffd3197ae2e0a88f4eeae8b315571fd2"
   integrity sha512-8V2NKuzGOFzEZeZVgF2is7gmuopdRbMZ064tzPDE0vN34iX6s3O8A4oxIT7SA3qtymwm3t1yEvTnT+0gfbmh4g==
 
-"@matrix-org/matrix-wysiwyg@^2.0.0":
+"@matrix-org/matrix-wysiwyg@^2.2.2":
-  version "2.2.1"
+  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-wysiwyg/-/matrix-wysiwyg-2.2.1.tgz#076b409c0ffe655938d663863b1ee546a7101da6"
+  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-wysiwyg/-/matrix-wysiwyg-2.2.2.tgz#911d0a9858a5a4b620f93777085daac8eff6a220"
-  integrity sha512-QF4dJsyqBMxZx+GhSdSiRSDIuwE5dxd7vffQ5i6hf67bd0EbVvtf4PzWmNopGHA+ckjMJIc5X1EPT+6DG/wM6Q==
+  integrity sha512-FprkgKiqEHoFUfaamKwTGBENqDxbORFgoPjiE1b9yPS3hgRswobVKRl4qrXgVgFj4qQ7gWeTqogiyrHXkm1myw==
 
 "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz":
   version "3.2.14"