PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Allow image pasting in plain mode in RTE (#11056)

Browse source 
* get rough funcitonality working

* try to tidy up types

* fix merge error

* fix signature change error

* type wrangling

* use onBeforeInput listener

* add onBeforeInput handler, add logic to onPaste

* fix type error

* bring plain text listeners in line with useInputEventProcessor

* extract common function to util file, move tests

* tidy comment

* tidy comments

* fix typo

* add util tests

* add text paste test
This commit is contained in:
alunturner 2023-06-12 12:28:00 +01:00 committed by GitHub
parent 47ab99f908
commit e32823e5fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 188 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx
View file

@ -50,10 +50,12 @@ export function PlainTextComposer({
initialContent,
leftComponent,
rightComponent,
eventRelation,
}: PlainTextComposerProps): JSX.Element {
const {
ref: editorRef,
autocompleteRef,
onBeforeInput,
onInput,
onPaste,
onKeyDown,
@ -63,7 +65,7 @@ export function PlainTextComposer({
onSelect,
handleCommand,
handleMention,
} = usePlainTextListeners(initialContent, onChange, onSend);
} = usePlainTextListeners(initialContent, onChange, onSend, eventRelation);
const composerFunctions = useComposerFunctions(editorRef, setContent);
usePlainTextInitialization(initialContent, editorRef);
@ -77,6 +79,7 @@ export function PlainTextComposer({
className={classNames(className, { [`${className}-focused`]: isFocused })}
onFocus={onFocus}
onBlur={onFocus}
onBeforeInput={onBeforeInput}
onInput={onInput}
onPaste={onPaste}
onKeyDown={onKeyDown}

103
src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts
View file

@ -33,10 +33,7 @@ import { isCaretAtEnd, isCaretAtStart } from "../utils/selection";
import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/event";
import { endEditing } from "../utils/editing";
import Autocomplete from "../../Autocomplete";
import { handleEventWithAutocomplete } from "./utils";
import ContentMessages from "../../../../../ContentMessages";
import { getBlobSafeMimeType } from "../../../../../utils/blobs";
import { isNotNull } from "../../../../../Typeguards";
import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils";
export function useInputEventProcessor(
onSend: () => void,
@ -61,17 +58,8 @@ export function useInputEventProcessor(
onSend();
};
// this is required to handle edge case image pasting in Safari, see
// https://github.com/vector-im/element-web/issues/25327 and it is caught by the
// `beforeinput` listener attached to the composer
const isInputEventForClipboard =
event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
const isClipboardEvent = event instanceof ClipboardEvent;
const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard;
if (shouldHandleAsClipboardEvent) {
const data = isClipboardEvent ? event.clipboardData : event.dataTransfer;
if (isEventToHandleAsClipboardEvent(event)) {
const data = event instanceof ClipboardEvent ? event.clipboardData : event.dataTransfer;
const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation);
return handled ? null : event;
}
@ -244,88 +232,3 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool
return event;
}
/**
* Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
* the event or not. Must accept either clipboard or input events in order to prevent issue:
* https://github.com/vector-im/element-web/issues/25327
*
* @param event - event to process
* @param roomContext - room in which the event occurs
* @param mxClient - current matrix client
* @param eventRelation - used to send the event to the correct place eg timeline vs thread
* @returns - boolean to show if the event was handled or not
*/
export function handleClipboardEvent(
event: ClipboardEvent | InputEvent,
data: DataTransfer | null,
roomContext: IRoomState,
mxClient: MatrixClient,
eventRelation?: IEventRelation,
): boolean {
// Logic in this function follows that of `SendMessageComposer.onPaste`
const { room, timelineRenderingType, replyToEvent } = roomContext;
function handleError(error: unknown): void {
if (error instanceof Error) {
console.log(error.message);
} else if (typeof error === "string") {
console.log(error);
}
}
if (event.type !== "paste" || data === null || room === undefined) {
return false;
}
// Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
// in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
// We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
// it puts the filename in as text/plain which we want to ignore.
if (data.files.length && !data.types.includes("text/rtf")) {
ContentMessages.sharedInstance()
.sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
.catch(handleError);
return true;
}
// Safari `Insert from iPhone or iPad`
// data.getData("text/html") returns a string like: <img src="blob:https://...">
if (data.types.includes("text/html")) {
const imgElementStr = data.getData("text/html");
const parser = new DOMParser();
const imgDoc = parser.parseFromString(imgElementStr, "text/html");
if (
imgDoc.getElementsByTagName("img").length !== 1 ||
!imgDoc.querySelector("img")?.src.startsWith("blob:") ||
imgDoc.childNodes.length !== 1
) {
handleError("Failed to handle pasted content as Safari inserted content");
return false;
}
const imgSrc = imgDoc.querySelector("img")!.src;
fetch(imgSrc)
.then((response) => {
response
.blob()
.then((imgBlob) => {
const type = imgBlob.type;
const safetype = getBlobSafeMimeType(type);
const ext = type.split("/")[1];
const parts = response.url.split("/");
const filename = parts[parts.length - 1];
const file = new File([imgBlob], filename + "." + ext, { type: safetype });
ContentMessages.sharedInstance()
.sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
.catch(handleError);
})
.catch(handleError);
})
.catch(handleError);
return true;
}
return false;
}

34
src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts
View file

@ -16,13 +16,16 @@ limitations under the License.
import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react";
import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg";
import { IEventRelation } from "matrix-js-sdk/src/matrix";
import { useSettingValue } from "../../../../../hooks/useSettings";
import { IS_MAC, Key } from "../../../../../Keyboard";
import Autocomplete from "../../Autocomplete";
import { handleEventWithAutocomplete } from "./utils";
import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils";
import { useSuggestion } from "./useSuggestion";
import { isNotNull, isNotUndefined } from "../../../../../Typeguards";
import { useRoomContext } from "../../../../../contexts/RoomContext";
import { useMatrixClientContext } from "../../../../../contexts/MatrixClientContext";
function isDivElement(target: EventTarget): target is HTMLDivElement {
return target instanceof HTMLDivElement;
@ -59,10 +62,12 @@ export function usePlainTextListeners(
initialContent?: string,
onChange?: (content: string) => void,
onSend?: () => void,
eventRelation?: IEventRelation,
): {
ref: RefObject<HTMLDivElement>;
autocompleteRef: React.RefObject<Autocomplete>;
content?: string;
onBeforeInput(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
onInput(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
onPaste(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
onKeyDown(event: KeyboardEvent<HTMLDivElement>): void;
@ -72,6 +77,9 @@ export function usePlainTextListeners(
onSelect: (event: SyntheticEvent<HTMLDivElement>) => void;
suggestion: MappedSuggestion | null;
} {
const roomContext = useRoomContext();
const mxClient = useMatrixClientContext();
const ref = useRef<HTMLDivElement | null>(null);
const autocompleteRef = useRef<Autocomplete | null>(null);
const [content, setContent] = useState<string | undefined>(initialContent);
@ -115,6 +123,27 @@ export function usePlainTextListeners(
[setText, enterShouldSend],
);
const onPaste = useCallback(
(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>) => {
const { nativeEvent } = event;
let imagePasteWasHandled = false;
if (isEventToHandleAsClipboardEvent(nativeEvent)) {
const data =
nativeEvent instanceof ClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer;
imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation);
}
// prevent default behaviour and skip call to onInput if the image paste event was handled
if (imagePasteWasHandled) {
event.preventDefault();
} else {
onInput(event);
}
},
[eventRelation, mxClient, onInput, roomContext],
);
const onKeyDown = useCallback(
(event: KeyboardEvent<HTMLDivElement>) => {
// we need autocomplete to take priority when it is open for using enter to select
@ -149,8 +178,9 @@ export function usePlainTextListeners(
return {
ref,
autocompleteRef,
onBeforeInput: onPaste,
onInput,
onPaste: onInput,
onPaste,
onKeyDown,
content,
setContent: setText,

110
src/components/views/rooms/wysiwyg_composer/hooks/utils.ts
View file

@ -15,12 +15,17 @@ limitations under the License.
*/
import { MutableRefObject, RefObject } from "react";
import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix";
import { WysiwygEvent } from "@matrix-org/matrix-wysiwyg";
import { TimelineRenderingType } from "../../../../../contexts/RoomContext";
import { IRoomState } from "../../../../structures/RoomView";
import Autocomplete from "../../Autocomplete";
import { getKeyBindingsManager } from "../../../../../KeyBindingsManager";
import { KeyBindingAction } from "../../../../../accessibility/KeyboardShortcuts";
import { getBlobSafeMimeType } from "../../../../../utils/blobs";
import ContentMessages from "../../../../../ContentMessages";
import { isNotNull } from "../../../../../Typeguards";
export function focusComposer(
composerElement: MutableRefObject<HTMLElement | null>,
@ -110,3 +115,108 @@ export function handleEventWithAutocomplete(
return handled;
}
/**
* Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
* the event or not. Must accept either clipboard or input events in order to prevent issue:
* https://github.com/vector-im/element-web/issues/25327
*
* @param event - event to process
* @param data - data from the event to process
* @param roomContext - room in which the event occurs
* @param mxClient - current matrix client
* @param eventRelation - used to send the event to the correct place eg timeline vs thread
* @returns - boolean to show if the event was handled or not
*/
export function handleClipboardEvent(
event: ClipboardEvent | InputEvent,
data: DataTransfer | null,
roomContext: IRoomState,
mxClient: MatrixClient,
eventRelation?: IEventRelation,
): boolean {
// Logic in this function follows that of `SendMessageComposer.onPaste`
const { room, timelineRenderingType, replyToEvent } = roomContext;
function handleError(error: unknown): void {
if (error instanceof Error) {
console.log(error.message);
} else if (typeof error === "string") {
console.log(error);
}
}
if (event.type !== "paste" || data === null || room === undefined) {
return false;
}
// Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
// in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
// We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
// it puts the filename in as text/plain which we want to ignore.
if (data.files.length && !data.types.includes("text/rtf")) {
ContentMessages.sharedInstance()
.sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
.catch(handleError);
return true;
}
// Safari `Insert from iPhone or iPad`
// data.getData("text/html") returns a string like: <img src="blob:https://...">
if (data.types.includes("text/html")) {
const imgElementStr = data.getData("text/html");
const parser = new DOMParser();
const imgDoc = parser.parseFromString(imgElementStr, "text/html");
if (
imgDoc.getElementsByTagName("img").length !== 1 ||
!imgDoc.querySelector("img")?.src.startsWith("blob:") ||
imgDoc.childNodes.length !== 1
) {
handleError("Failed to handle pasted content as Safari inserted content");
return false;
}
const imgSrc = imgDoc.querySelector("img")!.src;
fetch(imgSrc)
.then((response) => {
response
.blob()
.then((imgBlob) => {
const type = imgBlob.type;
const safetype = getBlobSafeMimeType(type);
const ext = type.split("/")[1];
const parts = response.url.split("/");
const filename = parts[parts.length - 1];
const file = new File([imgBlob], filename + "." + ext, { type: safetype });
ContentMessages.sharedInstance()
.sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
.catch(handleError);
})
.catch(handleError);
})
.catch(handleError);
return true;
}
return false;
}
/**
* Util to determine if an input event or clipboard event must be handled as a clipboard event.
* Due to https://github.com/vector-im/element-web/issues/25327, certain paste events
* must be listenened for with an onBeforeInput handler and so will be caught as input events.
*
* @param event - the event to test, can be a WysiwygEvent if it comes from the rich text editor, or
* input or clipboard events if from the plain text editor
* @returns - true if event should be handled as a clipboard event
*/
export function isEventToHandleAsClipboardEvent(
event: WysiwygEvent | InputEvent | ClipboardEvent,
): event is InputEvent | ClipboardEvent {
const isInputEventForClipboard =
event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
const isClipboardEvent = event instanceof ClipboardEvent;
return isClipboardEvent || isInputEventForClipboard;
}

12
test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx
View file

@ -290,4 +290,16 @@ describe("PlainTextComposer", () => {
expect(screen.getByTestId("autocomplete-wrapper")).toBeInTheDocument();
});
it("Should allow pasting of text values", async () => {
customRender();
const textBox = screen.getByRole("textbox");
await userEvent.click(textBox);
await userEvent.type(textBox, "hello");
await userEvent.paste(" world");
expect(textBox).toHaveTextContent("hello world");
});
});

28
test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx → test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx
View file

@ -16,11 +16,14 @@ limitations under the License.
import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix";
import { waitFor } from "@testing-library/react";
import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor";
import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext";
import { mkStubRoom, stubClient } from "../../../../../test-utils";
import ContentMessages from "../../../../../../src/ContentMessages";
import { IRoomState } from "../../../../../../src/components/structures/RoomView";
import {
handleClipboardEvent,
isEventToHandleAsClipboardEvent,
} from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/utils";
const mockClient = stubClient();
const mockRoom = mkStubRoom("mock room", "mock room", mockClient);
@ -285,3 +288,26 @@ describe("handleClipboardEvent", () => {
expect(output).toBe(true);
});
});
describe("isEventToHandleAsClipboardEvent", () => {
it("returns true for ClipboardEvent", () => {
const input = new ClipboardEvent("clipboard");
expect(isEventToHandleAsClipboardEvent(input)).toBe(true);
});
it("returns true for special case input", () => {
const input = new InputEvent("insertFromPaste", { inputType: "insertFromPaste" });
Object.assign(input, { dataTransfer: "not null" });
expect(isEventToHandleAsClipboardEvent(input)).toBe(true);
});
it("returns false for regular InputEvent", () => {
const input = new InputEvent("input");
expect(isEventToHandleAsClipboardEvent(input)).toBe(false);
});
it("returns false for other input", () => {
const input = new KeyboardEvent("keyboard");
expect(isEventToHandleAsClipboardEvent(input)).toBe(false);
});
});