[feature] Include sources in TLExternalContent (#1925)

This PR adds the source items from a paste event to the data shared with
external content handlers. This allows developers to customize the way
certain content is handled.

For example, pasting text sometimes incudes additional clipboard items,
such as the HTML representation of that text. We wouldn't want to create
two shapes—one for the text and one for the HTML—so we still treat this
as a single text paste. The `registerExternalContentHandler` API allows
a developer to change how that text is handled, and the new `sources`
API will now allow the developer to take into consideration all of the
items that were on the clipboard.
 
![Kapture 2023-09-19 at 12 25
52](https://github.com/tldraw/tldraw/assets/23072548/fa976320-cfec-4921-b481-10cae0d4043e)

### Change Type

- [x] `minor` — New feature

### Test Plan

1. Try the external content source example.
2. Paste text that includes HTML (e.g. from VS Code)

### Release Notes

- [editor / tldraw] add `sources` to `TLExternalContent`

apps/examples/src/examples/ExternalContentSourcesExample.tsx

@@ -0,0 +1,65 @@
+import { BaseBoxShapeUtil, Editor, HTMLContainer, TLBaseShape, Tldraw } from '@tldraw/tldraw'
+import '@tldraw/tldraw/tldraw.css'
+import { useCallback } from 'react'
+
+export type IDangerousHtmlShape = TLBaseShape<
+	'html',
+	{
+		w: number
+		h: number
+		html: string
+	}
+>
+
+class DangerousHtmlExample extends BaseBoxShapeUtil<IDangerousHtmlShape> {
+	static override type = 'html' as const
+
+	override getDefaultProps() {
+		return {
+			type: 'html',
+			w: 500,
+			h: 300,
+			html: '<div>hello</div>',
+		}
+	}
+
+	override component(shape: IDangerousHtmlShape) {
+		return (
+			<HTMLContainer style={{ overflow: 'auto' }}>
+				<div dangerouslySetInnerHTML={{ __html: shape.props.html }}></div>
+			</HTMLContainer>
+		)
+	}
+
+	override indicator(shape: IDangerousHtmlShape) {
+		return <rect width={shape.props.w} height={shape.props.h} />
+	}
+}
+
+export default function ExternalContentSourcesExample() {
+	const handleMount = useCallback((editor: Editor) => {
+		// When a user uploads a file, create an asset from it
+		editor.registerExternalContentHandler('text', async ({ point, sources }) => {
+			const htmlSource = sources?.find((s) => s.type === 'text' && s.subtype === 'html')
+
+			if (htmlSource) {
+				const center = point ?? editor.viewportPageCenter
+
+				editor.createShape({
+					type: 'html',
+					x: center.x - 250,
+					y: center.y - 150,
+					props: {
+						html: htmlSource.data,
+					},
+				})
+			}
+		})
+	}, [])
+
+	return (
+		<div className="tldraw__editor">
+			<Tldraw autoFocus onMount={handleMount} shapeUtils={[DangerousHtmlExample]} />
+		</div>
+	)
+}

apps/examples/src/index.tsx

@@ -22,6 +22,7 @@ import CustomStylesExample from './examples/CustomStylesExample/CustomStylesExam
 import CustomUiExample from './examples/CustomUiExample/CustomUiExample'
 import ErrorBoundaryExample from './examples/ErrorBoundaryExample/ErrorBoundaryExample'
 import ExplodedExample from './examples/ExplodedExample'
+import ExternalContentSourcesExample from './examples/ExternalContentSourcesExample'
 import HideUiExample from './examples/HideUiExample'
 import MultipleExample from './examples/MultipleExample'
 import PersistenceExample from './examples/PersistenceExample'
@@ -161,6 +162,11 @@ export const allExamples: Example[] = [
 		path: '/asset-props',
 		element: <AssetPropsExample />,
 	},
+	{
+		title: 'External content sources',
+		path: '/external-content-sources',
+		element: <ExternalContentSourcesExample />,
+	},
 	// not listed
 	{
 		path: '/end-to-end',

packages/editor/api-report.md

@@ -623,16 +623,16 @@ export class Editor extends EventEmitter<TLEventMap> {
     get erasingShapes(): NonNullable<TLShape | undefined>[];
     // @internal (undocumented)
     externalAssetContentHandlers: {
-        [K in TLExternalAssetContent_2['type']]: {
-            [Key in K]: ((info: TLExternalAssetContent_2 & {
+        [K in TLExternalAssetContent['type']]: {
+            [Key in K]: ((info: TLExternalAssetContent & {
                 type: Key;
             }) => Promise<TLAsset | undefined>) | null;
         }[K];
     };
     // @internal (undocumented)
     externalContentHandlers: {
-        [K in TLExternalContent_2['type']]: {
-            [Key in K]: ((info: TLExternalContent_2 & {
+        [K in TLExternalContent['type']]: {
+            [Key in K]: ((info: TLExternalContent & {
                 type: Key;
             }) => void) | null;
         }[K];
@@ -649,7 +649,7 @@ export class Editor extends EventEmitter<TLEventMap> {
         handleId: "end" | "start";
     }[];
     getAsset(asset: TLAsset | TLAssetId): TLAsset | undefined;
-    getAssetForExternalContent(info: TLExternalAssetContent_2): Promise<TLAsset | undefined>;
+    getAssetForExternalContent(info: TLExternalAssetContent): Promise<TLAsset | undefined>;
     getContainer: () => HTMLElement;
     getContentFromCurrentPage(shapes: TLShape[] | TLShapeId[]): TLContent | undefined;
     getDroppingOverShape(point: VecLike, droppingShapes?: TLShape[]): TLShape | undefined;
@@ -770,14 +770,14 @@ export class Editor extends EventEmitter<TLEventMap> {
         preservePosition?: boolean;
         preserveIds?: boolean;
     }): this;
-    putExternalContent(info: TLExternalContent_2): Promise<void>;
+    putExternalContent(info: TLExternalContent): Promise<void>;
     redo(): this;
-    registerExternalAssetHandler<T extends TLExternalAssetContent_2['type']>(type: T, handler: ((info: TLExternalAssetContent_2 & {
+    registerExternalAssetHandler<T extends TLExternalAssetContent['type']>(type: T, handler: ((info: TLExternalAssetContent & {
         type: T;
     }) => Promise<TLAsset>) | null): this;
-    registerExternalContentHandler<T extends TLExternalContent_2['type']>(type: T, handler: ((info: T extends TLExternalContent_2['type'] ? TLExternalContent_2 & {
+    registerExternalContentHandler<T extends TLExternalContent['type']>(type: T, handler: ((info: T extends TLExternalContent['type'] ? TLExternalContent & {
         type: T;
-    } : TLExternalContent_2) => void) | null): this;
+    } : TLExternalContent) => void) | null): this;
     renamePage(page: TLPage | TLPageId, name: string, historyOptions?: TLCommandHistoryOptions): this;
     get renderingBounds(): Box2d;
     get renderingBoundsExpanded(): Box2d;
@@ -2147,27 +2147,42 @@ export type TLExternalAssetContent = {
 
 // @public (undocumented)
 export type TLExternalContent = {
+    sources?: TLExternalContentSource[];
+    point?: VecLike;
+} & ({
     type: 'embed';
     url: string;
-    point?: VecLike;
     embed: EmbedDefinition;
 } | {
     type: 'files';
     files: File[];
-    point?: VecLike;
     ignoreParent: boolean;
 } | {
     type: 'svg-text';
     text: string;
-    point?: VecLike;
 } | {
     type: 'text';
-    point?: VecLike;
     text: string;
 } | {
     type: 'url';
     url: string;
-    point?: VecLike;
+});
+
+// @public (undocumented)
+export type TLExternalContentSource = {
+    type: 'error';
+    data: null | string;
+    reason: string;
+} | {
+    type: 'excalidraw';
+    data: any;
+} | {
+    type: 'text';
+    data: string;
+    subtype: 'html' | 'json' | 'text' | 'url';
+} | {
+    type: 'tldraw';
+    data: TLContent;
 };
 
 // @public (undocumented)

packages/editor/src/index.ts

@@ -230,6 +230,7 @@ export {
 export {
 	type TLExternalAssetContent,
 	type TLExternalContent,
+	type TLExternalContentSource,
 } from './lib/editor/types/external-content'
 export {
 	type TLCommand,

packages/editor/src/lib/editor/Editor.ts

@@ -2,7 +2,6 @@ import { EMPTY_ARRAY, atom, computed, transact } from '@tldraw/state'
 import { ComputedCache, RecordType } from '@tldraw/store'
 import {
 	CameraRecordType,
-	EmbedDefinition,
 	InstancePageStateRecordType,
 	PageRecordType,
 	StyleProp,
@@ -122,6 +121,7 @@ import { SvgExportContext, SvgExportDef } from './types/SvgExportContext'
 import { TLContent } from './types/clipboard-types'
 import { TLEventMap } from './types/emit-types'
 import { TLEventInfo, TLPinchEventInfo, TLPointerEventInfo } from './types/event-types'
+import { TLExternalAssetContent, TLExternalContent } from './types/external-content'
 import { TLCommandHistoryOptions } from './types/history-types'
 import { OptionalKeys, RequiredKeys } from './types/misc-types'
 import { TLResizeHandle } from './types/selection-types'
@@ -8908,36 +8908,3 @@ function alertMaxShapes(editor: Editor, pageId = editor.currentPageId) {
 	const name = editor.getPage(pageId)!.name
 	editor.emit('max-shapes', { name, pageId, count: MAX_SHAPES_PER_PAGE })
 }
-
-/** @public */
-export type TLExternalContent =
-	| {
-			type: 'text'
-			point?: VecLike
-			text: string
-	  }
-	| {
-			type: 'files'
-			files: File[]
-			point?: VecLike
-			ignoreParent: boolean
-	  }
-	| {
-			type: 'url'
-			url: string
-			point?: VecLike
-	  }
-	| {
-			type: 'svg-text'
-			text: string
-			point?: VecLike
-	  }
-	| {
-			type: 'embed'
-			url: string
-			point?: VecLike
-			embed: EmbedDefinition
-	  }
-
-/** @public */
-export type TLExternalAssetContent = { type: 'file'; file: File } | { type: 'url'; url: string }

packages/editor/src/lib/editor/types/external-content.ts

@@ -1,35 +1,56 @@
 import { EmbedDefinition } from '@tldraw/tlschema'
 import { VecLike } from '../../primitives/Vec2d'
+import { TLContent } from './clipboard-types'
 
 /** @public */
-export type TLExternalContent =
+export type TLExternalContentSource =
+	| {
+			type: 'tldraw'
+			data: TLContent
+	  }
+	| {
+			type: 'excalidraw'
+			data: any
+	  }
+	| {
+			type: 'text'
+			data: string
+			subtype: 'json' | 'html' | 'text' | 'url'
+	  }
+	| {
+			type: 'error'
+			data: string | null
+			reason: string
+	  }
+
+/** @public */
+export type TLExternalContent = {
+	sources?: TLExternalContentSource[]
+	point?: VecLike
+} & (
 	| {
 			type: 'text'
-			point?: VecLike
 			text: string
 	  }
 	| {
 			type: 'files'
 			files: File[]
-			point?: VecLike
 			ignoreParent: boolean
 	  }
 	| {
 			type: 'url'
 			url: string
-			point?: VecLike
 	  }
 	| {
 			type: 'svg-text'
 			text: string
-			point?: VecLike
 	  }
 	| {
 			type: 'embed'
 			url: string
-			point?: VecLike
 			embed: EmbedDefinition
 	  }
+)
 
 /** @public */
 export type TLExternalAssetContent = { type: 'file'; file: File } | { type: 'url'; url: string }

packages/tldraw/src/lib/ui/hooks/clipboard/pasteFiles.ts

@@ -1,4 +1,4 @@
-import { Editor, VecLike } from '@tldraw/editor'
+import { Editor, TLExternalContentSource, VecLike } from '@tldraw/editor'
 
 /**
  * When the clipboard has a file, create an image shape from the file and paste it into the scene
@@ -8,7 +8,12 @@ import { Editor, VecLike } from '@tldraw/editor'
  * @param point - (optional) The point at which to paste the file.
  * @internal
  */
-export async function pasteFiles(editor: Editor, urls: string[], point?: VecLike) {
+export async function pasteFiles(
+	editor: Editor,
+	urls: string[],
+	point?: VecLike,
+	sources?: TLExternalContentSource[]
+) {
 	const blobs = await Promise.all(urls.map(async (url) => await (await fetch(url)).blob()))
 	const files = blobs.map((blob) => new File([blob], 'tldrawFile', { type: blob.type }))
 
@@ -19,6 +24,7 @@ export async function pasteFiles(editor: Editor, urls: string[], point?: VecLike
 		files,
 		point,
 		ignoreParent: false,
+		sources,
 	})
 
 	urls.forEach((url) => URL.revokeObjectURL(url))

packages/tldraw/src/lib/ui/hooks/clipboard/pasteUrl.ts

@@ -1,4 +1,4 @@
-import { Editor, VecLike } from '@tldraw/editor'
+import { Editor, TLExternalContentSource, VecLike } from '@tldraw/editor'
 import { pasteFiles } from './pasteFiles'
 
 /**
@@ -10,7 +10,12 @@ import { pasteFiles } from './pasteFiles'
  * @param point - (optional) The point at which to paste the file.
  * @internal
  */
-export async function pasteUrl(editor: Editor, url: string, point?: VecLike) {
+export async function pasteUrl(
+	editor: Editor,
+	url: string,
+	point?: VecLike,
+	sources?: TLExternalContentSource[]
+) {
 	// Lets see if its an image and we have CORs
 	try {
 		const resp = await fetch(url)
@@ -31,5 +36,6 @@ export async function pasteUrl(editor: Editor, url: string, point?: VecLike) {
 		type: 'url',
 		point,
 		url,
+		sources,
 	})
 }

packages/tldraw/src/lib/ui/hooks/useClipboardEvents.ts

@@ -2,8 +2,8 @@ import {
 	Editor,
 	TLArrowShape,
 	TLBookmarkShape,
-	TLContent,
 	TLEmbedShape,
+	TLExternalContentSource,
 	TLGeoShape,
 	TLTextShape,
 	VecLike,
@@ -20,6 +20,18 @@ import { pasteTldrawContent } from './clipboard/pasteTldrawContent'
 import { pasteUrl } from './clipboard/pasteUrl'
 import { TLUiEventSource, useUiEvents } from './useEventsProvider'
 
+/**
+ * Strip HTML tags from a string.
+ * @param html - The HTML to strip.
+ * @internal
+ */
+function stripHtml(html: string) {
+	// See <https://github.com/developit/preact-markup/blob/4788b8d61b4e24f83688710746ee36e7464f7bbc/src/parse-markup.js#L60-L69>
+	const doc = document.implementation.createHTMLDocument('')
+	doc.documentElement.innerHTML = html.trim()
+	return doc.body.textContent || doc.body.innerText || ''
+}
+
 /** @public */
 export const isValidHttpURL = (url: string) => {
 	try {
@@ -89,18 +101,6 @@ async function blobAsString(blob: Blob) {
 	})
 }
 
-/**
- * Strip HTML tags from a string.
- * @param html - The HTML to strip.
- * @internal
- */
-function stripHtml(html: string) {
-	// See <https://github.com/developit/preact-markup/blob/4788b8d61b4e24f83688710746ee36e7464f7bbc/src/parse-markup.js#L60-L69>
-	const doc = document.implementation.createHTMLDocument('')
-	doc.documentElement.innerHTML = html.trim()
-	return doc.body.textContent || doc.body.innerText || ''
-}
-
 /**
  * Whether a ClipboardItem is a file.
  * @param item - The ClipboardItem to check.
@@ -117,7 +117,12 @@ const isFile = (item: ClipboardItem) => {
  * @param point - (optional) The point at which to paste the text.
  * @internal
  */
-const handleText = (editor: Editor, data: string, point?: VecLike) => {
+const handleText = (
+	editor: Editor,
+	data: string,
+	point?: VecLike,
+	sources?: TLExternalContentSource[]
+) => {
 	const validUrlList = getValidHttpURLList(data)
 	if (validUrlList) {
 		for (const url of validUrlList) {
@@ -131,6 +136,7 @@ const handleText = (editor: Editor, data: string, point?: VecLike) => {
 			type: 'svg-text',
 			text: data,
 			point,
+			sources,
 		})
 	} else {
 		editor.mark('paste')
@@ -138,6 +144,7 @@ const handleText = (editor: Editor, data: string, point?: VecLike) => {
 			type: 'text',
 			text: data,
 			point,
+			sources,
 		})
 	}
 }
@@ -172,30 +179,6 @@ type ClipboardThing =
 			source: Promise<string>
 	  }
 
-/**
- * The result of processing a `ClipboardThing`.
- * @internal
- */
-type ClipboardResult =
-	| {
-			type: 'tldraw'
-			data: TLContent
-	  }
-	| {
-			type: 'excalidraw'
-			data: any
-	  }
-	| {
-			type: 'text'
-			data: string
-			subtype: 'json' | 'html' | 'text' | 'url'
-	  }
-	| {
-			type: 'error'
-			data: string | null
-			reason: string
-	  }
-
 /**
  * Handle a paste using event clipboard data. This is the "original"
  * paste method that uses the clipboard data from the paste event.
@@ -339,7 +322,7 @@ async function handleClipboardThings(editor: Editor, things: ClipboardThing[], p
 	// we can't await them in a loop. So we'll map them to promises and await them all at once,
 	// then make decisions based on what we find.
 
-	const results = await Promise.all<ClipboardResult>(
+	const results = await Promise.all<TLExternalContentSource>(
 		things
 			.filter((t) => t.type !== 'file')
 			.map(
@@ -477,13 +460,13 @@ async function handleClipboardThings(editor: Editor, things: ClipboardThing[], p
 
 			if (isHtmlSingleLink) {
 				const href = bodyNode.firstElementChild.getAttribute('href')!
-				handleText(editor, href, point)
+				handleText(editor, href, point, results)
 				return
 			}
 
 			// If the html is NOT a link, and we have NO OTHER texty content, then paste the html as text
 			if (!results.some((r) => r.type === 'text' && r.subtype !== 'html') && result.data.trim()) {
-				handleText(editor, stripHtml(result.data), point)
+				handleText(editor, stripHtml(result.data), point, results)
 				return
 			}
 		}
@@ -492,7 +475,7 @@ async function handleClipboardThings(editor: Editor, things: ClipboardThing[], p
 	// Try to paste a link
 	for (const result of results) {
 		if (result.type === 'text' && result.subtype === 'url') {
-			pasteUrl(editor, result.data, point)
+			pasteUrl(editor, result.data, point, results)
 			return
 		}
 	}
@@ -501,7 +484,7 @@ async function handleClipboardThings(editor: Editor, things: ClipboardThing[], p
 	for (const result of results) {
 		if (result.type === 'text' && result.subtype === 'text' && result.data.trim()) {
 			// The clipboard may include multiple text items, but we only want to paste the first one
-			handleText(editor, result.data, point)
+			handleText(editor, result.data, point, results)
 			return
 		}
 	}