Flatten shapes to image(s) (#3933)

This PR adds some functionality for turning shapes into images.

![Kapture 2024-06-13 at 12 51
00](https://github.com/tldraw/tldraw/assets/23072548/78525e29-61b5-418f-889d-2f061f26f34d)

It adds:
- the `flattenShapesToImages`
- the `useFlatten` hook
- a `flatten-shapes-to-images` action (shift + f)
- adds `flattenImageBoundsExpand` option
- adds `flattenImageBoundsPadding` option

## Flatten shapes to images

The `flattenShapesToImages` helper method will 1) create an image for
the given shape ids, 2) add it to the canvas in the same location / size
as the source shapes, and then 3) delete the original shapes. The new
image will be placed correctly in the z index and in the correct
rotation of the root-most ancestor of the given shape ids.


![image](https://github.com/tldraw/tldraw/assets/23072548/fe888980-05a5-4369-863f-90c142f9f8b9)

It has an argument, `flattenImageBoundsExpand`, which if provided will
chunk the given shapes into images based on their overlapping (expanded)
bounding boxes.


![image](https://github.com/tldraw/tldraw/assets/23072548/c4799309-244d-4a2b-ac59-9c2fd100319c)

By default, the flatten action uses the editor's
`options.flattenImageBoundsExpand`. The `flattenImageBoundsPadding`
option is used as a value for how much larger the image should be than
the source image bounds (to account for large strokes, for example).

### Change Type

- [x] `sdk` — Changes the tldraw SDK
- [x] `feature` — New feature

### Test Plan

1. Select shapes
2. Select context menu > edit > flatten

- [ ] Unit Tests
- [ ] End to end tests

### Release Notes

- Add Flatten, a new menu item to flatten shapes into images

assets/translations/main.json

@@ -96,6 +96,7 @@
 	"action.toggle-grid.menu": "Show grid",
 	"action.toggle-grid": "Toggle grid",
 	"action.toggle-lock": "Toggle locked",
+	"action.flatten-to-image": "Flatten",
 	"action.toggle-snap-mode.menu": "Always snap",
 	"action.toggle-snap-mode": "Toggle always snap",
 	"action.toggle-tool-lock.menu": "Tool lock",
@@ -232,6 +233,7 @@
 	"menu.language": "Language",
 	"menu.preferences": "Preferences",
 	"menu.view": "View",
+	"context-menu.edit": "Edit",
 	"context-menu.arrange": "Arrange",
 	"context-menu.copy-as": "Copy as",
 	"context-menu.export-as": "Export as",

packages/editor/api-report.md

@@ -681,6 +681,8 @@ export const defaultTldrawOptions: {
     readonly dragDistanceSquared: 16;
     readonly edgeScrollDistance: 8;
     readonly edgeScrollSpeed: 20;
+    readonly flattenImageBoundsExpand: 64;
+    readonly flattenImageBoundsPadding: 16;
     readonly followChaseViewportSnap: 2;
     readonly gridSteps: readonly [{
         readonly mid: 0.15;
@@ -2571,6 +2573,10 @@ export interface TldrawOptions {
     // (undocumented)
     readonly edgeScrollSpeed: number;
     // (undocumented)
+    readonly flattenImageBoundsExpand: number;
+    // (undocumented)
+    readonly flattenImageBoundsPadding: number;
+    // (undocumented)
     readonly followChaseViewportSnap: number;
     // (undocumented)
     readonly gridSteps: readonly {

packages/editor/src/lib/options.ts

@@ -45,6 +45,8 @@ export interface TldrawOptions {
 	readonly longPressDurationMs: number
 	readonly textShadowLod: number
 	readonly adjacentShapeMargin: number
+	readonly flattenImageBoundsExpand: number
+	readonly flattenImageBoundsPadding: number
 }
 
 /** @public */
@@ -79,4 +81,6 @@ export const defaultTldrawOptions = {
 	longPressDurationMs: 500,
 	textShadowLod: 0.35,
 	adjacentShapeMargin: 10,
+	flattenImageBoundsExpand: 64,
+	flattenImageBoundsPadding: 16,
 } as const satisfies TldrawOptions

packages/tldraw/src/lib/ui/components/ContextMenu/DefaultContextMenuContent.tsx

@@ -3,18 +3,10 @@ import {
 	ArrangeMenuSubmenu,
 	ClipboardMenuGroup,
 	ConversionsMenuGroup,
-	ConvertToBookmarkMenuItem,
-	ConvertToEmbedMenuItem,
-	EditLinkMenuItem,
-	FitFrameToContentMenuItem,
-	GroupMenuItem,
+	EditMenuSubmenu,
 	MoveToPageMenu,
-	RemoveFrameMenuItem,
 	ReorderMenuSubmenu,
 	SelectAllMenuItem,
-	ToggleAutoSizeMenuItem,
-	ToggleLockMenuItem,
-	UngroupMenuItem,
 } from '../menu-items'
 import { TldrawUiMenuGroup } from '../primitives/menus/TldrawUiMenuGroup'
 
@@ -32,18 +24,8 @@ export function DefaultContextMenuContent() {
 
 	return (
 		<>
-			<TldrawUiMenuGroup id="misc">
-				<GroupMenuItem />
-				<UngroupMenuItem />
-				<EditLinkMenuItem />
-				<ToggleAutoSizeMenuItem />
-				<RemoveFrameMenuItem />
-				<FitFrameToContentMenuItem />
-				<ConvertToEmbedMenuItem />
-				<ConvertToBookmarkMenuItem />
-				<ToggleLockMenuItem />
-			</TldrawUiMenuGroup>
 			<TldrawUiMenuGroup id="modify">
+				<EditMenuSubmenu />
 				<ArrangeMenuSubmenu />
 				<ReorderMenuSubmenu />
 				<MoveToPageMenu />

packages/tldraw/src/lib/ui/components/MainMenu/DefaultMainMenuContent.tsx

@@ -9,6 +9,7 @@ import {
 	ConvertToEmbedMenuItem,
 	EditLinkMenuItem,
 	FitFrameToContentMenuItem,
+	FlattenMenuItem,
 	GroupMenuItem,
 	RemoveFrameMenuItem,
 	SelectAllMenuItem,
@@ -101,6 +102,7 @@ export function MiscMenuGroup() {
 			<FitFrameToContentMenuItem />
 			<ConvertToEmbedMenuItem />
 			<ConvertToBookmarkMenuItem />
+			<FlattenMenuItem />
 		</TldrawUiMenuGroup>
 	)
 }

packages/tldraw/src/lib/ui/components/menu-items.tsx

@@ -2,6 +2,7 @@ import {
 	TLBookmarkShape,
 	TLEmbedShape,
 	TLFrameShape,
+	TLImageShape,
 	TLPageId,
 	useEditor,
 	useValue,
@@ -51,6 +52,27 @@ export function DuplicateMenuItem() {
 	return <TldrawUiMenuItem {...actions['duplicate']} />
 }
 /** @public @react */
+export function FlattenMenuItem() {
+	const actions = useActions()
+	const editor = useEditor()
+	const shouldDisplay = useValue(
+		'should display flatten option',
+		() => {
+			const selectedShapeIds = editor.getSelectedShapeIds()
+			if (selectedShapeIds.length === 0) return false
+			const onlySelectedShape = editor.getOnlySelectedShape()
+			if (onlySelectedShape && editor.isShapeOfType<TLImageShape>(onlySelectedShape, 'image')) {
+				return false
+			}
+			return true
+		},
+		[editor]
+	)
+	if (!shouldDisplay) return null
+
+	return <TldrawUiMenuItem {...actions['flatten-to-image']} />
+}
+/** @public @react */
 export function GroupMenuItem() {
 	const actions = useActions()
 	const shouldDisplay = useAllowGroup()
@@ -305,6 +327,25 @@ export function DeleteMenuItem() {
 }
 
 /* --------------------- Modify --------------------- */
+
+/** @public @react */
+export function EditMenuSubmenu() {
+	return (
+		<TldrawUiMenuSubmenu id="edit" label="context-menu.edit" size="small">
+			<GroupMenuItem />
+			<UngroupMenuItem />
+			<FlattenMenuItem />
+			<EditLinkMenuItem />
+			<FitFrameToContentMenuItem />
+			<RemoveFrameMenuItem />
+			<ConvertToEmbedMenuItem />
+			<ConvertToBookmarkMenuItem />
+			<ToggleAutoSizeMenuItem />
+			<ToggleLockMenuItem />
+		</TldrawUiMenuSubmenu>
+	)
+}
+
 /** @public @react */
 export function ArrangeMenuSubmenu() {
 	const twoSelected = useUnlockedSelectedShapesCount(2)

packages/tldraw/src/lib/ui/context/actions.tsx

@@ -27,6 +27,7 @@ import { EmbedDialog } from '../components/EmbedDialog'
 import { useMenuClipboardEvents } from '../hooks/useClipboardEvents'
 import { useCopyAs } from '../hooks/useCopyAs'
 import { useExportAs } from '../hooks/useExportAs'
+import { flattenShapesToImages } from '../hooks/useFlatten'
 import { useInsertMedia } from '../hooks/useInsertMedia'
 import { usePrint } from '../hooks/usePrint'
 import { TLUiTranslationKey } from '../hooks/useTranslation/TLUiTranslationKey'
@@ -1341,6 +1342,28 @@ export function ActionsProvider({ overrides, children }: ActionsProviderProps) {
 					trackEvent('set-style', { source, id: style.id, value: 'white' })
 				},
 			},
+			{
+				id: 'flatten-to-image',
+				label: 'action.flatten-to-image',
+				kbd: '!f',
+				onSelect: async (source) => {
+					const ids = editor.getSelectedShapeIds()
+					if (ids.length === 0) return
+
+					editor.mark('flattening to image')
+					trackEvent('flatten-to-image', { source })
+
+					const newShapeIds = await flattenShapesToImages(
+						editor,
+						ids,
+						editor.options.flattenImageBoundsExpand
+					)
+
+					if (newShapeIds?.length) {
+						editor.setSelectedShapes(newShapeIds)
+					}
+				},
+			},
 		]
 
 		const actions = makeActions(actionItems)

packages/tldraw/src/lib/ui/context/events.tsx

@@ -96,6 +96,7 @@ export interface TLUiEventMap {
 	'open-cursor-chat': null
 	'zoom-tool': null
 	'unlock-all': null
+	'flatten-to-image': null
 }
 
 /** @public */

packages/tldraw/src/lib/ui/hooks/useFlatten.ts

@@ -0,0 +1,202 @@
+import {
+	AssetRecordType,
+	Box,
+	Editor,
+	IndexKey,
+	TLImageAsset,
+	TLImageShape,
+	TLShape,
+	TLShapeId,
+	Vec,
+	compact,
+	createShapeId,
+	isShapeId,
+	transact,
+	useEditor,
+} from '@tldraw/editor'
+import { useCallback } from 'react'
+
+export async function flattenShapesToImages(
+	editor: Editor,
+	shapeIds: TLShapeId[],
+	flattenImageBoundsExpand?: number
+) {
+	const shapes = compact(
+		shapeIds.map((id) => {
+			const shape = editor.getShape(id)
+			if (!shape) return
+			const util = editor.getShapeUtil(shape.type)
+			// skip shapes that don't have a toSvg method
+			if (util.toSvg === undefined) return
+			return shape
+		})
+	)
+
+	if (shapes.length === 0) return
+
+	// Don't flatten if it's just one image
+	if (shapes.length === 1) {
+		const shape = shapes[0]
+		if (!shape) return
+		if (editor.isShapeOfType(shape, 'image')) return
+	}
+
+	const groups: { shapes: TLShape[]; bounds: Box; asset?: TLImageAsset }[] = []
+
+	if (flattenImageBoundsExpand !== undefined) {
+		const expandedBounds = shapes.map((shape) => {
+			return {
+				shape,
+				bounds: editor.getShapeMaskedPageBounds(shape)!.clone().expandBy(flattenImageBoundsExpand),
+			}
+		})
+
+		for (let i = 0; i < expandedBounds.length; i++) {
+			const item = expandedBounds[i]
+			if (i === 0) {
+				groups[0] = {
+					shapes: [item.shape],
+					bounds: item.bounds,
+				}
+				continue
+			}
+
+			let didLand = false
+
+			for (const group of groups) {
+				if (group.bounds.includes(item.bounds)) {
+					group.shapes.push(item.shape)
+					group.bounds.expand(item.bounds)
+					didLand = true
+					break
+				}
+			}
+
+			if (!didLand) {
+				groups.push({
+					shapes: [item.shape],
+					bounds: item.bounds,
+				})
+			}
+		}
+	} else {
+		const bounds = Box.Common(shapes.map((shape) => editor.getShapeMaskedPageBounds(shape)!))
+		groups.push({
+			shapes,
+			bounds,
+		})
+	}
+
+	const padding = editor.options.flattenImageBoundsPadding
+
+	for (const group of groups) {
+		if (flattenImageBoundsExpand !== undefined) {
+			// shrink the bounds again, removing the expanded area
+			group.bounds.expandBy(-flattenImageBoundsExpand)
+		}
+
+		// get an image for the shapes
+		const svgResult = await editor.getSvgString(group.shapes, {
+			padding,
+		})
+		if (!svgResult?.svg) continue
+
+		// get an image asset for the image
+		const blob = new Blob([svgResult.svg], { type: 'image/svg+xml' })
+		const asset = (await editor.getAssetForExternalContent({
+			type: 'file',
+			file: new File([blob], 'asset.svg', { type: 'image/svg+xml' }),
+		})) as TLImageAsset
+		if (!asset) continue
+
+		// add it to the group
+		group.asset = asset
+	}
+
+	const createdShapeIds: TLShapeId[] = []
+
+	transact(() => {
+		for (const group of groups) {
+			const { asset, bounds, shapes } = group
+			if (!asset) continue
+
+			const assetId = AssetRecordType.createId()
+
+			const commonAncestorId = editor.findCommonAncestor(shapes) ?? editor.getCurrentPageId()
+			if (!commonAncestorId) continue
+
+			let index: IndexKey = 'a1' as IndexKey
+			for (const shape of shapes) {
+				if (shape.parentId === commonAncestorId) {
+					if (shape.index > index) {
+						index = shape.index
+					}
+					break
+				}
+			}
+
+			let x: number
+			let y: number
+			let rotation: number
+
+			if (isShapeId(commonAncestorId)) {
+				const commonAncestor = editor.getShape(commonAncestorId)
+				if (!commonAncestor) continue
+				// put the point in the parent's space
+				const point = editor.getPointInShapeSpace(commonAncestor, {
+					x: bounds.x,
+					y: bounds.y,
+				})
+				// get the parent's rotation
+				rotation = editor.getShapePageTransform(commonAncestorId).rotation()
+				// rotate the point against the parent's rotation
+				point.sub(new Vec(padding, padding).rot(-rotation))
+				x = point.x
+				y = point.y
+			} else {
+				// if the common ancestor is the page, then just adjust for the padding
+				x = bounds.x - padding
+				y = bounds.y - padding
+				rotation = 0
+			}
+
+			// delete the shapes
+			editor.deleteShapes(shapes)
+
+			// create the asset
+			editor.createAssets([{ ...asset, id: assetId }])
+
+			const shapeId = createShapeId()
+
+			// create an image shape in the same place as the shapes
+			editor.createShape<TLImageShape>({
+				id: shapeId,
+				type: 'image',
+				index,
+				parentId: commonAncestorId,
+				x,
+				y,
+				rotation: -rotation,
+				props: {
+					assetId,
+					w: bounds.w + padding * 2,
+					h: bounds.h + padding * 2,
+				},
+			})
+
+			createdShapeIds.push(shapeId)
+		}
+	})
+
+	return createdShapeIds
+}
+
+export function useFlatten() {
+	const editor = useEditor()
+	return useCallback(
+		(ids: TLShapeId[]) => {
+			return flattenShapesToImages(editor, ids)
+		},
+		[editor]
+	)
+}

packages/tldraw/src/lib/ui/hooks/useTranslation/TLUiTranslationKey.ts

@@ -100,6 +100,7 @@ export type TLUiTranslationKey =
 	| 'action.toggle-grid.menu'
 	| 'action.toggle-grid'
 	| 'action.toggle-lock'
+	| 'action.flatten-to-image'
 	| 'action.toggle-snap-mode.menu'
 	| 'action.toggle-snap-mode'
 	| 'action.toggle-tool-lock.menu'
@@ -236,6 +237,7 @@ export type TLUiTranslationKey =
 	| 'menu.language'
 	| 'menu.preferences'
 	| 'menu.view'
+	| 'context-menu.edit'
 	| 'context-menu.arrange'
 	| 'context-menu.copy-as'
 	| 'context-menu.export-as'

packages/tldraw/src/lib/ui/hooks/useTranslation/defaultTranslation.ts

@@ -100,6 +100,7 @@ export const DEFAULT_TRANSLATION = {
 	'action.toggle-grid.menu': 'Show grid',
 	'action.toggle-grid': 'Toggle grid',
 	'action.toggle-lock': 'Toggle locked',
+	'action.flatten-to-image': 'Flatten',
 	'action.toggle-snap-mode.menu': 'Always snap',
 	'action.toggle-snap-mode': 'Toggle always snap',
 	'action.toggle-tool-lock.menu': 'Tool lock',
@@ -236,6 +237,7 @@ export const DEFAULT_TRANSLATION = {
 	'menu.language': 'Language',
 	'menu.preferences': 'Preferences',
 	'menu.view': 'View',
+	'context-menu.edit': 'Edit',
 	'context-menu.arrange': 'Arrange',
 	'context-menu.copy-as': 'Copy as',
 	'context-menu.export-as': 'Export as',