[culling] minimal culled diff with webgl (#3377)

This PR extracts the #3344 changes to a smaller diff against main. It
does not include the changes to how / where culled shapes are
calculated, though I understand this could be much more efficiently
done!

### Change Type

- [x] `sdk` — Changes the tldraw SDK
- [x] `improvement` — Improving existing features

---------

Co-authored-by: Mitja Bezenšek <mitja.bezensek@gmail.com>

packages/editor/editor.css

@@ -24,6 +24,7 @@
 	/* Z Index */
 	--layer-background: 100;
 	--layer-grid: 150;
+	--layer-culled-shapes: 175;
 	--layer-canvas: 200;
 	--layer-shapes: 300;
 	--layer-overlays: 400;
@@ -236,6 +237,20 @@ input,
 	contain: strict;
 }
 
+.tl-culled-shapes {
+	width: 100%;
+	height: 100%;
+	z-index: var(--layer-culled-shapes);
+	position: absolute;
+	pointer-events: none;
+	contain: size layout;
+}
+
+.tl-culled-shapes__canvas {
+	width: 100%;
+	height: 100%;
+}
+
 .tl-shapes {
 	position: relative;
 	z-index: var(--layer-shapes);
@@ -269,13 +284,16 @@ input,
 
 /* ------------------- Background ------------------- */
 
+.tl-background__wrapper {
+	z-index: var(--layer-background);
+}
+
 .tl-background {
 	position: absolute;
 	background-color: var(--color-background);
 	inset: 0px;
 	height: 100%;
 	width: 100%;
-	z-index: var(--layer-background);
 }
 
 /* --------------------- Grid Layer --------------------- */

packages/editor/src/lib/components/CulledShapes.tsx

@@ -0,0 +1,178 @@
+import { computed, react } from '@tldraw/state'
+import { useEffect, useRef } from 'react'
+import { useEditor } from '../hooks/useEditor'
+import { useIsDarkMode } from '../hooks/useIsDarkMode'
+
+// Parts of the below code are taken from MIT licensed project:
+// https://github.com/sessamekesh/webgl-tutorials-2023
+function setupWebGl(canvas: HTMLCanvasElement | null, isDarkMode: boolean) {
+	if (!canvas) return
+
+	const context = canvas.getContext('webgl2')
+	if (!context) return
+
+	const vertexShaderSourceCode = `#version 300 es
+  precision mediump float;
+  
+  in vec2 shapeVertexPosition;
+  uniform vec2 viewportStart; 
+  uniform vec2 viewportEnd; 
+
+  void main() {
+	// We need to transform from page coordinates to something WebGl understands
+	float viewportWidth = viewportEnd.x - viewportStart.x;
+	float viewportHeight = viewportEnd.y - viewportStart.y;
+	vec2 finalPosition = vec2(
+		2.0 * (shapeVertexPosition.x - viewportStart.x) / viewportWidth - 1.0, 
+		1.0 - 2.0 * (shapeVertexPosition.y - viewportStart.y) / viewportHeight
+	);
+	gl_Position = vec4(finalPosition, 0.0, 1.0);
+  }`
+
+	const vertexShader = context.createShader(context.VERTEX_SHADER)
+	if (!vertexShader) return
+	context.shaderSource(vertexShader, vertexShaderSourceCode)
+	context.compileShader(vertexShader)
+	if (!context.getShaderParameter(vertexShader, context.COMPILE_STATUS)) {
+		return
+	}
+	// Dark = hsl(210, 11%, 19%)
+	// Light = hsl(204, 14%, 93%)
+	const color = isDarkMode ? 'vec4(0.169, 0.188, 0.212, 1.0)' : 'vec4(0.922, 0.933, 0.941, 1.0)'
+
+	const fragmentShaderSourceCode = `#version 300 es
+  precision mediump float;
+  
+  out vec4 outputColor;
+
+  void main() {
+	outputColor = ${color};
+  }`
+
+	const fragmentShader = context.createShader(context.FRAGMENT_SHADER)
+	if (!fragmentShader) return
+	context.shaderSource(fragmentShader, fragmentShaderSourceCode)
+	context.compileShader(fragmentShader)
+	if (!context.getShaderParameter(fragmentShader, context.COMPILE_STATUS)) {
+		return
+	}
+
+	const program = context.createProgram()
+	if (!program) return
+	context.attachShader(program, vertexShader)
+	context.attachShader(program, fragmentShader)
+	context.linkProgram(program)
+	if (!context.getProgramParameter(program, context.LINK_STATUS)) {
+		return
+	}
+	context.useProgram(program)
+
+	const shapeVertexPositionAttributeLocation = context.getAttribLocation(
+		program,
+		'shapeVertexPosition'
+	)
+	if (shapeVertexPositionAttributeLocation < 0) {
+		return
+	}
+	context.enableVertexAttribArray(shapeVertexPositionAttributeLocation)
+
+	const viewportStartUniformLocation = context.getUniformLocation(program, 'viewportStart')
+	const viewportEndUniformLocation = context.getUniformLocation(program, 'viewportEnd')
+	if (!viewportStartUniformLocation || !viewportEndUniformLocation) {
+		return
+	}
+	return {
+		context,
+		program,
+		shapeVertexPositionAttributeLocation,
+		viewportStartUniformLocation,
+		viewportEndUniformLocation,
+	}
+}
+
+export function CulledShapes() {
+	const editor = useEditor()
+	const isDarkMode = useIsDarkMode()
+	const canvasRef = useRef<HTMLCanvasElement>(null)
+
+	const isCullingOffScreenShapes = Number.isFinite(editor.renderingBoundsMargin)
+
+	useEffect(() => {
+		const webGl = setupWebGl(canvasRef.current, isDarkMode)
+		if (!webGl) return
+		if (!isCullingOffScreenShapes) return
+
+		const {
+			context,
+			shapeVertexPositionAttributeLocation,
+			viewportStartUniformLocation,
+			viewportEndUniformLocation,
+		} = webGl
+
+		const shapeVertices = computed('shape vertices', function calculateCulledShapeVertices() {
+			const results: number[] = []
+
+			for (const { isCulled, maskedPageBounds } of editor.getRenderingShapes()) {
+				if (isCulled && maskedPageBounds) {
+					results.push(
+						// triangle 1
+						maskedPageBounds.minX,
+						maskedPageBounds.minY,
+						maskedPageBounds.minX,
+						maskedPageBounds.maxY,
+						maskedPageBounds.maxX,
+						maskedPageBounds.maxY,
+						// triangle 2
+						maskedPageBounds.minX,
+						maskedPageBounds.minY,
+						maskedPageBounds.maxX,
+						maskedPageBounds.minY,
+						maskedPageBounds.maxX,
+						maskedPageBounds.maxY
+					)
+				}
+			}
+
+			return results
+		})
+
+		return react('render culled shapes ', function renderCulledShapes() {
+			const canvas = canvasRef.current
+			if (!canvas) return
+
+			const width = canvas.clientWidth
+			const height = canvas.clientHeight
+			if (width !== canvas.width || height !== canvas.height) {
+				canvas.width = width
+				canvas.height = height
+				context.viewport(0, 0, width, height)
+			}
+
+			const verticesArray = shapeVertices.get()
+
+			context.clear(context.COLOR_BUFFER_BIT | context.DEPTH_BUFFER_BIT)
+
+			if (verticesArray.length > 0) {
+				const viewport = editor.getViewportPageBounds() // when the viewport changes...
+				context.uniform2f(viewportStartUniformLocation, viewport.minX, viewport.minY)
+				context.uniform2f(viewportEndUniformLocation, viewport.maxX, viewport.maxY)
+				const triangleGeoCpuBuffer = new Float32Array(verticesArray)
+				const triangleGeoBuffer = context.createBuffer()
+				context.bindBuffer(context.ARRAY_BUFFER, triangleGeoBuffer)
+				context.bufferData(context.ARRAY_BUFFER, triangleGeoCpuBuffer, context.STATIC_DRAW)
+				context.vertexAttribPointer(
+					shapeVertexPositionAttributeLocation,
+					2,
+					context.FLOAT,
+					false,
+					2 * Float32Array.BYTES_PER_ELEMENT,
+					0
+				)
+				context.drawArrays(context.TRIANGLES, 0, verticesArray.length / 2)
+			}
+		})
+	}, [isCullingOffScreenShapes, isDarkMode, editor])
+	return isCullingOffScreenShapes ? (
+		<canvas ref={canvasRef} className="tl-culled-shapes__canvas" />
+	) : null
+}

packages/editor/src/lib/components/Shape.tsx

@@ -43,7 +43,6 @@ export const Shape = memo(function Shape({
 	const { ShapeErrorFallback } = useEditorComponents()
 
 	const containerRef = useRef<HTMLDivElement>(null)
-	const culledContainerRef = useRef<HTMLDivElement>(null)
 	const bgContainerRef = useRef<HTMLDivElement>(null)
 
 	const memoizedStuffRef = useRef({
@@ -67,7 +66,6 @@ export const Shape = memo(function Shape({
 			const clipPath = editor.getShapeClipPath(id) ?? 'none'
 			if (clipPath !== prev.clipPath) {
 				setStyleProperty(containerRef.current, 'clip-path', clipPath)
-				setStyleProperty(culledContainerRef.current, 'clip-path', clipPath)
 				setStyleProperty(bgContainerRef.current, 'clip-path', clipPath)
 				prev.clipPath = clipPath
 			}
@@ -81,11 +79,6 @@ export const Shape = memo(function Shape({
 			if (transform !== prev.transform) {
 				setStyleProperty(containerRef.current, 'transform', transform)
 				setStyleProperty(bgContainerRef.current, 'transform', transform)
-				setStyleProperty(
-					culledContainerRef.current,
-					'transform',
-					`${Mat.toCssString(pageTransform)} translate(${bounds.x}px, ${bounds.y}px)`
-				)
 				prev.transform = transform
 			}
 
@@ -100,8 +93,6 @@ export const Shape = memo(function Shape({
 			if (width !== prev.width || height !== prev.height) {
 				setStyleProperty(containerRef.current, 'width', Math.max(width, dprMultiple) + 'px')
 				setStyleProperty(containerRef.current, 'height', Math.max(height, dprMultiple) + 'px')
-				setStyleProperty(culledContainerRef.current, 'width', Math.max(width, dprMultiple) + 'px')
-				setStyleProperty(culledContainerRef.current, 'height', Math.max(height, dprMultiple) + 'px')
 				setStyleProperty(bgContainerRef.current, 'width', Math.max(width, dprMultiple) + 'px')
 				setStyleProperty(bgContainerRef.current, 'height', Math.max(height, dprMultiple) + 'px')
 				prev.width = width
@@ -132,10 +123,8 @@ export const Shape = memo(function Shape({
 	useLayoutEffect(() => {
 		const container = containerRef.current
 		const bgContainer = bgContainerRef.current
-		const culledContainer = culledContainerRef.current
 		setStyleProperty(container, 'display', isCulled ? 'none' : 'block')
 		setStyleProperty(bgContainer, 'display', isCulled ? 'none' : 'block')
-		setStyleProperty(culledContainer, 'display', isCulled ? 'block' : 'none')
 	}, [isCulled])
 
 	const annotateError = useCallback(
@@ -147,7 +136,6 @@ export const Shape = memo(function Shape({
 
 	return (
 		<>
-			<div ref={culledContainerRef} className="tl-shape__culled" draggable={false} />
 			{util.backgroundComponent && (
 				<div
 					ref={bgContainerRef}

packages/editor/src/lib/components/default-components/DefaultCanvas.tsx

@@ -20,6 +20,7 @@ import { toDomPrecision } from '../../primitives/utils'
 import { debugFlags } from '../../utils/debug-flags'
 import { setStyleProperty } from '../../utils/dom'
 import { nearestMultiple } from '../../utils/nearestMultiple'
+import { CulledShapes } from '../CulledShapes'
 import { GeometryDebuggingView } from '../GeometryDebuggingView'
 import { LiveCollaborators } from '../LiveCollaborators'
 import { Shape } from '../Shape'
@@ -46,7 +47,7 @@ export function DefaultCanvas({ className }: TLCanvasComponentProps) {
 
 	useQuickReactor(
 		'position layers',
-		() => {
+		function positionLayersWhenCameraMoves() {
 			const { x, y, z } = editor.getCamera()
 
 			// Because the html container has a width/height of 1px, we
@@ -105,9 +106,15 @@ export function DefaultCanvas({ className }: TLCanvasComponentProps) {
 					{SvgDefs && <SvgDefs />}
 				</defs>
 			</svg>
-			{Background && <Background />}
+			{Background && (
+				<div className="tl-background__wrapper">
+					<Background />
+				</div>
+			)}
 			<GridWrapper />
-
+			<div className="tl-culled-shapes">
+				<CulledShapes />
+			</div>
 			<div ref={rHtmlLayer} className="tl-html-layer tl-shapes" draggable={false}>
 				<OnTheCanvasWrapper />
 				<SelectionBackgroundWrapper />