Fix issues with clip paths for frames (#2406)

Fixes an issue with frame clipping paths. In fact, this should also
solve other issues we might have with intersect. Seems like our
`pointInPolygon` did not correctly detect that points in the corners or
on the edges of the polygon are in fact part of the polygon.

When calculating the intersection of two regular, intersecting
rectangles the`intersectPolygonPolygon` was returning a polygon with 2,
3, or sometimes even 0 points, which also could result in an error when
dragging one frame out of another frame.

It seems that for all intents and purposes the `pointInPolygon` function
should also consider corners and edges, but maybe we might want to
rename it?

Before:


https://github.com/tldraw/tldraw/assets/2523721/155d351d-8ceb-47c3-a263-024cab487d03

After:


https://github.com/tldraw/tldraw/assets/2523721/338b923a-f902-4dc4-a1b7-e954f906fb8d




Fixes https://github.com/tldraw/tldraw/issues/2387

### Change Type

- [x] `patch` — Bug fix
- [ ] `minor` — New feature
- [ ] `major` — Breaking change
- [ ] `dependencies` — Changes to package dependencies[^1]
- [ ] `documentation` — Changes to the documentation only[^2]
- [ ] `tests` — Changes to any test code only[^2]
- [ ] `internal` — Any other changes that don't affect the published
package[^2]
- [ ] I don't know

[^1]: publishes a `patch` release, for devDependencies use `internal`
[^2]: will not publish a new version

### Test Plan

1. Add a step-by-step description of how to test your PR here.
2.

- [ ] Unit Tests
- [ ] End to end tests

### Release Notes

- Add a brief release note for your PR here.

packages/editor/src/lib/editor/Editor.ts

@@ -4066,7 +4066,7 @@ export class Editor extends EventEmitter<TLEventMap> {
 			if (pageMask.length === 0) return undefined
 
 			const { corners } = pageBounds
-			if (corners.every((p, i) => Vec.Equals(p, pageMask[i]))) return pageBounds.clone()
+			if (corners.every((p, i) => p && Vec.Equals(p, pageMask[i]))) return pageBounds.clone()
 
 			// todo: find out why intersect polygon polygon for identical polygons produces zero w/h intersections
 			const intersection = intersectPolygonPolygon(pageMask, corners)

packages/editor/src/lib/primitives/intersect.ts

@@ -243,26 +243,32 @@ export function intersectPolygonPolygon(
 	polygonA: VecLike[],
 	polygonB: VecLike[]
 ): VecLike[] | null {
-	// Create an empty polygon as P
-	const result: VecLike[] = []
+	// Create an empty polygon as result
+	const result: Map<string, VecLike> = new Map()
 	let a: VecLike, b: VecLike, c: VecLike, d: VecLike
 
-	// Add all corners of PolygonA that is inside PolygonB to P
+	// Add all corners of PolygonA that is inside PolygonB to result
 	for (let i = 0, n = polygonA.length; i < n; i++) {
 		a = polygonA[i]
 		if (pointInPolygon(a, polygonB)) {
-			result.push(a)
+			const id = getPointId(a)
+			if (!result.has(id)) {
+				result.set(id, a)
 			}
 		}
-	// Add all corners of PolygonB that is inside PolygonA to P
+	}
+	// Add all corners of PolygonB that is inside PolygonA to result
 	for (let i = 0, n = polygonB.length; i < n; i++) {
 		a = polygonB[i]
 		if (pointInPolygon(a, polygonA)) {
-			result.push(a)
+			const id = getPointId(a)
+			if (!result.has(id)) {
+				result.set(id, a)
+			}
 		}
 	}
 
-	// Add all intersection points to P
+	// Add all intersection points to result
 	for (let i = 0, n = polygonA.length; i < n; i++) {
 		a = polygonA[i]
 		b = polygonA[(i + 1) % polygonA.length]
@@ -273,15 +279,22 @@ export function intersectPolygonPolygon(
 			const intersection = intersectLineSegmentLineSegment(a, b, c, d)
 
 			if (intersection !== null) {
-				result.push(intersection)
+				const id = getPointId(intersection)
+				if (!result.has(id)) {
+					result.set(id, intersection)
+				}
 			}
 		}
 	}
 
-	if (result.length === 0) return null // no intersection
+	if (result.size === 0) return null // no intersection
 
-	// Order all points in the P counter-clockwise.
-	return orderClockwise(result)
+	// Order all points in the result counter-clockwise.
+	return orderClockwise([...result.values()])
+}
+
+function getPointId(point: VecLike) {
+	return `${point.x},${point.y}`
 }
 
 function orderClockwise(points: VecLike[]): VecLike[] {

packages/editor/src/lib/primitives/utils.ts

@@ -313,8 +313,14 @@ export function pointInPolygon(A: VecLike, points: VecLike[]): boolean {
 
 	for (let i = 0; i < points.length; i++) {
 		a = points[i]
+		// Point is the same as one of the corners of the polygon
+		if (a.x === A.x && a.y === A.y) return true
+
 		b = points[(i + 1) % points.length]
 
+		// Point is on the polygon edge
+		if (Vec.Dist(A, a) + Vec.Dist(A, b) === Vec.Dist(a, b)) return true
+
 		if (a.y <= A.y) {
 			if (b.y > A.y && cross(a, b, A) > 0) {
 				windingNumber += 1