From a48a55d3defa422d91f9649f68eadf1a9ce6de9e Mon Sep 17 00:00:00 2001 From: David Sheldrick Date: Fri, 12 May 2023 12:39:36 +0100 Subject: [PATCH] [perf] make ensureStoreIsUsable scale better (#1362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new version of the sync engine is gonna be calling `ensureStoreIsUsable` on every sync message, so I took some time to make it scale better. At the moment it operates on a serialized version of the store, which is expensive and unnecessary. Here I changed it to use reactive queries for the data it needs, so it only operates on small bits of data and should not become more expensive as the number of shapes grows. ### Change Type - [x] `patch` — Bug Fix - [ ] `minor` — New Feature - [ ] `major` — Breaking Change - [ ] `dependencies` — Dependency Update (publishes a `patch` release, for devDependencies use `internal`) - [ ] `documentation` — Changes to the documentation only (will not publish a new version) - [ ] `tests` — Changes to any testing-related code only (will not publish a new version) - [ ] `internal` — Any other changes that don't affect the published package (will not publish a new version) ### Test Plan 1. Add a step-by-step description of how to test your PR here. 2. - [ ] Unit Tests - [ ] Webdriver tests ### Release Notes - Add a brief release note for your PR here. --- packages/tlschema/api-report.md | 6 +- packages/tlschema/src/TLStore.ts | 180 +++++++++++++----------- packages/tlschema/src/createTLSchema.ts | 4 +- packages/tlschema/src/index.ts | 2 +- packages/tlstore/api-report.md | 6 +- packages/tlstore/src/lib/Store.ts | 5 +- packages/tlstore/src/lib/StoreSchema.ts | 6 +- 7 files changed, 112 insertions(+), 97 deletions(-) diff --git a/packages/tlschema/api-report.md b/packages/tlschema/api-report.md index 0b34c0858..3ed19ad00 100644 --- a/packages/tlschema/api-report.md +++ b/packages/tlschema/api-report.md @@ -88,6 +88,9 @@ export function createAssetValidator( // @public (undocumented) export function createCustomShapeId(id: string): TLShapeId; +// @internal (undocumented) +export function createIntegrityChecker(store: TLStore): () => void; + // @public (undocumented) export function createShapeId(): TLShapeId; @@ -336,9 +339,6 @@ export const embedShapeMigrations: Migrations; // @public (undocumented) export const embedShapeTypeValidator: T.Validator; -// @internal (undocumented) -export function ensureStoreIsUsable(store: TLStore): void; - // @internal (undocumented) export const fillValidator: T.Validator<"none" | "pattern" | "semi" | "solid">; diff --git a/packages/tlschema/src/TLStore.ts b/packages/tlschema/src/TLStore.ts index ccfd47dd9..b9bd21417 100644 --- a/packages/tlschema/src/TLStore.ts +++ b/packages/tlschema/src/TLStore.ts @@ -99,104 +99,116 @@ function getDefaultPages() { } /** @internal */ -export function ensureStoreIsUsable(store: TLStore): void { - const { userId, instanceId: tabId } = store.props - // make sure we have exactly one document - if (!store.has(TLDOCUMENT_ID)) { - store.put([TLDocument.create({ id: TLDOCUMENT_ID })]) - return ensureStoreIsUsable(store) - } +export function createIntegrityChecker(store: TLStore): () => void { + const $pages = store.query.records('page') + const $userDocumentSettings = store.query.record('user_document', () => ({ + userId: { eq: store.props.userId }, + })) - const allRecords = store.allRecords() + const $instanceState = store.query.record('instance', () => ({ + id: { eq: store.props.instanceId }, + })) - // make sure we have document state for the current user - const userDocumentSettings = allRecords - .filter(TLUserDocument.isInstance) - .find((ud) => ud.userId === userId) + const $user = store.query.record('user', () => ({ id: { eq: store.props.userId } })) - if (!userDocumentSettings) { - store.put([TLUserDocument.create({ userId })]) - return ensureStoreIsUsable(store) - } + const $userPresences = store.query.records('user_presence') + const $instancePageStates = store.query.records('instance_page_state') - // make sure there is at least one page - const pages = allRecords.filter(TLPage.isInstance).sort(sortByIndex) - if (pages.length === 0) { - store.put(getDefaultPages()) - return ensureStoreIsUsable(store) - } + const ensureStoreIsUsable = (): void => { + const { userId, instanceId: tabId } = store.props + // make sure we have exactly one document + if (!store.has(TLDOCUMENT_ID)) { + store.put([TLDocument.create({ id: TLDOCUMENT_ID })]) + return ensureStoreIsUsable() + } - // make sure we have state for the current user's current tab - const instanceState = allRecords.filter(TLInstance.isInstance).find((t) => t.id === tabId) - if (!instanceState) { - // The tab props are either the the last used tab's props or undefined - const propsForNextShape = userDocumentSettings.lastUsedTabId - ? store.get(userDocumentSettings.lastUsedTabId)?.propsForNextShape - : undefined + // make sure we have document state for the current user + const userDocumentSettings = $userDocumentSettings.value - // The current page is either the last updated page or the first page - const currentPageId = userDocumentSettings?.lastUpdatedPageId ?? pages[0].id! + if (!userDocumentSettings) { + store.put([TLUserDocument.create({ userId })]) + return ensureStoreIsUsable() + } - store.put([ - TLInstance.create({ - id: tabId, - userId, - currentPageId, - propsForNextShape, - exportBackground: true, - }), - ]) + // make sure there is at least one page + const pages = $pages.value.sort(sortByIndex) + if (pages.length === 0) { + store.put(getDefaultPages()) + return ensureStoreIsUsable() + } - return ensureStoreIsUsable(store) - } + // make sure we have state for the current user's current tab + const instanceState = $instanceState.value + if (!instanceState) { + // The tab props are either the the last used tab's props or undefined + const propsForNextShape = userDocumentSettings.lastUsedTabId + ? store.get(userDocumentSettings.lastUsedTabId)?.propsForNextShape + : undefined - // make sure the user's currentPageId is still valid - let currentPageId = instanceState.currentPageId - if (!pages.find((p) => p.id === currentPageId)) { - currentPageId = pages[0].id! - store.put([{ ...instanceState, currentPageId }]) - return ensureStoreIsUsable(store) - } + // The current page is either the last updated page or the first page + const currentPageId = userDocumentSettings?.lastUpdatedPageId ?? pages[0].id! - // make sure we have a user state record for the current user - if (!allRecords.find((u) => u.id === userId)) { - store.put([TLUser.create({ id: userId })]) - return ensureStoreIsUsable(store) - } - - const userPresences = allRecords - .filter(TLUserPresence.isInstance) - .filter((r) => r.userId === userId) - if (userPresences.length === 0) { - store.put([TLUserPresence.create({ userId, color: getRandomColor() })]) - return ensureStoreIsUsable(store) - } else if (userPresences.length > 1) { - // make sure we don't duplicate user presences - store.remove(userPresences.slice(1).map((r) => r.id)) - } - - // make sure each page has a instancePageState and camera - for (const page of pages) { - const instancePageStates = allRecords - .filter(TLInstancePageState.isInstance) - .filter((tps) => tps.pageId === page.id && tps.instanceId === tabId) - if (instancePageStates.length > 1) { - // make sure we only have one instancePageState per instance per page - store.remove(instancePageStates.slice(1).map((ips) => ips.id)) - } else if (instancePageStates.length === 0) { - const camera = TLCamera.create({}) store.put([ - camera, - TLInstancePageState.create({ pageId: page.id, instanceId: tabId, cameraId: camera.id }), + TLInstance.create({ + id: tabId, + userId, + currentPageId, + propsForNextShape, + exportBackground: true, + }), ]) - return ensureStoreIsUsable(store) + + return ensureStoreIsUsable() } - // make sure the camera exists - const camera = store.get(instancePageStates[0].cameraId) - if (!camera) { - store.put([TLCamera.create({ id: instancePageStates[0].cameraId })]) - return ensureStoreIsUsable(store) + // make sure the user's currentPageId is still valid + let currentPageId = instanceState.currentPageId + if (!pages.find((p) => p.id === currentPageId)) { + currentPageId = pages[0].id! + store.put([{ ...instanceState, currentPageId }]) + return ensureStoreIsUsable() + } + + // make sure we have a user state record for the current user + if (!$user.value) { + store.put([TLUser.create({ id: userId })]) + return ensureStoreIsUsable() + } + + const userPresences = $userPresences.value.filter((r) => r.userId === userId) + if (userPresences.length === 0) { + store.put([TLUserPresence.create({ userId, color: getRandomColor() })]) + return ensureStoreIsUsable() + } else if (userPresences.length > 1) { + // make sure we don't duplicate user presences + store.remove(userPresences.slice(1).map((r) => r.id)) + } + + // make sure each page has a instancePageState and camera + for (const page of pages) { + const instancePageStates = $instancePageStates.value.filter( + (tps) => tps.pageId === page.id && tps.instanceId === tabId + ) + if (instancePageStates.length > 1) { + // make sure we only have one instancePageState per instance per page + store.remove(instancePageStates.slice(1).map((ips) => ips.id)) + } else if (instancePageStates.length === 0) { + const camera = TLCamera.create({}) + store.put([ + camera, + TLInstancePageState.create({ pageId: page.id, instanceId: tabId, cameraId: camera.id }), + ]) + return ensureStoreIsUsable() + } + + // make sure the camera exists + const camera = store.get(instancePageStates[0].cameraId) + if (!camera) { + store.put([TLCamera.create({ id: instancePageStates[0].cameraId })]) + return ensureStoreIsUsable() + } } } + + return ensureStoreIsUsable } diff --git a/packages/tlschema/src/createTLSchema.ts b/packages/tlschema/src/createTLSchema.ts index 7403d261f..6284204cd 100644 --- a/packages/tlschema/src/createTLSchema.ts +++ b/packages/tlschema/src/createTLSchema.ts @@ -2,7 +2,7 @@ import { StoreSchema, StoreValidator, createRecordType, defineMigrations } from import { T } from '@tldraw/tlvalidate' import { Signal } from 'signia' import { TLRecord } from './TLRecord' -import { TLStore, TLStoreProps, ensureStoreIsUsable, onValidationFailure } from './TLStore' +import { TLStore, TLStoreProps, createIntegrityChecker, onValidationFailure } from './TLStore' import { defaultDerivePresenceState } from './defaultDerivePresenceState' import { TLAsset } from './records/TLAsset' import { TLCamera } from './records/TLCamera' @@ -112,7 +112,7 @@ export function createTLSchema({ { snapshotMigrations: storeMigrations, onValidationFailure, - ensureStoreIsUsable, + createIntegrityChecker: createIntegrityChecker, derivePresenceState: derivePresenceState ?? defaultDerivePresenceState, } ) diff --git a/packages/tlschema/src/index.ts b/packages/tlschema/src/index.ts index eccd45cef..9b46da09e 100644 --- a/packages/tlschema/src/index.ts +++ b/packages/tlschema/src/index.ts @@ -1,7 +1,7 @@ export { type TLRecord } from './TLRecord' export { USER_COLORS, - ensureStoreIsUsable, + createIntegrityChecker, onValidationFailure, type TLStore, type TLStoreProps, diff --git a/packages/tlstore/api-report.md b/packages/tlstore/api-report.md index d6a4086f1..a3431f794 100644 --- a/packages/tlstore/api-report.md +++ b/packages/tlstore/api-report.md @@ -276,12 +276,12 @@ export class StoreSchema { createId: any; }; }, options?: StoreSchemaOptions): StoreSchema; + // @internal (undocumented) + createIntegrityChecker(store: Store): (() => void) | undefined; // (undocumented) get currentStoreVersion(): number; // @internal (undocumented) derivePresenceState(store: Store): Signal | undefined; - // @internal (undocumented) - ensureStoreIsUsable(store: Store): void; // (undocumented) migratePersistedRecord(record: R, persistedSchema: SerializedSchema, direction?: 'down' | 'up'): MigrationResult; // (undocumented) @@ -308,7 +308,7 @@ export type StoreSchemaOptions = { phase: 'createRecord' | 'initialize' | 'tests' | 'updateRecord'; recordBefore: null | R; }) => R; - ensureStoreIsUsable?: (store: Store) => void; + createIntegrityChecker?: (store: Store) => void; derivePresenceState?: (store: Store) => Signal; }; diff --git a/packages/tlstore/src/lib/Store.ts b/packages/tlstore/src/lib/Store.ts index 6a93df563..515833fce 100644 --- a/packages/tlstore/src/lib/Store.ts +++ b/packages/tlstore/src/lib/Store.ts @@ -622,9 +622,12 @@ export class Store { } } + private _integrityChecker?: () => void | undefined + /** @internal */ ensureStoreIsUsable() { - this.schema.ensureStoreIsUsable(this) + this._integrityChecker ??= this.schema.createIntegrityChecker(this) + this._integrityChecker?.() } private _isPossiblyCorrupted = false diff --git a/packages/tlstore/src/lib/StoreSchema.ts b/packages/tlstore/src/lib/StoreSchema.ts index e6c2b642a..5f6618822 100644 --- a/packages/tlstore/src/lib/StoreSchema.ts +++ b/packages/tlstore/src/lib/StoreSchema.ts @@ -48,7 +48,7 @@ export type StoreSchemaOptions = { recordBefore: R | null }) => R /** @internal */ - ensureStoreIsUsable?: (store: Store) => void + createIntegrityChecker?: (store: Store) => void /** @internal */ derivePresenceState?: (store: Store) => Signal } @@ -240,8 +240,8 @@ export class StoreSchema { } /** @internal */ - ensureStoreIsUsable(store: Store): void { - this.options.ensureStoreIsUsable?.(store) + createIntegrityChecker(store: Store): (() => void) | undefined { + return this.options.createIntegrityChecker?.(store) ?? undefined } /** @internal */