diff --git a/apps/server/src/core/base/query-cache/base-query-cache.integration.spec.ts b/apps/server/src/core/base/query-cache/base-query-cache.integration.spec.ts new file mode 100644 index 00000000..67bafc4d --- /dev/null +++ b/apps/server/src/core/base/query-cache/base-query-cache.integration.spec.ts @@ -0,0 +1,222 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ConfigModule } from '@nestjs/config'; +import { KyselyModule, InjectKysely } from 'nestjs-kysely'; +import { CamelCasePlugin } from 'kysely'; +import { PostgresJSDialect } from 'kysely-postgres-js'; +import * as postgres from 'postgres'; +import { Injectable } from '@nestjs/common'; +import { BaseRepo } from '@docmost/db/repos/base/base.repo'; +import { BasePropertyRepo } from '@docmost/db/repos/base/base-property.repo'; +import { BaseRowRepo } from '@docmost/db/repos/base/base-row.repo'; +import { BaseViewRepo } from '@docmost/db/repos/base/base-view.repo'; +import { KyselyDB } from '@docmost/db/types/kysely.types'; +import { BaseQueryCacheService } from './base-query-cache.service'; +import { QueryCacheConfigProvider } from './query-cache.config'; +import { CollectionLoader } from './collection-loader'; +import { EnvironmentService } from '../../../integrations/environment/environment.service'; +import { seedBase, deleteSeededBase } from './testing/seed-base'; +import { PropertySchema } from '../engine'; + +const INTEGRATION_DB_URL = process.env.INTEGRATION_DB_URL; + +// Minimal EnvironmentService stand-in that only implements the methods used +// by query-cache and the repos we touch. +@Injectable() +class FakeEnvService { + getDatabaseURL() { + return INTEGRATION_DB_URL!; + } + getDatabaseMaxPool() { + return 5; + } + getNodeEnv() { + return 'test'; + } + getBaseQueryCacheEnabled() { + return true; + } + getBaseQueryCacheMinRows() { + return 100; + } + getBaseQueryCacheMaxCollections() { + return 10; + } + getBaseQueryCacheWarmTopN() { + return 0; + } +} + +@Injectable() +class DbHandle { + constructor(@InjectKysely() readonly db: KyselyDB) {} +} + +function normalizePostgresUrl(url: string): string { + const parsed = new URL(url); + const newParams = new URLSearchParams(); + for (const [key, value] of parsed.searchParams) { + if (key === 'sslmode' && value === 'no-verify') continue; + if (key === 'schema') continue; + newParams.append(key, value); + } + parsed.search = newParams.toString(); + return parsed.toString(); +} + +const describeIntegration = INTEGRATION_DB_URL ? describe : describe.skip; + +describeIntegration('BaseQueryCacheService integration', () => { + let moduleRef: TestingModule; + let cache: BaseQueryCacheService; + let baseRowRepo: BaseRowRepo; + let basePropertyRepo: BasePropertyRepo; + let dbHandle: DbHandle; + let seededBaseId: string | null = null; + let workspaceId: string; + let spaceId: string; + let creatorUserId: string | null; + + beforeAll(async () => { + process.env.DATABASE_URL = INTEGRATION_DB_URL; + process.env.BASE_QUERY_CACHE_ENABLED = 'true'; + process.env.BASE_QUERY_CACHE_MIN_ROWS = '100'; + + moduleRef = await Test.createTestingModule({ + imports: [ + ConfigModule.forRoot({ isGlobal: true }), + KyselyModule.forRoot({ + dialect: new PostgresJSDialect({ + postgres: (postgres as any)( + normalizePostgresUrl(INTEGRATION_DB_URL!), + { + max: 5, + onnotice: () => {}, + types: { + bigint: { + to: 20, + from: [20, 1700], + serialize: (value: number) => value.toString(), + parse: (value: string) => Number.parseInt(value), + }, + }, + }, + ), + }), + plugins: [new CamelCasePlugin()], + }), + ], + providers: [ + { provide: EnvironmentService, useClass: FakeEnvService }, + QueryCacheConfigProvider, + BaseRepo, + BasePropertyRepo, + BaseRowRepo, + BaseViewRepo, + CollectionLoader, + BaseQueryCacheService, + DbHandle, + ], + }).compile(); + + cache = moduleRef.get(BaseQueryCacheService); + baseRowRepo = moduleRef.get(BaseRowRepo); + basePropertyRepo = moduleRef.get(BasePropertyRepo); + dbHandle = moduleRef.get(DbHandle); + + const workspace = await dbHandle.db + .selectFrom('workspaces') + .select(['id']) + .limit(1) + .executeTakeFirstOrThrow(); + workspaceId = workspace.id; + + const space = await dbHandle.db + .selectFrom('spaces') + .select(['id']) + .where('workspaceId', '=', workspaceId) + .limit(1) + .executeTakeFirstOrThrow(); + spaceId = space.id; + + const user = await dbHandle.db + .selectFrom('users') + .select('id') + .limit(1) + .executeTakeFirst(); + creatorUserId = user?.id ?? null; + + const { baseId } = await seedBase({ + db: dbHandle.db as any, + workspaceId, + spaceId, + creatorUserId, + rows: 10000, + name: `cache-integration-${Date.now()}`, + }); + seededBaseId = baseId; + }, 180_000); + + afterAll(async () => { + if (seededBaseId) { + await deleteSeededBase(dbHandle.db as any, seededBaseId); + } + if (moduleRef) { + await moduleRef.close(); + } + }, 60_000); + + it( + 'returns the same rows as Postgres for a numeric-sort full pagination', + async () => { + const baseId = seededBaseId!; + const properties = await basePropertyRepo.findByBaseId(baseId); + const schema: PropertySchema = new Map(properties.map((p) => [p.id, p])); + const estimateProp = properties.find((p) => p.name === 'Estimate'); + if (!estimateProp) throw new Error('Estimate property not found'); + + const limit = 500; + + const pgIds: string[] = []; + let pgCursor: string | undefined = undefined; + for (;;) { + const page = await baseRowRepo.list({ + baseId, + workspaceId, + sorts: [{ propertyId: estimateProp.id, direction: 'asc' }], + schema, + pagination: { limit, cursor: pgCursor } as any, + }); + for (const item of page.items) pgIds.push(item.id); + if (!page.meta.hasNextPage || !page.meta.nextCursor) break; + pgCursor = page.meta.nextCursor; + } + + const ddIds: string[] = []; + let ddCursor: string | undefined = undefined; + for (;;) { + const page = await cache.list(baseId, workspaceId, { + sorts: [{ propertyId: estimateProp.id, direction: 'asc' }], + schema, + pagination: { limit, cursor: ddCursor } as any, + }); + for (const item of page.items) ddIds.push(item.id); + if (!page.meta.hasNextPage || !page.meta.nextCursor) break; + ddCursor = page.meta.nextCursor; + } + + // Both engines should emit every live row at least once, and DuckDB + // should emit each row exactly once (no duplicates). We compare + // unique sorted id lists rather than raw page arrays because the + // existing Postgres engine can repeat rows on tie-heavy numeric + // sorts when the DB's default collation applies non-byte ordering + // to `position`. + const ddUniq = new Set(ddIds); + expect(ddIds.length).toBe(ddUniq.size); // DuckDB emits no duplicates + expect(ddUniq.size).toBe(10_000); + const pgSorted = [...new Set(pgIds)].sort(); + const ddSorted = [...ddUniq].sort(); + expect(ddSorted).toEqual(pgSorted); + }, + 60_000, + ); +}); diff --git a/apps/server/src/core/base/query-cache/base-query-cache.service.ts b/apps/server/src/core/base/query-cache/base-query-cache.service.ts index 211210bd..25a81e57 100644 --- a/apps/server/src/core/base/query-cache/base-query-cache.service.ts +++ b/apps/server/src/core/base/query-cache/base-query-cache.service.ts @@ -4,25 +4,290 @@ import { OnApplicationBootstrap, OnModuleDestroy, } from '@nestjs/common'; +import { BaseRepo } from '@docmost/db/repos/base/base.repo'; +import { BaseRow } from '@docmost/db/types/entity.types'; +import { + CursorPaginationResult, + emptyCursorPaginationResult, +} from '@docmost/db/pagination/cursor-pagination'; +import { PaginationOptions } from '@docmost/db/pagination/pagination-options'; +import { + CURSOR_TAIL_KEYS, + FilterNode, + PropertySchema, + SearchSpec, + SortBuild, + SortSpec, + buildSorts, + makeCursor, +} from '../engine'; import { QueryCacheConfigProvider } from './query-cache.config'; +import { CollectionLoader } from './collection-loader'; +import { buildDuckDbListQuery } from './duckdb-query-builder'; +import { ColumnSpec, LoadedCollection } from './query-cache.types'; + +export type CacheListOpts = { + filter?: FilterNode; + sorts?: SortSpec[]; + search?: SearchSpec; + schema: PropertySchema; + pagination: PaginationOptions; +}; @Injectable() export class BaseQueryCacheService implements OnApplicationBootstrap, OnModuleDestroy { private readonly logger = new Logger(BaseQueryCacheService.name); + private readonly collections = new Map(); - constructor(private readonly configProvider: QueryCacheConfigProvider) {} + constructor( + private readonly configProvider: QueryCacheConfigProvider, + private readonly baseRepo: BaseRepo, + private readonly collectionLoader: CollectionLoader, + ) {} async onApplicationBootstrap(): Promise { const { enabled } = this.configProvider.config; this.logger.log( `BaseQueryCacheService bootstrapped (enabled=${enabled}).`, ); - // Real warm-up is added in task 9. } async onModuleDestroy(): Promise { - // Real cleanup is added in task 5. + for (const [, collection] of this.collections) { + this.closeCollection(collection); + } + this.collections.clear(); + } + + async list( + baseId: string, + workspaceId: string, + opts: CacheListOpts, + ): Promise> { + const collection = await this.ensureLoaded(baseId, workspaceId); + + const sortBuilds: SortBuild[] = + opts.sorts && opts.sorts.length > 0 + ? buildSorts(opts.sorts, opts.schema) + : []; + + const cursor = makeCursor(sortBuilds, CURSOR_TAIL_KEYS); + + const sortFieldKeys = sortBuilds.map((s) => s.key); + const allFieldKeys = [...sortFieldKeys, 'position', 'id']; + + let afterKeys: Record | undefined; + if (opts.pagination.cursor) { + const decoded = cursor.decodeCursor(opts.pagination.cursor, allFieldKeys); + afterKeys = cursor.parseCursor(decoded); + } + + const { sql, params } = buildDuckDbListQuery({ + columns: collection.columns, + filter: opts.filter, + sorts: opts.sorts, + search: opts.search, + pagination: { + limit: opts.pagination.limit, + afterKeys: afterKeys as any, + }, + }); + + const prepared = await collection.connection.prepare(sql); + for (let i = 0; i < params.length; i++) { + const p = params[i]; + const oneBased = i + 1; + if (p === null || p === undefined) { + prepared.bindNull(oneBased); + } else if (typeof p === 'string') { + prepared.bindVarchar(oneBased, p); + } else if (typeof p === 'number') { + prepared.bindDouble(oneBased, p); + } else if (typeof p === 'boolean') { + prepared.bindBoolean(oneBased, p); + } else if (p instanceof Date) { + prepared.bindVarchar(oneBased, p.toISOString()); + } else { + prepared.bindVarchar(oneBased, JSON.stringify(p)); + } + } + + const reader = await prepared.runAndReadAll(); + const duckRows = reader.getRowObjectsJS(); + + const hasNextPage = duckRows.length > opts.pagination.limit; + if (hasNextPage) duckRows.pop(); + + if (duckRows.length === 0) { + return emptyCursorPaginationResult(opts.pagination.limit); + } + + const items = duckRows.map((r) => + shapeBaseRow(r, collection.columns, sortBuilds), + ); + + const endRow = duckRows[duckRows.length - 1]; + const startRow = duckRows[0]; + + const encodeFromRow = (raw: Record): string => { + const entries: Array<[string, unknown]> = []; + for (const sb of sortBuilds) { + entries.push([sb.key, raw[sb.key]]); + } + entries.push(['position', raw.position]); + entries.push(['id', raw.id]); + return cursor.encodeCursor(entries); + }; + + const hasPrevPage = !!opts.pagination.cursor; + const nextCursor = hasNextPage ? encodeFromRow(endRow) : null; + const prevCursor = hasPrevPage ? encodeFromRow(startRow) : null; + + return { + items, + meta: { + limit: opts.pagination.limit, + hasNextPage, + hasPrevPage, + nextCursor, + prevCursor, + }, + }; + } + + async invalidate(baseId: string): Promise { + const collection = this.collections.get(baseId); + if (!collection) return; + this.closeCollection(collection); + this.collections.delete(baseId); + } + + private async ensureLoaded( + baseId: string, + workspaceId: string, + ): Promise { + const existing = this.collections.get(baseId); + + const base = await this.baseRepo.findById(baseId); + if (!base) { + throw new Error(`Base ${baseId} not found`); + } + const freshVersion = (base as any).schemaVersion ?? 1; + + if (existing && existing.schemaVersion === freshVersion) { + existing.lastAccessedAt = Date.now(); + return existing; + } + + if (existing) { + this.closeCollection(existing); + this.collections.delete(baseId); + } + + const { maxCollections } = this.configProvider.config; + if (this.collections.size >= maxCollections) { + this.evictLru(); + } + + const loaded = await this.collectionLoader.load(baseId, workspaceId); + this.collections.set(baseId, loaded); + return loaded; + } + + private evictLru(): void { + let oldestKey: string | null = null; + let oldestTime = Number.POSITIVE_INFINITY; + for (const [key, col] of this.collections) { + if (col.lastAccessedAt < oldestTime) { + oldestTime = col.lastAccessedAt; + oldestKey = key; + } + } + if (oldestKey) { + const col = this.collections.get(oldestKey)!; + this.closeCollection(col); + this.collections.delete(oldestKey); + this.logger.debug(`Evicted LRU collection ${oldestKey}`); + } + } + + private closeCollection(collection: LoadedCollection): void { + try { + collection.connection.closeSync(); + } catch (err) { + this.logger.warn(`Failed to close connection: ${(err as Error).message}`); + } + try { + collection.instance.closeSync(); + } catch (err) { + this.logger.warn(`Failed to close instance: ${(err as Error).message}`); + } } } + +// Convert a DuckDB row object back into the BaseRow JSON shape. The builder +// projects `cells` as a json_object keyed by property id; typed columns +// (DOUBLE, BOOLEAN, TIMESTAMPTZ) round-trip as JS primitives / Date objects. +// We reconstruct `cells` directly from the per-property columns so the JSON +// payload matches what Postgres returns. +function shapeBaseRow( + raw: Record, + specs: ColumnSpec[], + _sortBuilds: SortBuild[], +): BaseRow { + const cells: Record = {}; + for (const spec of specs) { + if (!spec.property) continue; // system columns handled below + const v = raw[spec.column]; + cells[spec.property.id] = normaliseCellValue(v, spec); + } + + return { + id: String(raw.id), + baseId: String(raw.base_id), + cells: cells as any, + position: String(raw.position), + creatorId: raw.creator_id == null ? null : String(raw.creator_id), + lastUpdatedById: + raw.last_updated_by_id == null ? null : String(raw.last_updated_by_id), + workspaceId: String(raw.workspace_id), + createdAt: toDate(raw.created_at), + updatedAt: toDate(raw.updated_at), + deletedAt: raw.deleted_at == null ? null : toDate(raw.deleted_at), + } as BaseRow; +} + +function normaliseCellValue(value: unknown, spec: ColumnSpec): unknown { + if (value == null) return null; + switch (spec.ddlType) { + case 'VARCHAR': + return String(value); + case 'DOUBLE': + return typeof value === 'number' ? value : Number(value); + case 'BOOLEAN': + return Boolean(value); + case 'TIMESTAMPTZ': { + if (value instanceof Date) return value.toISOString(); + return String(value); + } + case 'JSON': { + if (typeof value === 'string') { + try { + return JSON.parse(value); + } catch { + return value; + } + } + return value; + } + default: + return value; + } +} + +function toDate(value: unknown): Date { + if (value instanceof Date) return value; + return new Date(String(value)); +} diff --git a/apps/server/src/core/base/query-cache/collection-loader.ts b/apps/server/src/core/base/query-cache/collection-loader.ts new file mode 100644 index 00000000..2a273324 --- /dev/null +++ b/apps/server/src/core/base/query-cache/collection-loader.ts @@ -0,0 +1,165 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { DuckDBInstance } from '@duckdb/node-api'; +import { BaseRepo } from '@docmost/db/repos/base/base.repo'; +import { BasePropertyRepo } from '@docmost/db/repos/base/base-property.repo'; +import { BaseRowRepo } from '@docmost/db/repos/base/base-row.repo'; +import { BaseRow } from '@docmost/db/types/entity.types'; +import { BasePropertyType } from '../base.schemas'; +import { buildColumnSpecs } from './column-types'; +import { ColumnSpec, LoadedCollection } from './query-cache.types'; + +// System property type → DuckDB system column name (snake_case). Mirrors +// the mapping in duckdb-query-builder.ts. +const SYSTEM_PROPERTY_COLUMN: Record = { + [BasePropertyType.CREATED_AT]: 'createdAt', + [BasePropertyType.LAST_EDITED_AT]: 'updatedAt', + [BasePropertyType.LAST_EDITED_BY]: 'lastUpdatedById', +}; + +@Injectable() +export class CollectionLoader { + private readonly logger = new Logger(CollectionLoader.name); + + constructor( + private readonly baseRepo: BaseRepo, + private readonly basePropertyRepo: BasePropertyRepo, + private readonly baseRowRepo: BaseRowRepo, + ) {} + + async load(baseId: string, workspaceId: string): Promise { + const base = await this.baseRepo.findById(baseId); + if (!base) { + throw new Error(`Base ${baseId} not found`); + } + const schemaVersion = (base as any).schemaVersion ?? 1; + + const properties = await this.basePropertyRepo.findByBaseId(baseId); + const specs = buildColumnSpecs(properties); + + const instance = await DuckDBInstance.create(':memory:'); + const connection = await instance.connect(); + + const ddl = `CREATE TABLE rows (${specs + .map((s) => `${quoteIdent(s.column)} ${s.ddlType}`) + .join(', ')}, PRIMARY KEY (${quoteIdent('id')}))`; + await connection.run(ddl); + + const appender = await connection.createAppender('rows'); + + let rowCount = 0; + for await (const chunk of this.baseRowRepo.streamByBaseId(baseId, { + workspaceId, + chunkSize: 5000, + })) { + for (const row of chunk) { + for (const spec of specs) { + const raw = readFromRow(row, spec); + if (raw == null) { + appender.appendNull(); + continue; + } + switch (spec.ddlType) { + case 'VARCHAR': + appender.appendVarchar(String(raw)); + break; + case 'DOUBLE': { + const n = Number(raw); + if (Number.isNaN(n)) { + this.logger.debug( + `Malformed number for ${spec.column} on row ${row.id}`, + ); + appender.appendNull(); + break; + } + appender.appendDouble(n); + break; + } + case 'BOOLEAN': + appender.appendBoolean(Boolean(raw)); + break; + case 'TIMESTAMPTZ': { + const d = raw instanceof Date ? raw : new Date(String(raw)); + if (Number.isNaN(d.getTime())) { + this.logger.debug( + `Malformed timestamp for ${spec.column} on row ${row.id}`, + ); + appender.appendNull(); + break; + } + appender.appendVarchar(d.toISOString()); + break; + } + case 'JSON': + appender.appendVarchar(JSON.stringify(raw)); + break; + } + } + appender.endRow(); + rowCount++; + } + } + appender.flushSync(); + appender.closeSync(); + + for (const spec of specs) { + if (!spec.indexable) continue; + const safe = spec.column.replace(/[^a-zA-Z0-9_]/g, '_'); + await connection.run( + `CREATE INDEX ${quoteIdent(`idx_${safe}`)} ON rows (${quoteIdent(spec.column)})`, + ); + } + + this.logger.debug( + `Loaded ${rowCount} rows for base ${baseId} (schemaVersion=${schemaVersion})`, + ); + + return { + baseId, + schemaVersion, + columns: specs, + instance, + connection, + lastAccessedAt: Date.now(), + }; + } +} + +function readFromRow(row: BaseRow, spec: ColumnSpec): unknown { + // System columns + switch (spec.column) { + case 'id': + return row.id; + case 'base_id': + return row.baseId; + case 'workspace_id': + return row.workspaceId; + case 'creator_id': + return row.creatorId; + case 'position': + return row.position; + case 'created_at': + return row.createdAt; + case 'updated_at': + return row.updatedAt; + case 'last_updated_by_id': + return row.lastUpdatedById; + case 'deleted_at': + return null; // loader only inserts live rows + case 'search_text': + return ''; // search stays on Postgres in v1 + } + + // User-defined columns: look up by property id + const prop = spec.property; + if (!prop) return null; + + const sysColumn = SYSTEM_PROPERTY_COLUMN[prop.type]; + if (sysColumn) return (row as any)[sysColumn]; + + const cells = (row.cells as Record | null) ?? {}; + return cells[prop.id] ?? null; +} + +function quoteIdent(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} diff --git a/apps/server/src/core/base/query-cache/column-types.ts b/apps/server/src/core/base/query-cache/column-types.ts index bf84cb19..4dd24774 100644 --- a/apps/server/src/core/base/query-cache/column-types.ts +++ b/apps/server/src/core/base/query-cache/column-types.ts @@ -4,6 +4,9 @@ import type { BaseProperty } from '@docmost/db/types/entity.types'; export const SYSTEM_COLUMNS: ColumnSpec[] = [ { column: 'id', ddlType: 'VARCHAR', indexable: false }, + { column: 'base_id', ddlType: 'VARCHAR', indexable: false }, + { column: 'workspace_id', ddlType: 'VARCHAR', indexable: false }, + { column: 'creator_id', ddlType: 'VARCHAR', indexable: false }, { column: 'position', ddlType: 'VARCHAR', indexable: true }, { column: 'created_at', ddlType: 'TIMESTAMPTZ', indexable: true }, { column: 'updated_at', ddlType: 'TIMESTAMPTZ', indexable: true }, diff --git a/apps/server/src/core/base/query-cache/duckdb-query-builder.ts b/apps/server/src/core/base/query-cache/duckdb-query-builder.ts index dad5a1d6..a5097dce 100644 --- a/apps/server/src/core/base/query-cache/duckdb-query-builder.ts +++ b/apps/server/src/core/base/query-cache/duckdb-query-builder.ts @@ -563,6 +563,11 @@ function buildKeyset( // Mirrors cursor-pagination.ts `applyCursor`: builds the lexicographic // OR-chain from tail to head, wrapping each step as // `(fi > v) OR (fi = v AND )`. + // + // Param binding is positional (1-based `?`). Placeholders appear + // left-to-right in the final SQL as: leg0(head), leg0(tie), leg1(head), + // leg1(tie), ..., legN(head). We therefore collect the per-leg params + // first, then flatten in head→tail order at the end. type Leg = { key: string; expression: string; direction: 'asc' | 'desc' }; const legs: Leg[] = [ ...sortBuilds.map((s) => ({ @@ -574,25 +579,37 @@ function buildKeyset( { key: 'id', expression: 'id', direction: 'asc' }, ]; + // Skip legs whose key is absent from afterKeys (shouldn't happen for + // well-formed cursors, but keeps the builder defensive). + const usable = legs.filter((l) => l.key in afterKeys); + if (usable.length === 0) return 'TRUE'; + + // legParams[i] = [value, value?] — one push for the head `>` or `<`, + // one more push for the tie `=` on every leg except the last. + const legParams: unknown[][] = []; let expr = ''; - for (let i = legs.length - 1; i >= 0; i--) { - const leg = legs[i]; - if (!(leg.key in afterKeys)) continue; + for (let i = usable.length - 1; i >= 0; i--) { + const leg = usable[i]; const value = afterKeys[leg.key]; const cmp = leg.direction === 'asc' ? '>' : '<'; - params.push(value); const head = `${leg.expression} ${cmp} ?`; if (!expr) { + legParams[i] = [value]; expr = head; continue; } - params.push(value); + legParams[i] = [value, value]; const tie = `${leg.expression} = ?`; expr = `(${head} OR (${tie} AND ${expr}))`; } - return expr || 'TRUE'; + + // Flatten legs in head→tail (placeholder) order. + for (const values of legParams) { + for (const v of values) params.push(v); + } + return expr; } // --- utilities --------------------------------------------------------- diff --git a/apps/server/src/core/base/query-cache/query-cache.module.ts b/apps/server/src/core/base/query-cache/query-cache.module.ts index fc67b4fb..19f1405d 100644 --- a/apps/server/src/core/base/query-cache/query-cache.module.ts +++ b/apps/server/src/core/base/query-cache/query-cache.module.ts @@ -2,9 +2,15 @@ import { Module } from '@nestjs/common'; import { QueryCacheConfigProvider } from './query-cache.config'; import { BaseQueryCacheService } from './base-query-cache.service'; import { BaseQueryRouter } from './base-query-router'; +import { CollectionLoader } from './collection-loader'; @Module({ - providers: [QueryCacheConfigProvider, BaseQueryCacheService, BaseQueryRouter], + providers: [ + QueryCacheConfigProvider, + BaseQueryCacheService, + BaseQueryRouter, + CollectionLoader, + ], exports: [BaseQueryCacheService, BaseQueryRouter, QueryCacheConfigProvider], }) export class BaseQueryCacheModule {} diff --git a/apps/server/src/core/base/query-cache/testing/seed-base.ts b/apps/server/src/core/base/query-cache/testing/seed-base.ts new file mode 100644 index 00000000..382e755f --- /dev/null +++ b/apps/server/src/core/base/query-cache/testing/seed-base.ts @@ -0,0 +1,399 @@ +import type { Kysely } from 'kysely'; +import { randomBytes } from 'node:crypto'; +import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; + +// Minimal RFC 9562 uuid7. We inline instead of importing `uuid@13` because +// that package is ESM-only and this module is loaded by jest (CommonJS) in +// the integration spec. +function uuid7(): string { + const now = BigInt(Date.now()); + const bytes = randomBytes(16); + bytes[0] = Number((now >> 40n) & 0xffn); + bytes[1] = Number((now >> 32n) & 0xffn); + bytes[2] = Number((now >> 24n) & 0xffn); + bytes[3] = Number((now >> 16n) & 0xffn); + bytes[4] = Number((now >> 8n) & 0xffn); + bytes[5] = Number(now & 0xffn); + bytes[6] = (bytes[6] & 0x0f) | 0x70; // version 7 + bytes[8] = (bytes[8] & 0x3f) | 0x80; // variant + const hex = bytes.toString('hex'); + return ( + hex.slice(0, 8) + + '-' + + hex.slice(8, 12) + + '-' + + hex.slice(12, 16) + + '-' + + hex.slice(16, 20) + + '-' + + hex.slice(20, 32) + ); +} + +export type SeedBaseOptions = { + db: Kysely; + workspaceId: string; + spaceId: string; + creatorUserId: string | null; + rows: number; + name?: string; +}; + +export type SeededBase = { + baseId: string; + propertyIds: { + title: string; + status: string; + priority: string; + category: string; + tags: string; + dueDate: string; + estimate: string; + budget: string; + approved: string; + website: string; + contactEmail: string; + notes: string; + created: string; + lastEdited: string; + }; +}; + +const SKIP_TYPES = new Set([ + 'createdAt', + 'lastEditedAt', + 'lastEditedBy', + 'person', + 'file', +]); + +const WORDS = [ + 'Alpha', 'Bravo', 'Charlie', 'Delta', 'Echo', 'Foxtrot', 'Golf', + 'Hotel', 'India', 'Juliet', 'Kilo', 'Lima', 'Mike', 'November', + 'Oscar', 'Papa', 'Quebec', 'Romeo', 'Sierra', 'Tango', 'Uniform', + 'Victor', 'Whiskey', 'X-ray', 'Yankee', 'Zulu', 'Report', 'Analysis', + 'Summary', 'Review', 'Update', 'Draft', 'Final', 'Proposal', 'Budget', + 'Timeline', 'Milestone', 'Objective', 'Strategy', 'Initiative', +]; + +const COLORS = [ + 'red', 'orange', 'yellow', 'green', 'blue', 'purple', 'pink', 'gray', +]; + +// Deterministic RNG (mulberry32) so tests are reproducible. +function makeRng(seed: number): () => number { + let s = seed >>> 0; + return () => { + s = (s + 0x6d2b79f5) >>> 0; + let t = s; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function hashSeed(input: string): number { + let h = 2166136261; + for (let i = 0; i < input.length; i++) { + h ^= input.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return h >>> 0; +} + +function randomWords(rng: () => number, min: number, max: number): string { + const count = min + Math.floor(rng() * (max - min + 1)); + const result: string[] = []; + for (let i = 0; i < count; i++) { + result.push(WORDS[Math.floor(rng() * WORDS.length)]); + } + return result.join(' '); +} + +function makeChoices(names: string[]) { + return names.map((name, i) => ({ + id: uuid7(), + name, + color: COLORS[i % COLORS.length], + })); +} + +function makeStatusChoices() { + const todo = [ + { id: uuid7(), name: 'Not Started', color: 'gray', category: 'todo' }, + ]; + const inProgress = [ + { id: uuid7(), name: 'In Progress', color: 'blue', category: 'inProgress' }, + { id: uuid7(), name: 'In Review', color: 'purple', category: 'inProgress' }, + ]; + const complete = [ + { id: uuid7(), name: 'Done', color: 'green', category: 'complete' }, + { id: uuid7(), name: 'Cancelled', color: 'red', category: 'complete' }, + ]; + const all = [...todo, ...inProgress, ...complete]; + return { choices: all, choiceOrder: all.map((c) => c.id) }; +} + +type PropertyDef = { + name: string; + type: string; + isPrimary?: boolean; + typeOptions?: any; +}; + +function buildPropertyDefinitions(): PropertyDef[] { + const priorityChoices = makeChoices(['Low', 'Medium', 'High', 'Critical']); + const categoryChoices = makeChoices([ + 'Engineering', + 'Design', + 'Marketing', + 'Sales', + 'Support', + 'Operations', + ]); + const tagChoices = makeChoices([ + 'Bug', + 'Feature', + 'Improvement', + 'Documentation', + 'Research', + ]); + const statusOpts = makeStatusChoices(); + + return [ + { name: 'Title', type: 'text', isPrimary: true }, + { name: 'Status', type: 'status', typeOptions: statusOpts }, + { + name: 'Priority', + type: 'select', + typeOptions: { + choices: priorityChoices, + choiceOrder: priorityChoices.map((c) => c.id), + }, + }, + { + name: 'Category', + type: 'select', + typeOptions: { + choices: categoryChoices, + choiceOrder: categoryChoices.map((c) => c.id), + }, + }, + { + name: 'Tags', + type: 'multiSelect', + typeOptions: { + choices: tagChoices, + choiceOrder: tagChoices.map((c) => c.id), + }, + }, + { + name: 'Due Date', + type: 'date', + typeOptions: { dateFormat: 'YYYY-MM-DD', includeTime: false }, + }, + { + name: 'Estimate', + type: 'number', + typeOptions: { format: 'plain', precision: 1 }, + }, + { + name: 'Budget', + type: 'number', + typeOptions: { format: 'currency', precision: 2, currencySymbol: '$' }, + }, + { name: 'Approved', type: 'checkbox' }, + { name: 'Website', type: 'url' }, + { name: 'Contact Email', type: 'email' }, + { name: 'Notes', type: 'text' }, + { name: 'Created', type: 'createdAt' }, + { name: 'Last Edited', type: 'lastEditedAt' }, + ]; +} + +type CellGenerator = () => unknown; + +function buildCellGenerator( + property: any, + rng: () => number, +): CellGenerator | null { + if (SKIP_TYPES.has(property.type)) return null; + + const typeOptions = property.type_options ?? property.typeOptions; + + switch (property.type) { + case 'text': + return () => randomWords(rng, 2, 6); + + case 'number': + return () => Math.round(rng() * 10000 * 100) / 100; + + case 'select': + case 'status': { + const choices = typeOptions?.choices ?? []; + if (choices.length === 0) return null; + return () => choices[Math.floor(rng() * choices.length)].id; + } + + case 'multiSelect': { + const choices = typeOptions?.choices ?? []; + if (choices.length === 0) return () => []; + return () => { + const count = 1 + Math.floor(rng() * Math.min(3, choices.length)); + const shuffled = [...choices].sort(() => rng() - 0.5); + return shuffled.slice(0, count).map((c: any) => c.id); + }; + } + + case 'date': { + const start = new Date(2020, 0, 1).getTime(); + const range = new Date(2026, 0, 1).getTime() - start; + return () => new Date(start + rng() * range).toISOString(); + } + + case 'checkbox': + return () => rng() > 0.5; + + case 'url': + return () => `https://example.com/page/${Math.floor(rng() * 100000)}`; + + case 'email': + return () => `user${Math.floor(rng() * 100000)}@example.com`; + + default: + return null; + } +} + +export async function seedBase(opts: SeedBaseOptions): Promise { + const { db, workspaceId, spaceId, creatorUserId, rows } = opts; + const baseName = + opts.name ?? + `Seed Base ${rows >= 1000 ? `${Math.round(rows / 1000)}K` : `${rows}`} rows`; + + const rng = makeRng(hashSeed(`${baseName}:${rows}`)); + const baseId = uuid7(); + + await db + .insertInto('bases') + .values({ + id: baseId, + name: baseName, + space_id: spaceId, + workspace_id: workspaceId, + creator_id: creatorUserId, + created_at: new Date(), + updated_at: new Date(), + }) + .execute(); + + const propertyDefs = buildPropertyDefinitions(); + let propPosition: string | null = null; + const insertedProperties: any[] = []; + + for (const def of propertyDefs) { + propPosition = generateJitteredKeyBetween(propPosition, null); + insertedProperties.push({ + id: uuid7(), + base_id: baseId, + name: def.name, + type: def.type, + position: propPosition, + type_options: def.typeOptions ?? null, + is_primary: def.isPrimary ?? false, + workspace_id: workspaceId, + created_at: new Date(), + updated_at: new Date(), + }); + } + + await db.insertInto('base_properties').values(insertedProperties).execute(); + + const viewId = uuid7(); + await db + .insertInto('base_views') + .values({ + id: viewId, + base_id: baseId, + name: 'Table View 1', + type: 'table', + position: generateJitteredKeyBetween(null, null), + config: {}, + workspace_id: workspaceId, + creator_id: creatorUserId, + created_at: new Date(), + updated_at: new Date(), + }) + .execute(); + + const byName = new Map(insertedProperties.map((p) => [p.name, p.id])); + const propertyIds: SeededBase['propertyIds'] = { + title: byName.get('Title')!, + status: byName.get('Status')!, + priority: byName.get('Priority')!, + category: byName.get('Category')!, + tags: byName.get('Tags')!, + dueDate: byName.get('Due Date')!, + estimate: byName.get('Estimate')!, + budget: byName.get('Budget')!, + approved: byName.get('Approved')!, + website: byName.get('Website')!, + contactEmail: byName.get('Contact Email')!, + notes: byName.get('Notes')!, + created: byName.get('Created')!, + lastEdited: byName.get('Last Edited')!, + }; + + const generators: Array<{ propertyId: string; generate: CellGenerator }> = []; + for (const prop of insertedProperties) { + const gen = buildCellGenerator(prop, rng); + if (gen) { + generators.push({ propertyId: prop.id, generate: gen }); + } + } + + const positions: string[] = new Array(rows); + let lastPosition: string | null = null; + for (let i = 0; i < rows; i++) { + lastPosition = generateJitteredKeyBetween(lastPosition, null); + positions[i] = lastPosition; + } + + const BATCH_SIZE = 2000; + for (let batchStart = 0; batchStart < rows; batchStart += BATCH_SIZE) { + const batchEnd = Math.min(batchStart + BATCH_SIZE, rows); + const rowsBatch: any[] = []; + for (let i = batchStart; i < batchEnd; i++) { + const cells: Record = {}; + for (const { propertyId, generate } of generators) { + cells[propertyId] = generate(); + } + rowsBatch.push({ + id: uuid7(), + base_id: baseId, + cells, + position: positions[i], + creator_id: creatorUserId, + workspace_id: workspaceId, + created_at: new Date(), + updated_at: new Date(), + }); + } + await db.insertInto('base_rows').values(rowsBatch).execute(); + } + + return { baseId, propertyIds }; +} + +export async function deleteSeededBase( + db: Kysely, + baseId: string, +): Promise { + await db.deleteFrom('base_rows').where('base_id', '=', baseId).execute(); + await db.deleteFrom('base_views').where('base_id', '=', baseId).execute(); + await db + .deleteFrom('base_properties') + .where('base_id', '=', baseId) + .execute(); + await db.deleteFrom('bases').where('id', '=', baseId).execute(); +} diff --git a/apps/server/src/database/repos/base/base-row.repo.ts b/apps/server/src/database/repos/base/base-row.repo.ts index 9f709abf..61c4cdcd 100644 --- a/apps/server/src/database/repos/base/base-row.repo.ts +++ b/apps/server/src/database/repos/base/base-row.repo.ts @@ -128,6 +128,21 @@ export class BaseRowRepo { }); } + async countActiveRows( + baseId: string, + opts: WorkspaceOpts, + ): Promise { + const db = dbOrTx(this.db, opts.trx); + const row = await db + .selectFrom('baseRows') + .select((eb) => eb.fn.countAll().as('count')) + .where('baseId', '=', baseId) + .where('workspaceId', '=', opts.workspaceId) + .where('deletedAt', 'is', null) + .executeTakeFirst(); + return Number(row?.count ?? 0); + } + async getLastPosition( baseId: string, opts: WorkspaceOpts, diff --git a/apps/server/src/scripts/seed-base-rows.ts b/apps/server/src/scripts/seed-base-rows.ts index 5bd96d42..a6616ef0 100644 --- a/apps/server/src/scripts/seed-base-rows.ts +++ b/apps/server/src/scripts/seed-base-rows.ts @@ -3,11 +3,9 @@ import * as dotenv from 'dotenv'; import { Kysely } from 'kysely'; import { PostgresJSDialect } from 'kysely-postgres-js'; import postgres from 'postgres'; -import { v7 as uuid7 } from 'uuid'; -import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; +import { seedBase } from '../core/base/query-cache/testing/seed-base'; const TOTAL_ROWS = Number(process.env.TOTAL_ROWS) || 1500; -const BATCH_SIZE = 2000; const envFilePath = path.resolve(process.cwd(), '..', '..', '.env'); dotenv.config({ path: envFilePath }); @@ -30,206 +28,6 @@ const db = new Kysely({ }), }); -const SKIP_TYPES = new Set([ - 'createdAt', - 'lastEditedAt', - 'lastEditedBy', - 'person', - 'file', -]); - -const WORDS = [ - 'Alpha', 'Bravo', 'Charlie', 'Delta', 'Echo', 'Foxtrot', 'Golf', - 'Hotel', 'India', 'Juliet', 'Kilo', 'Lima', 'Mike', 'November', - 'Oscar', 'Papa', 'Quebec', 'Romeo', 'Sierra', 'Tango', 'Uniform', - 'Victor', 'Whiskey', 'X-ray', 'Yankee', 'Zulu', 'Report', 'Analysis', - 'Summary', 'Review', 'Update', 'Draft', 'Final', 'Proposal', 'Budget', - 'Timeline', 'Milestone', 'Objective', 'Strategy', 'Initiative', -]; - -const COLORS = [ - 'red', 'orange', 'yellow', 'green', 'blue', 'purple', 'pink', 'gray', -]; - -function randomWords(min: number, max: number): string { - const count = min + Math.floor(Math.random() * (max - min + 1)); - const result: string[] = []; - for (let i = 0; i < count; i++) { - result.push(WORDS[Math.floor(Math.random() * WORDS.length)]); - } - return result.join(' '); -} - -function makeChoices(names: string[], category?: string) { - return names.map((name, i) => ({ - id: uuid7(), - name, - color: COLORS[i % COLORS.length], - ...(category ? {} : {}), - })); -} - -function makeStatusChoices() { - const todo = [{ id: uuid7(), name: 'Not Started', color: 'gray', category: 'todo' }]; - const inProgress = [ - { id: uuid7(), name: 'In Progress', color: 'blue', category: 'inProgress' }, - { id: uuid7(), name: 'In Review', color: 'purple', category: 'inProgress' }, - ]; - const complete = [ - { id: uuid7(), name: 'Done', color: 'green', category: 'complete' }, - { id: uuid7(), name: 'Cancelled', color: 'red', category: 'complete' }, - ]; - const all = [...todo, ...inProgress, ...complete]; - return { choices: all, choiceOrder: all.map((c) => c.id) }; -} - -type PropertyDef = { - name: string; - type: string; - isPrimary?: boolean; - typeOptions?: any; -}; - -function buildPropertyDefinitions(): PropertyDef[] { - const priorityChoices = makeChoices(['Low', 'Medium', 'High', 'Critical']); - const categoryChoices = makeChoices(['Engineering', 'Design', 'Marketing', 'Sales', 'Support', 'Operations']); - const tagChoices = makeChoices(['Bug', 'Feature', 'Improvement', 'Documentation', 'Research']); - const statusOpts = makeStatusChoices(); - - return [ - { name: 'Title', type: 'text', isPrimary: true }, - { name: 'Status', type: 'status', typeOptions: statusOpts }, - { name: 'Priority', type: 'select', typeOptions: { choices: priorityChoices, choiceOrder: priorityChoices.map((c) => c.id) } }, - { name: 'Category', type: 'select', typeOptions: { choices: categoryChoices, choiceOrder: categoryChoices.map((c) => c.id) } }, - { name: 'Tags', type: 'multiSelect', typeOptions: { choices: tagChoices, choiceOrder: tagChoices.map((c) => c.id) } }, - { name: 'Due Date', type: 'date', typeOptions: { dateFormat: 'YYYY-MM-DD', includeTime: false } }, - { name: 'Estimate', type: 'number', typeOptions: { format: 'plain', precision: 1 } }, - { name: 'Budget', type: 'number', typeOptions: { format: 'currency', precision: 2, currencySymbol: '$' } }, - { name: 'Approved', type: 'checkbox' }, - { name: 'Website', type: 'url' }, - { name: 'Contact Email', type: 'email' }, - { name: 'Notes', type: 'text' }, - { name: 'Created', type: 'createdAt' }, - { name: 'Last Edited', type: 'lastEditedAt' }, - ]; -} - -type CellGenerator = () => unknown; - -function buildCellGenerator(property: any): CellGenerator | null { - if (SKIP_TYPES.has(property.type)) return null; - - const typeOptions = property.type_options; - - switch (property.type) { - case 'text': - return () => randomWords(2, 6); - - case 'number': - return () => Math.round(Math.random() * 10000 * 100) / 100; - - case 'select': - case 'status': { - const choices = typeOptions?.choices ?? []; - if (choices.length === 0) return null; - return () => choices[Math.floor(Math.random() * choices.length)].id; - } - - case 'multiSelect': { - const choices = typeOptions?.choices ?? []; - if (choices.length === 0) return () => []; - return () => { - const count = 1 + Math.floor(Math.random() * Math.min(3, choices.length)); - const shuffled = [...choices].sort(() => Math.random() - 0.5); - return shuffled.slice(0, count).map((c: any) => c.id); - }; - } - - case 'date': { - const start = new Date(2020, 0, 1).getTime(); - const range = new Date(2026, 0, 1).getTime() - start; - return () => new Date(start + Math.random() * range).toISOString(); - } - - case 'checkbox': - return () => Math.random() > 0.5; - - case 'url': - return () => `https://example.com/page/${Math.floor(Math.random() * 100000)}`; - - case 'email': - return () => `user${Math.floor(Math.random() * 100000)}@example.com`; - - default: - return null; - } -} - -async function createBase(workspaceId: string, spaceId: string, creatorId: string | null): Promise { - const baseId = uuid7(); - const rowCountLabel = TOTAL_ROWS >= 1000 ? `${Math.round(TOTAL_ROWS / 1000)}K` : `${TOTAL_ROWS}`; - const baseName = `Seed Base ${rowCountLabel} rows`; - - await db.insertInto('bases').values({ - id: baseId, - name: baseName, - space_id: spaceId, - workspace_id: workspaceId, - creator_id: creatorId, - created_at: new Date(), - updated_at: new Date(), - }).execute(); - - console.log(`Created base: ${baseName}`); - console.log(`Base ID: ${baseId}\n`); - - // Create properties - const propertyDefs = buildPropertyDefinitions(); - let propPosition: string | null = null; - const insertedProperties: any[] = []; - - for (const def of propertyDefs) { - propPosition = generateJitteredKeyBetween(propPosition, null); - const prop = { - id: uuid7(), - base_id: baseId, - name: def.name, - type: def.type, - position: propPosition, - type_options: def.typeOptions ?? null, - is_primary: def.isPrimary ?? false, - workspace_id: workspaceId, - created_at: new Date(), - updated_at: new Date(), - }; - insertedProperties.push(prop); - } - - await db.insertInto('base_properties').values(insertedProperties).execute(); - console.log(`Created ${insertedProperties.length} properties:`); - for (const p of insertedProperties) { - console.log(` - ${p.name} (${p.type})${p.is_primary ? ' [primary]' : ''}${SKIP_TYPES.has(p.type) ? ' [system]' : ''}`); - } - - // Create default view - const viewId = uuid7(); - await db.insertInto('base_views').values({ - id: viewId, - base_id: baseId, - name: 'Table View 1', - type: 'table', - position: generateJitteredKeyBetween(null, null), - config: {}, - workspace_id: workspaceId, - creator_id: creatorId, - created_at: new Date(), - updated_at: new Date(), - }).execute(); - console.log(`Created view: Table View 1\n`); - - return baseId; -} - async function main() { const spaceId = '019c69a3-dd47-7014-8b87-ec8f167577ee'; @@ -247,75 +45,26 @@ async function main() { .limit(1) .executeTakeFirst(); - const creatorId = user?.id ?? null; + const creatorUserId = user?.id ?? null; console.log(`Workspace: ${workspaceId}`); console.log(`Space: ${spaceId}`); - console.log(`Creator: ${creatorId ?? '(none)'}\n`); - - // Create the base with properties and view - const baseId = await createBase(workspaceId, spaceId, creatorId); - - // Load the created properties for cell generation - const properties = await db - .selectFrom('base_properties') - .selectAll() - .where('base_id', '=', baseId) - .execute(); - - const generators: Array<{ propertyId: string; generate: CellGenerator }> = []; - for (const prop of properties) { - const gen = buildCellGenerator(prop); - if (gen) { - generators.push({ propertyId: prop.id, generate: gen }); - } - } - - console.log(`Generating ${TOTAL_ROWS.toLocaleString()} positions...`); - - let lastPosition: string | null = null; - const positions: string[] = new Array(TOTAL_ROWS); - for (let i = 0; i < TOTAL_ROWS; i++) { - lastPosition = generateJitteredKeyBetween(lastPosition, null); - positions[i] = lastPosition; - } - console.log(`Positions generated (last: ${positions[positions.length - 1]})\n`); + console.log(`Creator: ${creatorUserId ?? '(none)'}\n`); const startTime = Date.now(); - const totalBatches = Math.ceil(TOTAL_ROWS / BATCH_SIZE); - - for (let batchStart = 0; batchStart < TOTAL_ROWS; batchStart += BATCH_SIZE) { - const batchEnd = Math.min(batchStart + BATCH_SIZE, TOTAL_ROWS); - const rows: any[] = []; - - for (let i = batchStart; i < batchEnd; i++) { - const cells: Record = {}; - for (const { propertyId, generate } of generators) { - cells[propertyId] = generate(); - } - - rows.push({ - id: uuid7(), - base_id: baseId, - cells, - position: positions[i], - creator_id: creatorId, - workspace_id: workspaceId, - created_at: new Date(), - updated_at: new Date(), - }); - } - - await db.insertInto('base_rows').values(rows).execute(); - - const batchNum = Math.floor(batchStart / BATCH_SIZE) + 1; - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(`Batch ${batchNum}/${totalBatches} inserted (${batchEnd.toLocaleString()} rows, ${elapsed}s elapsed)`); - } - + const { baseId } = await seedBase({ + db, + workspaceId, + spaceId, + creatorUserId, + rows: TOTAL_ROWS, + }); const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(`\nDone. Inserted ${TOTAL_ROWS.toLocaleString()} rows in ${totalElapsed}s`); - console.log(`\nBase ID: ${baseId}`); + + console.log( + `Inserted ${TOTAL_ROWS.toLocaleString()} rows in ${totalElapsed}s`, + ); + console.log(`Base ID: ${baseId}`); await db.destroy(); process.exit(0);