diff --git a/apps/client/src/ee/ai/services/ai-search-service.ts b/apps/client/src/ee/ai/services/ai-search-service.ts index 8c2af64a8..d580a2090 100644 --- a/apps/client/src/ee/ai/services/ai-search-service.ts +++ b/apps/client/src/ee/ai/services/ai-search-service.ts @@ -15,6 +15,14 @@ export interface IAiSearchResponse { }>; } +export async function hintVectorCache(): Promise { + try { + await api.post("/ai/vector-cache-hint"); + } catch { + // best-effort cache hint + } +} + export async function aiAnswers( params: IPageSearchParams, onChunk?: (chunk: { content?: string; sources?: any[] }) => void, diff --git a/apps/client/src/features/search/components/search-spotlight.tsx b/apps/client/src/features/search/components/search-spotlight.tsx index 4c5269f15..3ada7ab1f 100644 --- a/apps/client/src/features/search/components/search-spotlight.tsx +++ b/apps/client/src/features/search/components/search-spotlight.tsx @@ -13,11 +13,16 @@ import { SearchResultItem } from "./search-result-item.tsx"; import { AiSearchResult } from "../../../ee/ai/components/ai-search-result.tsx"; import { useHasFeature } from "@/ee/hooks/use-feature"; import { Feature } from "@/ee/features"; +import { useAtomValue } from "jotai"; +import { workspaceAtom } from "@/features/user/atoms/current-user-atom.ts"; +import { hintVectorCache } from "@/ee/ai/services/ai-search-service.ts"; +import { getAiVectorDriver } from "@/lib/config.ts"; interface SearchSpotlightProps { spaceId?: string; } export function SearchSpotlight({ spaceId }: SearchSpotlightProps) { + const workspace = useAtomValue(workspaceAtom); const { t } = useTranslation(); const hasAiFeature = useHasFeature(Feature.AI); const hasAttachmentIndexing = useHasFeature(Feature.ATTACHMENT_INDEXING); @@ -96,6 +101,15 @@ export function SearchSpotlight({ spaceId }: SearchSpotlightProps) { /> )); + const handleSpotlightOpen = () => { + if ( + workspace?.settings?.ai?.search === true && + getAiVectorDriver() === "turbopuffer" + ) { + hintVectorCache(); + } + }; + const handleFiltersChange = (newFilters: any) => { setFilters(newFilters); }; @@ -115,6 +129,7 @@ export function SearchSpotlight({ spaceId }: SearchSpotlightProps) { { BILLING_TRIAL_DAYS, POSTHOG_HOST, POSTHOG_KEY, + AI_VECTOR_DRIVER, } = loadEnv(mode, envPath, ""); return { @@ -31,6 +32,7 @@ export default defineConfig(({ mode }) => { BILLING_TRIAL_DAYS, POSTHOG_HOST, POSTHOG_KEY, + AI_VECTOR_DRIVER, }, APP_VERSION: JSON.stringify(process.env.npm_package_version), }, diff --git a/apps/server/package.json b/apps/server/package.json index cb32657a7..3e5ae9592 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -66,6 +66,7 @@ "@nestjs/websockets": "11.1.28", "@node-saml/passport-saml": "5.1.0", "@socket.io/redis-adapter": "8.3.0", + "@turbopuffer/turbopuffer": "^2.8.0", "ai": "6.0.134", "ai-sdk-ollama": "3.8.1", "bcrypt": "6.0.0", diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index ed182cf7f..e5c55c77a 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -57,6 +57,7 @@ import { JSONContent, } from '@tiptap/core'; import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; +import { collapseBlankLines } from '../common/helpers'; // @tiptap/html library works best for generating prosemirror json state but not HTML // see: https://github.com/ueberdosis/tiptap/issues/5352 // see:https://github.com/ueberdosis/tiptap/issues/4089 @@ -146,7 +147,7 @@ export function htmlToJson(html: string) { } export function jsonToText(tiptapJson: JSONContent) { - return generateText(tiptapJson, tiptapExtensions); + return collapseBlankLines(generateText(tiptapJson, tiptapExtensions)); } export function jsonToNode(tiptapJson: JSONContent) { diff --git a/apps/server/src/common/helpers/index.ts b/apps/server/src/common/helpers/index.ts index 80e9a9028..c9fac94c5 100644 --- a/apps/server/src/common/helpers/index.ts +++ b/apps/server/src/common/helpers/index.ts @@ -1,4 +1,5 @@ export * from './utils'; +export * from './text.utils'; export * from './nanoid.utils'; export * from './file.helper'; export * from './constants'; diff --git a/apps/server/src/common/helpers/text.utils.spec.ts b/apps/server/src/common/helpers/text.utils.spec.ts new file mode 100644 index 000000000..6d049a541 --- /dev/null +++ b/apps/server/src/common/helpers/text.utils.spec.ts @@ -0,0 +1,14 @@ +import { collapseBlankLines } from './text.utils'; + +describe('collapseBlankLines', () => { + it.each([ + ['a\n\n\n\nb', 'a\n\nb'], + ['a\n\nb', 'a\n\nb'], + ['a\nb', 'a\nb'], + ['\n\n\n\na\n\n\n', '\n\na\n\n'], + ['no newlines', 'no newlines'], + ['', ''], + ])('collapses %j to %j', (input, expected) => { + expect(collapseBlankLines(input)).toBe(expected); + }); +}); diff --git a/apps/server/src/common/helpers/text.utils.ts b/apps/server/src/common/helpers/text.utils.ts new file mode 100644 index 000000000..8a9cd1a27 --- /dev/null +++ b/apps/server/src/common/helpers/text.utils.ts @@ -0,0 +1,3 @@ +export function collapseBlankLines(text: string): string { + return text.replace(/\n{2,}/g, '\n\n'); +} diff --git a/apps/server/src/core/page/services/page.service.ts b/apps/server/src/core/page/services/page.service.ts index 387abc5cc..cabfb0c74 100644 --- a/apps/server/src/core/page/services/page.service.ts +++ b/apps/server/src/core/page/services/page.service.ts @@ -496,10 +496,21 @@ export class PageService { }, ); - await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, { - pageIds: pageIdsToMove, - workspaceId: rootPage.workspaceId, - }); + await this.aiQueue.add( + QueueJob.PAGE_MOVED_TO_SPACE, + { + pageIds: pageIdsToMove, + spaceId, + workspaceId: rootPage.workspaceId, + }, + { + attempts: 2, + backoff: { + type: 'fixed', + delay: 2 * 60 * 1000, + }, + }, + ); } }); diff --git a/apps/server/src/database/listeners/space.listener.ts b/apps/server/src/database/listeners/space.listener.ts index af6e3a222..3a766b6ef 100644 --- a/apps/server/src/database/listeners/space.listener.ts +++ b/apps/server/src/database/listeners/space.listener.ts @@ -8,6 +8,7 @@ import { EnvironmentService } from '../../integrations/environment/environment.s export class SpaceEvent { spaceId: string; + workspaceId: string; } @Injectable() @@ -22,12 +23,12 @@ export class SpaceListener { @OnEvent(EventName.SPACE_DELETED) async handleSpaceDeleted(event: SpaceEvent) { - const { spaceId } = event; + const { spaceId, workspaceId } = event; if (this.isTypesense()) { await this.searchQueue.add(QueueJob.SPACE_DELETED, { spaceId }); } - await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId }); + await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId, workspaceId }); } isTypesense(): boolean { diff --git a/apps/server/src/database/repos/space/space.repo.ts b/apps/server/src/database/repos/space/space.repo.ts index 905d2fac6..82fc170bd 100644 --- a/apps/server/src/database/repos/space/space.repo.ts +++ b/apps/server/src/database/repos/space/space.repo.ts @@ -230,6 +230,7 @@ export class SpaceRepo { this.eventEmitter.emit(EventName.SPACE_DELETED, { spaceId, + workspaceId, }); } } diff --git a/apps/server/src/database/repos/workspace/workspace.repo.ts b/apps/server/src/database/repos/workspace/workspace.repo.ts index d0f9b5cee..a8323d7ce 100644 --- a/apps/server/src/database/repos/workspace/workspace.repo.ts +++ b/apps/server/src/database/repos/workspace/workspace.repo.ts @@ -211,6 +211,24 @@ export class WorkspaceRepo { .executeTakeFirst(); } + async updateAiEmbeddingFingerprint( + workspaceId: string, + fingerprint: { driver: string; model: string; dimensions: number }, + trx?: KyselyTransaction, + ) { + const db = dbOrTx(this.db, trx); + return db + .updateTable('workspaces') + .set({ + settings: sql`COALESCE(settings, '{}'::jsonb) + || jsonb_build_object('ai', COALESCE(settings->'ai', '{}'::jsonb) + || jsonb_build_object('embedding', ${JSON.stringify(fingerprint)}::text::jsonb))`, + updatedAt: new Date(), + }) + .where('id', '=', workspaceId) + .execute(); + } + async updateSharingSettings( workspaceId: string, prefKey: string, diff --git a/apps/server/src/ee b/apps/server/src/ee index 22a5f200d..660418ac2 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit 22a5f200dfb7d042e88c6a1397568da5bdf9477b +Subproject commit 660418ac2c175b37b2491ab3cd2560c2fd406412 diff --git a/apps/server/src/integrations/environment/environment.service.ts b/apps/server/src/integrations/environment/environment.service.ts index 5667bf5a6..c483df626 100644 --- a/apps/server/src/integrations/environment/environment.service.ts +++ b/apps/server/src/integrations/environment/environment.service.ts @@ -310,6 +310,31 @@ export class EnvironmentService { return val === 'true'; } + getAiVectorDriver(): string { + return this.configService + .get('AI_VECTOR_DRIVER', 'pgvector') + .toLowerCase(); + } + + getTurbopufferApiKey(): string { + return this.configService.get('TURBOPUFFER_API_KEY'); + } + + getTurbopufferRegion(): string { + return this.configService.get('TURBOPUFFER_REGION'); + } + + getTurbopufferBaseUrl(): string { + return this.configService.get('TURBOPUFFER_BASE_URL'); + } + + getTurbopufferNamespacePrefix(): string { + return this.configService.get( + 'TURBOPUFFER_NAMESPACE_PREFIX', + 'docmost', + ); + } + getOpenAiApiKey(): string { return this.configService.get('OPENAI_API_KEY'); } diff --git a/apps/server/src/integrations/environment/environment.validation.ts b/apps/server/src/integrations/environment/environment.validation.ts index ef3c420cf..d431d7e3a 100644 --- a/apps/server/src/integrations/environment/environment.validation.ts +++ b/apps/server/src/integrations/environment/environment.validation.ts @@ -5,6 +5,7 @@ import { IsOptional, IsString, IsUrl, + Matches, MinLength, ValidateIf, validateSync, @@ -108,6 +109,41 @@ export class EnvironmentVariables { @IsString() AI_DRIVER: string; + @IsOptional() + @ValidateIf((obj) => obj.AI_VECTOR_DRIVER) + @IsIn(['pgvector', 'turbopuffer']) + @IsString() + AI_VECTOR_DRIVER: string; + + @ValidateIf((obj) => obj.AI_VECTOR_DRIVER === 'turbopuffer') + @IsNotEmpty() + @IsString() + TURBOPUFFER_API_KEY: string; + + @ValidateIf( + (obj) => + obj.AI_VECTOR_DRIVER === 'turbopuffer' && !obj.TURBOPUFFER_BASE_URL, + ) + @IsNotEmpty({ + message: + 'TURBOPUFFER_REGION is required when AI_VECTOR_DRIVER is turbopuffer, unless TURBOPUFFER_BASE_URL is set', + }) + @IsString() + TURBOPUFFER_REGION: string; + + @IsOptional() + @ValidateIf((obj) => obj.TURBOPUFFER_BASE_URL != '' && obj.TURBOPUFFER_BASE_URL != null) + @IsUrl({ protocols: ['http', 'https'], require_tld: false }) + TURBOPUFFER_BASE_URL: string; + + @IsOptional() + @IsString() + @Matches(/^[A-Za-z0-9\-_.]{1,90}$/, { + message: + 'TURBOPUFFER_NAMESPACE_PREFIX may only contain letters, digits, dot, dash, underscore (max 90 chars)', + }) + TURBOPUFFER_NAMESPACE_PREFIX: string; + @IsOptional() @IsString() AI_EMBEDDING_MODEL: string; diff --git a/apps/server/src/integrations/queue/constants/queue.constants.ts b/apps/server/src/integrations/queue/constants/queue.constants.ts index 546553438..8b7c03a1f 100644 --- a/apps/server/src/integrations/queue/constants/queue.constants.ts +++ b/apps/server/src/integrations/queue/constants/queue.constants.ts @@ -61,6 +61,7 @@ export enum QueueJob { WORKSPACE_DELETED = 'workspace-deleted', WORKSPACE_CREATE_EMBEDDINGS = 'workspace-create-embeddings', WORKSPACE_DELETE_EMBEDDINGS = 'workspace-delete-embeddings', + WORKSPACE_RESET_EMBEDDINGS = 'workspace-reset-embeddings', GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings', DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings', diff --git a/apps/server/src/integrations/static/static.module.ts b/apps/server/src/integrations/static/static.module.ts index f0b7e8319..8805804f3 100644 --- a/apps/server/src/integrations/static/static.module.ts +++ b/apps/server/src/integrations/static/static.module.ts @@ -49,6 +49,10 @@ export class StaticModule implements OnModuleInit { : undefined, POSTHOG_HOST: this.environmentService.getPostHogHost(), POSTHOG_KEY: this.environmentService.getPostHogKey(), + AI_VECTOR_DRIVER: + this.environmentService.getAiVectorDriver() === 'turbopuffer' + ? 'turbopuffer' + : undefined, }; const windowScriptContent = ``; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e26d0a45d..3a6ca01a7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -593,6 +593,9 @@ importers: '@socket.io/redis-adapter': specifier: 8.3.0 version: 8.3.0(socket.io-adapter@2.5.4(supports-color@10.2.2))(supports-color@10.2.2) + '@turbopuffer/turbopuffer': + specifier: ^2.8.0 + version: 2.8.0 ai: specifier: 6.0.134 version: 6.0.134(zod@4.3.6) @@ -4382,6 +4385,9 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + '@turbopuffer/turbopuffer@2.8.0': + resolution: {integrity: sha512-lurmVByLynuWtPtVl7SBEJ0bNakb8bs4KsafKzeLWnXA6BsQiGS9obT/7ExQVb9/yecyRn/grj8VgwUzi2xMyg==} + '@tybys/wasm-util@0.10.2': resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} @@ -5698,6 +5704,7 @@ packages: cron-parser@4.9.0: resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} engines: {node: '>=12.0.0'} + deprecated: v4 is no longer maintained, upgrade to v5 cron@4.4.0: resolution: {integrity: sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==} @@ -8168,6 +8175,9 @@ packages: pako@2.0.3: resolution: {integrity: sha512-WjR1hOeg+kki3ZIOjaf4b5WVcay1jaliKSYiEaB1XzwhMQZJxRdQRv0V31EKBYlxb4T7SK3hjfc/jxyU64BoSw==} + pako@2.2.0: + resolution: {integrity: sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -14341,6 +14351,11 @@ snapshots: '@tsconfig/node16@1.0.4': {} + '@turbopuffer/turbopuffer@2.8.0': + dependencies: + pako: 2.2.0 + undici: 7.29.0 + '@tybys/wasm-util@0.10.2': dependencies: tslib: 2.8.1 @@ -18749,6 +18764,8 @@ snapshots: pako@2.0.3: {} + pako@2.2.0: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0