feat(ee): turbopuffer ai vector search driver (#2402)

* feat(ai): add AI_VECTOR_DRIVER and turbopuffer configuration

* feat(ai): carry workspace and target space ids on vector lifecycle events

* feat(ai): add vector driver interface and turbopuffer request helpers

* feat(ai): add pgvector driver behind the vector driver interface

* refactor(ai): route vector reads and writes through the vector driver

* feat(ai): add turbopuffer vector driver

* feat(ai): rebuild turbopuffer namespaces when the embedding model changes

* feat(ai): warm the vector namespace cache on session start

* fix(ai): harden turbopuffer misconfiguration and reset failure paths

* fix(ai): collapse blank-line runs in extracted page text

* fix(ai): skip full re-embed when ai search is re-enabled within the delete grace window

* sync

* fix(ai): store real embedding dimensions instead of serialized vector length

* fix(ai): filter search hits by the page's current space at query time

* fix(ai): retry the page moved-to-space vector patch job

* sync

* feat(ai): pre-warm the vector namespace

* fix(ai): pass AI_VECTOR_DRIVER through the client build config

apps/server/src/integrations/environment/environment.service.ts

@@ -310,6 +310,31 @@ export class EnvironmentService {
     return val === 'true';
   }
 
+  getAiVectorDriver(): string {
+    return this.configService
+      .get<string>('AI_VECTOR_DRIVER', 'pgvector')
+      .toLowerCase();
+  }
+
+  getTurbopufferApiKey(): string {
+    return this.configService.get<string>('TURBOPUFFER_API_KEY');
+  }
+
+  getTurbopufferRegion(): string {
+    return this.configService.get<string>('TURBOPUFFER_REGION');
+  }
+
+  getTurbopufferBaseUrl(): string {
+    return this.configService.get<string>('TURBOPUFFER_BASE_URL');
+  }
+
+  getTurbopufferNamespacePrefix(): string {
+    return this.configService.get<string>(
+      'TURBOPUFFER_NAMESPACE_PREFIX',
+      'docmost',
+    );
+  }
+
   getOpenAiApiKey(): string {
     return this.configService.get<string>('OPENAI_API_KEY');
   }

apps/server/src/integrations/environment/environment.validation.ts

@@ -5,6 +5,7 @@ import {
   IsOptional,
   IsString,
   IsUrl,
+  Matches,
   MinLength,
   ValidateIf,
   validateSync,
@@ -108,6 +109,41 @@ export class EnvironmentVariables {
   @IsString()
   AI_DRIVER: string;
 
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_VECTOR_DRIVER)
+  @IsIn(['pgvector', 'turbopuffer'])
+  @IsString()
+  AI_VECTOR_DRIVER: string;
+
+  @ValidateIf((obj) => obj.AI_VECTOR_DRIVER === 'turbopuffer')
+  @IsNotEmpty()
+  @IsString()
+  TURBOPUFFER_API_KEY: string;
+
+  @ValidateIf(
+    (obj) =>
+      obj.AI_VECTOR_DRIVER === 'turbopuffer' && !obj.TURBOPUFFER_BASE_URL,
+  )
+  @IsNotEmpty({
+    message:
+      'TURBOPUFFER_REGION is required when AI_VECTOR_DRIVER is turbopuffer, unless TURBOPUFFER_BASE_URL is set',
+  })
+  @IsString()
+  TURBOPUFFER_REGION: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.TURBOPUFFER_BASE_URL != '' && obj.TURBOPUFFER_BASE_URL != null)
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  TURBOPUFFER_BASE_URL: string;
+
+  @IsOptional()
+  @IsString()
+  @Matches(/^[A-Za-z0-9\-_.]{1,90}$/, {
+    message:
+      'TURBOPUFFER_NAMESPACE_PREFIX may only contain letters, digits, dot, dash, underscore (max 90 chars)',
+  })
+  TURBOPUFFER_NAMESPACE_PREFIX: string;
+
   @IsOptional()
   @IsString()
   AI_EMBEDDING_MODEL: string;

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -61,6 +61,7 @@ export enum QueueJob {
   WORKSPACE_DELETED = 'workspace-deleted',
   WORKSPACE_CREATE_EMBEDDINGS = 'workspace-create-embeddings',
   WORKSPACE_DELETE_EMBEDDINGS = 'workspace-delete-embeddings',
+  WORKSPACE_RESET_EMBEDDINGS = 'workspace-reset-embeddings',
 
   GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
   DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',

apps/server/src/integrations/static/static.module.ts

@@ -49,6 +49,10 @@ export class StaticModule implements OnModuleInit {
           : undefined,
         POSTHOG_HOST: this.environmentService.getPostHogHost(),
         POSTHOG_KEY: this.environmentService.getPostHogKey(),
+        AI_VECTOR_DRIVER:
+          this.environmentService.getAiVectorDriver() === 'turbopuffer'
+            ? 'turbopuffer'
+            : undefined,
       };
 
       const windowScriptContent = `<script>window.CONFIG=${JSON.stringify(configString)};</script>`;