feat(EE): AI vector search (#1691)

* WIP

* AI module - init

* WIP

* sync

* WIP

* refactor naming

* new columns

* sync

* sync

* fix search bug

* stream response

* WIP

* feat embeddings sync

* refine

* Add workspaceId to page events

* refine

* WIP

* add translation string

* sync

* reset ai answer on query change

* hide AI search in cloud

* capture streaming error

* sync

apps/server/src/integrations/environment/environment.service.ts

@@ -10,6 +10,10 @@ export class EnvironmentService {
     return this.configService.get<string>('NODE_ENV', 'development');
   }
 
+  isDevelopment(): boolean {
+    return this.getNodeEnv() === 'development';
+  }
+
   getAppUrl(): string {
     const rawUrl =
       this.configService.get<string>('APP_URL') ||
@@ -231,6 +235,46 @@ export class EnvironmentService {
   }
 
   getTypesenseLocale(): string {
-    return this.configService.get<string>('TYPESENSE_LOCALE', 'en').toLowerCase();
+    return this.configService
+      .get<string>('TYPESENSE_LOCALE', 'en')
+      .toLowerCase();
+  }
+
+  getAiDriver(): string {
+    return this.configService.get<string>('AI_DRIVER');
+  }
+
+  getAiEmbeddingModel(): string {
+    return this.configService.get<string>('AI_EMBEDDING_MODEL');
+  }
+
+  getAiCompletionModel(): string {
+    return this.configService.get<string>('AI_COMPLETION_MODEL');
+  }
+
+  getAiEmbeddingDimension(): number {
+    return parseInt(
+      this.configService.get<string>('AI_EMBEDDING_DIMENSION'),
+      10,
+    );
+  }
+
+  getOpenAiApiKey(): string {
+    return this.configService.get<string>('OPENAI_API_KEY');
+  }
+
+  getOpenAiApiUrl(): string {
+    return this.configService.get<string>('OPENAI_API_URL');
+  }
+
+  getGeminiApiKey(): string {
+    return this.configService.get<string>('GEMINI_API_KEY');
+  }
+
+  getOllamaApiUrl(): string {
+    return this.configService.get<string>(
+      'OLLAMA_API_URL',
+      'http://localhost:11434',
+    );
   }
 }

apps/server/src/integrations/environment/environment.validation.ts

@@ -93,6 +93,7 @@ export class EnvironmentVariables {
 
   @IsOptional()
   @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsNotEmpty()
   @IsString()
   TYPESENSE_API_KEY: string;
 
@@ -101,6 +102,53 @@ export class EnvironmentVariables {
   @IsISO6391()
   @IsString()
   TYPESENSE_LOCALE: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsIn(['openai', 'gemini', 'ollama'])
+  @IsString()
+  AI_DRIVER: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsString()
+  @IsNotEmpty()
+  AI_EMBEDDING_MODEL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_EMBEDDING_DIMENSION)
+  @IsIn(['768', '1024', '1536'])
+  @IsString()
+  AI_EMBEDDING_DIMENSION: string;
+
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsString()
+  @IsNotEmpty()
+  AI_COMPLETION_MODEL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'openai')
+  @IsString()
+  @IsNotEmpty()
+  OPENAI_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.OPENAI_API_URL && obj.AI_DRIVER === 'openai')
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  OPENAI_API_URL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'gemini')
+  @IsString()
+  @IsNotEmpty()
+  GEMINI_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'ollama')
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  OLLAMA_API_URL: string;
 }
 
 export function validate(config: Record<string, any>) {

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -473,6 +473,7 @@ export class FileImportTaskService {
         if (validPageIds.size > 0) {
           this.eventEmitter.emit(EventName.PAGE_CREATED, {
             pageIds: Array.from(validPageIds),
+            workspaceId: fileTask.workspaceId,
           });
         }
 

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -5,6 +5,7 @@ export enum QueueName {
   BILLING_QUEUE = '{billing-queue}',
   FILE_TASK_QUEUE = '{file-task-queue}',
   SEARCH_QUEUE = '{search-queue}',
+  AI_QUEUE = '{ai-queue}',
 }
 
 export enum QueueJob {
@@ -13,7 +14,6 @@ export enum QueueJob {
   ATTACHMENT_INDEX_CONTENT = 'attachment-index-content',
   ATTACHMENT_INDEXING = 'attachment-indexing',
   DELETE_PAGE_ATTACHMENTS = 'delete-page-attachments',
-  PAGE_CONTENT_UPDATE = 'page-content-update',
 
   DELETE_USER_AVATARS = 'delete-user-avatars',
 
@@ -39,8 +39,23 @@ export enum QueueJob {
   TYPESENSE_FLUSH = 'typesense-flush',
 
   PAGE_CREATED = 'page-created',
+  PAGE_CONTENT_UPDATED = 'page-content-updated',
+  PAGE_MOVED_TO_SPACE = 'page-moved-to-space',
   PAGE_UPDATED = 'page-updated',
   PAGE_SOFT_DELETED = 'page-soft-deleted',
   PAGE_RESTORED = 'page-restored',
   PAGE_DELETED = 'page-deleted',
+
+  SPACE_CREATED = 'space-created',
+  SPACE_UPDATED = 'space-updated',
+  SPACE_DELETED = 'space-deleted',
+
+  WORKSPACE_CREATED = 'workspace-created',
+  WORKSPACE_SPACE_UPDATED = 'workspace-updated',
+  WORKSPACE_DELETED = 'workspace-deleted',
+  WORKSPACE_CREATE_EMBEDDINGS = 'workspace-create-embeddings',
+  WORKSPACE_DELETE_EMBEDDINGS = 'workspace-delete-embeddings',
+
+  GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
+  DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',
 }

apps/server/src/integrations/queue/queue.module.ts

@@ -65,6 +65,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
         attempts: 2,
       },
     }),
+    BullModule.registerQueue({
+      name: QueueName.AI_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: 1,
+      },
+    }),
   ],
   exports: [BullModule],
   providers: [BacklinksProcessor],