feat(EE): AI vector search (#1691)

* WIP

* AI module - init

* WIP

* sync

* WIP

* refactor naming

* new columns

* sync

* sync

* fix search bug

* stream response

* WIP

* feat embeddings sync

* refine

* Add workspaceId to page events

* refine

* WIP

* add translation string

* sync

* reset ai answer on query change

* hide AI search in cloud

* capture streaming error

* sync

apps/server/package.json

@@ -30,6 +30,9 @@
     "test:e2e": "jest --config test/jest-e2e.json"
   },
   "dependencies": {
+    "@ai-sdk/azure": "^2.0.47",
+    "@ai-sdk/google": "^2.0.18",
+    "@ai-sdk/openai": "^2.0.46",
     "@aws-sdk/client-s3": "3.701.0",
     "@aws-sdk/lib-storage": "3.701.0",
     "@aws-sdk/s3-request-presigner": "3.701.0",
@@ -37,6 +40,7 @@
     "@fastify/cookie": "^11.0.2",
     "@fastify/multipart": "^9.0.3",
     "@fastify/static": "^8.2.0",
+    "@langchain/textsplitters": "^0.1.0",
     "@nestjs-labs/nestjs-ioredis": "^11.0.4",
     "@nestjs/bullmq": "^11.0.4",
     "@nestjs/common": "^11.1.9",
@@ -55,6 +59,8 @@
     "@react-email/components": "0.0.28",
     "@react-email/render": "1.0.2",
     "@socket.io/redis-adapter": "^8.3.0",
+    "ai": "^5.0.65",
+    "ai-sdk-ollama": "^0.12.0",
     "bcrypt": "^6.0.0",
     "bullmq": "^5.65.0",
     "cache-manager": "^6.4.3",
@@ -82,6 +88,7 @@
     "pdfjs-dist": "^5.4.394",
     "pg": "^8.16.3",
     "pg-tsquery": "^8.4.2",
+    "pgvector": "^0.2.1",
     "postmark": "^4.0.5",
     "react": "^18.3.1",
     "reflect-metadata": "^0.2.2",

apps/server/src/collaboration/extensions/persistence.extension.ts

@@ -35,6 +35,7 @@ export class PersistenceExtension implements Extension {
     @InjectKysely() private readonly db: KyselyDB,
     private eventEmitter: EventEmitter2,
     @InjectQueue(QueueName.GENERAL_QUEUE) private generalQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
   ) {}
 
   async onLoadDocument(data: onLoadDocumentPayload) {
@@ -168,6 +169,11 @@ export class PersistenceExtension implements Extension {
         workspaceId: page.workspaceId,
         mentions: pageMentions,
       } as IPageBacklinkJob);
+
+      await this.aiQueue.add(QueueJob.PAGE_CONTENT_UPDATED, {
+        pageIds: [pageId],
+        workspaceId: page.workspaceId,
+      });
     }
   }
 

apps/server/src/common/events/event.contants.ts

@@ -2,7 +2,17 @@ export enum EventName {
   COLLAB_PAGE_UPDATED = 'collab.page.updated',
   PAGE_CREATED = 'page.created',
   PAGE_UPDATED = 'page.updated',
+  PAGE_CONTENT_UPDATED = 'page-content-updated',
+  PAGE_MOVED_TO_SPACE = 'page-moved-to-space',
   PAGE_DELETED = 'page.deleted',
   PAGE_SOFT_DELETED = 'page.soft_deleted',
   PAGE_RESTORED = 'page.restored',
+
+  SPACE_CREATED = 'space.created',
+  SPACE_UPDATED = 'space.updated',
+  SPACE_DELETED = 'space.deleted',
+
+  WORKSPACE_CREATED = 'workspace.created',
+  WORKSPACE_UPDATED = 'workspace.updated',
+  WORKSPACE_DELETED = 'workspace.deleted',
 }

apps/server/src/core/page/page.controller.ts

@@ -1,23 +1,23 @@
 import {
-  Controller,
-  Post,
+  BadRequestException,
   Body,
+  Controller,
+  ForbiddenException,
   HttpCode,
   HttpStatus,
-  UseGuards,
-  ForbiddenException,
   NotFoundException,
-  BadRequestException,
+  Post,
+  UseGuards,
 } from '@nestjs/common';
 import { PageService } from './services/page.service';
 import { CreatePageDto } from './dto/create-page.dto';
 import { UpdatePageDto } from './dto/update-page.dto';
 import { MovePageDto, MovePageToSpaceDto } from './dto/move-page.dto';
 import {
+  DeletePageDto,
   PageHistoryIdDto,
   PageIdDto,
   PageInfoDto,
-  DeletePageDto,
 } from './dto/page.dto';
 import { PageHistoryService } from './services/page-history.service';
 import { AuthUser } from '../../common/decorators/auth-user.decorator';
@@ -106,7 +106,11 @@ export class PageController {
 
   @HttpCode(HttpStatus.OK)
   @Post('delete')
-  async delete(@Body() deletePageDto: DeletePageDto, @AuthUser() user: User) {
+  async delete(
+    @Body() deletePageDto: DeletePageDto,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
     const page = await this.pageRepo.findById(deletePageDto.pageId);
 
     if (!page) {
@@ -122,19 +126,27 @@ export class PageController {
           'Only space admins can permanently delete pages',
         );
       }
-      await this.pageService.forceDelete(deletePageDto.pageId);
+      await this.pageService.forceDelete(deletePageDto.pageId, workspace.id);
     } else {
       // Soft delete requires page manage permissions
       if (ability.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Page)) {
         throw new ForbiddenException();
       }
-      await this.pageService.remove(deletePageDto.pageId, user.id);
+      await this.pageService.removePage(
+        deletePageDto.pageId,
+        user.id,
+        workspace.id,
+      );
     }
   }
 
   @HttpCode(HttpStatus.OK)
   @Post('restore')
-  async restore(@Body() pageIdDto: PageIdDto, @AuthUser() user: User) {
+  async restore(
+    @Body() pageIdDto: PageIdDto,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
     const page = await this.pageRepo.findById(pageIdDto.pageId);
 
     if (!page) {
@@ -146,13 +158,11 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    await this.pageRepo.restorePage(pageIdDto.pageId);
+    await this.pageRepo.restorePage(pageIdDto.pageId, workspace.id);
 
-    // Return the restored page data with hasChildren info
-    const restoredPage = await this.pageRepo.findById(pageIdDto.pageId, {
+    return this.pageRepo.findById(pageIdDto.pageId, {
       includeHasChildren: true,
     });
-    return restoredPage;
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/page/services/page.service.ts

@@ -51,6 +51,7 @@ export class PageService {
     @InjectKysely() private readonly db: KyselyDB,
     private readonly storageService: StorageService,
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
     private eventEmitter: EventEmitter2,
   ) {}
 
@@ -255,6 +256,11 @@ export class PageService {
           pageIds,
           trx,
         );
+
+        await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
+          pageId: pageIds,
+          workspaceId: rootPage.workspaceId
+        });
       }
     });
   }
@@ -393,6 +399,7 @@ export class PageService {
     const insertedPageIds = insertablePages.map((page) => page.id);
     this.eventEmitter.emit(EventName.PAGE_CREATED, {
       pageIds: insertedPageIds,
+      workspaceId: authUser.workspaceId,
     });
 
     //TODO: best to handle this in a queue
@@ -580,7 +587,7 @@ export class PageService {
     return await this.pageRepo.getDeletedPagesInSpace(spaceId, pagination);
   }
 
-  async forceDelete(pageId: string): Promise<void> {
+  async forceDelete(pageId: string, workspaceId: string): Promise<void> {
     // Get all descendant IDs (including the page itself) using recursive CTE
     const descendants = await this.db
       .withRecursive('page_descendants', (db) =>
@@ -623,11 +630,16 @@ export class PageService {
       await this.db.deleteFrom('pages').where('id', 'in', pageIds).execute();
       this.eventEmitter.emit(EventName.PAGE_DELETED, {
         pageIds: pageIds,
+        workspaceId,
       });
     }
   }
 
-  async remove(pageId: string, userId: string): Promise<void> {
-    await this.pageRepo.removePage(pageId, userId);
+  async removePage(
+    pageId: string,
+    userId: string,
+    workspaceId: string,
+  ): Promise<void> {
+    await this.pageRepo.removePage(pageId, userId, workspaceId);
   }
 }

apps/server/src/core/search/search.service.ts

@@ -62,7 +62,7 @@ export class SearchService {
       )
       .where('deletedAt', 'is', null)
       .orderBy('rank', 'desc')
-      .limit(searchParams.limit | 20)
+      .limit(searchParams.limit | 25)
       .offset(searchParams.offset || 0);
 
     if (!searchParams.shareId) {

apps/server/src/core/workspace/dto/update-workspace.dto.ts

@@ -22,4 +22,12 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
   @IsOptional()
   @IsBoolean()
   restrictApiToAdmins: boolean;
+
+  @IsOptional()
+  @IsBoolean()
+  aiSearch: boolean;
+
+  @IsOptional()
+  @IsBoolean()
+  generativeAi: boolean;
 }

apps/server/src/core/workspace/services/workspace.service.ts

@@ -33,6 +33,7 @@ import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { Queue } from 'bullmq';
 import { generateRandomSuffixNumbers } from '../../../common/helpers';
+import { isPageEmbeddingsTableExists } from '@docmost/db/helpers/helpers';
 
 @Injectable()
 export class WorkspaceService {
@@ -50,6 +51,7 @@ export class WorkspaceService {
     @InjectKysely() private readonly db: KyselyDB,
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
     @InjectQueue(QueueName.BILLING_QUEUE) private billingQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
   ) {}
 
   async findById(workspaceId: string) {
@@ -312,6 +314,51 @@ export class WorkspaceService {
       delete updateWorkspaceDto.restrictApiToAdmins;
     }
 
+    if (typeof updateWorkspaceDto.aiSearch !== 'undefined') {
+      await this.workspaceRepo.updateAiSettings(
+        workspaceId,
+        'search',
+        updateWorkspaceDto.aiSearch,
+      );
+
+      if (updateWorkspaceDto.aiSearch) {
+        const tableExists = await isPageEmbeddingsTableExists(this.db);
+        if (!tableExists) {
+          throw new BadRequestException(
+            'Failed to activate. Make sure pgvector postgres extension is installed.',
+          );
+        }
+
+        await this.aiQueue.add(QueueJob.WORKSPACE_CREATE_EMBEDDINGS, {
+          workspaceId,
+        });
+      } else {
+        // Schedule deletion after 24 hours
+        const deleteJobId = `ai-search-disabled-${workspaceId}`;
+        await this.aiQueue.add(
+          QueueJob.WORKSPACE_DELETE_EMBEDDINGS,
+          { workspaceId },
+          {
+            jobId: deleteJobId,
+            delay: 24 * 60 * 60 * 1000,
+            removeOnComplete: true,
+            removeOnFail: true,
+          },
+        );
+      }
+
+      delete updateWorkspaceDto.aiSearch;
+    }
+
+    if (typeof updateWorkspaceDto.generativeAi !== 'undefined') {
+      await this.workspaceRepo.updateAiSettings(
+        workspaceId,
+        'generative',
+        updateWorkspaceDto.generativeAi,
+      );
+      delete updateWorkspaceDto.generativeAi;
+    }
+
     await this.workspaceRepo.updateWorkspace(updateWorkspaceDto, workspaceId);
 
     const workspace = await this.workspaceRepo.findById(workspaceId, {

apps/server/src/database/helpers/helpers.ts

@@ -0,0 +1,22 @@
+import { sql } from 'kysely';
+import { KyselyDB } from '@docmost/db/types/kysely.types';
+
+export async function isPageEmbeddingsTableExists(db: KyselyDB) {
+  return tableExists({ db, tableName: 'page_embeddings' });
+}
+
+export async function tableExists(opts: {
+  db: KyselyDB;
+  tableName: string;
+}): Promise<boolean> {
+  const { db, tableName } = opts;
+  const result = await sql<{ exists: boolean }>`
+      SELECT EXISTS (
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = COALESCE(current_schema(), 'public')
+        AND table_name = ${tableName}
+      ) as exists
+    `.execute(db);
+
+  return result.rows[0]?.exists ?? false;
+}

apps/server/src/database/listeners/page.listener.ts

@@ -4,9 +4,11 @@ import { EventName } from '../../common/events/event.contants';
 import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../integrations/queue/constants';
 import { Queue } from 'bullmq';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
 
 export class PageEvent {
   pageIds: string[];
+  workspaceId: string;
 }
 
 @Injectable()
@@ -14,36 +16,65 @@ export class PageListener {
   private readonly logger = new Logger(PageListener.name);
 
   constructor(
+    private readonly environmentService: EnvironmentService,
     @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
   ) {}
 
   @OnEvent(EventName.PAGE_CREATED)
   async handlePageCreated(event: PageEvent) {
-    const { pageIds } = event;
-    await this.searchQueue.add(QueueJob.PAGE_CREATED, { pageIds });
+    const { pageIds, workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_CREATED, {
+        pageIds,
+      });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_CREATED, { pageIds, workspaceId });
   }
 
   @OnEvent(EventName.PAGE_UPDATED)
   async handlePageUpdated(event: PageEvent) {
     const { pageIds } = event;
+
     await this.searchQueue.add(QueueJob.PAGE_UPDATED, { pageIds });
   }
 
   @OnEvent(EventName.PAGE_DELETED)
   async handlePageDeleted(event: PageEvent) {
-    const { pageIds } = event;
-    await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds });
+    const { pageIds, workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_DELETED, { pageIds, workspaceId });
   }
 
   @OnEvent(EventName.PAGE_SOFT_DELETED)
   async handlePageSoftDeleted(event: PageEvent) {
-    const { pageIds } = event;
-    await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds });
+    const { pageIds, workspaceId } = event;
+
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_SOFT_DELETED, {
+      pageIds,
+      workspaceId,
+    });
   }
 
   @OnEvent(EventName.PAGE_RESTORED)
   async handlePageRestored(event: PageEvent) {
-    const { pageIds } = event;
-    await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds });
+    const { pageIds, workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_RESTORED, { pageIds, workspaceId });
+  }
+
+  isTypesense(): boolean {
+    return this.environmentService.getSearchDriver() === 'typesense';
   }
 }

apps/server/src/database/listeners/space.listener.ts

@@ -0,0 +1,36 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+import { EventName } from '../../common/events/event.contants';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../integrations/queue/constants';
+import { Queue } from 'bullmq';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+
+export class SpaceEvent {
+  spaceId: string;
+}
+
+@Injectable()
+export class SpaceListener {
+  private readonly logger = new Logger(SpaceListener.name);
+
+  constructor(
+    private readonly environmentService: EnvironmentService,
+    @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
+  ) {}
+
+  @OnEvent(EventName.SPACE_DELETED)
+  async handleSpaceDeleted(event: SpaceEvent) {
+    const { spaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.SPACE_DELETED, { spaceId });
+    }
+
+    await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId });
+  }
+
+  isTypesense(): boolean {
+    return this.environmentService.getSearchDriver() === 'typesense';
+  }
+}

apps/server/src/database/listeners/workspace.listener.ts

@@ -0,0 +1,36 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+import { EventName } from '../../common/events/event.contants';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../integrations/queue/constants';
+import { Queue } from 'bullmq';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+
+export class WorkspaceEvent {
+  workspaceId: string;
+}
+
+@Injectable()
+export class WorkspaceListener {
+  private readonly logger = new Logger(WorkspaceListener.name);
+
+  constructor(
+    private readonly environmentService: EnvironmentService,
+    @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
+  ) {}
+
+  @OnEvent(EventName.WORKSPACE_DELETED)
+  async handlePageDeleted(event: WorkspaceEvent) {
+    const { workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.WORKSPACE_DELETED, { workspaceId });
+    }
+
+    await this.aiQueue.add(QueueJob.WORKSPACE_DELETED, { workspaceId });
+  }
+
+  isTypesense(): boolean {
+    return this.environmentService.getSearchDriver() === 'typesense';
+  }
+}

apps/server/src/database/repos/page/page.repo.ts

@@ -125,6 +125,7 @@ export class PageRepo {
 
     this.eventEmitter.emit(EventName.PAGE_UPDATED, {
       pageIds: pageIds,
+      workspaceId: updatePageData.workspaceId,
     });
 
     return result;
@@ -143,6 +144,7 @@ export class PageRepo {
 
     this.eventEmitter.emit(EventName.PAGE_CREATED, {
       pageIds: [result.id],
+      workspaceId: result.workspaceId,
     });
 
     return result;
@@ -160,7 +162,11 @@ export class PageRepo {
     await query.execute();
   }
 
-  async removePage(pageId: string, deletedById: string): Promise<void> {
+  async removePage(
+    pageId: string,
+    deletedById: string,
+    workspaceId: string,
+  ): Promise<void> {
     const currentDate = new Date();
 
     const descendants = await this.db
@@ -195,13 +201,15 @@ export class PageRepo {
 
         await trx.deleteFrom('shares').where('pageId', 'in', pageIds).execute();
       });
+
       this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, {
         pageIds: pageIds,
+        workspaceId,
       });
     }
   }
 
-  async restorePage(pageId: string): Promise<void> {
+  async restorePage(pageId: string, workspaceId: string): Promise<void> {
     // First, check if the page being restored has a deleted parent
     const pageToRestore = await this.db
       .selectFrom('pages')
@@ -263,6 +271,7 @@ export class PageRepo {
     }
     this.eventEmitter.emit(EventName.PAGE_RESTORED, {
       pageIds: pageIds,
+      workspaceId: workspaceId,
     });
   }
 

apps/server/src/database/repos/space/space.repo.ts

@@ -12,10 +12,15 @@ import { PaginationOptions } from '../../pagination/pagination-options';
 import { executeWithPagination } from '@docmost/db/pagination/pagination';
 import { DB } from '@docmost/db/types/db';
 import { validate as isValidUUID } from 'uuid';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
 
 @Injectable()
 export class SpaceRepo {
-  constructor(@InjectKysely() private readonly db: KyselyDB) {}
+  constructor(
+    @InjectKysely() private readonly db: KyselyDB,
+    private eventEmitter: EventEmitter2,
+  ) {}
 
   async findById(
     spaceId: string,
@@ -110,7 +115,11 @@ export class SpaceRepo {
 
     if (pagination.query) {
       query = query.where((eb) =>
-        eb(sql`f_unaccent(name)`, 'ilike', sql`f_unaccent(${'%' + pagination.query + '%'})`).or(
+        eb(
+          sql`f_unaccent(name)`,
+          'ilike',
+          sql`f_unaccent(${'%' + pagination.query + '%'})`,
+        ).or(
           sql`f_unaccent(description)`,
           'ilike',
           sql`f_unaccent(${'%' + pagination.query + '%'})`,
@@ -155,5 +164,9 @@ export class SpaceRepo {
       .where('id', '=', spaceId)
       .where('workspaceId', '=', workspaceId)
       .execute();
+
+    this.eventEmitter.emit(EventName.SPACE_DELETED, {
+      spaceId,
+    });
   }
 }

apps/server/src/database/repos/workspace/workspace.repo.ts

@@ -175,4 +175,22 @@ export class WorkspaceRepo {
       .returning(this.baseFields)
       .executeTakeFirst();
   }
+
+  async updateAiSettings(
+    workspaceId: string,
+    prefKey: string,
+    prefValue: string | boolean,
+  ) {
+    return this.db
+      .updateTable('workspaces')
+      .set({
+        settings: sql`COALESCE(settings, '{}'::jsonb)
+                || jsonb_build_object('ai', COALESCE(settings->'ai', '{}'::jsonb) 
+                || jsonb_build_object('${sql.raw(prefKey)}', ${sql.lit(prefValue)}))`,
+        updatedAt: new Date(),
+      })
+      .where('id', '=', workspaceId)
+      .returning(this.baseFields)
+      .executeTakeFirst();
+  }
 }

apps/server/src/database/types/db.interface.ts

@@ -0,0 +1,47 @@
+import {
+  ApiKeys,
+  Attachments,
+  AuthAccounts,
+  AuthProviders,
+  Backlinks,
+  Billing,
+  Comments,
+  FileTasks,
+  Groups,
+  GroupUsers,
+  PageHistory,
+  Pages,
+  Shares,
+  SpaceMembers,
+  Spaces,
+  UserMfa,
+  Users,
+  UserTokens,
+  WorkspaceInvitations,
+  Workspaces,
+} from '@docmost/db/types/db';
+import { PageEmbeddings } from '@docmost/db/types/embeddings.types';
+
+export interface DbInterface {
+  attachments: Attachments;
+  authAccounts: AuthAccounts;
+  authProviders: AuthProviders;
+  backlinks: Backlinks;
+  billing: Billing;
+  comments: Comments;
+  fileTasks: FileTasks;
+  groups: Groups;
+  groupUsers: GroupUsers;
+  pageEmbeddings: PageEmbeddings;
+  pageHistory: PageHistory;
+  pages: Pages;
+  shares: Shares;
+  spaceMembers: SpaceMembers;
+  spaces: Spaces;
+  userMfa: UserMfa;
+  users: Users;
+  userTokens: UserTokens;
+  workspaceInvitations: WorkspaceInvitations;
+  workspaces: Workspaces;
+  apiKeys: ApiKeys;
+}

apps/server/src/database/types/embeddings.types.ts

@@ -0,0 +1,20 @@
+import { Json, Timestamp, Generated } from '@docmost/db/types/db';
+
+// embeddings type
+export interface PageEmbeddings {
+  id: Generated<string>;
+  pageId: string;
+  spaceId: string;
+  modelName: string;
+  modelDimensions: number;
+  workspaceId: string;
+  attachmentId: string;
+  embedding: number[];
+  chunkIndex: Generated<number>;
+  chunkStart: Generated<number>;
+  chunkLength: Generated<number>;
+  metadata: Generated<Json>;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Generated<Timestamp>;
+  deletedAt: Timestamp | null;
+}

apps/server/src/database/types/entity.types.ts

@@ -21,6 +21,7 @@ import {
   UserMfa as _UserMFA,
   ApiKeys,
 } from './db';
+import { PageEmbeddings } from '@docmost/db/types/embeddings.types';
 
 // Workspace
 export type Workspace = Selectable<Workspaces>;
@@ -125,3 +126,8 @@ export type UpdatableUserMFA = Updateable<Omit<_UserMFA, 'id'>>;
 export type ApiKey = Selectable<ApiKeys>;
 export type InsertableApiKey = Insertable<ApiKeys>;
 export type UpdatableApiKey = Updateable<Omit<ApiKeys, 'id'>>;
+
+// Page Embedding
+export type PageEmbedding = Selectable<PageEmbeddings>;
+export type InsertablePageEmbedding = Insertable<PageEmbeddings>;
+export type UpdatablePageEmbedding = Updateable<Omit<PageEmbeddings, 'id'>>;

apps/server/src/database/types/kysely.types.ts

@@ -1,5 +1,5 @@
-import { DB } from './db';
 import { Kysely, Transaction } from 'kysely';
+import { DbInterface } from '@docmost/db/types/db.interface';
 
-export type KyselyDB = Kysely<DB>;
-export type KyselyTransaction = Transaction<DB>;
+export type KyselyDB = Kysely<DbInterface>;
+export type KyselyTransaction = Transaction<DbInterface>;

apps/server/src/integrations/environment/environment.service.ts

@@ -10,6 +10,10 @@ export class EnvironmentService {
     return this.configService.get<string>('NODE_ENV', 'development');
   }
 
+  isDevelopment(): boolean {
+    return this.getNodeEnv() === 'development';
+  }
+
   getAppUrl(): string {
     const rawUrl =
       this.configService.get<string>('APP_URL') ||
@@ -231,6 +235,46 @@ export class EnvironmentService {
   }
 
   getTypesenseLocale(): string {
-    return this.configService.get<string>('TYPESENSE_LOCALE', 'en').toLowerCase();
+    return this.configService
+      .get<string>('TYPESENSE_LOCALE', 'en')
+      .toLowerCase();
+  }
+
+  getAiDriver(): string {
+    return this.configService.get<string>('AI_DRIVER');
+  }
+
+  getAiEmbeddingModel(): string {
+    return this.configService.get<string>('AI_EMBEDDING_MODEL');
+  }
+
+  getAiCompletionModel(): string {
+    return this.configService.get<string>('AI_COMPLETION_MODEL');
+  }
+
+  getAiEmbeddingDimension(): number {
+    return parseInt(
+      this.configService.get<string>('AI_EMBEDDING_DIMENSION'),
+      10,
+    );
+  }
+
+  getOpenAiApiKey(): string {
+    return this.configService.get<string>('OPENAI_API_KEY');
+  }
+
+  getOpenAiApiUrl(): string {
+    return this.configService.get<string>('OPENAI_API_URL');
+  }
+
+  getGeminiApiKey(): string {
+    return this.configService.get<string>('GEMINI_API_KEY');
+  }
+
+  getOllamaApiUrl(): string {
+    return this.configService.get<string>(
+      'OLLAMA_API_URL',
+      'http://localhost:11434',
+    );
   }
 }

apps/server/src/integrations/environment/environment.validation.ts

@@ -93,6 +93,7 @@ export class EnvironmentVariables {
 
   @IsOptional()
   @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsNotEmpty()
   @IsString()
   TYPESENSE_API_KEY: string;
 
@@ -101,6 +102,53 @@ export class EnvironmentVariables {
   @IsISO6391()
   @IsString()
   TYPESENSE_LOCALE: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsIn(['openai', 'gemini', 'ollama'])
+  @IsString()
+  AI_DRIVER: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsString()
+  @IsNotEmpty()
+  AI_EMBEDDING_MODEL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_EMBEDDING_DIMENSION)
+  @IsIn(['768', '1024', '1536'])
+  @IsString()
+  AI_EMBEDDING_DIMENSION: string;
+
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsString()
+  @IsNotEmpty()
+  AI_COMPLETION_MODEL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'openai')
+  @IsString()
+  @IsNotEmpty()
+  OPENAI_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.OPENAI_API_URL && obj.AI_DRIVER === 'openai')
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  OPENAI_API_URL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'gemini')
+  @IsString()
+  @IsNotEmpty()
+  GEMINI_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'ollama')
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  OLLAMA_API_URL: string;
 }
 
 export function validate(config: Record<string, any>) {

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -473,6 +473,7 @@ export class FileImportTaskService {
         if (validPageIds.size > 0) {
           this.eventEmitter.emit(EventName.PAGE_CREATED, {
             pageIds: Array.from(validPageIds),
+            workspaceId: fileTask.workspaceId,
           });
         }
 

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -5,6 +5,7 @@ export enum QueueName {
   BILLING_QUEUE = '{billing-queue}',
   FILE_TASK_QUEUE = '{file-task-queue}',
   SEARCH_QUEUE = '{search-queue}',
+  AI_QUEUE = '{ai-queue}',
 }
 
 export enum QueueJob {
@@ -13,7 +14,6 @@ export enum QueueJob {
   ATTACHMENT_INDEX_CONTENT = 'attachment-index-content',
   ATTACHMENT_INDEXING = 'attachment-indexing',
   DELETE_PAGE_ATTACHMENTS = 'delete-page-attachments',
-  PAGE_CONTENT_UPDATE = 'page-content-update',
 
   DELETE_USER_AVATARS = 'delete-user-avatars',
 
@@ -39,8 +39,23 @@ export enum QueueJob {
   TYPESENSE_FLUSH = 'typesense-flush',
 
   PAGE_CREATED = 'page-created',
+  PAGE_CONTENT_UPDATED = 'page-content-updated',
+  PAGE_MOVED_TO_SPACE = 'page-moved-to-space',
   PAGE_UPDATED = 'page-updated',
   PAGE_SOFT_DELETED = 'page-soft-deleted',
   PAGE_RESTORED = 'page-restored',
   PAGE_DELETED = 'page-deleted',
+
+  SPACE_CREATED = 'space-created',
+  SPACE_UPDATED = 'space-updated',
+  SPACE_DELETED = 'space-deleted',
+
+  WORKSPACE_CREATED = 'workspace-created',
+  WORKSPACE_SPACE_UPDATED = 'workspace-updated',
+  WORKSPACE_DELETED = 'workspace-deleted',
+  WORKSPACE_CREATE_EMBEDDINGS = 'workspace-create-embeddings',
+  WORKSPACE_DELETE_EMBEDDINGS = 'workspace-delete-embeddings',
+
+  GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
+  DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',
 }

apps/server/src/integrations/queue/queue.module.ts

@@ -65,6 +65,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
         attempts: 2,
       },
     }),
+    BullModule.registerQueue({
+      name: QueueName.AI_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: 1,
+      },
+    }),
   ],
   exports: [BullModule],
   providers: [BacklinksProcessor],