Merge branch 'main' into permissions-2

apps/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "server",
-  "version": "0.22.2",
+  "version": "0.24.1",
   "description": "",
   "author": "",
   "private": true,
@@ -30,6 +30,9 @@
     "test:e2e": "jest --config test/jest-e2e.json"
   },
   "dependencies": {
+    "@ai-sdk/azure": "^2.0.47",
+    "@ai-sdk/google": "^2.0.18",
+    "@ai-sdk/openai": "^2.0.46",
     "@aws-sdk/client-s3": "3.701.0",
     "@aws-sdk/lib-storage": "3.701.0",
     "@aws-sdk/s3-request-presigner": "3.701.0",
@@ -37,55 +40,66 @@
     "@fastify/cookie": "^11.0.2",
     "@fastify/multipart": "^9.0.3",
     "@fastify/static": "^8.2.0",
-    "@nestjs/bullmq": "^11.0.2",
-    "@nestjs/common": "^11.1.3",
+    "@langchain/textsplitters": "^0.1.0",
+    "@nestjs-labs/nestjs-ioredis": "^11.0.4",
+    "@nestjs/bullmq": "^11.0.4",
+    "@nestjs/common": "^11.1.9",
     "@nestjs/config": "^4.0.2",
-    "@nestjs/core": "^11.1.3",
+    "@nestjs/core": "^11.1.9",
     "@nestjs/event-emitter": "^3.0.1",
-    "@nestjs/jwt": "^11.0.0",
+    "@nestjs/jwt": "11.0.0",
     "@nestjs/mapped-types": "^2.1.0",
     "@nestjs/passport": "^11.0.5",
-    "@nestjs/platform-fastify": "^11.1.3",
-    "@nestjs/platform-socket.io": "^11.1.3",
-    "@nestjs/schedule": "^6.0.0",
+    "@nestjs/platform-fastify": "^11.1.9",
+    "@nestjs/platform-socket.io": "^11.1.9",
+    "@nestjs/schedule": "^6.0.1",
     "@nestjs/terminus": "^11.0.0",
-    "@nestjs/websockets": "^11.1.3",
+    "@nestjs/websockets": "^11.1.9",
     "@node-saml/passport-saml": "^5.1.0",
     "@react-email/components": "0.0.28",
     "@react-email/render": "1.0.2",
     "@socket.io/redis-adapter": "^8.3.0",
-    "bcrypt": "^5.1.1",
-    "bullmq": "^5.53.2",
+    "ai": "^5.0.65",
+    "ai-sdk-ollama": "^0.12.0",
+    "bcrypt": "^6.0.0",
+    "bullmq": "^5.65.0",
     "cache-manager": "^6.4.3",
     "cheerio": "^1.1.0",
     "class-transformer": "^0.5.1",
-    "class-validator": "^0.14.1",
+    "class-validator": "^0.14.3",
     "cookie": "^1.0.2",
     "fs-extra": "^11.3.0",
-    "happy-dom": "^15.11.6",
+    "happy-dom": "20.0.10",
+    "ioredis": "^5.4.1",
     "jsonwebtoken": "^9.0.2",
     "kysely": "^0.28.2",
     "kysely-migration-cli": "^0.4.2",
+    "ldapts": "^7.4.0",
+    "mammoth": "^1.11.0",
     "mime-types": "^2.1.35",
     "nanoid": "3.3.11",
     "nestjs-kysely": "^1.2.0",
-    "nodemailer": "^7.0.3",
+    "nodemailer": "^7.0.11",
     "openid-client": "^5.7.1",
     "otpauth": "^9.4.0",
     "p-limit": "^6.2.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-jwt": "^4.0.1",
-    "pg": "^8.16.0",
+    "pdfjs-dist": "^5.4.394",
+    "pg": "^8.16.3",
     "pg-tsquery": "^8.4.2",
+    "pgvector": "^0.2.1",
     "postmark": "^4.0.5",
     "react": "^18.3.1",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.2",
-    "sanitize-filename-ts": "^1.0.2",
+    "sanitize-filename-ts": "1.0.2",
+    "sharp": "0.34.3",
     "socket.io": "^4.8.1",
     "stripe": "^17.5.0",
     "tmp-promise": "^3.0.3",
-    "ws": "^8.18.2",
+    "typesense": "^2.1.0",
+    "ws": "^8.18.3",
     "yauzl": "^3.2.0"
   },
   "devDependencies": {

apps/server/src/app.module.ts

@@ -16,6 +16,8 @@ import { ExportModule } from './integrations/export/export.module';
 import { ImportModule } from './integrations/import/import.module';
 import { SecurityModule } from './integrations/security/security.module';
 import { TelemetryModule } from './integrations/telemetry/telemetry.module';
+import { RedisModule } from '@nestjs-labs/nestjs-ioredis';
+import { RedisConfigService } from './integrations/redis/redis-config.service';
 
 const enterpriseModules = [];
 try {
@@ -36,6 +38,9 @@ try {
     CoreModule,
     DatabaseModule,
     EnvironmentModule,
+    RedisModule.forRootAsync({
+      useClass: RedisConfigService,
+    }),
     CollaborationModule,
     WsModule,
     QueueModule,

apps/server/src/collaboration/collaboration.util.ts

@@ -5,12 +5,12 @@ import { TaskItem } from '@tiptap/extension-task-item';
 import { Underline } from '@tiptap/extension-underline';
 import { Superscript } from '@tiptap/extension-superscript';
 import SubScript from '@tiptap/extension-subscript';
-import { Highlight } from '@tiptap/extension-highlight';
 import { Typography } from '@tiptap/extension-typography';
 import { TextStyle } from '@tiptap/extension-text-style';
 import { Color } from '@tiptap/extension-color';
 import { Youtube } from '@tiptap/extension-youtube';
 import {
+  Heading,
   Callout,
   Comment,
   CustomCodeBlock,
@@ -32,18 +32,26 @@ import {
   Excalidraw,
   Embed,
   Mention,
+  Subpages,
+  Highlight,
+  UniqueID,
+  addUniqueIdsToDoc,
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
-import { generateHTML } from '../common/helpers/prosemirror/html';
+import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
 // @tiptap/html library works best for generating prosemirror json state but not HTML
 // see: https://github.com/ueberdosis/tiptap/issues/5352
 // see:https://github.com/ueberdosis/tiptap/issues/4089
-import { generateJSON } from '@tiptap/html';
 import { Node } from '@tiptap/pm/model';
 
 export const tiptapExtensions = [
   StarterKit.configure({
     codeBlock: false,
+    heading: false,
+  }),
+  Heading,
+  UniqueID.configure({
+    types: ['heading', 'paragraph'],
   }),
   Comment,
   TextAlign.configure({ types: ['heading', 'paragraph'] }),
@@ -79,6 +87,7 @@ export const tiptapExtensions = [
   Excalidraw,
   Embed,
   Mention,
+  Subpages,
 ] as any;
 
 export function jsonToHtml(tiptapJson: any) {
@@ -86,7 +95,14 @@ export function jsonToHtml(tiptapJson: any) {
 }
 
 export function htmlToJson(html: string) {
-  return generateJSON(html, tiptapExtensions);
+  const pmJson = generateJSON(html, tiptapExtensions);
+
+  try {
+    return addUniqueIdsToDoc(pmJson, tiptapExtensions);
+  } catch (error) {
+    console.warn('failed to add unique ids to doc', error);
+    return pmJson;
+  }
 }
 
 export function jsonToText(tiptapJson: JSONContent) {

apps/server/src/collaboration/extensions/persistence.extension.ts

@@ -35,6 +35,7 @@ export class PersistenceExtension implements Extension {
     @InjectKysely() private readonly db: KyselyDB,
     private eventEmitter: EventEmitter2,
     @InjectQueue(QueueName.GENERAL_QUEUE) private generalQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
   ) {}
 
   async onLoadDocument(data: onLoadDocumentPayload) {
@@ -168,6 +169,11 @@ export class PersistenceExtension implements Extension {
         workspaceId: page.workspaceId,
         mentions: pageMentions,
       } as IPageBacklinkJob);
+
+      await this.aiQueue.add(QueueJob.PAGE_CONTENT_UPDATED, {
+        pageIds: [pageId],
+        workspaceId: page.workspaceId,
+      });
     }
   }
 

apps/server/src/common/events/event.contants.ts

@@ -1,3 +1,19 @@
 export enum EventName {
   COLLAB_PAGE_UPDATED = 'collab.page.updated',
+
+  PAGE_CREATED = 'page.created',
+  PAGE_UPDATED = 'page.updated',
+  PAGE_CONTENT_UPDATED = 'page-content-updated',
+  PAGE_MOVED_TO_SPACE = 'page-moved-to-space',
+  PAGE_DELETED = 'page.deleted',
+  PAGE_SOFT_DELETED = 'page.soft_deleted',
+  PAGE_RESTORED = 'page.restored',
+
+  SPACE_CREATED = 'space.created',
+  SPACE_UPDATED = 'space.updated',
+  SPACE_DELETED = 'space.deleted',
+
+  WORKSPACE_CREATED = 'workspace.created',
+  WORKSPACE_UPDATED = 'workspace.updated',
+  WORKSPACE_DELETED = 'workspace.deleted',
 }

apps/server/src/common/helpers/prosemirror/html/generateHTML.ts

@@ -1,21 +1,29 @@
-import { Extensions, getSchema, JSONContent } from '@tiptap/core';
-import { DOMSerializer, Node } from '@tiptap/pm/model';
-import { Window } from 'happy-dom';
+import { type Extensions, type JSONContent, getSchema } from '@tiptap/core';
+import { Node } from '@tiptap/pm/model';
+import { getHTMLFromFragment } from './getHTMLFromFragment';
 
+/**
+ * This function generates HTML from a ProseMirror JSON content object.
+ *
+ * @remarks **Important**: This function requires `happy-dom` to be installed in your project.
+ * @param doc - The ProseMirror JSON content object.
+ * @param extensions - The Tiptap extensions used to build the schema.
+ * @returns The generated HTML string.
+ * @example
+ * ```js
+ * const html = generateHTML(doc, extensions)
+ * console.log(html)
+ * ```
+ */
 export function generateHTML(doc: JSONContent, extensions: Extensions): string {
+  if (typeof window !== 'undefined') {
+    throw new Error(
+      'generateHTML can only be used in a Node environment\nIf you want to use this in a browser environment, use the `@tiptap/html` import instead.',
+    );
+  }
+
   const schema = getSchema(extensions);
   const contentNode = Node.fromJSON(schema, doc);
 
-  const window = new Window();
-
-  const fragment = DOMSerializer.fromSchema(schema).serializeFragment(
-    contentNode.content,
-    {
-      document: window.document as unknown as Document,
-    },
-  );
-
-  const serializer = new window.XMLSerializer();
-  // @ts-ignore
-  return serializer.serializeToString(fragment as unknown as Node);
+  return getHTMLFromFragment(contentNode, schema);
 }

apps/server/src/common/helpers/prosemirror/html/generateJSON.ts

@@ -1,21 +1,55 @@
-import { Extensions, getSchema } from '@tiptap/core';
-import { DOMParser, ParseOptions } from '@tiptap/pm/model';
+import type { Extensions } from '@tiptap/core';
+import { getSchema } from '@tiptap/core';
+import { type ParseOptions, DOMParser as PMDOMParser } from '@tiptap/pm/model';
 import { Window } from 'happy-dom';
 
-// this function does not work as intended
-// it has issues with closing tags
+/**
+ * Generates a JSON object from the given HTML string and converts it into a Prosemirror node with content.
+ * @remarks **Important**: This function requires `happy-dom` to be installed in your project.
+ * @param {string} html - The HTML string to be converted into a Prosemirror node.
+ * @param {Extensions} extensions - The extensions to be used for generating the schema.
+ * @param {ParseOptions} options - The options to be supplied to the parser.
+ * @returns {Promise<Record<string, any>>} - A promise with the generated JSON object.
+ * @example
+ * const html = '<p>Hello, world!</p>'
+ * const extensions = [...]
+ * const json = generateJSON(html, extensions)
+ * console.log(json) // { type: 'doc', content: [{ type: 'paragraph', content: [{ type: 'text', text: 'Hello, world!' }] }] }
+ */
 export function generateJSON(
   html: string,
   extensions: Extensions,
   options?: ParseOptions,
 ): Record<string, any> {
-  const schema = getSchema(extensions);
+  if (typeof window !== 'undefined') {
+    throw new Error(
+      'generateJSON can only be used in a Node environment\nIf you want to use this in a browser environment, use the `@tiptap/html` import instead.',
+    );
+  }
 
-  const window = new Window();
-  const document = window.document;
-  document.body.innerHTML = html;
+  const localWindow = new Window();
+  const localDOMParser = new localWindow.DOMParser();
+  let result: Record<string, any>;
 
-  return DOMParser.fromSchema(schema)
-    .parse(document as never, options)
-    .toJSON();
+  try {
+    const schema = getSchema(extensions);
+    let doc: ReturnType<typeof localDOMParser.parseFromString> | null = null;
+
+    const htmlString = `<!DOCTYPE html><html><body>${html}</body></html>`;
+    doc = localDOMParser.parseFromString(htmlString, 'text/html');
+
+    if (!doc) {
+      throw new Error('Failed to parse HTML string');
+    }
+
+    result = PMDOMParser.fromSchema(schema)
+      .parse(doc.body as unknown as Node, options)
+      .toJSON();
+  } finally {
+    // clean up happy-dom to avoid memory leaks
+    localWindow.happyDOM.abort();
+    localWindow.happyDOM.close();
+  }
+
+  return result;
 }

apps/server/src/common/helpers/prosemirror/html/getHTMLFromFragment.ts

@@ -0,0 +1,54 @@
+import type { Node, Schema } from '@tiptap/pm/model';
+import { DOMSerializer } from '@tiptap/pm/model';
+import { Window } from 'happy-dom';
+
+/**
+ * Returns the HTML string representation of a given document node.
+ *
+ * @remarks **Important**: This function requires `happy-dom` to be installed in your project.
+ * @param doc - The document node to serialize.
+ * @param schema - The Prosemirror schema to use for serialization.
+ * @returns A promise containing the HTML string representation of the document fragment.
+ *
+ * @example
+ * ```typescript
+ * const html = getHTMLFromFragment(doc, schema)
+ * ```
+ */
+export function getHTMLFromFragment(
+  doc: Node,
+  schema: Schema,
+  options?: { document?: Document },
+): string {
+  if (options?.document) {
+    const wrap = options.document.createElement('div');
+
+    DOMSerializer.fromSchema(schema).serializeFragment(
+      doc.content,
+      { document: options.document },
+      wrap,
+    );
+    return wrap.innerHTML;
+  }
+
+  const localWindow = new Window();
+  let result: string;
+
+  try {
+    const fragment = DOMSerializer.fromSchema(schema).serializeFragment(
+      doc.content,
+      {
+        document: localWindow.document as unknown as Document,
+      },
+    );
+
+    const serializer = new localWindow.XMLSerializer();
+    result = serializer.serializeToString(fragment as any);
+  } finally {
+    // clean up happy-dom to avoid memory leaks
+    localWindow.happyDOM.abort();
+    localWindow.happyDOM.close();
+  }
+
+  return result;
+}

apps/server/src/common/helpers/utils.ts

@@ -72,7 +72,9 @@ export function extractDateFromUuid7(uuid7: string) {
 }
 
 export function sanitizeFileName(fileName: string): string {
-  const sanitizedFilename = sanitize(fileName).replace(/ /g, '_');
+  const sanitizedFilename = sanitize(fileName)
+    .replace(/ /g, '_')
+    .replace(/#/g, '_');
   return sanitizedFilename.slice(0, 255);
 }
 
@@ -87,3 +89,12 @@ export function extractBearerTokenFromHeader(
   const [type, token] = request.headers.authorization?.split(' ') ?? [];
   return type === 'Bearer' ? token : undefined;
 }
+
+export function hasLicenseOrEE(opts: {
+  licenseKey: string;
+  plan: string;
+  isCloud: boolean;
+}): boolean {
+  const { licenseKey, plan, isCloud } = opts;
+  return Boolean(licenseKey) || (isCloud && plan === 'business');
+}

apps/server/src/common/validator/is-iso6391.ts

@@ -0,0 +1,34 @@
+// MIT - https://github.com/typestack/class-validator/pull/2626
+import isISO6391Validator from 'validator/lib/isISO6391';
+import { buildMessage, ValidateBy, ValidationOptions } from 'class-validator';
+
+export const IS_ISO6391 = 'isISO6391';
+
+/**
+ * Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code.
+ */
+export function isISO6391(value: unknown): boolean {
+  return typeof value === 'string' && isISO6391Validator(value);
+}
+
+/**
+ * Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code.
+ */
+export function IsISO6391(
+  validationOptions?: ValidationOptions,
+): PropertyDecorator {
+  return ValidateBy(
+    {
+      name: IS_ISO6391,
+      validator: {
+        validate: (value, args): boolean => isISO6391(value),
+        defaultMessage: buildMessage(
+          (eachPrefix) =>
+            eachPrefix + '$property must be a valid ISO 639-1 language code',
+          validationOptions,
+        ),
+      },
+    },
+    validationOptions,
+  );
+}

apps/server/src/core/attachment/attachment.constants.ts

@@ -1,12 +1,12 @@
 export enum AttachmentType {
   Avatar = 'avatar',
-  WorkspaceLogo = 'workspace-logo',
-  SpaceLogo = 'space-logo',
+  WorkspaceIcon = 'workspace-icon',
+  SpaceIcon = 'space-icon',
   File = 'file',
 }
 
 export const validImageExtensions = ['.jpg', '.png', '.jpeg'];
-export const MAX_AVATAR_SIZE = '5MB';
+export const MAX_AVATAR_SIZE = '10MB';
 
 export const inlineFileExtensions = [
   '.jpg',

apps/server/src/core/attachment/attachment.controller.ts

@@ -1,5 +1,6 @@
 import {
   BadRequestException,
+  Body,
   Controller,
   ForbiddenException,
   Get,
@@ -51,6 +52,7 @@ import { EnvironmentService } from '../../integrations/environment/environment.s
 import { TokenService } from '../auth/services/token.service';
 import { JwtAttachmentPayload, JwtType } from '../auth/dto/jwt-payload';
 import * as path from 'path';
+import { RemoveIconDto } from './dto/attachment.dto';
 
 @Controller()
 export class AttachmentController {
@@ -302,7 +304,7 @@ export class AttachmentController {
       throw new BadRequestException('Invalid image attachment type');
     }
 
-    if (attachmentType === AttachmentType.WorkspaceLogo) {
+    if (attachmentType === AttachmentType.WorkspaceIcon) {
       const ability = this.workspaceAbility.createForUser(user, workspace);
       if (
         ability.cannot(
@@ -314,7 +316,7 @@ export class AttachmentController {
       }
     }
 
-    if (attachmentType === AttachmentType.SpaceLogo) {
+    if (attachmentType === AttachmentType.SpaceIcon) {
       if (!spaceId) {
         throw new BadRequestException('spaceId is required');
       }
@@ -372,8 +374,59 @@ export class AttachmentController {
       });
       return res.send(fileStream);
     } catch (err) {
-      this.logger.error(err);
+     // this.logger.error(err);
       throw new NotFoundException('File not found');
     }
   }
+
+  @UseGuards(JwtAuthGuard)
+  @HttpCode(HttpStatus.OK)
+  @Post('attachments/remove-icon')
+  async removeIcon(
+    @Body() dto: RemoveIconDto,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
+    const { type, spaceId } = dto;
+
+    // remove current user avatar
+    if (type === AttachmentType.Avatar) {
+      await this.attachmentService.removeUserAvatar(user);
+      return;
+    }
+
+    // remove space icon
+    if (type === AttachmentType.SpaceIcon) {
+      if (!spaceId) {
+        throw new BadRequestException(
+          'spaceId is required to change space icons',
+        );
+      }
+
+      const spaceAbility = await this.spaceAbility.createForUser(user, spaceId);
+      if (
+        spaceAbility.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Settings)
+      ) {
+        throw new ForbiddenException();
+      }
+
+      await this.attachmentService.removeSpaceIcon(spaceId, workspace.id);
+      return;
+    }
+
+    // remove workspace icon
+    if (type === AttachmentType.WorkspaceIcon) {
+      const ability = this.workspaceAbility.createForUser(user, workspace);
+      if (
+        ability.cannot(
+          WorkspaceCaslAction.Manage,
+          WorkspaceCaslSubject.Settings,
+        )
+      ) {
+        throw new ForbiddenException();
+      }
+      await this.attachmentService.removeWorkspaceIcon(workspace);
+      return;
+    }
+  }
 }

apps/server/src/core/attachment/attachment.utils.ts

@@ -1,8 +1,8 @@
 import { MultipartFile } from '@fastify/multipart';
-import { randomBytes } from 'crypto';
-import { sanitize } from 'sanitize-filename-ts';
 import * as path from 'path';
 import { AttachmentType } from './attachment.constants';
+import { sanitizeFileName } from '../../common/helpers';
+import * as sharp from 'sharp';
 
 export interface PreparedFile {
   buffer: Buffer;
@@ -22,10 +22,8 @@ export async function prepareFile(
   }
 
   try {
-    const rand = randomBytes(8).toString('hex');
-
     const buffer = await file.toBuffer();
-    const sanitizedFilename = sanitize(file.filename).replace(/ /g, '_');
+    const sanitizedFilename = sanitizeFileName(file.filename);
     const fileName = sanitizedFilename.slice(0, 255);
     const fileSize = buffer.length;
     const fileExtension = path.extname(file.filename).toLowerCase();
@@ -58,9 +56,9 @@ export function getAttachmentFolderPath(
   switch (type) {
     case AttachmentType.Avatar:
       return `${workspaceId}/avatars`;
-    case AttachmentType.WorkspaceLogo:
-      return `${workspaceId}/workspace-logo`;
-    case AttachmentType.SpaceLogo:
+    case AttachmentType.WorkspaceIcon:
+      return `${workspaceId}/workspace-logos`;
+    case AttachmentType.SpaceIcon:
       return `${workspaceId}/space-logos`;
     case AttachmentType.File:
       return `${workspaceId}/files`;
@@ -70,3 +68,51 @@ export function getAttachmentFolderPath(
 }
 
 export const validAttachmentTypes = Object.values(AttachmentType);
+
+export async function compressAndResizeIcon(
+  buffer: Buffer,
+  attachmentType?: AttachmentType,
+): Promise<Buffer> {
+  try {
+    let sharpInstance = sharp(buffer);
+    const metadata = await sharpInstance.metadata();
+
+    const targetWidth = 300;
+    const targetHeight = 300;
+
+    // Only resize if image is larger than target dimensions
+    if (metadata.width > targetWidth || metadata.height > targetHeight) {
+      sharpInstance = sharpInstance.resize(targetWidth, targetHeight, {
+        fit: 'inside',
+        withoutEnlargement: true,
+      });
+    }
+
+    // Handle based on original format
+    if (metadata.format === 'png') {
+      // Only flatten avatars to remove transparency
+      if (attachmentType === AttachmentType.Avatar) {
+        sharpInstance = sharpInstance.flatten({
+          background: { r: 255, g: 255, b: 255 },
+        });
+      }
+
+      return await sharpInstance
+        .png({
+          quality: 85,
+          compressionLevel: 6,
+        })
+        .toBuffer();
+    } else {
+      return await sharpInstance
+        .jpeg({
+          quality: 85,
+          progressive: true,
+          mozjpeg: true,
+        })
+        .toBuffer();
+    }
+  } catch (err) {
+    throw err;
+  }
+}

apps/server/src/core/attachment/dto/attachment.dto.ts

@@ -0,0 +1,17 @@
+import { IsEnum, IsIn, IsNotEmpty, IsOptional, IsUUID } from 'class-validator';
+import { AttachmentType } from '../attachment.constants';
+
+export class RemoveIconDto {
+  @IsEnum(AttachmentType)
+  @IsIn([
+    AttachmentType.Avatar,
+    AttachmentType.SpaceIcon,
+    AttachmentType.WorkspaceIcon,
+  ])
+  @IsNotEmpty()
+  type: AttachmentType;
+
+  @IsOptional()
+  @IsUUID()
+  spaceId: string;
+}

apps/server/src/core/attachment/dto/avatar-upload.dto.ts

@@ -1,3 +0,0 @@
-import { IsOptional, IsString, IsUUID } from 'class-validator';
-
-export class AvatarUploadDto {}

apps/server/src/core/attachment/dto/get-file.dto.ts

@@ -1,7 +0,0 @@
-import { IsNotEmpty, IsString } from 'class-validator';
-
-export class GetFileDto {
-  @IsString()
-  @IsNotEmpty()
-  attachmentId: string;
-}

apps/server/src/core/attachment/dto/upload-file.dto.ts

@@ -1,20 +0,0 @@
-import {
-  IsDefined,
-  IsNotEmpty,
-  IsOptional,
-  IsString,
-  IsUUID,
-} from 'class-validator';
-
-export class UploadFileDto {
-  @IsString()
-  @IsNotEmpty()
-  attachmentType: string;
-
-  @IsOptional()
-  @IsUUID()
-  pageId: string;
-
-  @IsDefined()
-  file: any;
-}

apps/server/src/core/attachment/processors/attachment.processor.ts

@@ -3,12 +3,15 @@ import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { AttachmentService } from '../services/attachment.service';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
-import { Space } from '@docmost/db/types/entity.types';
+import { ModuleRef } from '@nestjs/core';
 
 @Processor(QueueName.ATTACHMENT_QUEUE)
 export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(AttachmentProcessor.name);
-  constructor(private readonly attachmentService: AttachmentService) {
+  constructor(
+    private readonly attachmentService: AttachmentService,
+    private moduleRef: ModuleRef,
+  ) {
     super();
   }
 
@@ -25,6 +28,33 @@ export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
           job.data.pageId,
         );
       }
+      if (
+        job.name === QueueJob.ATTACHMENT_INDEX_CONTENT ||
+        job.name === QueueJob.ATTACHMENT_INDEXING
+      ) {
+        let AttachmentEeModule: any;
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-require-imports
+          AttachmentEeModule = require('./../../../ee/attachments-ee/attachment-ee.service');
+        } catch (err) {
+          this.logger.debug(
+            'Attachment enterprise module requested but EE module not bundled in this build',
+          );
+          return;
+        }
+        const attachmentEeService = this.moduleRef.get(
+          AttachmentEeModule.AttachmentEeService,
+          { strict: false },
+        );
+
+        if (job.name === QueueJob.ATTACHMENT_INDEX_CONTENT) {
+          await attachmentEeService.indexAttachment(job.data.attachmentId);
+        } else if (job.name === QueueJob.ATTACHMENT_INDEXING) {
+          await attachmentEeService.indexAttachments(
+            job.data.workspaceId,
+          );
+        }
+      }
     } catch (err) {
       throw err;
     }
@@ -37,9 +67,15 @@ export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
 
   @OnWorkerEvent('failed')
   onError(job: Job) {
-    this.logger.error(
-      `Error processing ${job.name} job. Reason: ${job.failedReason}`,
-    );
+    if (job.name === QueueJob.ATTACHMENT_INDEX_CONTENT) {
+      this.logger.debug(
+        `Error processing ${job.name} job for attachment ${job.data?.attachmentId}. Reason: ${job.failedReason}`,
+      );
+    } else {
+      this.logger.error(
+        `Error processing ${job.name} job. Reason: ${job.failedReason}`,
+      );
+    }
   }
 
   @OnWorkerEvent('completed')

apps/server/src/core/attachment/services/attachment.service.ts

@@ -7,6 +7,7 @@ import {
 import { StorageService } from '../../../integrations/storage/storage.service';
 import { MultipartFile } from '@fastify/multipart';
 import {
+  compressAndResizeIcon,
   getAttachmentFolderPath,
   PreparedFile,
   prepareFile,
@@ -16,12 +17,15 @@ import { v4 as uuid4, v7 as uuid7 } from 'uuid';
 import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
 import { AttachmentType, validImageExtensions } from '../attachment.constants';
 import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
-import { Attachment } from '@docmost/db/types/entity.types';
+import { Attachment, User, Workspace } from '@docmost/db/types/entity.types';
 import { InjectKysely } from 'nestjs-kysely';
 import { executeTx } from '@docmost/db/utils';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../../integrations/queue/constants';
+import { Queue } from 'bullmq';
 
 @Injectable()
 export class AttachmentService {
@@ -33,6 +37,7 @@ export class AttachmentService {
     private readonly workspaceRepo: WorkspaceRepo,
     private readonly spaceRepo: SpaceRepo,
     @InjectKysely() private readonly db: KyselyDB,
+    @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
   async uploadFile(opts: {
@@ -99,6 +104,23 @@ export class AttachmentService {
           pageId,
         });
       }
+
+      // Only index PDFs and DOCX files
+      if (['.pdf', '.docx'].includes(attachment.fileExt.toLowerCase())) {
+        await this.attachmentQueue.add(
+          QueueJob.ATTACHMENT_INDEX_CONTENT,
+          {
+            attachmentId: attachmentId,
+          },
+          {
+            attempts: 2,
+            backoff: {
+              type: 'exponential',
+              delay: 10000,
+            },
+          },
+        );
+      }
     } catch (err) {
       // delete uploaded file on error
       this.logger.error(err);
@@ -111,8 +133,8 @@ export class AttachmentService {
     filePromise: Promise<MultipartFile>,
     type:
       | AttachmentType.Avatar
-      | AttachmentType.WorkspaceLogo
-      | AttachmentType.SpaceLogo,
+      | AttachmentType.WorkspaceIcon
+      | AttachmentType.SpaceIcon,
     userId: string,
     workspaceId: string,
     spaceId?: string,
@@ -120,6 +142,9 @@ export class AttachmentService {
     const preparedFile: PreparedFile = await prepareFile(filePromise);
     validateFileType(preparedFile.fileExtension, validImageExtensions);
 
+    const processedBuffer = await compressAndResizeIcon(preparedFile.buffer, type);
+    preparedFile.buffer = processedBuffer;
+    preparedFile.fileSize = processedBuffer.length;
     preparedFile.fileName = uuid4() + preparedFile.fileExtension;
 
     const filePath = `${getAttachmentFolderPath(type, workspaceId)}/${preparedFile.fileName}`;
@@ -153,7 +178,7 @@ export class AttachmentService {
             workspaceId,
             trx,
           );
-        } else if (type === AttachmentType.WorkspaceLogo) {
+        } else if (type === AttachmentType.WorkspaceIcon) {
           const workspace = await this.workspaceRepo.findById(workspaceId, {
             trx,
           });
@@ -165,7 +190,7 @@ export class AttachmentService {
             workspaceId,
             trx,
           );
-        } else if (type === AttachmentType.SpaceLogo && spaceId) {
+        } else if (type === AttachmentType.SpaceIcon && spaceId) {
           const space = await this.spaceRepo.findById(spaceId, workspaceId, {
             trx,
           });
@@ -184,7 +209,6 @@ export class AttachmentService {
       });
     } catch (err) {
       // delete uploaded file on db update failure
-      this.logger.error('Image upload error:', err);
       await this.deleteRedundantFile(filePath);
       throw new BadRequestException('Failed to upload image');
     }
@@ -367,4 +391,41 @@ export class AttachmentService {
       throw err;
     }
   }
+
+  async removeUserAvatar(user: User) {
+    if (user.avatarUrl && !user.avatarUrl.toLowerCase().startsWith('http')) {
+      const filePath = `${getAttachmentFolderPath(AttachmentType.Avatar, user.workspaceId)}/${user.avatarUrl}`;
+      await this.deleteRedundantFile(filePath);
+    }
+
+    await this.userRepo.updateUser(
+      { avatarUrl: null },
+      user.id,
+      user.workspaceId,
+    );
+  }
+
+  async removeSpaceIcon(spaceId: string, workspaceId: string) {
+    const space = await this.spaceRepo.findById(spaceId, workspaceId);
+
+    if (!space) {
+      throw new NotFoundException('Space not found');
+    }
+
+    if (space.logo && !space.logo.toLowerCase().startsWith('http')) {
+      const filePath = `${getAttachmentFolderPath(AttachmentType.SpaceIcon, workspaceId)}/${space.logo}`;
+      await this.deleteRedundantFile(filePath);
+    }
+
+    await this.spaceRepo.updateSpace({ logo: null }, spaceId, workspaceId);
+  }
+
+  async removeWorkspaceIcon(workspace: Workspace) {
+    if (workspace.logo && !workspace.logo.toLowerCase().startsWith('http')) {
+      const filePath = `${getAttachmentFolderPath(AttachmentType.WorkspaceIcon, workspace.id)}/${workspace.logo}`;
+      await this.deleteRedundantFile(filePath);
+    }
+
+    await this.workspaceRepo.updateWorkspace({ logo: null }, workspace.id);
+  }
 }

apps/server/src/core/auth/dto/jwt-payload.ts

@@ -4,6 +4,7 @@ export enum JwtType {
   EXCHANGE = 'exchange',
   ATTACHMENT = 'attachment',
   MFA_TOKEN = 'mfa_token',
+  API_KEY = 'api_key',
 }
 export type JwtPayload = {
   sub: string;
@@ -36,3 +37,10 @@ export interface JwtMfaTokenPayload {
   workspaceId: string;
   type: 'mfa_token';
 }
+
+export type JwtApiKeyPayload = {
+  sub: string;
+  workspaceId: string;
+  apiKeyId: string;
+  type: 'api_key';
+};

apps/server/src/core/auth/services/auth.service.ts

@@ -106,6 +106,7 @@ export class AuthService {
     await this.userRepo.updateUser(
       {
         password: newPasswordHash,
+        hasGeneratedPassword: false,
       },
       userId,
       workspaceId,
@@ -186,6 +187,7 @@ export class AuthService {
       await this.userRepo.updateUser(
         {
           password: newPasswordHash,
+          hasGeneratedPassword: false,
         },
         user.id,
         workspace.id,

apps/server/src/core/auth/services/token.service.ts

@@ -6,6 +6,7 @@ import {
 import { JwtService } from '@nestjs/jwt';
 import { EnvironmentService } from '../../../integrations/environment/environment.service';
 import {
+  JwtApiKeyPayload,
   JwtAttachmentPayload,
   JwtCollabPayload,
   JwtExchangePayload,
@@ -90,6 +91,27 @@ export class TokenService {
     return this.jwtService.sign(payload, { expiresIn: '5m' });
   }
 
+  async generateApiToken(opts: {
+    apiKeyId: string;
+    user: User;
+    workspaceId: string;
+    expiresIn?: string | number;
+  }): Promise<string> {
+    const { apiKeyId, user, workspaceId, expiresIn } = opts;
+    if (user.deactivatedAt || user.deletedAt) {
+      throw new ForbiddenException();
+    }
+
+    const payload: JwtApiKeyPayload = {
+      sub: user.id,
+      apiKeyId: apiKeyId,
+      workspaceId,
+      type: JwtType.API_KEY,
+    };
+
+    return this.jwtService.sign(payload, expiresIn ? { expiresIn } : {});
+  }
+
   async verifyJwt(token: string, tokenType: string) {
     const payload = await this.jwtService.verifyAsync(token, {
       secret: this.environmentService.getAppSecret(),

apps/server/src/core/auth/strategies/jwt.strategy.ts

@@ -2,11 +2,12 @@ import { Injectable, Logger, UnauthorizedException } from '@nestjs/common';
 import { PassportStrategy } from '@nestjs/passport';
 import { Strategy } from 'passport-jwt';
 import { EnvironmentService } from '../../../integrations/environment/environment.service';
-import { JwtPayload, JwtType } from '../dto/jwt-payload';
+import { JwtApiKeyPayload, JwtPayload, JwtType } from '../dto/jwt-payload';
 import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { FastifyRequest } from 'fastify';
 import { extractBearerTokenFromHeader } from '../../../common/helpers';
+import { ModuleRef } from '@nestjs/core';
 
 @Injectable()
 export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
@@ -16,6 +17,7 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
     private userRepo: UserRepo,
     private workspaceRepo: WorkspaceRepo,
     private readonly environmentService: EnvironmentService,
+    private moduleRef: ModuleRef,
   ) {
     super({
       jwtFromRequest: (req: FastifyRequest) => {
@@ -27,8 +29,8 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
     });
   }
 
-  async validate(req: any, payload: JwtPayload) {
-    if (!payload.workspaceId || payload.type !== JwtType.ACCESS) {
+  async validate(req: any, payload: JwtPayload | JwtApiKeyPayload) {
+    if (!payload.workspaceId) {
       throw new UnauthorizedException();
     }
 
@@ -36,6 +38,14 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
       throw new UnauthorizedException('Workspace does not match');
     }
 
+    if (payload.type === JwtType.API_KEY) {
+      return this.validateApiKey(req, payload as JwtApiKeyPayload);
+    }
+
+    if (payload.type !== JwtType.ACCESS) {
+      throw new UnauthorizedException();
+    }
+
     const workspace = await this.workspaceRepo.findById(payload.workspaceId);
 
     if (!workspace) {
@@ -49,4 +59,30 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
 
     return { user, workspace };
   }
+
+  private async validateApiKey(req: any, payload: JwtApiKeyPayload) {
+    let ApiKeyModule: any;
+    let isApiKeyModuleReady = false;
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      ApiKeyModule = require('./../../../ee/api-key/api-key.service');
+      isApiKeyModuleReady = true;
+    } catch (err) {
+      this.logger.debug(
+        'API Key module requested but enterprise module not bundled in this build',
+      );
+      isApiKeyModuleReady = false;
+    }
+
+    if (isApiKeyModuleReady) {
+      const ApiKeyService = this.moduleRef.get(ApiKeyModule.ApiKeyService, {
+        strict: false,
+      });
+
+      return ApiKeyService.validateApiKey(payload);
+    }
+
+    throw new UnauthorizedException('Enterprise API Key module missing');
+  }
 }

apps/server/src/core/casl/abilities/workspace-ability.factory.ts

@@ -40,6 +40,7 @@ function buildWorkspaceOwnerAbility() {
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Group);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Member);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Attachment);
+  can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.API);
 
   return build();
 }
@@ -55,6 +56,7 @@ function buildWorkspaceAdminAbility() {
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Group);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Member);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Attachment);
+  can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.API);
 
   return build();
 }
@@ -68,6 +70,7 @@ function buildWorkspaceMemberAbility() {
   can(WorkspaceCaslAction.Read, WorkspaceCaslSubject.Space);
   can(WorkspaceCaslAction.Read, WorkspaceCaslSubject.Group);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Attachment);
+  can(WorkspaceCaslAction.Create, WorkspaceCaslSubject.API);
 
   return build();
 }

apps/server/src/core/casl/interfaces/workspace-ability.type.ts

@@ -11,6 +11,7 @@ export enum WorkspaceCaslSubject {
   Space = 'space',
   Group = 'group',
   Attachment = 'attachment',
+  API = 'api_key',
 }
 
 export type IWorkspaceAbility =
@@ -18,4 +19,5 @@ export type IWorkspaceAbility =
   | [WorkspaceCaslAction, WorkspaceCaslSubject.Member]
   | [WorkspaceCaslAction, WorkspaceCaslSubject.Space]
   | [WorkspaceCaslAction, WorkspaceCaslSubject.Group]
-  | [WorkspaceCaslAction, WorkspaceCaslSubject.Attachment];
+  | [WorkspaceCaslAction, WorkspaceCaslSubject.Attachment]
+  | [WorkspaceCaslAction, WorkspaceCaslSubject.API];

apps/server/src/core/page/dto/sidebar-page.dto.ts

@@ -1,7 +1,11 @@
-import { IsOptional, IsString } from 'class-validator';
+import { IsOptional, IsString, IsUUID } from 'class-validator';
 import { SpaceIdDto } from './page.dto';
 
-export class SidebarPageDto extends SpaceIdDto {
+export class SidebarPageDto {
+  @IsOptional()
+  @IsUUID()
+  spaceId: string;
+
   @IsOptional()
   @IsString()
   pageId: string;

apps/server/src/core/page/page.controller.ts

@@ -1,23 +1,23 @@
 import {
-  Controller,
-  Post,
+  BadRequestException,
   Body,
+  Controller,
+  ForbiddenException,
   HttpCode,
   HttpStatus,
-  UseGuards,
-  ForbiddenException,
   NotFoundException,
-  BadRequestException,
+  Post,
+  UseGuards,
 } from '@nestjs/common';
 import { PageService } from './services/page.service';
 import { CreatePageDto } from './dto/create-page.dto';
 import { UpdatePageDto } from './dto/update-page.dto';
 import { MovePageDto, MovePageToSpaceDto } from './dto/move-page.dto';
 import {
+  DeletePageDto,
   PageHistoryIdDto,
   PageIdDto,
   PageInfoDto,
-  DeletePageDto,
 } from './dto/page.dto';
 import { PageHistoryService } from './services/page-history.service';
 import { AuthUser } from '../../common/decorators/auth-user.decorator';
@@ -134,7 +134,11 @@ export class PageController {
 
   @HttpCode(HttpStatus.OK)
   @Post('delete')
-  async delete(@Body() deletePageDto: DeletePageDto, @AuthUser() user: User) {
+  async delete(
+    @Body() deletePageDto: DeletePageDto,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
     const page = await this.pageRepo.findById(deletePageDto.pageId);
 
     if (!page) {
@@ -150,19 +154,27 @@ export class PageController {
           'Only space admins can permanently delete pages',
         );
       }
-      await this.pageService.forceDelete(deletePageDto.pageId);
+      await this.pageService.forceDelete(deletePageDto.pageId, workspace.id);
     } else {
       // Soft delete requires page manage permissions
       if (ability.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Page)) {
         throw new ForbiddenException();
       }
-      await this.pageService.remove(deletePageDto.pageId, user.id);
+      await this.pageService.removePage(
+        deletePageDto.pageId,
+        user.id,
+        workspace.id,
+      );
     }
   }
 
   @HttpCode(HttpStatus.OK)
   @Post('restore')
-  async restore(@Body() pageIdDto: PageIdDto, @AuthUser() user: User) {
+  async restore(
+    @Body() pageIdDto: PageIdDto,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
     const page = await this.pageRepo.findById(pageIdDto.pageId);
 
     if (!page) {
@@ -174,13 +186,11 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    await this.pageRepo.restorePage(pageIdDto.pageId);
+    await this.pageRepo.restorePage(pageIdDto.pageId, workspace.id);
 
-    // Return the restored page data with hasChildren info
-    const restoredPage = await this.pageRepo.findById(pageIdDto.pageId, {
+    return this.pageRepo.findById(pageIdDto.pageId, {
       includeHasChildren: true,
     });
-    return restoredPage;
   }
 
   @HttpCode(HttpStatus.OK)
@@ -282,21 +292,28 @@ export class PageController {
     @Body() pagination: PaginationOptions,
     @AuthUser() user: User,
   ) {
-    const ability = await this.spaceAbility.createForUser(user, dto.spaceId);
+    if (!dto.spaceId && !dto.pageId) {
+      throw new BadRequestException(
+        'Either spaceId or pageId must be provided',
+      );
+    }
+    let spaceId = dto.spaceId;
+
+    if (dto.pageId) {
+      const page = await this.pageRepo.findById(dto.pageId);
+      if (!page) {
+        throw new ForbiddenException();
+      }
+
+      spaceId = page.spaceId;
+    }
+
+    const ability = await this.spaceAbility.createForUser(user, spaceId);
     if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
       throw new ForbiddenException();
     }
 
-    let pageId = null;
-    if (dto.pageId) {
-      const page = await this.pageRepo.findById(dto.pageId);
-      if (page.spaceId !== dto.spaceId) {
-        throw new ForbiddenException();
-      }
-      pageId = page.id;
-    }
-
-    return this.pageService.getSidebarPages(dto.spaceId, pagination, pageId);
+    return this.pageService.getSidebarPages(spaceId, pagination, dto.pageId);
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/page/services/page.service.ts

@@ -38,6 +38,8 @@ import { StorageService } from '../../../integrations/storage/storage.service';
 import { InjectQueue } from '@nestjs/bullmq';
 import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
+import { EventName } from '../../../common/events/event.contants';
+import { EventEmitter2 } from '@nestjs/event-emitter';
 
 @Injectable()
 export class PageService {
@@ -49,6 +51,8 @@ export class PageService {
     @InjectKysely() private readonly db: KyselyDB,
     private readonly storageService: StorageService,
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
+    private eventEmitter: EventEmitter2,
   ) {}
 
   async findById(
@@ -231,21 +235,33 @@ export class PageService {
         );
       }
 
-      // update spaceId in shares
       if (pageIds.length > 0) {
+        // update spaceId in shares
         await trx
           .updateTable('shares')
           .set({ spaceId: spaceId })
           .where('pageId', 'in', pageIds)
           .execute();
-      }
 
-      // Update attachments
-      await this.attachmentRepo.updateAttachmentsByPageId(
-        { spaceId },
-        pageIds,
-        trx,
-      );
+        // Update comments
+        await trx
+          .updateTable('comments')
+          .set({ spaceId: spaceId })
+          .where('pageId', 'in', pageIds)
+          .execute();
+
+        // Update attachments
+        await this.attachmentRepo.updateAttachmentsByPageId(
+          { spaceId },
+          pageIds,
+          trx,
+        );
+
+        await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
+          pageId: pageIds,
+          workspaceId: rootPage.workspaceId
+        });
+      }
     });
   }
 
@@ -371,15 +387,21 @@ export class PageService {
           workspaceId: page.workspaceId,
           creatorId: authUser.id,
           lastUpdatedById: authUser.id,
-          parentPageId: page.parentPageId
-            ? pageMap.get(page.parentPageId)?.newPageId
-            : null,
+          parentPageId: page.id === rootPage.id
+            ? (isDuplicateInSameSpace ? rootPage.parentPageId : null)
+            : (page.parentPageId ? pageMap.get(page.parentPageId)?.newPageId : null),
         };
       }),
     );
 
     await this.db.insertInto('pages').values(insertablePages).execute();
 
+    const insertedPageIds = insertablePages.map((page) => page.id);
+    this.eventEmitter.emit(EventName.PAGE_CREATED, {
+      pageIds: insertedPageIds,
+      workspaceId: authUser.workspaceId,
+    });
+
     //TODO: best to handle this in a queue
     const attachmentsIds = Array.from(attachmentMap.keys());
     if (attachmentsIds.length > 0) {
@@ -565,7 +587,7 @@ export class PageService {
     return await this.pageRepo.getDeletedPagesInSpace(spaceId, pagination);
   }
 
-  async forceDelete(pageId: string): Promise<void> {
+  async forceDelete(pageId: string, workspaceId: string): Promise<void> {
     // Get all descendant IDs (including the page itself) using recursive CTE
     const descendants = await this.db
       .withRecursive('page_descendants', (db) =>
@@ -606,10 +628,18 @@ export class PageService {
 
     if (pageIds.length > 0) {
       await this.db.deleteFrom('pages').where('id', 'in', pageIds).execute();
+      this.eventEmitter.emit(EventName.PAGE_DELETED, {
+        pageIds: pageIds,
+        workspaceId,
+      });
     }
   }
 
-  async remove(pageId: string, userId: string): Promise<void> {
-    await this.pageRepo.removePage(pageId, userId);
+  async removePage(
+    pageId: string,
+    userId: string,
+    workspaceId: string,
+  ): Promise<void> {
+    await this.pageRepo.removePage(pageId, userId, workspaceId);
   }
 }

apps/server/src/core/search/dto/search-response.dto.ts

@@ -1,3 +1,5 @@
+import { Space } from '@docmost/db/types/entity.types';
+
 export class SearchResponseDto {
   id: string;
   title: string;
@@ -8,4 +10,5 @@ export class SearchResponseDto {
   highlight: string;
   createdAt: Date;
   updatedAt: Date;
+  space: Partial<Space>;
 }

apps/server/src/core/search/dto/search.dto.ts

@@ -5,15 +5,13 @@ import {
   IsOptional,
   IsString,
 } from 'class-validator';
-import { PartialType } from '@nestjs/mapped-types';
-import { CreateWorkspaceDto } from '../../workspace/dto/create-workspace.dto';
 
 export class SearchDTO {
   @IsNotEmpty()
   @IsString()
   query: string;
 
-  @IsNotEmpty()
+  @IsOptional()
   @IsString()
   spaceId: string;
 

apps/server/src/core/search/search.controller.ts

@@ -5,6 +5,7 @@ import {
   ForbiddenException,
   HttpCode,
   HttpStatus,
+  Logger,
   Post,
   UseGuards,
 } from '@nestjs/common';
@@ -24,13 +25,19 @@ import {
 } from '../casl/interfaces/space-ability.type';
 import { AuthUser } from '../../common/decorators/auth-user.decorator';
 import { Public } from 'src/common/decorators/public.decorator';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+import { ModuleRef } from '@nestjs/core';
 
 @UseGuards(JwtAuthGuard)
 @Controller('search')
 export class SearchController {
+  private readonly logger = new Logger(SearchController.name);
+
   constructor(
     private readonly searchService: SearchService,
     private readonly spaceAbility: SpaceAbilityFactory,
+    private readonly environmentService: EnvironmentService,
+    private moduleRef: ModuleRef,
   ) {}
 
   @HttpCode(HttpStatus.OK)
@@ -53,7 +60,14 @@ export class SearchController {
       }
     }
 
-    return this.searchService.searchPage(searchDto.query, searchDto, {
+    if (this.environmentService.getSearchDriver() === 'typesense') {
+      return this.searchTypesense(searchDto, {
+        userId: user.id,
+        workspaceId: workspace.id,
+      });
+    }
+
+    return this.searchService.searchPage(searchDto, {
       userId: user.id,
       workspaceId: workspace.id,
     });
@@ -81,8 +95,47 @@ export class SearchController {
       throw new BadRequestException('shareId is required');
     }
 
-    return this.searchService.searchPage(searchDto.query, searchDto, {
+    if (this.environmentService.getSearchDriver() === 'typesense') {
+      return this.searchTypesense(searchDto, {
+        workspaceId: workspace.id,
+      });
+    }
+
+    return this.searchService.searchPage(searchDto, {
       workspaceId: workspace.id,
     });
   }
+
+  async searchTypesense(
+    searchParams: SearchDTO,
+    opts: {
+      userId?: string;
+      workspaceId: string;
+    },
+  ) {
+    const { userId, workspaceId } = opts;
+    let TypesenseModule: any;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      TypesenseModule = require('./../../ee/typesense/services/page-search.service');
+
+      const PageSearchService = this.moduleRef.get(
+        TypesenseModule.PageSearchService,
+        {
+          strict: false,
+        },
+      );
+
+      return PageSearchService.searchPage(searchParams, {
+        userId: userId,
+        workspaceId,
+      });
+    } catch (err) {
+      this.logger.debug(
+        'Typesense module requested but enterprise module not bundled in this build',
+      );
+    }
+
+    throw new BadRequestException('Enterprise Typesense search module missing');
+  }
 }

apps/server/src/core/search/search.service.ts

@@ -21,13 +21,14 @@ export class SearchService {
   ) {}
 
   async searchPage(
-    query: string,
     searchParams: SearchDTO,
     opts: {
       userId?: string;
       workspaceId: string;
     },
   ): Promise<SearchResponseDto[]> {
+    const { query } = searchParams;
+
     if (query.length < 1) {
       return;
     }
@@ -61,7 +62,7 @@ export class SearchService {
       )
       .where('deletedAt', 'is', null)
       .orderBy('rank', 'desc')
-      .limit(searchParams.limit | 20)
+      .limit(searchParams.limit | 25)
       .offset(searchParams.offset || 0);
 
     if (!searchParams.shareId) {

apps/server/src/core/share/share.controller.ts

@@ -31,6 +31,7 @@ import { Public } from '../../common/decorators/public.decorator';
 import { ShareRepo } from '@docmost/db/repos/share/share.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import { EnvironmentService } from '../../integrations/environment/environment.service';
+import { hasLicenseOrEE } from '../../common/helpers';
 
 @UseGuards(JwtAuthGuard)
 @Controller('shares')
@@ -65,9 +66,11 @@ export class ShareController {
 
     return {
       ...(await this.shareService.getSharedPage(dto, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 
@@ -175,9 +178,11 @@ export class ShareController {
   ) {
     return {
       ...(await this.shareService.getShareTree(dto.shareId, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 }

apps/server/src/core/share/share.service.ts

@@ -69,8 +69,8 @@ export class ShareService {
       return await this.shareRepo.insertShare({
         key: nanoIdGen().toLowerCase(),
         pageId: page.id,
-        includeSubPages: createShareDto.includeSubPages || true,
-        searchIndexing: createShareDto.searchIndexing || true,
+        includeSubPages: createShareDto.includeSubPages ?? false,
+        searchIndexing: createShareDto.searchIndexing ?? false,
         creatorId: authUserId,
         spaceId: page.spaceId,
         workspaceId,

apps/server/src/core/workspace/dto/update-workspace.dto.ts

@@ -18,4 +18,16 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
   @IsOptional()
   @IsBoolean()
   enforceMfa: boolean;
+
+  @IsOptional()
+  @IsBoolean()
+  restrictApiToAdmins: boolean;
+
+  @IsOptional()
+  @IsBoolean()
+  aiSearch: boolean;
+
+  @IsOptional()
+  @IsBoolean()
+  generativeAi: boolean;
 }

apps/server/src/core/workspace/services/workspace.service.ts

@@ -33,6 +33,7 @@ import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { Queue } from 'bullmq';
 import { generateRandomSuffixNumbers } from '../../../common/helpers';
+import { isPageEmbeddingsTableExists } from '@docmost/db/helpers/helpers';
 
 @Injectable()
 export class WorkspaceService {
@@ -50,6 +51,7 @@ export class WorkspaceService {
     @InjectKysely() private readonly db: KyselyDB,
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
     @InjectQueue(QueueName.BILLING_QUEUE) private billingQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
   ) {}
 
   async findById(workspaceId: string) {
@@ -303,6 +305,60 @@ export class WorkspaceService {
       }
     }
 
+    if (typeof updateWorkspaceDto.restrictApiToAdmins !== 'undefined') {
+      await this.workspaceRepo.updateApiSettings(
+        workspaceId,
+        'restrictToAdmins',
+        updateWorkspaceDto.restrictApiToAdmins,
+      );
+      delete updateWorkspaceDto.restrictApiToAdmins;
+    }
+
+    if (typeof updateWorkspaceDto.aiSearch !== 'undefined') {
+      await this.workspaceRepo.updateAiSettings(
+        workspaceId,
+        'search',
+        updateWorkspaceDto.aiSearch,
+      );
+
+      if (updateWorkspaceDto.aiSearch) {
+        const tableExists = await isPageEmbeddingsTableExists(this.db);
+        if (!tableExists) {
+          throw new BadRequestException(
+            'Failed to activate. Make sure pgvector postgres extension is installed.',
+          );
+        }
+
+        await this.aiQueue.add(QueueJob.WORKSPACE_CREATE_EMBEDDINGS, {
+          workspaceId,
+        });
+      } else {
+        // Schedule deletion after 24 hours
+        const deleteJobId = `ai-search-disabled-${workspaceId}`;
+        await this.aiQueue.add(
+          QueueJob.WORKSPACE_DELETE_EMBEDDINGS,
+          { workspaceId },
+          {
+            jobId: deleteJobId,
+            delay: 24 * 60 * 60 * 1000,
+            removeOnComplete: true,
+            removeOnFail: true,
+          },
+        );
+      }
+
+      delete updateWorkspaceDto.aiSearch;
+    }
+
+    if (typeof updateWorkspaceDto.generativeAi !== 'undefined') {
+      await this.workspaceRepo.updateAiSettings(
+        workspaceId,
+        'generative',
+        updateWorkspaceDto.generativeAi,
+      );
+      delete updateWorkspaceDto.generativeAi;
+    }
+
     await this.workspaceRepo.updateWorkspace(updateWorkspaceDto, workspaceId);
 
     const workspace = await this.workspaceRepo.findById(workspaceId, {

apps/server/src/database/database.module.ts

@@ -25,6 +25,7 @@ import { MigrationService } from '@docmost/db/services/migration.service';
 import { UserTokenRepo } from './repos/user-token/user-token.repo';
 import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
 import { ShareRepo } from '@docmost/db/repos/share/share.repo';
+import { PageListener } from '@docmost/db/listeners/page.listener';
 import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission-repo.service';
 
 // https://github.com/brianc/node-postgres/issues/811
@@ -77,6 +78,7 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
     UserTokenRepo,
     BacklinkRepo,
     ShareRepo,
+    PageListener,
     PagePermissionRepo,
   ],
   exports: [

apps/server/src/database/helpers/helpers.ts

@@ -0,0 +1,22 @@
+import { sql } from 'kysely';
+import { KyselyDB } from '@docmost/db/types/kysely.types';
+
+export async function isPageEmbeddingsTableExists(db: KyselyDB) {
+  return tableExists({ db, tableName: 'page_embeddings' });
+}
+
+export async function tableExists(opts: {
+  db: KyselyDB;
+  tableName: string;
+}): Promise<boolean> {
+  const { db, tableName } = opts;
+  const result = await sql<{ exists: boolean }>`
+      SELECT EXISTS (
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = COALESCE(current_schema(), 'public')
+        AND table_name = ${tableName}
+      ) as exists
+    `.execute(db);
+
+  return result.rows[0]?.exists ?? false;
+}

apps/server/src/database/listeners/page.listener.ts

@@ -0,0 +1,80 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+import { EventName } from '../../common/events/event.contants';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../integrations/queue/constants';
+import { Queue } from 'bullmq';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+
+export class PageEvent {
+  pageIds: string[];
+  workspaceId: string;
+}
+
+@Injectable()
+export class PageListener {
+  private readonly logger = new Logger(PageListener.name);
+
+  constructor(
+    private readonly environmentService: EnvironmentService,
+    @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
+  ) {}
+
+  @OnEvent(EventName.PAGE_CREATED)
+  async handlePageCreated(event: PageEvent) {
+    const { pageIds, workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_CREATED, {
+        pageIds,
+      });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_CREATED, { pageIds, workspaceId });
+  }
+
+  @OnEvent(EventName.PAGE_UPDATED)
+  async handlePageUpdated(event: PageEvent) {
+    const { pageIds } = event;
+
+    await this.searchQueue.add(QueueJob.PAGE_UPDATED, { pageIds });
+  }
+
+  @OnEvent(EventName.PAGE_DELETED)
+  async handlePageDeleted(event: PageEvent) {
+    const { pageIds, workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_DELETED, { pageIds, workspaceId });
+  }
+
+  @OnEvent(EventName.PAGE_SOFT_DELETED)
+  async handlePageSoftDeleted(event: PageEvent) {
+    const { pageIds, workspaceId } = event;
+
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_SOFT_DELETED, {
+      pageIds,
+      workspaceId,
+    });
+  }
+
+  @OnEvent(EventName.PAGE_RESTORED)
+  async handlePageRestored(event: PageEvent) {
+    const { pageIds, workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds });
+    }
+
+    await this.aiQueue.add(QueueJob.PAGE_RESTORED, { pageIds, workspaceId });
+  }
+
+  isTypesense(): boolean {
+    return this.environmentService.getSearchDriver() === 'typesense';
+  }
+}

apps/server/src/database/listeners/space.listener.ts

@@ -0,0 +1,36 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+import { EventName } from '../../common/events/event.contants';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../integrations/queue/constants';
+import { Queue } from 'bullmq';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+
+export class SpaceEvent {
+  spaceId: string;
+}
+
+@Injectable()
+export class SpaceListener {
+  private readonly logger = new Logger(SpaceListener.name);
+
+  constructor(
+    private readonly environmentService: EnvironmentService,
+    @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
+  ) {}
+
+  @OnEvent(EventName.SPACE_DELETED)
+  async handleSpaceDeleted(event: SpaceEvent) {
+    const { spaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.SPACE_DELETED, { spaceId });
+    }
+
+    await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId });
+  }
+
+  isTypesense(): boolean {
+    return this.environmentService.getSearchDriver() === 'typesense';
+  }
+}

apps/server/src/database/listeners/workspace.listener.ts

@@ -0,0 +1,36 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+import { EventName } from '../../common/events/event.contants';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../integrations/queue/constants';
+import { Queue } from 'bullmq';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+
+export class WorkspaceEvent {
+  workspaceId: string;
+}
+
+@Injectable()
+export class WorkspaceListener {
+  private readonly logger = new Logger(WorkspaceListener.name);
+
+  constructor(
+    private readonly environmentService: EnvironmentService,
+    @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+    @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
+  ) {}
+
+  @OnEvent(EventName.WORKSPACE_DELETED)
+  async handlePageDeleted(event: WorkspaceEvent) {
+    const { workspaceId } = event;
+    if (this.isTypesense()) {
+      await this.searchQueue.add(QueueJob.WORKSPACE_DELETED, { workspaceId });
+    }
+
+    await this.aiQueue.add(QueueJob.WORKSPACE_DELETED, { workspaceId });
+  }
+
+  isTypesense(): boolean {
+    return this.environmentService.getSearchDriver() === 'typesense';
+  }
+}

apps/server/src/database/migrations/20250831T191600-add-group-sync-to-auth-providers.ts

@@ -0,0 +1,15 @@
+import { type Kysely } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('auth_providers')
+    .addColumn('group_sync', 'boolean', (col) => col.defaultTo(false).notNull())
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('auth_providers')
+    .dropColumn('group_sync')
+    .execute();
+}

apps/server/src/database/migrations/20250831T202306-ldap-auth.ts

@@ -0,0 +1,68 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  // switch type to text column since you can't add value to PG types in a transaction
+  await db.schema
+    .alterTable('auth_providers')
+    .alterColumn('type', (col) => col.setDataType('text'))
+    .execute();
+
+  await db.schema.dropType('auth_provider_type').ifExists().execute();
+
+  await db.schema
+    .alterTable('users')
+    .addColumn('has_generated_password', 'boolean', (col) =>
+      col.notNull().defaultTo(false).ifNotExists(),
+    )
+    .execute();
+
+  await db.schema
+    .alterTable('auth_providers')
+    .addColumn('ldap_url', 'varchar', (col) => col)
+    .addColumn('ldap_bind_dn', 'varchar', (col) => col)
+    .addColumn('ldap_bind_password', 'varchar', (col) => col)
+    .addColumn('ldap_base_dn', 'varchar', (col) => col)
+    .addColumn('ldap_user_search_filter', 'varchar', (col) => col)
+    .addColumn('ldap_user_attributes', 'jsonb', (col) =>
+      col.defaultTo(sql`'{}'::jsonb`),
+    )
+    .addColumn('ldap_tls_enabled', 'boolean', (col) => col.defaultTo(false))
+    .addColumn('ldap_tls_ca_cert', 'text', (col) => col)
+    .addColumn('ldap_config', 'jsonb', (col) => col.defaultTo(sql`'{}'::jsonb`))
+    .addColumn('settings', 'jsonb', (col) => col.defaultTo(sql`'{}'::jsonb`))
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('users')
+    .dropColumn('has_generated_password')
+    .execute();
+
+  await db.schema
+    .alterTable('auth_providers')
+    .dropColumn('ldap_url')
+    .dropColumn('ldap_bind_dn')
+    .dropColumn('ldap_bind_password')
+    .dropColumn('ldap_base_dn')
+    .dropColumn('ldap_user_search_filter')
+    .dropColumn('ldap_user_attributes')
+    .dropColumn('ldap_tls_enabled')
+    .dropColumn('ldap_tls_ca_cert')
+    .dropColumn('ldap_config')
+    .dropColumn('settings')
+    .execute();
+
+  await db.schema
+    .createType('auth_provider_type')
+    .asEnum(['saml', 'oidc', 'google'])
+    .execute();
+
+  await db.deleteFrom('auth_providers').where('type', '=', 'ldap').execute();
+
+  await sql`
+    ALTER TABLE auth_providers 
+    ALTER COLUMN type TYPE auth_provider_type 
+    USING type::auth_provider_type
+  `.execute(db);
+}

apps/server/src/database/migrations/20250901T184612-attachments-search.ts

@@ -0,0 +1,29 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .addColumn('text_content', 'text', (col) => col)
+    .addColumn('tsv', sql`tsvector`, (col) => col)
+    .execute();
+
+  await db.schema
+    .createIndex('attachments_tsv_idx')
+    .on('attachments')
+    .using('GIN')
+    .column('tsv')
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .dropIndex('attachments_tsv_idx')
+    .execute();
+
+  await db.schema
+    .alterTable('attachments')
+    .dropColumn('text_content')
+    .dropColumn('tsv')
+    .execute();
+}

apps/server/src/database/migrations/20250912T101500-api-keys.ts

@@ -0,0 +1,30 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .createTable('api_keys')
+    .addColumn('id', 'uuid', (col) =>
+      col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
+    )
+    .addColumn('name', 'text', (col) => col)
+    .addColumn('creator_id', 'uuid', (col) =>
+      col.notNull().references('users.id').onDelete('cascade'),
+    )
+    .addColumn('workspace_id', 'uuid', (col) =>
+      col.notNull().references('workspaces.id').onDelete('cascade'),
+    )
+    .addColumn('expires_at', 'timestamptz')
+    .addColumn('last_used_at', 'timestamptz')
+    .addColumn('created_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`),
+    )
+    .addColumn('updated_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`),
+    )
+    .addColumn('deleted_at', 'timestamptz', (col) => col)
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.dropTable('api_keys').execute();
+}

apps/server/src/database/pagination/pagination-options.ts

@@ -1,4 +1,5 @@
 import {
+  IsBoolean,
   IsNumber,
   IsOptional,
   IsPositive,
@@ -23,4 +24,8 @@ export class PaginationOptions {
   @IsOptional()
   @IsString()
   query?: string;
+
+  @IsOptional()
+  @IsBoolean()
+  adminView: boolean;
 }

apps/server/src/database/repos/attachment/attachment.repo.ts

@@ -12,6 +12,23 @@ import {
 export class AttachmentRepo {
   constructor(@InjectKysely() private readonly db: KyselyDB) {}
 
+  private baseFields: Array<keyof Attachment> = [
+    'id',
+    'fileName',
+    'filePath',
+    'fileSize',
+    'fileExt',
+    'mimeType',
+    'type',
+    'creatorId',
+    'pageId',
+    'spaceId',
+    'workspaceId',
+    'createdAt',
+    'updatedAt',
+    'deletedAt',
+  ];
+
   async findById(
     attachmentId: string,
     opts?: {
@@ -22,7 +39,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('id', '=', attachmentId)
       .executeTakeFirst();
   }
@@ -36,7 +53,7 @@ export class AttachmentRepo {
     return db
       .insertInto('attachments')
       .values(insertableAttachment)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -50,7 +67,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('spaceId', '=', spaceId)
       .execute();
   }
@@ -64,6 +81,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('pageId', 'in', pageIds)
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -75,7 +93,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('id', '=', attachmentId)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 

apps/server/src/database/repos/page/page.repo.ts

@@ -14,12 +14,15 @@ import { ExpressionBuilder, sql } from 'kysely';
 import { DB } from '@docmost/db/types/db';
 import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
 import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
 
 @Injectable()
 export class PageRepo {
   constructor(
     @InjectKysely() private readonly db: KyselyDB,
     private spaceMemberRepo: SpaceMemberRepo,
+    private eventEmitter: EventEmitter2,
   ) {}
 
   private baseFields: Array<keyof Page> = [
@@ -45,6 +48,7 @@ export class PageRepo {
     pageId: string,
     opts?: {
       includeContent?: boolean;
+      includeTextContent?: boolean;
       includeYdoc?: boolean;
       includeSpace?: boolean;
       includeCreator?: boolean;
@@ -62,6 +66,7 @@ export class PageRepo {
       .select(this.baseFields)
       .$if(opts?.includeContent, (qb) => qb.select('content'))
       .$if(opts?.includeYdoc, (qb) => qb.select('ydoc'))
+      .$if(opts?.includeTextContent, (qb) => qb.select('textContent'))
       .$if(opts?.includeHasChildren, (qb) =>
         qb.select((eb) => this.withHasChildren(eb)),
       );
@@ -108,7 +113,7 @@ export class PageRepo {
     pageIds: string[],
     trx?: KyselyTransaction,
   ) {
-    return dbOrTx(this.db, trx)
+    const result = await dbOrTx(this.db, trx)
       .updateTable('pages')
       .set({ ...updatePageData, updatedAt: new Date() })
       .where(
@@ -117,6 +122,13 @@ export class PageRepo {
         pageIds,
       )
       .executeTakeFirst();
+
+    this.eventEmitter.emit(EventName.PAGE_UPDATED, {
+      pageIds: pageIds,
+      workspaceId: updatePageData.workspaceId,
+    });
+
+    return result;
   }
 
   async insertPage(
@@ -124,11 +136,18 @@ export class PageRepo {
     trx?: KyselyTransaction,
   ): Promise<Page> {
     const db = dbOrTx(this.db, trx);
-    return db
+    const result = await db
       .insertInto('pages')
       .values(insertablePage)
       .returning(this.baseFields)
       .executeTakeFirst();
+
+    this.eventEmitter.emit(EventName.PAGE_CREATED, {
+      pageIds: [result.id],
+      workspaceId: result.workspaceId,
+    });
+
+    return result;
   }
 
   async deletePage(pageId: string): Promise<void> {
@@ -143,7 +162,11 @@ export class PageRepo {
     await query.execute();
   }
 
-  async removePage(pageId: string, deletedById: string): Promise<void> {
+  async removePage(
+    pageId: string,
+    deletedById: string,
+    workspaceId: string,
+  ): Promise<void> {
     const currentDate = new Date();
 
     const descendants = await this.db
@@ -178,10 +201,15 @@ export class PageRepo {
 
         await trx.deleteFrom('shares').where('pageId', 'in', pageIds).execute();
       });
+
+      this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, {
+        pageIds: pageIds,
+        workspaceId,
+      });
     }
   }
 
-  async restorePage(pageId: string): Promise<void> {
+  async restorePage(pageId: string, workspaceId: string): Promise<void> {
     // First, check if the page being restored has a deleted parent
     const pageToRestore = await this.db
       .selectFrom('pages')
@@ -241,6 +269,10 @@ export class PageRepo {
         .where('id', '=', pageId)
         .execute();
     }
+    this.eventEmitter.emit(EventName.PAGE_RESTORED, {
+      pageIds: pageIds,
+      workspaceId: workspaceId,
+    });
   }
 
   async getRecentPagesInSpace(spaceId: string, pagination: PaginationOptions) {
@@ -399,6 +431,7 @@ export class PageRepo {
           ])
           .$if(opts?.includeContent, (qb) => qb.select('content'))
           .where('id', '=', parentPageId)
+          .where('deletedAt', 'is', null)
           .unionAll((exp) =>
             exp
               .selectFrom('pages as p')
@@ -413,7 +446,8 @@ export class PageRepo {
                 'p.workspaceId',
               ])
               .$if(opts?.includeContent, (qb) => qb.select('p.content'))
-              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id'),
+              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id')
+              .where('p.deletedAt', 'is', null),
           ),
       )
       .selectFrom('page_hierarchy')

apps/server/src/database/repos/space/space.repo.ts

@@ -12,10 +12,15 @@ import { PaginationOptions } from '../../pagination/pagination-options';
 import { executeWithPagination } from '@docmost/db/pagination/pagination';
 import { DB } from '@docmost/db/types/db';
 import { validate as isValidUUID } from 'uuid';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
 
 @Injectable()
 export class SpaceRepo {
-  constructor(@InjectKysely() private readonly db: KyselyDB) {}
+  constructor(
+    @InjectKysely() private readonly db: KyselyDB,
+    private eventEmitter: EventEmitter2,
+  ) {}
 
   async findById(
     spaceId: string,
@@ -159,5 +164,9 @@ export class SpaceRepo {
       .where('id', '=', spaceId)
       .where('workspaceId', '=', workspaceId)
       .execute();
+
+    this.eventEmitter.emit(EventName.SPACE_DELETED, {
+      spaceId,
+    });
   }
 }

apps/server/src/database/repos/user/user.repo.ts

@@ -34,6 +34,7 @@ export class UserRepo {
     'createdAt',
     'updatedAt',
     'deletedAt',
+    'hasGeneratedPassword',
   ];
 
   async findById(

apps/server/src/database/repos/workspace/workspace.repo.ts

@@ -157,4 +157,40 @@ export class WorkspaceRepo {
 
     return activeUsers.length;
   }
+
+  async updateApiSettings(
+    workspaceId: string,
+    prefKey: string,
+    prefValue: string | boolean,
+  ) {
+    return this.db
+      .updateTable('workspaces')
+      .set({
+        settings: sql`COALESCE(settings, '{}'::jsonb)
+                || jsonb_build_object('api', COALESCE(settings->'api', '{}'::jsonb) 
+                || jsonb_build_object('${sql.raw(prefKey)}', ${sql.lit(prefValue)}))`,
+        updatedAt: new Date(),
+      })
+      .where('id', '=', workspaceId)
+      .returning(this.baseFields)
+      .executeTakeFirst();
+  }
+
+  async updateAiSettings(
+    workspaceId: string,
+    prefKey: string,
+    prefValue: string | boolean,
+  ) {
+    return this.db
+      .updateTable('workspaces')
+      .set({
+        settings: sql`COALESCE(settings, '{}'::jsonb)
+                || jsonb_build_object('ai', COALESCE(settings->'ai', '{}'::jsonb) 
+                || jsonb_build_object('${sql.raw(prefKey)}', ${sql.lit(prefValue)}))`,
+        updatedAt: new Date(),
+      })
+      .where('id', '=', workspaceId)
+      .returning(this.baseFields)
+      .executeTakeFirst();
+  }
 }

apps/server/src/database/types/db.d.ts

@@ -30,6 +30,18 @@ export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
 
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
+export interface ApiKeys {
+  createdAt: Generated<Timestamp>;
+  deletedAt: Timestamp | null;
+  expiresAt: Timestamp | null;
+  id: Generated<string>;
+  lastUsedAt: Timestamp | null;
+  name: string | null;
+  updatedAt: Generated<Timestamp>;
+  creatorId: string;
+  workspaceId: string;
+}
+
 export interface Attachments {
   createdAt: Generated<Timestamp>;
   creatorId: string;
@@ -42,6 +54,8 @@ export interface Attachments {
   mimeType: string | null;
   pageId: string | null;
   spaceId: string | null;
+  textContent: string | null;
+  tsv: string | null;
   type: string | null;
   updatedAt: Generated<Timestamp>;
   workspaceId: string;
@@ -65,6 +79,7 @@ export interface AuthProviders {
   deletedAt: Timestamp | null;
   id: Generated<string>;
   isEnabled: Generated<boolean>;
+  groupSync: Generated<boolean>;
   ldapBaseDn: string | null;
   ldapBindDn: string | null;
   ldapBindPassword: string | null;
@@ -73,6 +88,8 @@ export interface AuthProviders {
   ldapUrl: string | null;
   ldapUserAttributes: Json | null;
   ldapUserSearchFilter: string | null;
+  ldapConfig: Json | null;
+  settings: Json | null;
   name: string;
   oidcClientId: string | null;
   oidcClientSecret: string | null;
@@ -301,6 +318,7 @@ export interface Users {
   lastActiveAt: Timestamp | null;
   lastLoginAt: Timestamp | null;
   locale: string | null;
+  hasGeneratedPassword: Generated<boolean | null>;
   name: string | null;
   password: string | null;
   role: string | null;
@@ -364,6 +382,7 @@ export interface Workspaces {
 }
 
 export interface DB {
+  apiKeys: ApiKeys;
   attachments: Attachments;
   authAccounts: AuthAccounts;
   authProviders: AuthProviders;

apps/server/src/database/types/db.interface.ts

@@ -0,0 +1,51 @@
+import {
+  ApiKeys,
+  Attachments,
+  AuthAccounts,
+  AuthProviders,
+  Backlinks,
+  Billing,
+  Comments,
+  FileTasks,
+  Groups,
+  GroupUsers,
+  PageHistory,
+  PagePermissions,
+  Pages,
+  Shares,
+  SpaceMembers,
+  Spaces,
+  UserMfa,
+  Users,
+  UserSharedPages,
+  UserTokens,
+  WorkspaceInvitations,
+  Workspaces,
+} from '@docmost/db/types/db';
+import { PageEmbeddings } from '@docmost/db/types/embeddings.types';
+
+export interface DbInterface {
+  attachments: Attachments;
+  authAccounts: AuthAccounts;
+  authProviders: AuthProviders;
+  backlinks: Backlinks;
+  billing: Billing;
+  comments: Comments;
+  fileTasks: FileTasks;
+  groups: Groups;
+  groupUsers: GroupUsers;
+  pageEmbeddings: PageEmbeddings;
+  pagePermissions: PagePermissions;
+  pageHistory: PageHistory;
+  pages: Pages;
+  shares: Shares;
+  spaceMembers: SpaceMembers;
+  spaces: Spaces;
+  userMfa: UserMfa;
+  users: Users;
+  userSharedPages: UserSharedPages;
+  userTokens: UserTokens;
+  workspaceInvitations: WorkspaceInvitations;
+  workspaces: Workspaces;
+  apiKeys: ApiKeys;
+}

apps/server/src/database/types/embeddings.types.ts

@@ -0,0 +1,20 @@
+import { Json, Timestamp, Generated } from '@docmost/db/types/db';
+
+// embeddings type
+export interface PageEmbeddings {
+  id: Generated<string>;
+  pageId: string;
+  spaceId: string;
+  modelName: string;
+  modelDimensions: number;
+  workspaceId: string;
+  attachmentId: string;
+  embedding: number[];
+  chunkIndex: Generated<number>;
+  chunkStart: Generated<number>;
+  chunkLength: Generated<number>;
+  metadata: Generated<Json>;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Generated<Timestamp>;
+  deletedAt: Timestamp | null;
+}

apps/server/src/database/types/entity.types.ts

@@ -21,7 +21,9 @@ import {
   Shares,
   FileTasks,
   UserMfa as _UserMFA,
+  ApiKeys,
 } from './db';
+import { PageEmbeddings } from '@docmost/db/types/embeddings.types';
 
 // Workspace
 export type Workspace = Selectable<Workspaces>;
@@ -130,3 +132,13 @@ export type UpdatableFileTask = Updateable<Omit<FileTasks, 'id'>>;
 export type UserMFA = Selectable<_UserMFA>;
 export type InsertableUserMFA = Insertable<_UserMFA>;
 export type UpdatableUserMFA = Updateable<Omit<_UserMFA, 'id'>>;
+
+// Api Keys
+export type ApiKey = Selectable<ApiKeys>;
+export type InsertableApiKey = Insertable<ApiKeys>;
+export type UpdatableApiKey = Updateable<Omit<ApiKeys, 'id'>>;
+
+// Page Embedding
+export type PageEmbedding = Selectable<PageEmbeddings>;
+export type InsertablePageEmbedding = Insertable<PageEmbeddings>;
+export type UpdatablePageEmbedding = Updateable<Omit<PageEmbeddings, 'id'>>;

apps/server/src/database/types/kysely.types.ts

@@ -1,5 +1,5 @@
-import { DB } from './db';
 import { Kysely, Transaction } from 'kysely';
+import { DbInterface } from '@docmost/db/types/db.interface';
 
-export type KyselyDB = Kysely<DB>;
-export type KyselyTransaction = Transaction<DB>;
+export type KyselyDB = Kysely<DbInterface>;
+export type KyselyTransaction = Transaction<DbInterface>;

apps/server/src/integrations/environment/environment.service.ts

@@ -10,6 +10,10 @@ export class EnvironmentService {
     return this.configService.get<string>('NODE_ENV', 'development');
   }
 
+  isDevelopment(): boolean {
+    return this.getNodeEnv() === 'development';
+  }
+
   getAppUrl(): string {
     const rawUrl =
       this.configService.get<string>('APP_URL') ||
@@ -213,4 +217,64 @@ export class EnvironmentService {
   getPostHogKey(): string {
     return this.configService.get<string>('POSTHOG_KEY');
   }
+
+  getSearchDriver(): string {
+    return this.configService
+      .get<string>('SEARCH_DRIVER', 'database')
+      .toLowerCase();
+  }
+
+  getTypesenseUrl(): string {
+    return this.configService
+      .get<string>('TYPESENSE_URL', 'http://localhost:8108')
+      .toLowerCase();
+  }
+
+  getTypesenseApiKey(): string {
+    return this.configService.get<string>('TYPESENSE_API_KEY');
+  }
+
+  getTypesenseLocale(): string {
+    return this.configService
+      .get<string>('TYPESENSE_LOCALE', 'en')
+      .toLowerCase();
+  }
+
+  getAiDriver(): string {
+    return this.configService.get<string>('AI_DRIVER');
+  }
+
+  getAiEmbeddingModel(): string {
+    return this.configService.get<string>('AI_EMBEDDING_MODEL');
+  }
+
+  getAiCompletionModel(): string {
+    return this.configService.get<string>('AI_COMPLETION_MODEL');
+  }
+
+  getAiEmbeddingDimension(): number {
+    return parseInt(
+      this.configService.get<string>('AI_EMBEDDING_DIMENSION'),
+      10,
+    );
+  }
+
+  getOpenAiApiKey(): string {
+    return this.configService.get<string>('OPENAI_API_KEY');
+  }
+
+  getOpenAiApiUrl(): string {
+    return this.configService.get<string>('OPENAI_API_URL');
+  }
+
+  getGeminiApiKey(): string {
+    return this.configService.get<string>('GEMINI_API_KEY');
+  }
+
+  getOllamaApiUrl(): string {
+    return this.configService.get<string>(
+      'OLLAMA_API_URL',
+      'http://localhost:11434',
+    );
+  }
 }

apps/server/src/integrations/environment/environment.validation.ts

@@ -3,12 +3,14 @@ import {
   IsNotEmpty,
   IsNotIn,
   IsOptional,
+  IsString,
   IsUrl,
   MinLength,
   ValidateIf,
   validateSync,
 } from 'class-validator';
 import { plainToInstance } from 'class-transformer';
+import { IsISO6391 } from '../../common/validator/is-iso6391';
 
 export class EnvironmentVariables {
   @IsNotEmpty()
@@ -68,6 +70,85 @@ export class EnvironmentVariables {
   )
   @ValidateIf((obj) => obj.CLOUD === 'true'.toLowerCase())
   SUBDOMAIN_HOST: string;
+
+  @IsOptional()
+  @IsIn(['database', 'typesense'])
+  @IsString()
+  SEARCH_DRIVER: string;
+
+  @IsOptional()
+  @IsUrl(
+    {
+      protocols: ['http', 'https'],
+      require_tld: false,
+      allow_underscores: true,
+    },
+    {
+      message:
+        'TYPESENSE_URL must be a valid typesense url e.g http://localhost:8108',
+    },
+  )
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  TYPESENSE_URL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsNotEmpty()
+  @IsString()
+  TYPESENSE_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsISO6391()
+  @IsString()
+  TYPESENSE_LOCALE: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsIn(['openai', 'gemini', 'ollama'])
+  @IsString()
+  AI_DRIVER: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsString()
+  @IsNotEmpty()
+  AI_EMBEDDING_MODEL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_EMBEDDING_DIMENSION)
+  @IsIn(['768', '1024', '1536', '2000'])
+  @IsString()
+  AI_EMBEDDING_DIMENSION: string;
+
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER)
+  @IsString()
+  @IsNotEmpty()
+  AI_COMPLETION_MODEL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'openai')
+  @IsString()
+  @IsNotEmpty()
+  OPENAI_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.OPENAI_API_URL && obj.AI_DRIVER === 'openai')
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  OPENAI_API_URL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'gemini')
+  @IsString()
+  @IsNotEmpty()
+  GEMINI_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.AI_DRIVER && obj.AI_DRIVER === 'ollama')
+  @IsUrl({ protocols: ['http', 'https'], require_tld: false })
+  OLLAMA_API_URL: string;
 }
 
 export function validate(config: Record<string, any>) {

apps/server/src/integrations/export/export.controller.ts

@@ -46,7 +46,7 @@ export class ExportController {
       includeContent: true,
     });
 
-    if (!page) {
+    if (!page || page.deletedAt) {
       throw new NotFoundException('Page not found');
     }
 

apps/server/src/integrations/import/import.module.ts

@@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
 import { ImportService } from './services/import.service';
 import { ImportController } from './import.controller';
 import { StorageModule } from '../storage/storage.module';
-import { FileTaskService } from './services/file-task.service';
+import { FileImportTaskService } from './services/file-import-task.service';
 import { FileTaskProcessor } from './processors/file-task.processor';
 import { ImportAttachmentService } from './services/import-attachment.service';
 import { FileTaskController } from './file-task.controller';
@@ -11,7 +11,7 @@ import { PageModule } from '../../core/page/page.module';
 @Module({
   providers: [
     ImportService,
-    FileTaskService,
+    FileImportTaskService,
     FileTaskProcessor,
     ImportAttachmentService,
   ],

apps/server/src/integrations/import/processors/file-task.processor.ts

@@ -2,7 +2,7 @@ import { Logger, OnModuleDestroy } from '@nestjs/common';
 import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
-import { FileTaskService } from '../services/file-task.service';
+import { FileImportTaskService } from '../services/file-import-task.service';
 import { FileTaskStatus } from '../utils/file.utils';
 import { StorageService } from '../../storage/storage.service';
 
@@ -11,7 +11,7 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(FileTaskProcessor.name);
 
   constructor(
-    private readonly fileTaskService: FileTaskService,
+    private readonly fileTaskService: FileImportTaskService,
     private readonly storageService: StorageService,
   ) {
     super();
@@ -41,15 +41,40 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   @OnWorkerEvent('failed')
   async onFailed(job: Job) {
     this.logger.error(
-      `Error processing ${job.name} job. Reason: ${job.failedReason}`,
+      `Error processing ${job.name} job. Import Task ID: ${job.data.fileTaskId}. Reason: ${job.failedReason}`,
+    );
+
+    await this.handleFailedJob(job);
+  }
+
+  @OnWorkerEvent('completed')
+  async onCompleted(job: Job) {
+    this.logger.log(
+      `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`,
     );
 
+    try {
+      const fileTask = await this.fileTaskService.getFileTask(
+        job.data.fileTaskId,
+      );
+      if (fileTask) {
+        await this.storageService.delete(fileTask.filePath);
+        this.logger.debug(`Deleted imported zip file: ${fileTask.filePath}`);
+      }
+    } catch (err) {
+      this.logger.error(`Failed to delete imported zip file:`, err);
+    }
+  }
+
+  private async handleFailedJob(job: Job) {
     try {
       const fileTaskId = job.data.fileTaskId;
+      const reason = job.failedReason || 'Unknown error';
+
       await this.fileTaskService.updateTaskStatus(
         fileTaskId,
         FileTaskStatus.Failed,
-        job.failedReason,
+        reason,
       );
 
       const fileTask = await this.fileTaskService.getFileTask(fileTaskId);
@@ -61,13 +86,6 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
     }
   }
 
-  @OnWorkerEvent('completed')
-  onCompleted(job: Job) {
-    this.logger.log(
-      `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`,
-    );
-  }
-
   async onModuleDestroy(): Promise<void> {
     if (this.worker) {
       await this.worker.close();

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -0,0 +1,516 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as path from 'path';
+import { jsonToText } from '../../../collaboration/collaboration.util';
+import { InjectKysely } from 'nestjs-kysely';
+import { KyselyDB } from '@docmost/db/types/kysely.types';
+import {
+  extractZip,
+  FileImportSource,
+  FileTaskStatus,
+} from '../utils/file.utils';
+import { StorageService } from '../../storage/storage.service';
+import * as tmp from 'tmp-promise';
+import { pipeline } from 'node:stream/promises';
+import { createWriteStream } from 'node:fs';
+import { ImportService } from './import.service';
+import { promises as fs } from 'fs';
+import { generateSlugId } from '../../../common/helpers';
+import { v7 } from 'uuid';
+import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
+import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
+import { markdownToHtml } from '@docmost/editor-ext';
+import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils';
+import { formatImportHtml } from '../utils/import-formatter';
+import {
+  buildAttachmentCandidates,
+  collectMarkdownAndHtmlFiles,
+  stripNotionID,
+} from '../utils/import.utils';
+import { executeTx } from '@docmost/db/utils';
+import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
+import { ImportAttachmentService } from './import-attachment.service';
+import { ModuleRef } from '@nestjs/core';
+import { PageService } from '../../../core/page/services/page.service';
+import { ImportPageNode } from '../dto/file-task-dto';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
+
+@Injectable()
+export class FileImportTaskService {
+  private readonly logger = new Logger(FileImportTaskService.name);
+
+  constructor(
+    private readonly storageService: StorageService,
+    private readonly importService: ImportService,
+    private readonly pageService: PageService,
+    private readonly backlinkRepo: BacklinkRepo,
+    @InjectKysely() private readonly db: KyselyDB,
+    private readonly importAttachmentService: ImportAttachmentService,
+    private moduleRef: ModuleRef,
+    private eventEmitter: EventEmitter2,
+  ) {}
+
+  async processZIpImport(fileTaskId: string): Promise<void> {
+    const fileTask = await this.db
+      .selectFrom('fileTasks')
+      .selectAll()
+      .where('id', '=', fileTaskId)
+      .executeTakeFirst();
+
+    if (!fileTask) {
+      this.logger.log(`Import file task with ID ${fileTaskId} not found`);
+      return;
+    }
+
+    if (fileTask.status === FileTaskStatus.Failed) {
+      return;
+    }
+
+    if (fileTask.status === FileTaskStatus.Success) {
+      this.logger.log('Imported task already processed.');
+      return;
+    }
+
+    const { path: tmpZipPath, cleanup: cleanupTmpFile } = await tmp.file({
+      prefix: 'docmost-import',
+      postfix: '.zip',
+      discardDescriptor: true,
+    });
+
+    const { path: tmpExtractDir, cleanup: cleanupTmpDir } = await tmp.dir({
+      prefix: 'docmost-extract-',
+      unsafeCleanup: true,
+    });
+
+    try {
+      const fileStream = await this.storageService.readStream(
+        fileTask.filePath,
+      );
+      await pipeline(fileStream, createWriteStream(tmpZipPath));
+      await extractZip(tmpZipPath, tmpExtractDir);
+    } catch (err) {
+      await cleanupTmpFile();
+      await cleanupTmpDir();
+
+      throw err;
+    }
+
+    try {
+      if (
+        fileTask.source === FileImportSource.Generic ||
+        fileTask.source === FileImportSource.Notion
+      ) {
+        await this.processGenericImport({
+          extractDir: tmpExtractDir,
+          fileTask,
+        });
+      }
+
+      if (fileTask.source === FileImportSource.Confluence) {
+        let ConfluenceModule: any;
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-require-imports
+          ConfluenceModule = require('./../../../ee/confluence-import/confluence-import.service');
+        } catch (err) {
+          this.logger.error(
+            'Confluence import requested but EE module not bundled in this build',
+          );
+          return;
+        }
+        const confluenceImportService = this.moduleRef.get(
+          ConfluenceModule.ConfluenceImportService,
+          { strict: false },
+        );
+
+        await confluenceImportService.processConfluenceImport({
+          extractDir: tmpExtractDir,
+          fileTask,
+        });
+      }
+      try {
+        await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success, null);
+        await cleanupTmpFile();
+        await cleanupTmpDir();
+        // delete stored file on success
+        await this.storageService.delete(fileTask.filePath);
+      } catch (err) {
+        this.logger.error(
+          `Failed to delete import file from storage. Task ID: ${fileTaskId}`,
+          err,
+        );
+      }
+    } catch (err) {
+      await cleanupTmpFile();
+      await cleanupTmpDir();
+
+      throw err;
+    }
+  }
+
+  async processGenericImport(opts: {
+    extractDir: string;
+    fileTask: FileTask;
+  }): Promise<void> {
+    const { extractDir, fileTask } = opts;
+    const allFiles = await collectMarkdownAndHtmlFiles(extractDir);
+    const attachmentCandidates = await buildAttachmentCandidates(extractDir);
+
+    const pagesMap = new Map<string, ImportPageNode>();
+
+    for (const absPath of allFiles) {
+      const relPath = path
+        .relative(extractDir, absPath)
+        .split(path.sep)
+        .join('/'); // normalize to forward-slashes
+      const ext = path.extname(relPath).toLowerCase();
+
+      pagesMap.set(relPath, {
+        id: v7(),
+        slugId: generateSlugId(),
+        name: stripNotionID(path.basename(relPath, ext)),
+        content: '',
+        parentPageId: null,
+        fileExtension: ext,
+        filePath: relPath,
+      });
+    }
+
+    // Create placeholder pages for folders without corresponding files
+    const foldersWithContent = new Set<string>();
+
+    pagesMap.forEach((page) => {
+      const segments = page.filePath.split('/');
+      segments.pop(); // remove filename
+
+      // Build up all folder paths and mark them as having content
+      let currentPath = '';
+      for (const segment of segments) {
+        currentPath = currentPath ? `${currentPath}/${segment}` : segment;
+        foldersWithContent.add(currentPath); // All ancestor folders have content
+      }
+    });
+
+    // Determine if there's a single root container folder
+    const rootLevelItems = new Set<string>();
+    pagesMap.forEach((page) => {
+      const firstSegment = page.filePath.split('/')[0];
+      rootLevelItems.add(firstSegment);
+    });
+
+    // If all files are in a single root folder and no files at root level exist
+    let skipRootFolder: string | null = null;
+    if (rootLevelItems.size === 1) {
+      const onlyRootItem = Array.from(rootLevelItems)[0];
+      // Check if this is a folder (not a file at root)
+      const hasRootFiles = Array.from(pagesMap.keys()).some(
+        (filePath) => !filePath.includes('/'),
+      );
+      if (!hasRootFiles) {
+        skipRootFolder = onlyRootItem;
+      }
+    }
+
+    // For each folder with content, create a placeholder page if no corresponding .md or .html exists
+    foldersWithContent.forEach((folderPath) => {
+      if (
+        skipRootFolder &&
+        folderPath?.toLowerCase() === skipRootFolder?.toLowerCase()
+      ) {
+        return;
+      }
+
+      const mdPath = `${folderPath}.md`;
+      const htmlPath = `${folderPath}.html`;
+
+      if (!pagesMap.has(mdPath) && !pagesMap.has(htmlPath)) {
+        const folderName = path.basename(folderPath);
+        pagesMap.set(mdPath, {
+          id: v7(),
+          slugId: generateSlugId(),
+          name: stripNotionID(folderName),
+          content: '',
+          parentPageId: null,
+          fileExtension: '.md',
+          filePath: mdPath,
+        });
+      }
+    });
+
+    // parent/child linking
+    pagesMap.forEach((page, filePath) => {
+      const segments = filePath.split('/');
+      segments.pop();
+      let parentPage = null;
+      while (segments.length) {
+        const tryMd = segments.join('/') + '.md';
+        const tryHtml = segments.join('/') + '.html';
+        if (pagesMap.has(tryMd)) {
+          parentPage = pagesMap.get(tryMd)!;
+          break;
+        }
+        if (pagesMap.has(tryHtml)) {
+          parentPage = pagesMap.get(tryHtml)!;
+          break;
+        }
+        segments.pop();
+      }
+      if (parentPage) page.parentPageId = parentPage.id;
+    });
+
+    // generate position keys
+    const siblingsMap = new Map<string | null, ImportPageNode[]>();
+
+    pagesMap.forEach((page) => {
+      const group = siblingsMap.get(page.parentPageId) ?? [];
+      group.push(page);
+      siblingsMap.set(page.parentPageId, group);
+    });
+
+    // get root pages
+    const rootSibs = siblingsMap.get(null);
+
+    if (rootSibs?.length) {
+      rootSibs.sort((a, b) => a.name.localeCompare(b.name));
+
+      // get first position key from the server
+      const nextPosition = await this.pageService.nextPagePosition(
+        fileTask.spaceId,
+      );
+
+      let prevPos: string | null = null;
+      rootSibs.forEach((page, idx) => {
+        if (idx === 0) {
+          page.position = nextPosition;
+        } else {
+          page.position = generateJitteredKeyBetween(prevPos, null);
+        }
+        prevPos = page.position;
+      });
+    }
+
+    // non-root buckets (children & deeper levels)
+    siblingsMap.forEach((sibs, parentId) => {
+      if (parentId === null) return; // root already done
+
+      sibs.sort((a, b) => a.name.localeCompare(b.name));
+
+      let prevPos: string | null = null;
+      for (const page of sibs) {
+        page.position = generateJitteredKeyBetween(prevPos, null);
+        prevPos = page.position;
+      }
+    });
+
+    // internal page links
+    const filePathToPageMetaMap = new Map<
+      string,
+      { id: string; title: string; slugId: string }
+    >();
+    pagesMap.forEach((page) => {
+      filePathToPageMetaMap.set(page.filePath, {
+        id: page.id,
+        title: page.name,
+        slugId: page.slugId,
+      });
+    });
+
+    // Group pages by level (topological sort for parent-child relationships)
+    const pagesByLevel = new Map<number, Array<[string, ImportPageNode]>>();
+    const pageLevel = new Map<string, number>();
+
+    // Calculate levels using BFS
+    const calculateLevels = () => {
+      const queue: Array<{ filePath: string; level: number }> = [];
+
+      // Start with root pages (no parent)
+      for (const [filePath, page] of pagesMap.entries()) {
+        if (!page.parentPageId) {
+          queue.push({ filePath, level: 0 });
+          pageLevel.set(filePath, 0);
+        }
+      }
+
+      // BFS to assign levels
+      while (queue.length > 0) {
+        const { filePath, level } = queue.shift()!;
+        const currentPage = pagesMap.get(filePath)!;
+
+        // Find children of current page
+        for (const [childFilePath, childPage] of pagesMap.entries()) {
+          if (
+            childPage.parentPageId === currentPage.id &&
+            !pageLevel.has(childFilePath)
+          ) {
+            pageLevel.set(childFilePath, level + 1);
+            queue.push({ filePath: childFilePath, level: level + 1 });
+          }
+        }
+      }
+
+      // Group pages by level
+      for (const [filePath, page] of pagesMap.entries()) {
+        const level = pageLevel.get(filePath) || 0;
+        if (!pagesByLevel.has(level)) {
+          pagesByLevel.set(level, []);
+        }
+        pagesByLevel.get(level)!.push([filePath, page]);
+      }
+    };
+
+    calculateLevels();
+
+    if (pagesMap.size < 1) return;
+
+    // Process pages level by level sequentially to respect foreign key constraints
+    const allBacklinks: any[] = [];
+    const validPageIds = new Set<string>();
+    let totalPagesProcessed = 0;
+
+    // Sort levels to process in order
+    const sortedLevels = Array.from(pagesByLevel.keys()).sort((a, b) => a - b);
+
+    try {
+      await executeTx(this.db, async (trx) => {
+        // Process pages level by level sequentially within the transaction
+        for (const level of sortedLevels) {
+          const levelPages = pagesByLevel.get(level)!;
+
+          for (const [filePath, page] of levelPages) {
+            const absPath = path.join(extractDir, filePath);
+            let content = '';
+
+            // Check if file exists (placeholder pages won't have physical files)
+            try {
+              await fs.access(absPath);
+              content = await fs.readFile(absPath, 'utf-8');
+
+              if (page.fileExtension.toLowerCase() === '.md') {
+                content = await markdownToHtml(content);
+              }
+            } catch (err: any) {
+              if (err?.code === 'ENOENT') {
+                // Use empty content, title will be the folder name
+                content = '';
+              } else {
+                throw err;
+              }
+            }
+
+            const htmlContent =
+              await this.importAttachmentService.processAttachments({
+                html: content,
+                pageRelativePath: page.filePath,
+                extractDir,
+                pageId: page.id,
+                fileTask,
+                attachmentCandidates,
+              });
+
+            const { html, backlinks, pageIcon } = await formatImportHtml({
+              html: htmlContent,
+              currentFilePath: page.filePath,
+              filePathToPageMetaMap: filePathToPageMetaMap,
+              creatorId: fileTask.creatorId,
+              sourcePageId: page.id,
+              workspaceId: fileTask.workspaceId,
+            });
+
+            const pmState = getProsemirrorContent(
+              await this.importService.processHTML(html),
+            );
+
+            const { title, prosemirrorJson } =
+              this.importService.extractTitleAndRemoveHeading(pmState);
+
+            const insertablePage: InsertablePage = {
+              id: page.id,
+              slugId: page.slugId,
+              title: title || page.name,
+              icon: pageIcon || null,
+              content: prosemirrorJson,
+              textContent: jsonToText(prosemirrorJson),
+              ydoc: await this.importService.createYdoc(prosemirrorJson),
+              position: page.position!,
+              spaceId: fileTask.spaceId,
+              workspaceId: fileTask.workspaceId,
+              creatorId: fileTask.creatorId,
+              lastUpdatedById: fileTask.creatorId,
+              parentPageId: page.parentPageId,
+            };
+
+            await trx.insertInto('pages').values(insertablePage).execute();
+
+            // Track valid page IDs and collect backlinks
+            validPageIds.add(insertablePage.id);
+            allBacklinks.push(...backlinks);
+            totalPagesProcessed++;
+
+            // Log progress periodically
+            if (totalPagesProcessed % 50 === 0) {
+              this.logger.debug(`Processed ${totalPagesProcessed} pages...`);
+            }
+          }
+        }
+
+        const filteredBacklinks = allBacklinks.filter(
+          ({ sourcePageId, targetPageId }) =>
+            validPageIds.has(sourcePageId) && validPageIds.has(targetPageId),
+        );
+
+        // Insert backlinks in batches
+        if (filteredBacklinks.length > 0) {
+          const BACKLINK_BATCH_SIZE = 100;
+          for (
+            let i = 0;
+            i < filteredBacklinks.length;
+            i += BACKLINK_BATCH_SIZE
+          ) {
+            const backlinkChunk = filteredBacklinks.slice(
+              i,
+              Math.min(i + BACKLINK_BATCH_SIZE, filteredBacklinks.length),
+            );
+            await this.backlinkRepo.insertBacklink(backlinkChunk, trx);
+          }
+        }
+
+        if (validPageIds.size > 0) {
+          this.eventEmitter.emit(EventName.PAGE_CREATED, {
+            pageIds: Array.from(validPageIds),
+            workspaceId: fileTask.workspaceId,
+          });
+        }
+
+        this.logger.log(
+          `Successfully imported ${totalPagesProcessed} pages with ${filteredBacklinks.length} backlinks`,
+        );
+      });
+    } catch (error) {
+      this.logger.error('Failed to import files:', error);
+      throw new Error(`File import failed: ${error?.['message']}`);
+    }
+  }
+
+  async getFileTask(fileTaskId: string) {
+    return this.db
+      .selectFrom('fileTasks')
+      .selectAll()
+      .where('id', '=', fileTaskId)
+      .executeTakeFirst();
+  }
+
+  async updateTaskStatus(
+    fileTaskId: string,
+    status: FileTaskStatus,
+    errorMessage?: string,
+  ) {
+    try {
+      await this.db
+        .updateTable('fileTasks')
+        .set({ status: status, errorMessage, updatedAt: new Date() })
+        .where('id', '=', fileTaskId)
+        .execute();
+    } catch (err) {
+      this.logger.error(err);
+    }
+  }
+}

apps/server/src/integrations/import/services/file-task.service.ts

@@ -1,346 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import * as path from 'path';
-import { jsonToText } from '../../../collaboration/collaboration.util';
-import { InjectKysely } from 'nestjs-kysely';
-import { KyselyDB } from '@docmost/db/types/kysely.types';
-import {
-  extractZip,
-  FileImportSource,
-  FileTaskStatus,
-} from '../utils/file.utils';
-import { StorageService } from '../../storage/storage.service';
-import * as tmp from 'tmp-promise';
-import { pipeline } from 'node:stream/promises';
-import { createWriteStream } from 'node:fs';
-import { ImportService } from './import.service';
-import { promises as fs } from 'fs';
-import { generateSlugId } from '../../../common/helpers';
-import { v7 } from 'uuid';
-import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
-import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
-import { markdownToHtml } from '@docmost/editor-ext';
-import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils';
-import { formatImportHtml } from '../utils/import-formatter';
-import {
-  buildAttachmentCandidates,
-  collectMarkdownAndHtmlFiles,
-} from '../utils/import.utils';
-import { executeTx } from '@docmost/db/utils';
-import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
-import { ImportAttachmentService } from './import-attachment.service';
-import { ModuleRef } from '@nestjs/core';
-import { PageService } from '../../../core/page/services/page.service';
-import { ImportPageNode } from '../dto/file-task-dto';
-
-@Injectable()
-export class FileTaskService {
-  private readonly logger = new Logger(FileTaskService.name);
-
-  constructor(
-    private readonly storageService: StorageService,
-    private readonly importService: ImportService,
-    private readonly pageService: PageService,
-    private readonly backlinkRepo: BacklinkRepo,
-    @InjectKysely() private readonly db: KyselyDB,
-    private readonly importAttachmentService: ImportAttachmentService,
-    private moduleRef: ModuleRef,
-  ) {}
-
-  async processZIpImport(fileTaskId: string): Promise<void> {
-    const fileTask = await this.db
-      .selectFrom('fileTasks')
-      .selectAll()
-      .where('id', '=', fileTaskId)
-      .executeTakeFirst();
-
-    if (!fileTask) {
-      this.logger.log(`Import file task with ID ${fileTaskId} not found`);
-      return;
-    }
-
-    if (fileTask.status === FileTaskStatus.Failed) {
-      return;
-    }
-
-    if (fileTask.status === FileTaskStatus.Success) {
-      this.logger.log('Imported task already processed.');
-      return;
-    }
-
-    const { path: tmpZipPath, cleanup: cleanupTmpFile } = await tmp.file({
-      prefix: 'docmost-import',
-      postfix: '.zip',
-      discardDescriptor: true,
-    });
-
-    const { path: tmpExtractDir, cleanup: cleanupTmpDir } = await tmp.dir({
-      prefix: 'docmost-extract-',
-      unsafeCleanup: true,
-    });
-
-    try {
-      const fileStream = await this.storageService.readStream(
-        fileTask.filePath,
-      );
-      await pipeline(fileStream, createWriteStream(tmpZipPath));
-      await extractZip(tmpZipPath, tmpExtractDir);
-    } catch (err) {
-      await cleanupTmpFile();
-      await cleanupTmpDir();
-
-      throw err;
-    }
-
-    try {
-      if (
-        fileTask.source === FileImportSource.Generic ||
-        fileTask.source === FileImportSource.Notion
-      ) {
-        await this.processGenericImport({
-          extractDir: tmpExtractDir,
-          fileTask,
-        });
-      }
-
-      if (fileTask.source === FileImportSource.Confluence) {
-        let ConfluenceModule: any;
-        try {
-          // eslint-disable-next-line @typescript-eslint/no-require-imports
-          ConfluenceModule = require('./../../../ee/confluence-import/confluence-import.service');
-        } catch (err) {
-          this.logger.error(
-            'Confluence import requested but EE module not bundled in this build',
-          );
-          return;
-        }
-        const confluenceImportService = this.moduleRef.get(
-          ConfluenceModule.ConfluenceImportService,
-          { strict: false },
-        );
-
-        await confluenceImportService.processConfluenceImport({
-          extractDir: tmpExtractDir,
-          fileTask,
-        });
-      }
-      try {
-        await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success, null);
-        await cleanupTmpFile();
-        await cleanupTmpDir();
-        // delete stored file on success
-        await this.storageService.delete(fileTask.filePath);
-      } catch (err) {
-        this.logger.error(
-          `Failed to delete import file from storage. Task ID: ${fileTaskId}`,
-          err,
-        );
-      }
-    } catch (err) {
-      await cleanupTmpFile();
-      await cleanupTmpDir();
-
-      throw err;
-    }
-  }
-
-  async processGenericImport(opts: {
-    extractDir: string;
-    fileTask: FileTask;
-  }): Promise<void> {
-    const { extractDir, fileTask } = opts;
-    const allFiles = await collectMarkdownAndHtmlFiles(extractDir);
-    const attachmentCandidates = await buildAttachmentCandidates(extractDir);
-
-    const pagesMap = new Map<string, ImportPageNode>();
-
-    for (const absPath of allFiles) {
-      const relPath = path
-        .relative(extractDir, absPath)
-        .split(path.sep)
-        .join('/'); // normalize to forward-slashes
-      const ext = path.extname(relPath).toLowerCase();
-      let content = await fs.readFile(absPath, 'utf-8');
-
-      if (ext.toLowerCase() === '.md') {
-        content = await markdownToHtml(content);
-      }
-
-      pagesMap.set(relPath, {
-        id: v7(),
-        slugId: generateSlugId(),
-        name: path.basename(relPath, ext),
-        content,
-        parentPageId: null,
-        fileExtension: ext,
-        filePath: relPath,
-      });
-    }
-
-    // parent/child linking
-    pagesMap.forEach((page, filePath) => {
-      const segments = filePath.split('/');
-      segments.pop();
-      let parentPage = null;
-      while (segments.length) {
-        const tryMd = segments.join('/') + '.md';
-        const tryHtml = segments.join('/') + '.html';
-        if (pagesMap.has(tryMd)) {
-          parentPage = pagesMap.get(tryMd)!;
-          break;
-        }
-        if (pagesMap.has(tryHtml)) {
-          parentPage = pagesMap.get(tryHtml)!;
-          break;
-        }
-        segments.pop();
-      }
-      if (parentPage) page.parentPageId = parentPage.id;
-    });
-
-    // generate position keys
-    const siblingsMap = new Map<string | null, ImportPageNode[]>();
-
-    pagesMap.forEach((page) => {
-      const group = siblingsMap.get(page.parentPageId) ?? [];
-      group.push(page);
-      siblingsMap.set(page.parentPageId, group);
-    });
-
-    // get root pages
-    const rootSibs = siblingsMap.get(null);
-
-    if (rootSibs?.length) {
-      rootSibs.sort((a, b) => a.name.localeCompare(b.name));
-
-      // get first position key from the server
-      const nextPosition = await this.pageService.nextPagePosition(
-        fileTask.spaceId,
-      );
-
-      let prevPos: string | null = null;
-      rootSibs.forEach((page, idx) => {
-        if (idx === 0) {
-          page.position = nextPosition;
-        } else {
-          page.position = generateJitteredKeyBetween(prevPos, null);
-        }
-        prevPos = page.position;
-      });
-    }
-
-    // non-root buckets (children & deeper levels)
-    siblingsMap.forEach((sibs, parentId) => {
-      if (parentId === null) return; // root already done
-
-      sibs.sort((a, b) => a.name.localeCompare(b.name));
-
-      let prevPos: string | null = null;
-      for (const page of sibs) {
-        page.position = generateJitteredKeyBetween(prevPos, null);
-        prevPos = page.position;
-      }
-    });
-
-    // internal page links
-    const filePathToPageMetaMap = new Map<
-      string,
-      { id: string; title: string; slugId: string }
-    >();
-    pagesMap.forEach((page) => {
-      filePathToPageMetaMap.set(page.filePath, {
-        id: page.id,
-        title: page.name,
-        slugId: page.slugId,
-      });
-    });
-
-    const pageResults = await Promise.all(
-      Array.from(pagesMap.values()).map(async (page) => {
-        const htmlContent =
-          await this.importAttachmentService.processAttachments({
-            html: page.content,
-            pageRelativePath: page.filePath,
-            extractDir,
-            pageId: page.id,
-            fileTask,
-            attachmentCandidates,
-          });
-
-        const { html, backlinks } = await formatImportHtml({
-          html: htmlContent,
-          currentFilePath: page.filePath,
-          filePathToPageMetaMap: filePathToPageMetaMap,
-          creatorId: fileTask.creatorId,
-          sourcePageId: page.id,
-          workspaceId: fileTask.workspaceId,
-        });
-
-        const pmState = getProsemirrorContent(
-          await this.importService.processHTML(html),
-        );
-
-        const { title, prosemirrorJson } =
-          this.importService.extractTitleAndRemoveHeading(pmState);
-
-        const insertablePage: InsertablePage = {
-          id: page.id,
-          slugId: page.slugId,
-          title: title || page.name,
-          content: prosemirrorJson,
-          textContent: jsonToText(prosemirrorJson),
-          ydoc: await this.importService.createYdoc(prosemirrorJson),
-          position: page.position!,
-          spaceId: fileTask.spaceId,
-          workspaceId: fileTask.workspaceId,
-          creatorId: fileTask.creatorId,
-          lastUpdatedById: fileTask.creatorId,
-          parentPageId: page.parentPageId,
-        };
-
-        return { insertablePage, backlinks };
-      }),
-    );
-
-    const insertablePages = pageResults.map((r) => r.insertablePage);
-    const insertableBacklinks = pageResults.flatMap((r) => r.backlinks);
-
-    if (insertablePages.length < 1) return;
-    const validPageIds = new Set(insertablePages.map((row) => row.id));
-    const filteredBacklinks = insertableBacklinks.filter(
-      ({ sourcePageId, targetPageId }) =>
-        validPageIds.has(sourcePageId) && validPageIds.has(targetPageId),
-    );
-
-    await executeTx(this.db, async (trx) => {
-      await trx.insertInto('pages').values(insertablePages).execute();
-
-      if (filteredBacklinks.length > 0) {
-        await this.backlinkRepo.insertBacklink(filteredBacklinks, trx);
-      }
-    });
-  }
-
-  async getFileTask(fileTaskId: string) {
-    return this.db
-      .selectFrom('fileTasks')
-      .selectAll()
-      .where('id', '=', fileTaskId)
-      .executeTakeFirst();
-  }
-
-  async updateTaskStatus(
-    fileTaskId: string,
-    status: FileTaskStatus,
-    errorMessage?: string,
-  ) {
-    try {
-      await this.db
-        .updateTable('fileTasks')
-        .set({ status: status, errorMessage, updatedAt: new Date() })
-        .where('id', '=', fileTaskId)
-        .execute();
-    } catch (err) {
-      this.logger.error(err);
-    }
-  }
-}

apps/server/src/integrations/import/services/import-attachment.service.ts

@@ -6,6 +6,7 @@ import { cleanUrlString } from '../utils/file.utils';
 import { StorageService } from '../../storage/storage.service';
 import { createReadStream } from 'node:fs';
 import { promises as fs } from 'fs';
+import { Readable } from 'stream';
 import { getMimeType, sanitizeFileName } from '../../../common/helpers';
 import { v7 } from 'uuid';
 import { FileTask } from '@docmost/db/types/entity.types';
@@ -15,6 +16,21 @@ import { unwrapFromParagraph } from '../utils/import-formatter';
 import { resolveRelativeAttachmentPath } from '../utils/import.utils';
 import { load } from 'cheerio';
 import pLimit from 'p-limit';
+import { InjectQueue } from '@nestjs/bullmq';
+import { Queue } from 'bullmq';
+import { QueueJob, QueueName } from '../../queue/constants';
+
+interface AttachmentInfo {
+  href: string;
+  fileName: string;
+  mimeType: string;
+}
+
+interface DrawioPair {
+  drawioFile?: AttachmentInfo;
+  pngFile?: AttachmentInfo;
+  baseName: string;
+}
 
 @Injectable()
 export class ImportAttachmentService {
@@ -26,6 +42,7 @@ export class ImportAttachmentService {
   constructor(
     private readonly storageService: StorageService,
     @InjectKysely() private readonly db: KyselyDB,
+    @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
   async processAttachments(opts: {
@@ -35,6 +52,8 @@ export class ImportAttachmentService {
     pageId: string;
     fileTask: FileTask;
     attachmentCandidates: Map<string, string>;
+    pageAttachments?: AttachmentInfo[];
+    isConfluenceImport?: boolean;
   }): Promise<string> {
     const {
       html,
@@ -43,6 +62,8 @@ export class ImportAttachmentService {
       pageId,
       fileTask,
       attachmentCandidates,
+      pageAttachments = [],
+      isConfluenceImport,
     } = opts;
 
     const attachmentTasks: (() => Promise<void>)[] = [];
@@ -57,7 +78,7 @@ export class ImportAttachmentService {
     /**
      * Cache keyed by the *relative* path that appears in the HTML.
      * Ensures we upload (and DB-insert) each attachment at most once,
-     * even if it’s referenced multiple times on the page.
+     * even if it's referenced multiple times on the page.
      */
     const processed = new Map<
       string,
@@ -70,6 +91,104 @@ export class ImportAttachmentService {
       }
     >();
 
+    // Analyze attachments to identify Draw.io pairs
+    const { drawioPairs, skipFiles } = this.analyzeAttachments(
+      pageAttachments,
+      isConfluenceImport,
+    );
+
+    // Map to store processed Draw.io SVGs
+    const drawioSvgMap = new Map<
+      string,
+      {
+        attachmentId: string;
+        apiFilePath: string;
+        fileName: string;
+      }
+    >();
+
+    //this.logger.debug(`Found ${drawioPairs.size} Draw.io pairs to process`);
+
+    // Process Draw.io pairs and create combined SVG files
+    for (const [drawioHref, pair] of drawioPairs) {
+      if (!pair.drawioFile) continue;
+
+      const drawioAbsPath = attachmentCandidates.get(drawioHref);
+      if (!drawioAbsPath) continue;
+
+      const pngAbsPath = pair.pngFile
+        ? attachmentCandidates.get(pair.pngFile.href)
+        : undefined;
+
+      try {
+        // Create combined SVG with Draw.io data and PNG image
+        const svgBuffer = await this.createDrawioSvg(drawioAbsPath, pngAbsPath);
+
+        // Generate file details - always use "diagram.drawio.svg" as filename
+        const attachmentId = v7();
+        const fileName = 'diagram.drawio.svg';
+        const storageFilePath = `${getAttachmentFolderPath(
+          AttachmentType.File,
+          fileTask.workspaceId,
+        )}/${attachmentId}/${fileName}`;
+        const apiFilePath = `/api/files/${attachmentId}/${fileName}`;
+
+        // Upload the SVG file
+        attachmentTasks.push(async () => {
+          try {
+            const stream = Readable.from(svgBuffer);
+
+            // Upload to storage
+            await this.storageService.uploadStream(storageFilePath, stream, {
+              recreateClient: true,
+            });
+
+            // Insert into database
+            await this.db
+              .insertInto('attachments')
+              .values({
+                id: attachmentId,
+                filePath: storageFilePath,
+                fileName: fileName,
+                fileSize: svgBuffer.length,
+                mimeType: 'image/svg+xml',
+                type: 'file',
+                fileExt: '.svg',
+                creatorId: fileTask.creatorId,
+                workspaceId: fileTask.workspaceId,
+                pageId,
+                spaceId: fileTask.spaceId,
+              })
+              .execute();
+
+            uploadStats.completed++;
+          } catch (error) {
+            uploadStats.failed++;
+            uploadStats.failedFiles.push(fileName);
+            this.logger.error(
+              `Failed to upload Draw.io SVG ${fileName}:`,
+              error,
+            );
+          }
+        });
+
+        // Store the mapping for both Draw.io and PNG references
+        drawioSvgMap.set(drawioHref, { attachmentId, apiFilePath, fileName });
+        if (pair.pngFile) {
+          drawioSvgMap.set(pair.pngFile.href, {
+            attachmentId,
+            apiFilePath,
+            fileName,
+          });
+        }
+      } catch (error) {
+        this.logger.error(
+          `Failed to process Draw.io pair ${pair.baseName}:`,
+          error,
+        );
+      }
+    }
+
     const uploadOnce = (relPath: string) => {
       const abs = attachmentCandidates.get(relPath)!;
       const attachmentId = v7();
@@ -136,8 +255,23 @@ export class ImportAttachmentService {
       );
       if (!relPath) continue;
 
-      const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
+      // Check if this image is part of a Draw.io pair
+      const drawioSvg = drawioSvgMap.get(relPath);
+      if (drawioSvg) {
+        const $drawio = $('<div>')
+          .attr('data-type', 'drawio')
+          .attr('data-src', drawioSvg.apiFilePath)
+          .attr('data-title', 'diagram')
+          .attr('data-width', '100%')
+          .attr('data-align', 'center')
+          .attr('data-attachment-id', drawioSvg.attachmentId);
+
+        $img.replaceWith($drawio);
+        unwrapFromParagraph($, $drawio);
+        continue;
+      }
+
+      const { attachmentId, apiFilePath } = processFile(relPath);
 
       const width = $img.attr('width') ?? '100%';
       const align = $img.attr('data-align') ?? 'center';
@@ -145,7 +279,6 @@ export class ImportAttachmentService {
       $img
         .attr('src', apiFilePath)
         .attr('data-attachment-id', attachmentId)
-        .attr('data-size', stat.size.toString())
         .attr('width', width)
         .attr('data-align', align);
 
@@ -165,8 +298,7 @@ export class ImportAttachmentService {
       );
       if (!relPath) continue;
 
-      const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
+      const { attachmentId, apiFilePath } = processFile(relPath);
 
       const width = $vid.attr('width') ?? '100%';
       const align = $vid.attr('data-align') ?? 'center';
@@ -174,7 +306,6 @@ export class ImportAttachmentService {
       $vid
         .attr('src', apiFilePath)
         .attr('data-attachment-id', attachmentId)
-        .attr('data-size', stat.size.toString())
         .attr('width', width)
         .attr('data-align', align);
 
@@ -195,7 +326,6 @@ export class ImportAttachmentService {
       if (!relPath) continue;
 
       const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
       const fileName = path.basename(abs);
       const mime = getMimeType(abs);
 
@@ -204,7 +334,6 @@ export class ImportAttachmentService {
         .attr('data-attachment-url', apiFilePath)
         .attr('data-attachment-name', fileName)
         .attr('data-attachment-mime', mime)
-        .attr('data-attachment-size', stat.size.toString())
         .attr('data-attachment-id', attachmentId);
 
       $oldDiv.replaceWith($newDiv);
@@ -224,15 +353,35 @@ export class ImportAttachmentService {
       );
       if (!relPath) continue;
 
+      // Check if this is a Draw.io file
+      const drawioSvg = drawioSvgMap.get(relPath);
+      if (drawioSvg) {
+        const $drawio = $('<div>')
+          .attr('data-type', 'drawio')
+          .attr('data-src', drawioSvg.apiFilePath)
+          .attr('data-title', 'diagram')
+          .attr('data-width', '100%')
+          .attr('data-align', 'center')
+          .attr('data-attachment-id', drawioSvg.attachmentId);
+
+        $a.replaceWith($drawio);
+        unwrapFromParagraph($, $drawio);
+        continue;
+      }
+
+      // Skip files that should be ignored
+      if (skipFiles.has(relPath)) {
+        $a.remove();
+        continue;
+      }
+
       const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
       const ext = path.extname(relPath).toLowerCase();
 
       if (ext === '.mp4') {
         const $video = $('<video>')
           .attr('src', apiFilePath)
           .attr('data-attachment-id', attachmentId)
-          .attr('data-size', stat.size.toString())
           .attr('width', '100%')
           .attr('data-align', 'center');
         $a.replaceWith($video);
@@ -247,7 +396,6 @@ export class ImportAttachmentService {
           .attr('data-attachment-url', apiFilePath)
           .attr('data-attachment-name', attachmentName)
           .attr('data-attachment-mime', getMimeType(abs))
-          .attr('data-attachment-size', stat.size.toString())
           .attr('data-attachment-id', attachmentId);
 
         $a.replaceWith($div);
@@ -270,7 +418,6 @@ export class ImportAttachmentService {
         if (!relPath) continue;
 
         const { attachmentId, apiFilePath, abs } = processFile(relPath);
-        const stat = await fs.stat(abs);
         const fileName = path.basename(abs);
 
         const width = $oldDiv.attr('data-width') || '100%';
@@ -281,7 +428,6 @@ export class ImportAttachmentService {
           .attr('data-src', apiFilePath)
           .attr('data-title', fileName)
           .attr('data-width', width)
-          .attr('data-size', stat.size.toString())
           .attr('data-align', align)
           .attr('data-attachment-id', attachmentId);
 
@@ -290,14 +436,81 @@ export class ImportAttachmentService {
       }
     }
 
+    // Collect all attachment IDs in the HTML in a single DOM traversal - O(n)
+    const usedAttachmentIds = new Set<string>();
+    $.root()
+      .find('[data-attachment-id]')
+      .each((_, el) => {
+        const attachmentId = $(el).attr('data-attachment-id');
+        if (attachmentId) {
+          usedAttachmentIds.add(attachmentId);
+        }
+      });
+
+    // Add Draw.io diagrams that weren't referenced in the HTML content
+    for (const [drawioHref, pair] of drawioPairs) {
+      const drawioSvg = drawioSvgMap.get(drawioHref);
+      if (!drawioSvg) continue;
+
+      if (usedAttachmentIds.has(drawioSvg.attachmentId)) {
+        continue; // Already in content
+      }
+
+      const $drawio = $('<div>')
+        .attr('data-type', 'drawio')
+        .attr('data-src', drawioSvg.apiFilePath)
+        .attr('data-title', 'diagram')
+        .attr('data-width', '100%')
+        .attr('data-align', 'center')
+        .attr('data-attachment-id', drawioSvg.attachmentId);
+
+      $.root().append($drawio);
+    }
+
+    // Process attachments from the attachment section that weren't referenced in HTML
+    // These need to be added as attachment nodes so they get uploaded
+    for (const attachment of pageAttachments) {
+      const { href, fileName, mimeType } = attachment;
+
+      // Skip temporary files or files that should be ignored
+      if (skipFiles.has(href)) {
+        continue;
+      }
+
+      // Check if this was part of a Draw.io pair that was already handled
+      if (drawioSvgMap.has(href)) {
+        continue;
+      }
+
+      // Check if already processed (was referenced in HTML)
+      if (processed.has(href)) {
+        continue;
+      }
+
+      // Skip if the file doesn't exist
+      if (!attachmentCandidates.has(href)) {
+        continue;
+      }
+
+      // This attachment was in the list but not referenced in HTML - add it
+      const { attachmentId, apiFilePath, abs } = processFile(href);
+      const mime = mimeType || getMimeType(abs);
+
+      // Add as attachment node at the end
+      const $attachmentDiv = $('<div>')
+        .attr('data-type', 'attachment')
+        .attr('data-attachment-url', apiFilePath)
+        .attr('data-attachment-name', fileName)
+        .attr('data-attachment-mime', mime)
+        .attr('data-attachment-id', attachmentId);
+
+      $.root().append($attachmentDiv);
+    }
+
     // wait for all uploads & DB inserts
     uploadStats.total = attachmentTasks.length;
 
     if (uploadStats.total > 0) {
-      this.logger.debug(
-        `Starting upload of ${uploadStats.total} attachments...`,
-      );
-
       try {
         await Promise.all(attachmentTasks.map((task) => limit(task)));
       } catch (err) {
@@ -316,9 +529,250 @@ export class ImportAttachmentService {
       }
     }
 
+    // Post-process DOM elements to add file sizes after uploads complete
+    // This avoids blocking file operations during initial DOM processing
+    const elementsNeedingSize = $('[data-attachment-id]:not([data-size])');
+    for (const element of elementsNeedingSize.toArray()) {
+      const $el = $(element);
+      const attachmentId = $el.attr('data-attachment-id');
+      if (!attachmentId) continue;
+
+      // Find the corresponding processed file info
+      const processedEntry = Array.from(processed.values()).find(
+        (entry) => entry.attachmentId === attachmentId,
+      );
+
+      if (processedEntry) {
+        try {
+          const stat = await fs.stat(processedEntry.abs);
+          $el.attr('data-size', stat.size.toString());
+        } catch (error) {
+          this.logger.debug(
+            `Could not get size for ${processedEntry.abs}:`,
+            error,
+          );
+        }
+      }
+    }
+
     return $.root().html() || '';
   }
 
+  private analyzeAttachments(
+    attachments: AttachmentInfo[],
+    isConfluenceImport?: boolean,
+  ): {
+    drawioPairs: Map<string, DrawioPair>;
+    skipFiles: Set<string>;
+  } {
+    const drawioPairs = new Map<string, DrawioPair>();
+    const skipFiles = new Set<string>();
+
+    if (!isConfluenceImport) {
+      return { drawioPairs, skipFiles };
+    }
+
+    // Group attachments by type
+    const drawioFiles: AttachmentInfo[] = [];
+    const pngByBaseName = new Map<string, AttachmentInfo[]>();
+
+    const nonDrawioExtensions = new Set([
+      '.png',
+      '.jpg',
+      '.jpeg',
+      '.gif',
+      '.svg',
+      '.txt',
+      '.pdf',
+      '.doc',
+      '.docx',
+      '.xls',
+      '.xlsx',
+      '.csv',
+      '.zip',
+      '.tar',
+      '.gz',
+    ]);
+
+    // Single pass through attachments
+    for (const attachment of attachments) {
+      const { fileName, mimeType, href } = attachment;
+      const fileNameLower = fileName.toLowerCase();
+
+      // Skip temporary files
+      if (fileName.endsWith('.tmp') || fileName.includes('~drawio~')) {
+        skipFiles.add(href);
+        continue;
+      }
+
+      // Check for Draw.io files
+      if (mimeType === 'application/vnd.jgraph.mxfile') {
+        const ext = fileNameLower.substring(fileNameLower.lastIndexOf('.'));
+        if (!nonDrawioExtensions.has(ext)) {
+          drawioFiles.push(attachment);
+        } else {
+          //Skipped non-Draw.io file with mxfile MIME.
+        }
+      }
+
+      if (mimeType === 'image/png' || fileNameLower.endsWith('.png')) {
+        const baseNames: string[] = [];
+
+        if (fileName.endsWith('.drawio.png')) {
+          // Cloud format: "name.drawio.png" -> base is "name"
+          baseNames.push(fileName.slice(0, -11)); // Remove .drawio.png
+        } else if (fileName.endsWith('.png')) {
+          // Server format: "name.png" -> base is "name"
+          baseNames.push(fileName.slice(0, -4)); // Remove .png
+        }
+
+        for (const baseName of baseNames) {
+          if (!pngByBaseName.has(baseName)) {
+            pngByBaseName.set(baseName, []);
+          }
+          pngByBaseName.get(baseName)!.push(attachment);
+        }
+      }
+    }
+
+    // Match Draw.io files with PNG counterparts
+    for (const drawio of drawioFiles) {
+      let baseName: string;
+
+      if (drawio.fileName.endsWith('.drawio')) {
+        baseName = drawio.fileName.slice(0, -7); // Remove .drawio
+      } else {
+        // Confluence Server: no extension
+        baseName = drawio.fileName;
+      }
+
+      const candidatePngs = pngByBaseName.get(baseName) || [];
+      let matchingPng: AttachmentInfo | undefined;
+
+      // Extract the attachment ID from the Draw.io href
+      // Format: attachments/16941088/36044817.png -> ID is 36044817
+      const drawioIdMatch = drawio.href.match(/\/(\d+)\.\w+$/);
+      const drawioId = drawioIdMatch ? drawioIdMatch[1] : null;
+
+      if (drawioId) {
+        // Look for PNG with adjacent ID (usually PNG ID = Draw.io ID + small increment)
+        // In Confluence, related files often have sequential or near-sequential IDs
+        for (const png of candidatePngs) {
+          const pngIdMatch = png.href.match(/\/(\d+)\.png$/);
+          const pngId = pngIdMatch ? pngIdMatch[1] : null;
+
+          //TODO: should revisit this
+          // but seem to be the best option for now
+          // to prevent reusing the first drawio preview image if there are more with the same name
+          if (pngId && drawioId) {
+            const idDiff = Math.abs(parseInt(pngId) - parseInt(drawioId));
+            // PNG is usually within ~30 IDs of the Draw.io file
+            if (idDiff <= 30) {
+              // Verify filename match
+              if (
+                png.fileName === `${baseName}.drawio.png` ||
+                (!drawio.fileName.endsWith('.drawio') &&
+                  png.fileName === `${baseName}.png`)
+              ) {
+                matchingPng = png;
+                break;
+              }
+            }
+          }
+        }
+      }
+
+      // Fallback to name-only matching if ID-based matching fails
+      if (!matchingPng) {
+        for (const png of candidatePngs) {
+          if (png.fileName === `${baseName}.drawio.png`) {
+            matchingPng = png;
+            break;
+          }
+          if (
+            !drawio.fileName.endsWith('.drawio') &&
+            png.fileName === `${baseName}.png`
+          ) {
+            matchingPng = png;
+            break;
+          }
+        }
+      }
+
+      if (matchingPng) {
+        this.logger.debug(
+          `Found Draw.io pair: ${drawio.fileName} -> ${matchingPng.fileName}`,
+        );
+      } else {
+        this.logger.debug(`No PNG found for Draw.io file: ${drawio.fileName}`);
+      }
+
+      const pair: DrawioPair = {
+        drawioFile: drawio,
+        pngFile: matchingPng,
+        baseName,
+      };
+
+      drawioPairs.set(drawio.href, pair);
+      skipFiles.add(drawio.href);
+      if (matchingPng) {
+        skipFiles.add(matchingPng.href);
+        // Remove the matched PNG from the candidates to prevent reuse
+        const remainingPngs = pngByBaseName
+          .get(baseName)
+          ?.filter((png) => png.href !== matchingPng.href);
+        if (remainingPngs && remainingPngs.length > 0) {
+          pngByBaseName.set(baseName, remainingPngs);
+        } else {
+          pngByBaseName.delete(baseName);
+        }
+      }
+    }
+
+    return { drawioPairs, skipFiles };
+  }
+
+  private async createDrawioSvg(
+    drawioPath: string,
+    pngPath?: string,
+  ): Promise<Buffer> {
+    try {
+      const drawioContent = await fs.readFile(drawioPath, 'utf-8');
+      const drawioBase64 = Buffer.from(drawioContent).toString('base64');
+
+      let imageElement = '';
+      // If we have a PNG, include it in the SVG
+      if (pngPath) {
+        try {
+          const pngBuffer = await fs.readFile(pngPath);
+          const pngBase64 = pngBuffer.toString('base64');
+
+          imageElement = `<image href="data:image/png;base64,${pngBase64}" width="100%" height="100%"/>`;
+        } catch (error) {
+          this.logger.warn(
+            `Could not read PNG file for Draw.io diagram: ${pngPath}`,
+            error,
+          );
+        }
+      }
+
+      // Create the SVG with embedded Draw.io data and image
+      // Default dimensions for Draw.io diagrams if no image is provided
+      const svgContent = `<?xml version="1.0" encoding="UTF-8"?>
+      <svg xmlns="http://www.w3.org/2000/svg" 
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      width="600"
+      height="400"
+      viewBox="0 0 600 400"
+      content="${drawioBase64}">${imageElement}</svg>`;
+
+      return Buffer.from(svgContent, 'utf-8');
+    } catch (error) {
+      this.logger.error(`Failed to create Draw.io SVG: ${error}`);
+      throw error;
+    }
+  }
+
   private async uploadWithRetry(opts: {
     abs: string;
     storageFilePath: string;
@@ -350,7 +804,10 @@ export class ImportAttachmentService {
     for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
       try {
         const fileStream = createReadStream(abs);
-        await this.storageService.uploadStream(storageFilePath, fileStream);
+        await this.storageService.uploadStream(storageFilePath, fileStream, {
+          recreateClient: true,
+        });
+
         const stat = await fs.stat(abs);
 
         await this.db
@@ -370,6 +827,36 @@ export class ImportAttachmentService {
           })
           .execute();
 
+        // Queue PDF and DOCX files for indexing
+        const supportedExtensions = ['.pdf', '.docx'];
+        if (supportedExtensions.includes(ext.toLowerCase())) {
+          try {
+            await this.attachmentQueue.add(
+              QueueJob.ATTACHMENT_INDEX_CONTENT,
+              { attachmentId },
+              {
+                attempts: 1,
+                backoff: {
+                  type: 'exponential',
+                  delay: 3 * 60 * 1000,
+                },
+                deduplication: {
+                  id: attachmentId,
+                },
+                removeOnComplete: true,
+                removeOnFail: false,
+              },
+            );
+            this.logger.debug(
+              `Queued ${fileNameWithExt} for indexing (attachment ID: ${attachmentId})`,
+            );
+          } catch (err) {
+            this.logger.error(
+              `Failed to queue indexing for imported attachment ${attachmentId}: ${err}`,
+            );
+          }
+        }
+
         uploadStats.completed++;
 
         if (uploadStats.completed % 10 === 0) {

apps/server/src/integrations/import/utils/file.utils.ts

@@ -103,6 +103,14 @@ function extractZipInternal(
         zipfile.on('entry', (entry) => {
           const name = entry.fileName.toString('utf8');
           const safe = name.replace(/^\/+/, '');
+
+          const validationError = yauzl.validateFileName(safe);
+          if (validationError) {
+            console.warn(`Skipping invalid entry (${validationError})`);
+            zipfile.readEntry();
+            return;
+          }
+
           if (safe.startsWith('__MACOSX/')) {
             zipfile.readEntry();
             return;
@@ -110,6 +118,15 @@ function extractZipInternal(
 
           const fullPath = path.join(target, safe);
 
+          const resolved = path.resolve(fullPath);
+          const targetResolved = path.resolve(target);
+
+          if (!resolved.startsWith(targetResolved + path.sep)) {
+            console.warn(`Skipping entry (path outside target): ${safe}`);
+            zipfile.readEntry();
+            return;
+          }
+
           // Handle directories
           if (/\/$/.test(name)) {
             try {

apps/server/src/integrations/import/utils/import-formatter.ts

@@ -4,6 +4,11 @@ import { v7 } from 'uuid';
 import { InsertableBacklink } from '@docmost/db/types/entity.types';
 import { Cheerio, CheerioAPI, load } from 'cheerio';
 
+// Check if text contains Unicode characters (for emojis/icons)
+function isUnicodeCharacter(text: string): boolean {
+  return text.length > 0 && text.codePointAt(0)! > 127; // Non-ASCII characters
+}
+
 export async function formatImportHtml(opts: {
   html: string;
   currentFilePath: string;
@@ -16,7 +21,11 @@ export async function formatImportHtml(opts: {
   workspaceId: string;
   pageDir?: string;
   attachmentCandidates?: string[];
-}): Promise<{ html: string; backlinks: InsertableBacklink[] }> {
+}): Promise<{
+  html: string;
+  backlinks: InsertableBacklink[];
+  pageIcon?: string;
+}> {
   const {
     html,
     currentFilePath,
@@ -28,6 +37,17 @@ export async function formatImportHtml(opts: {
   const $: CheerioAPI = load(html);
   const $root: Cheerio<any> = $.root();
 
+  let pageIcon: string | null = null;
+  // extract notion page icon
+  const headerIconSpan = $root.find('header .page-header-icon .icon');
+
+  if (headerIconSpan.length > 0) {
+    const iconText = headerIconSpan.text().trim();
+    if (iconText && isUnicodeCharacter(iconText)) {
+      pageIcon = iconText;
+    }
+  }
+
   notionFormatter($, $root);
   defaultHtmlFormatter($, $root);
 
@@ -44,6 +64,7 @@ export async function formatImportHtml(opts: {
   return {
     html: $root.html() || '',
     backlinks,
+    pageIcon: pageIcon || undefined,
   };
 }
 
@@ -69,6 +90,10 @@ export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) {
 }
 
 export function notionFormatter($: CheerioAPI, $root: Cheerio<any>) {
+  // remove page header icon and cover image
+  $root.find('.page-header-icon').remove();
+  $root.find('.page-cover-image').remove();
+
   // remove empty description paragraphs
   $root.find('p.page-description').each((_, el) => {
     if (!$(el).text().trim()) $(el).remove();
@@ -189,22 +214,48 @@ export function notionFormatter($: CheerioAPI, $root: Cheerio<any>) {
       $fig.replaceWith($newAnchor);
     });
 
+  // remove user icons
+  $root.find('span.user img.user-icon').remove();
+
   // remove toc
   $root.find('nav.table_of_contents').remove();
 }
 
 export function unwrapFromParagraph($: CheerioAPI, $node: Cheerio<any>) {
-  // find the nearest <p> or <a> ancestor
-  let $wrapper = $node.closest('p, a');
+  // Keep track of processed wrappers to avoid infinite loops
+  const processedWrappers = new Set<any>();
 
+  let $wrapper = $node.closest('p, a');
   while ($wrapper.length) {
-    // if the wrapper has only our node inside, replace it entirely
-    if ($wrapper.contents().length === 1) {
+    const wrapperElement = $wrapper.get(0);
+
+    // If we've already processed this wrapper, break to avoid infinite loop
+    if (processedWrappers.has(wrapperElement)) {
+      break;
+    }
+
+    processedWrappers.add(wrapperElement);
+
+    // Check if the wrapper contains only whitespace and our target node
+    const hasOnlyTargetNode =
+      $wrapper.contents().filter((_, el) => {
+        const $el = $(el);
+        // Skip whitespace-only text nodes. NodeType 3 = text node
+        if (el.nodeType === 3 && !$el.text().trim()) {
+          return false;
+        }
+        // Return true if this is not our target node
+        return !$el.is($node) && !$node.is($el);
+      }).length === 0;
+
+    if (hasOnlyTargetNode) {
+      // Replace the wrapper entirely with our node
       $wrapper.replaceWith($node);
     } else {
-      // otherwise just move the node to before the wrapper
+      // Move the node to before the wrapper, preserving other content
       $wrapper.before($node);
     }
+
     // look again for any new wrapper around $node
     $wrapper = $node.closest('p, a');
   }

apps/server/src/integrations/import/utils/import.utils.ts

@@ -64,3 +64,9 @@ export async function collectMarkdownAndHtmlFiles(
   await walk(dir);
   return results;
 }
+
+export function stripNotionID(fileName: string): string {
+  // Handle optional separator (space or dash) + 32 alphanumeric chars at end
+  const notionIdPattern = /[ -]?[a-z0-9]{32}$/i;
+  return fileName.replace(notionIdPattern, '').trim();
+}

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -4,13 +4,16 @@ export enum QueueName {
   GENERAL_QUEUE = '{general-queue}',
   BILLING_QUEUE = '{billing-queue}',
   FILE_TASK_QUEUE = '{file-task-queue}',
+  SEARCH_QUEUE = '{search-queue}',
+  AI_QUEUE = '{ai-queue}',
 }
 
 export enum QueueJob {
   SEND_EMAIL = 'send-email',
   DELETE_SPACE_ATTACHMENTS = 'delete-space-attachments',
+  ATTACHMENT_INDEX_CONTENT = 'attachment-index-content',
+  ATTACHMENT_INDEXING = 'attachment-indexing',
   DELETE_PAGE_ATTACHMENTS = 'delete-page-attachments',
-  PAGE_CONTENT_UPDATE = 'page-content-update',
 
   DELETE_USER_AVATARS = 'delete-user-avatars',
 
@@ -23,4 +26,36 @@ export enum QueueJob {
 
   IMPORT_TASK = 'import-task',
   EXPORT_TASK = 'export-task',
+
+  SEARCH_INDEX_PAGE = 'search-index-page',
+  SEARCH_INDEX_PAGES = 'search-index-pages',
+  SEARCH_INDEX_COMMENT = 'search-index-comment',
+  SEARCH_INDEX_COMMENTS = 'search-index-comments',
+  SEARCH_INDEX_ATTACHMENT = 'search-index-attachment',
+  SEARCH_INDEX_ATTACHMENTS = 'search-index-attachments',
+  SEARCH_REMOVE_PAGE = 'search-remove-page',
+  SEARCH_REMOVE_ASSET = 'search-remove-attachment',
+  SEARCH_REMOVE_FACE = 'search-remove-comment',
+  TYPESENSE_FLUSH = 'typesense-flush',
+
+  PAGE_CREATED = 'page-created',
+  PAGE_CONTENT_UPDATED = 'page-content-updated',
+  PAGE_MOVED_TO_SPACE = 'page-moved-to-space',
+  PAGE_UPDATED = 'page-updated',
+  PAGE_SOFT_DELETED = 'page-soft-deleted',
+  PAGE_RESTORED = 'page-restored',
+  PAGE_DELETED = 'page-deleted',
+
+  SPACE_CREATED = 'space-created',
+  SPACE_UPDATED = 'space-updated',
+  SPACE_DELETED = 'space-deleted',
+
+  WORKSPACE_CREATED = 'workspace-created',
+  WORKSPACE_SPACE_UPDATED = 'workspace-updated',
+  WORKSPACE_DELETED = 'workspace-deleted',
+  WORKSPACE_CREATE_EMBEDDINGS = 'workspace-create-embeddings',
+  WORKSPACE_DELETE_EMBEDDINGS = 'workspace-delete-embeddings',
+
+  GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
+  DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',
 }

apps/server/src/integrations/queue/queue.module.ts

@@ -57,6 +57,22 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
         attempts: 1,
       },
     }),
+    BullModule.registerQueue({
+      name: QueueName.SEARCH_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: 2,
+      },
+    }),
+    BullModule.registerQueue({
+      name: QueueName.AI_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: 1,
+      },
+    }),
   ],
   exports: [BullModule],
   providers: [BacklinksProcessor],

apps/server/src/integrations/redis/redis-config.service.ts

@@ -0,0 +1,26 @@
+import { Injectable } from '@nestjs/common';
+import {
+  RedisModuleOptions,
+  RedisOptionsFactory,
+} from '@nestjs-labs/nestjs-ioredis';
+import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
+import { EnvironmentService } from '../environment/environment.service';
+
+@Injectable()
+export class RedisConfigService implements RedisOptionsFactory {
+  constructor(private readonly environmentService: EnvironmentService) {}
+  createRedisOptions(): RedisModuleOptions {
+    const redisConfig = parseRedisUrl(this.environmentService.getRedisUrl());
+    return {
+      readyLog: true,
+      config: {
+        host: redisConfig.host,
+        port: redisConfig.port,
+        password: redisConfig.password,
+        db: redisConfig.db,
+        family: redisConfig.family,
+        retryStrategy: createRetryStrategy(),
+      },
+    };
+  }
+}

apps/server/src/integrations/storage/drivers/local.driver.ts

@@ -28,7 +28,7 @@ export class LocalDriver implements StorageDriver {
     }
   }
 
-  async uploadStream(filePath: string, file: Readable): Promise<void> {
+  async uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise<void> {
     try {
       const fullPath = this._fullPath(filePath);
       await fs.mkdir(dirname(fullPath), { recursive: true });

apps/server/src/integrations/storage/drivers/s3.driver.ts

@@ -41,12 +41,26 @@ export class S3Driver implements StorageDriver {
     }
   }
 
-  async uploadStream(filePath: string, file: Readable): Promise<void> {
+  async uploadStream(
+    filePath: string,
+    file: Readable,
+    options?: { recreateClient?: boolean },
+  ): Promise<void> {
+    let clientToUse = this.s3Client;
+    let shouldDestroyClient = false;
+
+    // optionally recreate client to avoid socket hang errors
+    // (during multi-attachments imports)
+    if (options?.recreateClient) {
+      clientToUse = new S3Client(this.config as any);
+      shouldDestroyClient = true;
+    }
+
     try {
       const contentType = getMimeType(filePath);
 
       const upload = new Upload({
-        client: this.s3Client,
+        client: clientToUse,
         params: {
           Bucket: this.config.bucket,
           Key: filePath,
@@ -58,6 +72,10 @@ export class S3Driver implements StorageDriver {
       await upload.done();
     } catch (err) {
       throw new Error(`Failed to upload file: ${(err as Error).message}`);
+    } finally {
+      if (shouldDestroyClient && clientToUse) {
+        clientToUse.destroy();
+      }
     }
   }
 

apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts

@@ -3,7 +3,7 @@ import { Readable } from 'stream';
 export interface StorageDriver {
   upload(filePath: string, file: Buffer): Promise<void>;
 
-  uploadStream(filePath: string, file: Readable): Promise<void>;
+  uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise<void>;
 
   copy(fromFilePath: string, toFilePath: string): Promise<void>;
 

apps/server/src/integrations/storage/storage.service.ts

@@ -15,8 +15,8 @@ export class StorageService {
     this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
   }
 
-  async uploadStream(filePath: string, fileContent: Readable) {
-    await this.storageDriver.uploadStream(filePath, fileContent);
+  async uploadStream(filePath: string, fileContent: Readable, options?: { recreateClient?: boolean }) {
+    await this.storageDriver.uploadStream(filePath, fileContent, options);
     this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
   }
 

apps/server/src/main.ts

@@ -15,10 +15,12 @@ async function bootstrap() {
   const app = await NestFactory.create<NestFastifyApplication>(
     AppModule,
     new FastifyAdapter({
-      ignoreTrailingSlash: true,
-      ignoreDuplicateSlashes: true,
-      maxParamLength: 1000,
       trustProxy: true,
+      routerOptions: {
+        maxParamLength: 1000,
+        ignoreTrailingSlash: true,
+        ignoreDuplicateSlashes: true,
+      },
     }),
     {
       rawBody: true,