Merge branch 'main' into anchor-link

apps/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "server",
-  "version": "0.22.2",
+  "version": "0.23.1",
   "description": "",
   "author": "",
   "private": true,
@@ -66,6 +66,8 @@
     "jsonwebtoken": "^9.0.2",
     "kysely": "^0.28.2",
     "kysely-migration-cli": "^0.4.2",
+    "ldapts": "^7.4.0",
+    "mammoth": "^1.10.0",
     "mime-types": "^2.1.35",
     "nanoid": "3.3.11",
     "nestjs-kysely": "^1.2.0",
@@ -75,13 +77,15 @@
     "p-limit": "^6.2.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-jwt": "^4.0.1",
+    "pdfjs-dist": "^5.4.54",
     "pg": "^8.16.0",
     "pg-tsquery": "^8.4.2",
     "postmark": "^4.0.5",
     "react": "^18.3.1",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.2",
-    "sanitize-filename-ts": "^1.0.2",
+    "sanitize-filename-ts": "1.0.2",
+    "sharp": "0.34.3",
     "socket.io": "^4.8.1",
     "stripe": "^17.5.0",
     "tmp-promise": "^3.0.3",

apps/server/src/collaboration/collaboration.util.ts

@@ -33,6 +33,7 @@ import {
   Excalidraw,
   Embed,
   Mention,
+  Subpages,
   HeadingAnchors
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
@@ -83,6 +84,7 @@ export const tiptapExtensions = [
   Excalidraw,
   Embed,
   Mention,
+  Subpages,
   UniqueID.configure({
     types: ['heading'],
     attributeName: 'nodeId',

apps/server/src/common/helpers/utils.ts

@@ -72,7 +72,9 @@ export function extractDateFromUuid7(uuid7: string) {
 }
 
 export function sanitizeFileName(fileName: string): string {
-  const sanitizedFilename = sanitize(fileName).replace(/ /g, '_');
+  const sanitizedFilename = sanitize(fileName)
+    .replace(/ /g, '_')
+    .replace(/#/g, '_');
   return sanitizedFilename.slice(0, 255);
 }
 
@@ -87,3 +89,12 @@ export function extractBearerTokenFromHeader(
   const [type, token] = request.headers.authorization?.split(' ') ?? [];
   return type === 'Bearer' ? token : undefined;
 }
+
+export function hasLicenseOrEE(opts: {
+  licenseKey: string;
+  plan: string;
+  isCloud: boolean;
+}): boolean {
+  const { licenseKey, plan, isCloud } = opts;
+  return Boolean(licenseKey) || (isCloud && plan === 'business');
+}

apps/server/src/core/attachment/attachment.constants.ts

@@ -1,12 +1,12 @@
 export enum AttachmentType {
   Avatar = 'avatar',
-  WorkspaceLogo = 'workspace-logo',
-  SpaceLogo = 'space-logo',
+  WorkspaceIcon = 'workspace-icon',
+  SpaceIcon = 'space-icon',
   File = 'file',
 }
 
 export const validImageExtensions = ['.jpg', '.png', '.jpeg'];
-export const MAX_AVATAR_SIZE = '5MB';
+export const MAX_AVATAR_SIZE = '10MB';
 
 export const inlineFileExtensions = [
   '.jpg',

apps/server/src/core/attachment/attachment.controller.ts

@@ -1,5 +1,6 @@
 import {
   BadRequestException,
+  Body,
   Controller,
   ForbiddenException,
   Get,
@@ -51,6 +52,7 @@ import { EnvironmentService } from '../../integrations/environment/environment.s
 import { TokenService } from '../auth/services/token.service';
 import { JwtAttachmentPayload, JwtType } from '../auth/dto/jwt-payload';
 import * as path from 'path';
+import { RemoveIconDto } from './dto/attachment.dto';
 
 @Controller()
 export class AttachmentController {
@@ -302,7 +304,7 @@ export class AttachmentController {
       throw new BadRequestException('Invalid image attachment type');
     }
 
-    if (attachmentType === AttachmentType.WorkspaceLogo) {
+    if (attachmentType === AttachmentType.WorkspaceIcon) {
       const ability = this.workspaceAbility.createForUser(user, workspace);
       if (
         ability.cannot(
@@ -314,7 +316,7 @@ export class AttachmentController {
       }
     }
 
-    if (attachmentType === AttachmentType.SpaceLogo) {
+    if (attachmentType === AttachmentType.SpaceIcon) {
       if (!spaceId) {
         throw new BadRequestException('spaceId is required');
       }
@@ -372,8 +374,59 @@ export class AttachmentController {
       });
       return res.send(fileStream);
     } catch (err) {
-      this.logger.error(err);
+     // this.logger.error(err);
       throw new NotFoundException('File not found');
     }
   }
+
+  @UseGuards(JwtAuthGuard)
+  @HttpCode(HttpStatus.OK)
+  @Post('attachments/remove-icon')
+  async removeIcon(
+    @Body() dto: RemoveIconDto,
+    @AuthUser() user: User,
+    @AuthWorkspace() workspace: Workspace,
+  ) {
+    const { type, spaceId } = dto;
+
+    // remove current user avatar
+    if (type === AttachmentType.Avatar) {
+      await this.attachmentService.removeUserAvatar(user);
+      return;
+    }
+
+    // remove space icon
+    if (type === AttachmentType.SpaceIcon) {
+      if (!spaceId) {
+        throw new BadRequestException(
+          'spaceId is required to change space icons',
+        );
+      }
+
+      const spaceAbility = await this.spaceAbility.createForUser(user, spaceId);
+      if (
+        spaceAbility.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Settings)
+      ) {
+        throw new ForbiddenException();
+      }
+
+      await this.attachmentService.removeSpaceIcon(spaceId, workspace.id);
+      return;
+    }
+
+    // remove workspace icon
+    if (type === AttachmentType.WorkspaceIcon) {
+      const ability = this.workspaceAbility.createForUser(user, workspace);
+      if (
+        ability.cannot(
+          WorkspaceCaslAction.Manage,
+          WorkspaceCaslSubject.Settings,
+        )
+      ) {
+        throw new ForbiddenException();
+      }
+      await this.attachmentService.removeWorkspaceIcon(workspace);
+      return;
+    }
+  }
 }

apps/server/src/core/attachment/attachment.utils.ts

@@ -1,8 +1,8 @@
 import { MultipartFile } from '@fastify/multipart';
-import { randomBytes } from 'crypto';
-import { sanitize } from 'sanitize-filename-ts';
 import * as path from 'path';
 import { AttachmentType } from './attachment.constants';
+import { sanitizeFileName } from '../../common/helpers';
+import * as sharp from 'sharp';
 
 export interface PreparedFile {
   buffer: Buffer;
@@ -22,10 +22,8 @@ export async function prepareFile(
   }
 
   try {
-    const rand = randomBytes(8).toString('hex');
-
     const buffer = await file.toBuffer();
-    const sanitizedFilename = sanitize(file.filename).replace(/ /g, '_');
+    const sanitizedFilename = sanitizeFileName(file.filename);
     const fileName = sanitizedFilename.slice(0, 255);
     const fileSize = buffer.length;
     const fileExtension = path.extname(file.filename).toLowerCase();
@@ -58,9 +56,9 @@ export function getAttachmentFolderPath(
   switch (type) {
     case AttachmentType.Avatar:
       return `${workspaceId}/avatars`;
-    case AttachmentType.WorkspaceLogo:
-      return `${workspaceId}/workspace-logo`;
-    case AttachmentType.SpaceLogo:
+    case AttachmentType.WorkspaceIcon:
+      return `${workspaceId}/workspace-logos`;
+    case AttachmentType.SpaceIcon:
       return `${workspaceId}/space-logos`;
     case AttachmentType.File:
       return `${workspaceId}/files`;
@@ -70,3 +68,51 @@ export function getAttachmentFolderPath(
 }
 
 export const validAttachmentTypes = Object.values(AttachmentType);
+
+export async function compressAndResizeIcon(
+  buffer: Buffer,
+  attachmentType?: AttachmentType,
+): Promise<Buffer> {
+  try {
+    let sharpInstance = sharp(buffer);
+    const metadata = await sharpInstance.metadata();
+
+    const targetWidth = 300;
+    const targetHeight = 300;
+
+    // Only resize if image is larger than target dimensions
+    if (metadata.width > targetWidth || metadata.height > targetHeight) {
+      sharpInstance = sharpInstance.resize(targetWidth, targetHeight, {
+        fit: 'inside',
+        withoutEnlargement: true,
+      });
+    }
+
+    // Handle based on original format
+    if (metadata.format === 'png') {
+      // Only flatten avatars to remove transparency
+      if (attachmentType === AttachmentType.Avatar) {
+        sharpInstance = sharpInstance.flatten({
+          background: { r: 255, g: 255, b: 255 },
+        });
+      }
+
+      return await sharpInstance
+        .png({
+          quality: 85,
+          compressionLevel: 6,
+        })
+        .toBuffer();
+    } else {
+      return await sharpInstance
+        .jpeg({
+          quality: 85,
+          progressive: true,
+          mozjpeg: true,
+        })
+        .toBuffer();
+    }
+  } catch (err) {
+    throw err;
+  }
+}

apps/server/src/core/attachment/dto/attachment.dto.ts

@@ -0,0 +1,17 @@
+import { IsEnum, IsIn, IsNotEmpty, IsOptional, IsUUID } from 'class-validator';
+import { AttachmentType } from '../attachment.constants';
+
+export class RemoveIconDto {
+  @IsEnum(AttachmentType)
+  @IsIn([
+    AttachmentType.Avatar,
+    AttachmentType.SpaceIcon,
+    AttachmentType.WorkspaceIcon,
+  ])
+  @IsNotEmpty()
+  type: AttachmentType;
+
+  @IsOptional()
+  @IsUUID()
+  spaceId: string;
+}

apps/server/src/core/attachment/dto/avatar-upload.dto.ts

@@ -1,3 +0,0 @@
-import { IsOptional, IsString, IsUUID } from 'class-validator';
-
-export class AvatarUploadDto {}

apps/server/src/core/attachment/dto/get-file.dto.ts

@@ -1,7 +0,0 @@
-import { IsNotEmpty, IsString } from 'class-validator';
-
-export class GetFileDto {
-  @IsString()
-  @IsNotEmpty()
-  attachmentId: string;
-}

apps/server/src/core/attachment/dto/upload-file.dto.ts

@@ -1,20 +0,0 @@
-import {
-  IsDefined,
-  IsNotEmpty,
-  IsOptional,
-  IsString,
-  IsUUID,
-} from 'class-validator';
-
-export class UploadFileDto {
-  @IsString()
-  @IsNotEmpty()
-  attachmentType: string;
-
-  @IsOptional()
-  @IsUUID()
-  pageId: string;
-
-  @IsDefined()
-  file: any;
-}

apps/server/src/core/attachment/processors/attachment.processor.ts

@@ -3,12 +3,15 @@ import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { AttachmentService } from '../services/attachment.service';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
-import { Space } from '@docmost/db/types/entity.types';
+import { ModuleRef } from '@nestjs/core';
 
 @Processor(QueueName.ATTACHMENT_QUEUE)
 export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(AttachmentProcessor.name);
-  constructor(private readonly attachmentService: AttachmentService) {
+  constructor(
+    private readonly attachmentService: AttachmentService,
+    private moduleRef: ModuleRef,
+  ) {
     super();
   }
 
@@ -25,6 +28,33 @@ export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
           job.data.pageId,
         );
       }
+      if (
+        job.name === QueueJob.ATTACHMENT_INDEX_CONTENT ||
+        job.name === QueueJob.ATTACHMENT_INDEXING
+      ) {
+        let AttachmentEeModule: any;
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-require-imports
+          AttachmentEeModule = require('./../../../ee/attachments-ee/attachment-ee.service');
+        } catch (err) {
+          this.logger.debug(
+            'Attachment enterprise module requested but EE module not bundled in this build',
+          );
+          return;
+        }
+        const attachmentEeService = this.moduleRef.get(
+          AttachmentEeModule.AttachmentEeService,
+          { strict: false },
+        );
+
+        if (job.name === QueueJob.ATTACHMENT_INDEX_CONTENT) {
+          await attachmentEeService.indexAttachment(job.data.attachmentId);
+        } else if (job.name === QueueJob.ATTACHMENT_INDEXING) {
+          await attachmentEeService.indexAttachments(
+            job.data.workspaceId,
+          );
+        }
+      }
     } catch (err) {
       throw err;
     }
@@ -37,9 +67,15 @@ export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
 
   @OnWorkerEvent('failed')
   onError(job: Job) {
-    this.logger.error(
-      `Error processing ${job.name} job. Reason: ${job.failedReason}`,
-    );
+    if (job.name === QueueJob.ATTACHMENT_INDEX_CONTENT) {
+      this.logger.debug(
+        `Error processing ${job.name} job for attachment ${job.data?.attachmentId}. Reason: ${job.failedReason}`,
+      );
+    } else {
+      this.logger.error(
+        `Error processing ${job.name} job. Reason: ${job.failedReason}`,
+      );
+    }
   }
 
   @OnWorkerEvent('completed')

apps/server/src/core/attachment/services/attachment.service.ts

@@ -7,6 +7,7 @@ import {
 import { StorageService } from '../../../integrations/storage/storage.service';
 import { MultipartFile } from '@fastify/multipart';
 import {
+  compressAndResizeIcon,
   getAttachmentFolderPath,
   PreparedFile,
   prepareFile,
@@ -16,12 +17,15 @@ import { v4 as uuid4, v7 as uuid7 } from 'uuid';
 import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
 import { AttachmentType, validImageExtensions } from '../attachment.constants';
 import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
-import { Attachment } from '@docmost/db/types/entity.types';
+import { Attachment, User, Workspace } from '@docmost/db/types/entity.types';
 import { InjectKysely } from 'nestjs-kysely';
 import { executeTx } from '@docmost/db/utils';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../../integrations/queue/constants';
+import { Queue } from 'bullmq';
 
 @Injectable()
 export class AttachmentService {
@@ -33,6 +37,7 @@ export class AttachmentService {
     private readonly workspaceRepo: WorkspaceRepo,
     private readonly spaceRepo: SpaceRepo,
     @InjectKysely() private readonly db: KyselyDB,
+    @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
   async uploadFile(opts: {
@@ -99,6 +104,23 @@ export class AttachmentService {
           pageId,
         });
       }
+
+      // Only index PDFs and DOCX files
+      if (['.pdf', '.docx'].includes(attachment.fileExt.toLowerCase())) {
+        await this.attachmentQueue.add(
+          QueueJob.ATTACHMENT_INDEX_CONTENT,
+          {
+            attachmentId: attachmentId,
+          },
+          {
+            attempts: 2,
+            backoff: {
+              type: 'exponential',
+              delay: 10000,
+            },
+          },
+        );
+      }
     } catch (err) {
       // delete uploaded file on error
       this.logger.error(err);
@@ -111,8 +133,8 @@ export class AttachmentService {
     filePromise: Promise<MultipartFile>,
     type:
       | AttachmentType.Avatar
-      | AttachmentType.WorkspaceLogo
-      | AttachmentType.SpaceLogo,
+      | AttachmentType.WorkspaceIcon
+      | AttachmentType.SpaceIcon,
     userId: string,
     workspaceId: string,
     spaceId?: string,
@@ -120,6 +142,9 @@ export class AttachmentService {
     const preparedFile: PreparedFile = await prepareFile(filePromise);
     validateFileType(preparedFile.fileExtension, validImageExtensions);
 
+    const processedBuffer = await compressAndResizeIcon(preparedFile.buffer, type);
+    preparedFile.buffer = processedBuffer;
+    preparedFile.fileSize = processedBuffer.length;
     preparedFile.fileName = uuid4() + preparedFile.fileExtension;
 
     const filePath = `${getAttachmentFolderPath(type, workspaceId)}/${preparedFile.fileName}`;
@@ -153,7 +178,7 @@ export class AttachmentService {
             workspaceId,
             trx,
           );
-        } else if (type === AttachmentType.WorkspaceLogo) {
+        } else if (type === AttachmentType.WorkspaceIcon) {
           const workspace = await this.workspaceRepo.findById(workspaceId, {
             trx,
           });
@@ -165,7 +190,7 @@ export class AttachmentService {
             workspaceId,
             trx,
           );
-        } else if (type === AttachmentType.SpaceLogo && spaceId) {
+        } else if (type === AttachmentType.SpaceIcon && spaceId) {
           const space = await this.spaceRepo.findById(spaceId, workspaceId, {
             trx,
           });
@@ -184,7 +209,6 @@ export class AttachmentService {
       });
     } catch (err) {
       // delete uploaded file on db update failure
-      this.logger.error('Image upload error:', err);
       await this.deleteRedundantFile(filePath);
       throw new BadRequestException('Failed to upload image');
     }
@@ -367,4 +391,41 @@ export class AttachmentService {
       throw err;
     }
   }
+
+  async removeUserAvatar(user: User) {
+    if (user.avatarUrl && !user.avatarUrl.toLowerCase().startsWith('http')) {
+      const filePath = `${getAttachmentFolderPath(AttachmentType.Avatar, user.workspaceId)}/${user.avatarUrl}`;
+      await this.deleteRedundantFile(filePath);
+    }
+
+    await this.userRepo.updateUser(
+      { avatarUrl: null },
+      user.id,
+      user.workspaceId,
+    );
+  }
+
+  async removeSpaceIcon(spaceId: string, workspaceId: string) {
+    const space = await this.spaceRepo.findById(spaceId, workspaceId);
+
+    if (!space) {
+      throw new NotFoundException('Space not found');
+    }
+
+    if (space.logo && !space.logo.toLowerCase().startsWith('http')) {
+      const filePath = `${getAttachmentFolderPath(AttachmentType.SpaceIcon, workspaceId)}/${space.logo}`;
+      await this.deleteRedundantFile(filePath);
+    }
+
+    await this.spaceRepo.updateSpace({ logo: null }, spaceId, workspaceId);
+  }
+
+  async removeWorkspaceIcon(workspace: Workspace) {
+    if (workspace.logo && !workspace.logo.toLowerCase().startsWith('http')) {
+      const filePath = `${getAttachmentFolderPath(AttachmentType.WorkspaceIcon, workspace.id)}/${workspace.logo}`;
+      await this.deleteRedundantFile(filePath);
+    }
+
+    await this.workspaceRepo.updateWorkspace({ logo: null }, workspace.id);
+  }
 }

apps/server/src/core/auth/services/auth.service.ts

@@ -106,6 +106,7 @@ export class AuthService {
     await this.userRepo.updateUser(
       {
         password: newPasswordHash,
+        hasGeneratedPassword: false,
       },
       userId,
       workspaceId,
@@ -186,6 +187,7 @@ export class AuthService {
       await this.userRepo.updateUser(
         {
           password: newPasswordHash,
+          hasGeneratedPassword: false,
         },
         user.id,
         workspace.id,

apps/server/src/core/page/dto/sidebar-page.dto.ts

@@ -1,7 +1,11 @@
-import { IsOptional, IsString } from 'class-validator';
+import { IsOptional, IsString, IsUUID } from 'class-validator';
 import { SpaceIdDto } from './page.dto';
 
-export class SidebarPageDto extends SpaceIdDto {
+export class SidebarPageDto {
+  @IsOptional()
+  @IsUUID()
+  spaceId: string;
+
   @IsOptional()
   @IsString()
   pageId: string;

apps/server/src/core/page/page.controller.ts

@@ -254,21 +254,28 @@ export class PageController {
     @Body() pagination: PaginationOptions,
     @AuthUser() user: User,
   ) {
-    const ability = await this.spaceAbility.createForUser(user, dto.spaceId);
+    if (!dto.spaceId && !dto.pageId) {
+      throw new BadRequestException(
+        'Either spaceId or pageId must be provided',
+      );
+    }
+    let spaceId = dto.spaceId;
+
+    if (dto.pageId) {
+      const page = await this.pageRepo.findById(dto.pageId);
+      if (!page) {
+        throw new ForbiddenException();
+      }
+
+      spaceId = page.spaceId;
+    }
+
+    const ability = await this.spaceAbility.createForUser(user, spaceId);
     if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
       throw new ForbiddenException();
     }
 
-    let pageId = null;
-    if (dto.pageId) {
-      const page = await this.pageRepo.findById(dto.pageId);
-      if (page.spaceId !== dto.spaceId) {
-        throw new ForbiddenException();
-      }
-      pageId = page.id;
-    }
-
-    return this.pageService.getSidebarPages(dto.spaceId, pagination, pageId);
+    return this.pageService.getSidebarPages(spaceId, pagination, dto.pageId);
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/search/dto/search.dto.ts

@@ -5,15 +5,13 @@ import {
   IsOptional,
   IsString,
 } from 'class-validator';
-import { PartialType } from '@nestjs/mapped-types';
-import { CreateWorkspaceDto } from '../../workspace/dto/create-workspace.dto';
 
 export class SearchDTO {
   @IsNotEmpty()
   @IsString()
   query: string;
 
-  @IsNotEmpty()
+  @IsOptional()
   @IsString()
   spaceId: string;
 

apps/server/src/core/share/share.controller.ts

@@ -31,6 +31,7 @@ import { Public } from '../../common/decorators/public.decorator';
 import { ShareRepo } from '@docmost/db/repos/share/share.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import { EnvironmentService } from '../../integrations/environment/environment.service';
+import { hasLicenseOrEE } from '../../common/helpers';
 
 @UseGuards(JwtAuthGuard)
 @Controller('shares')
@@ -65,9 +66,11 @@ export class ShareController {
 
     return {
       ...(await this.shareService.getSharedPage(dto, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 
@@ -175,9 +178,11 @@ export class ShareController {
   ) {
     return {
       ...(await this.shareService.getShareTree(dto.shareId, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 }

apps/server/src/database/migrations/20250831T191600-add-group-sync-to-auth-providers.ts

@@ -0,0 +1,15 @@
+import { type Kysely } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('auth_providers')
+    .addColumn('group_sync', 'boolean', (col) => col.defaultTo(false).notNull())
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('auth_providers')
+    .dropColumn('group_sync')
+    .execute();
+}

apps/server/src/database/migrations/20250831T202306-ldap-auth.ts

@@ -0,0 +1,68 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  // switch type to text column since you can't add value to PG types in a transaction
+  await db.schema
+    .alterTable('auth_providers')
+    .alterColumn('type', (col) => col.setDataType('text'))
+    .execute();
+
+  await db.schema.dropType('auth_provider_type').ifExists().execute();
+
+  await db.schema
+    .alterTable('users')
+    .addColumn('has_generated_password', 'boolean', (col) =>
+      col.notNull().defaultTo(false).ifNotExists(),
+    )
+    .execute();
+
+  await db.schema
+    .alterTable('auth_providers')
+    .addColumn('ldap_url', 'varchar', (col) => col)
+    .addColumn('ldap_bind_dn', 'varchar', (col) => col)
+    .addColumn('ldap_bind_password', 'varchar', (col) => col)
+    .addColumn('ldap_base_dn', 'varchar', (col) => col)
+    .addColumn('ldap_user_search_filter', 'varchar', (col) => col)
+    .addColumn('ldap_user_attributes', 'jsonb', (col) =>
+      col.defaultTo(sql`'{}'::jsonb`),
+    )
+    .addColumn('ldap_tls_enabled', 'boolean', (col) => col.defaultTo(false))
+    .addColumn('ldap_tls_ca_cert', 'text', (col) => col)
+    .addColumn('ldap_config', 'jsonb', (col) => col.defaultTo(sql`'{}'::jsonb`))
+    .addColumn('settings', 'jsonb', (col) => col.defaultTo(sql`'{}'::jsonb`))
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('users')
+    .dropColumn('has_generated_password')
+    .execute();
+
+  await db.schema
+    .alterTable('auth_providers')
+    .dropColumn('ldap_url')
+    .dropColumn('ldap_bind_dn')
+    .dropColumn('ldap_bind_password')
+    .dropColumn('ldap_base_dn')
+    .dropColumn('ldap_user_search_filter')
+    .dropColumn('ldap_user_attributes')
+    .dropColumn('ldap_tls_enabled')
+    .dropColumn('ldap_tls_ca_cert')
+    .dropColumn('ldap_config')
+    .dropColumn('settings')
+    .execute();
+
+  await db.schema
+    .createType('auth_provider_type')
+    .asEnum(['saml', 'oidc', 'google'])
+    .execute();
+
+  await db.deleteFrom('auth_providers').where('type', '=', 'ldap').execute();
+
+  await sql`
+    ALTER TABLE auth_providers 
+    ALTER COLUMN type TYPE auth_provider_type 
+    USING type::auth_provider_type
+  `.execute(db);
+}

apps/server/src/database/migrations/20250901T184612-attachments-search.ts

@@ -0,0 +1,29 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .addColumn('text_content', 'text', (col) => col)
+    .addColumn('tsv', sql`tsvector`, (col) => col)
+    .execute();
+
+  await db.schema
+    .createIndex('attachments_tsv_idx')
+    .on('attachments')
+    .using('GIN')
+    .column('tsv')
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .dropIndex('attachments_tsv_idx')
+    .execute();
+
+  await db.schema
+    .alterTable('attachments')
+    .dropColumn('text_content')
+    .dropColumn('tsv')
+    .execute();
+}

apps/server/src/database/repos/attachment/attachment.repo.ts

@@ -12,6 +12,23 @@ import {
 export class AttachmentRepo {
   constructor(@InjectKysely() private readonly db: KyselyDB) {}
 
+  private baseFields: Array<keyof Attachment> = [
+    'id',
+    'fileName',
+    'filePath',
+    'fileSize',
+    'fileExt',
+    'mimeType',
+    'type',
+    'creatorId',
+    'pageId',
+    'spaceId',
+    'workspaceId',
+    'createdAt',
+    'updatedAt',
+    'deletedAt',
+  ];
+
   async findById(
     attachmentId: string,
     opts?: {
@@ -22,7 +39,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('id', '=', attachmentId)
       .executeTakeFirst();
   }
@@ -36,7 +53,7 @@ export class AttachmentRepo {
     return db
       .insertInto('attachments')
       .values(insertableAttachment)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -50,7 +67,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('spaceId', '=', spaceId)
       .execute();
   }
@@ -64,6 +81,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('pageId', 'in', pageIds)
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -75,7 +93,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('id', '=', attachmentId)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 

apps/server/src/database/repos/page/page.repo.ts

@@ -399,6 +399,7 @@ export class PageRepo {
           ])
           .$if(opts?.includeContent, (qb) => qb.select('content'))
           .where('id', '=', parentPageId)
+          .where('deletedAt', 'is', null)
           .unionAll((exp) =>
             exp
               .selectFrom('pages as p')
@@ -413,7 +414,8 @@ export class PageRepo {
                 'p.workspaceId',
               ])
               .$if(opts?.includeContent, (qb) => qb.select('p.content'))
-              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id'),
+              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id')
+              .where('p.deletedAt', 'is', null),
           ),
       )
       .selectFrom('page_hierarchy')

apps/server/src/database/repos/user/user.repo.ts

@@ -34,6 +34,7 @@ export class UserRepo {
     'createdAt',
     'updatedAt',
     'deletedAt',
+    'hasGeneratedPassword',
   ];
 
   async findById(

apps/server/src/database/types/db.d.ts

@@ -5,8 +5,6 @@
 
 import type { ColumnType } from "kysely";
 
-export type AuthProviderType = "google" | "oidc" | "saml";
-
 export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   ? ColumnType<S, I | undefined, U>
   : ColumnType<T, T | undefined, T>;
@@ -39,6 +37,8 @@ export interface Attachments {
   mimeType: string | null;
   pageId: string | null;
   spaceId: string | null;
+  textContent: string | null;
+  tsv: string | null;
   type: string | null;
   updatedAt: Generated<Timestamp>;
   workspaceId: string;
@@ -62,13 +62,24 @@ export interface AuthProviders {
   deletedAt: Timestamp | null;
   id: Generated<string>;
   isEnabled: Generated<boolean>;
+  groupSync: Generated<boolean>;
+  ldapBaseDn: string | null;
+  ldapBindDn: string | null;
+  ldapBindPassword: string | null;
+  ldapTlsCaCert: string | null;
+  ldapTlsEnabled: Generated<boolean | null>;
+  ldapUrl: string | null;
+  ldapUserAttributes: Json | null;
+  ldapUserSearchFilter: string | null;
+  ldapConfig: Json | null;
+  settings: Json | null;
   name: string;
   oidcClientId: string | null;
   oidcClientSecret: string | null;
   oidcIssuer: string | null;
   samlCertificate: string | null;
   samlUrl: string | null;
-  type: AuthProviderType;
+  type: string;
   updatedAt: Generated<Timestamp>;
   workspaceId: string;
 }
@@ -275,6 +286,7 @@ export interface Users {
   lastActiveAt: Timestamp | null;
   lastLoginAt: Timestamp | null;
   locale: string | null;
+  hasGeneratedPassword: Generated<boolean | null>;
   name: string | null;
   password: string | null;
   role: string | null;

apps/server/src/integrations/export/export.controller.ts

@@ -46,7 +46,7 @@ export class ExportController {
       includeContent: true,
     });
 
-    if (!page) {
+    if (!page || page.deletedAt) {
       throw new NotFoundException('Page not found');
     }
 

apps/server/src/integrations/import/import.module.ts

@@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
 import { ImportService } from './services/import.service';
 import { ImportController } from './import.controller';
 import { StorageModule } from '../storage/storage.module';
-import { FileTaskService } from './services/file-task.service';
+import { FileImportTaskService } from './services/file-import-task.service';
 import { FileTaskProcessor } from './processors/file-task.processor';
 import { ImportAttachmentService } from './services/import-attachment.service';
 import { FileTaskController } from './file-task.controller';
@@ -11,7 +11,7 @@ import { PageModule } from '../../core/page/page.module';
 @Module({
   providers: [
     ImportService,
-    FileTaskService,
+    FileImportTaskService,
     FileTaskProcessor,
     ImportAttachmentService,
   ],

apps/server/src/integrations/import/processors/file-task.processor.ts

@@ -2,7 +2,7 @@ import { Logger, OnModuleDestroy } from '@nestjs/common';
 import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
-import { FileTaskService } from '../services/file-task.service';
+import { FileImportTaskService } from '../services/file-import-task.service';
 import { FileTaskStatus } from '../utils/file.utils';
 import { StorageService } from '../../storage/storage.service';
 
@@ -11,7 +11,7 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(FileTaskProcessor.name);
 
   constructor(
-    private readonly fileTaskService: FileTaskService,
+    private readonly fileTaskService: FileImportTaskService,
     private readonly storageService: StorageService,
   ) {
     super();
@@ -41,15 +41,32 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   @OnWorkerEvent('failed')
   async onFailed(job: Job) {
     this.logger.error(
-      `Error processing ${job.name} job. Reason: ${job.failedReason}`,
+      `Error processing ${job.name} job. Import Task ID: ${job.data.fileTaskId}. Reason: ${job.failedReason}`,
     );
 
+    await this.handleFailedJob(job);
+  }
+
+  @OnWorkerEvent('stalled')
+  async onStalled(job: Job) {
+    this.logger.error(
+      `Job ${job.name} stalled. . Import Task ID: ${job.data.fileTaskId}.. Job ID: ${job.id}`,
+    );
+
+    // Set failedReason for stalled jobs since it's not automatically set
+    job.failedReason = 'Job stalled and was marked as failed';
+    await this.handleFailedJob(job);
+  }
+
+  private async handleFailedJob(job: Job) {
     try {
       const fileTaskId = job.data.fileTaskId;
+      const reason = job.failedReason || 'Unknown error';
+
       await this.fileTaskService.updateTaskStatus(
         fileTaskId,
         FileTaskStatus.Failed,
-        job.failedReason,
+        reason,
       );
 
       const fileTask = await this.fileTaskService.getFileTask(fileTaskId);
@@ -62,10 +79,22 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   }
 
   @OnWorkerEvent('completed')
-  onCompleted(job: Job) {
+  async onCompleted(job: Job) {
     this.logger.log(
       `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`,
     );
+
+    try {
+      const fileTask = await this.fileTaskService.getFileTask(
+        job.data.fileTaskId,
+      );
+      if (fileTask) {
+        await this.storageService.delete(fileTask.filePath);
+        this.logger.debug(`Deleted imported zip file: ${fileTask.filePath}`);
+      }
+    } catch (err) {
+      this.logger.error(`Failed to delete imported zip file:`, err);
+    }
   }
 
   async onModuleDestroy(): Promise<void> {

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -33,8 +33,8 @@ import { PageService } from '../../../core/page/services/page.service';
 import { ImportPageNode } from '../dto/file-task-dto';
 
 @Injectable()
-export class FileTaskService {
-  private readonly logger = new Logger(FileTaskService.name);
+export class FileImportTaskService {
+  private readonly logger = new Logger(FileImportTaskService.name);
 
   constructor(
     private readonly storageService: StorageService,
@@ -266,7 +266,7 @@ export class FileTaskService {
             attachmentCandidates,
           });
 
-        const { html, backlinks } = await formatImportHtml({
+        const { html, backlinks, pageIcon } = await formatImportHtml({
           html: htmlContent,
           currentFilePath: page.filePath,
           filePathToPageMetaMap: filePathToPageMetaMap,
@@ -286,6 +286,7 @@ export class FileTaskService {
           id: page.id,
           slugId: page.slugId,
           title: title || page.name,
+          icon: pageIcon || null,
           content: prosemirrorJson,
           textContent: jsonToText(prosemirrorJson),
           ydoc: await this.importService.createYdoc(prosemirrorJson),

apps/server/src/integrations/import/services/import-attachment.service.ts

@@ -6,6 +6,7 @@ import { cleanUrlString } from '../utils/file.utils';
 import { StorageService } from '../../storage/storage.service';
 import { createReadStream } from 'node:fs';
 import { promises as fs } from 'fs';
+import { Readable } from 'stream';
 import { getMimeType, sanitizeFileName } from '../../../common/helpers';
 import { v7 } from 'uuid';
 import { FileTask } from '@docmost/db/types/entity.types';
@@ -15,17 +16,33 @@ import { unwrapFromParagraph } from '../utils/import-formatter';
 import { resolveRelativeAttachmentPath } from '../utils/import.utils';
 import { load } from 'cheerio';
 import pLimit from 'p-limit';
+import { InjectQueue } from '@nestjs/bullmq';
+import { Queue } from 'bullmq';
+import { QueueJob, QueueName } from '../../queue/constants';
+
+interface AttachmentInfo {
+  href: string;
+  fileName: string;
+  mimeType: string;
+}
+
+interface DrawioPair {
+  drawioFile?: AttachmentInfo;
+  pngFile?: AttachmentInfo;
+  baseName: string;
+}
 
 @Injectable()
 export class ImportAttachmentService {
   private readonly logger = new Logger(ImportAttachmentService.name);
-  private readonly CONCURRENT_UPLOADS = 3;
+  private readonly CONCURRENT_UPLOADS = 1;
   private readonly MAX_RETRIES = 2;
   private readonly RETRY_DELAY = 2000;
 
   constructor(
     private readonly storageService: StorageService,
     @InjectKysely() private readonly db: KyselyDB,
+    @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
   async processAttachments(opts: {
@@ -35,6 +52,8 @@ export class ImportAttachmentService {
     pageId: string;
     fileTask: FileTask;
     attachmentCandidates: Map<string, string>;
+    pageAttachments?: AttachmentInfo[];
+    isConfluenceImport?: boolean;
   }): Promise<string> {
     const {
       html,
@@ -43,6 +62,8 @@ export class ImportAttachmentService {
       pageId,
       fileTask,
       attachmentCandidates,
+      pageAttachments = [],
+      isConfluenceImport,
     } = opts;
 
     const attachmentTasks: (() => Promise<void>)[] = [];
@@ -57,7 +78,7 @@ export class ImportAttachmentService {
     /**
      * Cache keyed by the *relative* path that appears in the HTML.
      * Ensures we upload (and DB-insert) each attachment at most once,
-     * even if it’s referenced multiple times on the page.
+     * even if it's referenced multiple times on the page.
      */
     const processed = new Map<
       string,
@@ -70,6 +91,104 @@ export class ImportAttachmentService {
       }
     >();
 
+    // Analyze attachments to identify Draw.io pairs
+    const { drawioPairs, skipFiles } = this.analyzeAttachments(
+      pageAttachments,
+      isConfluenceImport,
+    );
+
+    // Map to store processed Draw.io SVGs
+    const drawioSvgMap = new Map<
+      string,
+      {
+        attachmentId: string;
+        apiFilePath: string;
+        fileName: string;
+      }
+    >();
+
+    //this.logger.debug(`Found ${drawioPairs.size} Draw.io pairs to process`);
+
+    // Process Draw.io pairs and create combined SVG files
+    for (const [drawioHref, pair] of drawioPairs) {
+      if (!pair.drawioFile) continue;
+
+      const drawioAbsPath = attachmentCandidates.get(drawioHref);
+      if (!drawioAbsPath) continue;
+
+      const pngAbsPath = pair.pngFile
+        ? attachmentCandidates.get(pair.pngFile.href)
+        : undefined;
+
+      try {
+        // Create combined SVG with Draw.io data and PNG image
+        const svgBuffer = await this.createDrawioSvg(drawioAbsPath, pngAbsPath);
+
+        // Generate file details - always use "diagram.drawio.svg" as filename
+        const attachmentId = v7();
+        const fileName = 'diagram.drawio.svg';
+        const storageFilePath = `${getAttachmentFolderPath(
+          AttachmentType.File,
+          fileTask.workspaceId,
+        )}/${attachmentId}/${fileName}`;
+        const apiFilePath = `/api/files/${attachmentId}/${fileName}`;
+
+        // Upload the SVG file
+        attachmentTasks.push(async () => {
+          try {
+            const stream = Readable.from(svgBuffer);
+
+            // Upload to storage
+            await this.storageService.uploadStream(storageFilePath, stream, {
+              recreateClient: true,
+            });
+
+            // Insert into database
+            await this.db
+              .insertInto('attachments')
+              .values({
+                id: attachmentId,
+                filePath: storageFilePath,
+                fileName: fileName,
+                fileSize: svgBuffer.length,
+                mimeType: 'image/svg+xml',
+                type: 'file',
+                fileExt: '.svg',
+                creatorId: fileTask.creatorId,
+                workspaceId: fileTask.workspaceId,
+                pageId,
+                spaceId: fileTask.spaceId,
+              })
+              .execute();
+
+            uploadStats.completed++;
+          } catch (error) {
+            uploadStats.failed++;
+            uploadStats.failedFiles.push(fileName);
+            this.logger.error(
+              `Failed to upload Draw.io SVG ${fileName}:`,
+              error,
+            );
+          }
+        });
+
+        // Store the mapping for both Draw.io and PNG references
+        drawioSvgMap.set(drawioHref, { attachmentId, apiFilePath, fileName });
+        if (pair.pngFile) {
+          drawioSvgMap.set(pair.pngFile.href, {
+            attachmentId,
+            apiFilePath,
+            fileName,
+          });
+        }
+      } catch (error) {
+        this.logger.error(
+          `Failed to process Draw.io pair ${pair.baseName}:`,
+          error,
+        );
+      }
+    }
+
     const uploadOnce = (relPath: string) => {
       const abs = attachmentCandidates.get(relPath)!;
       const attachmentId = v7();
@@ -85,16 +204,18 @@ export class ImportAttachmentService {
 
       const apiFilePath = `/api/files/${attachmentId}/${fileNameWithExt}`;
 
-      attachmentTasks.push(() => this.uploadWithRetry({
-        abs,
-        storageFilePath,
-        attachmentId,
-        fileNameWithExt,
-        ext,
-        pageId,
-        fileTask,
-        uploadStats,
-      }));
+      attachmentTasks.push(() =>
+        this.uploadWithRetry({
+          abs,
+          storageFilePath,
+          attachmentId,
+          fileNameWithExt,
+          ext,
+          pageId,
+          fileTask,
+          uploadStats,
+        }),
+      );
 
       return {
         attachmentId,
@@ -134,8 +255,23 @@ export class ImportAttachmentService {
       );
       if (!relPath) continue;
 
-      const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
+      // Check if this image is part of a Draw.io pair
+      const drawioSvg = drawioSvgMap.get(relPath);
+      if (drawioSvg) {
+        const $drawio = $('<div>')
+          .attr('data-type', 'drawio')
+          .attr('data-src', drawioSvg.apiFilePath)
+          .attr('data-title', 'diagram')
+          .attr('data-width', '100%')
+          .attr('data-align', 'center')
+          .attr('data-attachment-id', drawioSvg.attachmentId);
+
+        $img.replaceWith($drawio);
+        unwrapFromParagraph($, $drawio);
+        continue;
+      }
+
+      const { attachmentId, apiFilePath } = processFile(relPath);
 
       const width = $img.attr('width') ?? '100%';
       const align = $img.attr('data-align') ?? 'center';
@@ -143,7 +279,6 @@ export class ImportAttachmentService {
       $img
         .attr('src', apiFilePath)
         .attr('data-attachment-id', attachmentId)
-        .attr('data-size', stat.size.toString())
         .attr('width', width)
         .attr('data-align', align);
 
@@ -163,8 +298,7 @@ export class ImportAttachmentService {
       );
       if (!relPath) continue;
 
-      const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
+      const { attachmentId, apiFilePath } = processFile(relPath);
 
       const width = $vid.attr('width') ?? '100%';
       const align = $vid.attr('data-align') ?? 'center';
@@ -172,7 +306,6 @@ export class ImportAttachmentService {
       $vid
         .attr('src', apiFilePath)
         .attr('data-attachment-id', attachmentId)
-        .attr('data-size', stat.size.toString())
         .attr('width', width)
         .attr('data-align', align);
 
@@ -193,7 +326,6 @@ export class ImportAttachmentService {
       if (!relPath) continue;
 
       const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
       const fileName = path.basename(abs);
       const mime = getMimeType(abs);
 
@@ -202,7 +334,6 @@ export class ImportAttachmentService {
         .attr('data-attachment-url', apiFilePath)
         .attr('data-attachment-name', fileName)
         .attr('data-attachment-mime', mime)
-        .attr('data-attachment-size', stat.size.toString())
         .attr('data-attachment-id', attachmentId);
 
       $oldDiv.replaceWith($newDiv);
@@ -222,15 +353,35 @@ export class ImportAttachmentService {
       );
       if (!relPath) continue;
 
+      // Check if this is a Draw.io file
+      const drawioSvg = drawioSvgMap.get(relPath);
+      if (drawioSvg) {
+        const $drawio = $('<div>')
+          .attr('data-type', 'drawio')
+          .attr('data-src', drawioSvg.apiFilePath)
+          .attr('data-title', 'diagram')
+          .attr('data-width', '100%')
+          .attr('data-align', 'center')
+          .attr('data-attachment-id', drawioSvg.attachmentId);
+
+        $a.replaceWith($drawio);
+        unwrapFromParagraph($, $drawio);
+        continue;
+      }
+
+      // Skip files that should be ignored
+      if (skipFiles.has(relPath)) {
+        $a.remove();
+        continue;
+      }
+
       const { attachmentId, apiFilePath, abs } = processFile(relPath);
-      const stat = await fs.stat(abs);
       const ext = path.extname(relPath).toLowerCase();
 
       if (ext === '.mp4') {
         const $video = $('<video>')
           .attr('src', apiFilePath)
           .attr('data-attachment-id', attachmentId)
-          .attr('data-size', stat.size.toString())
           .attr('width', '100%')
           .attr('data-align', 'center');
         $a.replaceWith($video);
@@ -245,7 +396,6 @@ export class ImportAttachmentService {
           .attr('data-attachment-url', apiFilePath)
           .attr('data-attachment-name', attachmentName)
           .attr('data-attachment-mime', getMimeType(abs))
-          .attr('data-attachment-size', stat.size.toString())
           .attr('data-attachment-id', attachmentId);
 
         $a.replaceWith($div);
@@ -268,7 +418,6 @@ export class ImportAttachmentService {
         if (!relPath) continue;
 
         const { attachmentId, apiFilePath, abs } = processFile(relPath);
-        const stat = await fs.stat(abs);
         const fileName = path.basename(abs);
 
         const width = $oldDiv.attr('data-width') || '100%';
@@ -279,7 +428,6 @@ export class ImportAttachmentService {
           .attr('data-src', apiFilePath)
           .attr('data-title', fileName)
           .attr('data-width', width)
-          .attr('data-size', stat.size.toString())
           .attr('data-align', align)
           .attr('data-attachment-id', attachmentId);
 
@@ -288,35 +436,343 @@ export class ImportAttachmentService {
       }
     }
 
+    // Collect all attachment IDs in the HTML in a single DOM traversal - O(n)
+    const usedAttachmentIds = new Set<string>();
+    $.root()
+      .find('[data-attachment-id]')
+      .each((_, el) => {
+        const attachmentId = $(el).attr('data-attachment-id');
+        if (attachmentId) {
+          usedAttachmentIds.add(attachmentId);
+        }
+      });
+
+    // Add Draw.io diagrams that weren't referenced in the HTML content
+    for (const [drawioHref, pair] of drawioPairs) {
+      const drawioSvg = drawioSvgMap.get(drawioHref);
+      if (!drawioSvg) continue;
+
+      if (usedAttachmentIds.has(drawioSvg.attachmentId)) {
+        continue; // Already in content
+      }
+
+      const $drawio = $('<div>')
+        .attr('data-type', 'drawio')
+        .attr('data-src', drawioSvg.apiFilePath)
+        .attr('data-title', 'diagram')
+        .attr('data-width', '100%')
+        .attr('data-align', 'center')
+        .attr('data-attachment-id', drawioSvg.attachmentId);
+
+      $.root().append($drawio);
+    }
+
+    // Process attachments from the attachment section that weren't referenced in HTML
+    // These need to be added as attachment nodes so they get uploaded
+    for (const attachment of pageAttachments) {
+      const { href, fileName, mimeType } = attachment;
+
+      // Skip temporary files or files that should be ignored
+      if (skipFiles.has(href)) {
+        continue;
+      }
+
+      // Check if this was part of a Draw.io pair that was already handled
+      if (drawioSvgMap.has(href)) {
+        continue;
+      }
+
+      // Check if already processed (was referenced in HTML)
+      if (processed.has(href)) {
+        continue;
+      }
+
+      // Skip if the file doesn't exist
+      if (!attachmentCandidates.has(href)) {
+        continue;
+      }
+
+      // This attachment was in the list but not referenced in HTML - add it
+      const { attachmentId, apiFilePath, abs } = processFile(href);
+      const mime = mimeType || getMimeType(abs);
+
+      // Add as attachment node at the end
+      const $attachmentDiv = $('<div>')
+        .attr('data-type', 'attachment')
+        .attr('data-attachment-url', apiFilePath)
+        .attr('data-attachment-name', fileName)
+        .attr('data-attachment-mime', mime)
+        .attr('data-attachment-id', attachmentId);
+
+      $.root().append($attachmentDiv);
+    }
+
     // wait for all uploads & DB inserts
     uploadStats.total = attachmentTasks.length;
-    
+
     if (uploadStats.total > 0) {
-      this.logger.debug(`Starting upload of ${uploadStats.total} attachments...`);
-      
       try {
-        await Promise.all(
-          attachmentTasks.map(task => limit(task))
-        );
+        await Promise.all(attachmentTasks.map((task) => limit(task)));
       } catch (err) {
         this.logger.error('Import attachment upload error', err);
       }
-      
+
       this.logger.debug(
-        `Upload completed: ${uploadStats.completed}/${uploadStats.total} successful, ${uploadStats.failed} failed`
+        `Upload completed: ${uploadStats.completed}/${uploadStats.total} successful, ${uploadStats.failed} failed`,
       );
-      
+
       if (uploadStats.failed > 0) {
         this.logger.warn(
           `Failed to upload ${uploadStats.failed} files:`,
-          uploadStats.failedFiles
+          uploadStats.failedFiles,
         );
       }
     }
 
+    // Post-process DOM elements to add file sizes after uploads complete
+    // This avoids blocking file operations during initial DOM processing
+    const elementsNeedingSize = $('[data-attachment-id]:not([data-size])');
+    for (const element of elementsNeedingSize.toArray()) {
+      const $el = $(element);
+      const attachmentId = $el.attr('data-attachment-id');
+      if (!attachmentId) continue;
+
+      // Find the corresponding processed file info
+      const processedEntry = Array.from(processed.values()).find(
+        (entry) => entry.attachmentId === attachmentId,
+      );
+
+      if (processedEntry) {
+        try {
+          const stat = await fs.stat(processedEntry.abs);
+          $el.attr('data-size', stat.size.toString());
+        } catch (error) {
+          this.logger.debug(
+            `Could not get size for ${processedEntry.abs}:`,
+            error,
+          );
+        }
+      }
+    }
+
     return $.root().html() || '';
   }
 
+  private analyzeAttachments(
+    attachments: AttachmentInfo[],
+    isConfluenceImport?: boolean,
+  ): {
+    drawioPairs: Map<string, DrawioPair>;
+    skipFiles: Set<string>;
+  } {
+    const drawioPairs = new Map<string, DrawioPair>();
+    const skipFiles = new Set<string>();
+
+    if (!isConfluenceImport) {
+      return { drawioPairs, skipFiles };
+    }
+
+    // Group attachments by type
+    const drawioFiles: AttachmentInfo[] = [];
+    const pngByBaseName = new Map<string, AttachmentInfo[]>();
+
+    const nonDrawioExtensions = new Set([
+      '.png',
+      '.jpg',
+      '.jpeg',
+      '.gif',
+      '.svg',
+      '.txt',
+      '.pdf',
+      '.doc',
+      '.docx',
+      '.xls',
+      '.xlsx',
+      '.csv',
+      '.zip',
+      '.tar',
+      '.gz',
+    ]);
+
+    // Single pass through attachments
+    for (const attachment of attachments) {
+      const { fileName, mimeType, href } = attachment;
+      const fileNameLower = fileName.toLowerCase();
+
+      // Skip temporary files
+      if (fileName.endsWith('.tmp') || fileName.includes('~drawio~')) {
+        skipFiles.add(href);
+        continue;
+      }
+
+      // Check for Draw.io files
+      if (mimeType === 'application/vnd.jgraph.mxfile') {
+        const ext = fileNameLower.substring(fileNameLower.lastIndexOf('.'));
+        if (!nonDrawioExtensions.has(ext)) {
+          drawioFiles.push(attachment);
+        } else {
+          //Skipped non-Draw.io file with mxfile MIME.
+        }
+      }
+
+      if (mimeType === 'image/png' || fileNameLower.endsWith('.png')) {
+        const baseNames: string[] = [];
+
+        if (fileName.endsWith('.drawio.png')) {
+          // Cloud format: "name.drawio.png" -> base is "name"
+          baseNames.push(fileName.slice(0, -11)); // Remove .drawio.png
+        } else if (fileName.endsWith('.png')) {
+          // Server format: "name.png" -> base is "name"
+          baseNames.push(fileName.slice(0, -4)); // Remove .png
+        }
+
+        for (const baseName of baseNames) {
+          if (!pngByBaseName.has(baseName)) {
+            pngByBaseName.set(baseName, []);
+          }
+          pngByBaseName.get(baseName)!.push(attachment);
+        }
+      }
+    }
+
+    // Match Draw.io files with PNG counterparts
+    for (const drawio of drawioFiles) {
+      let baseName: string;
+
+      if (drawio.fileName.endsWith('.drawio')) {
+        baseName = drawio.fileName.slice(0, -7); // Remove .drawio
+      } else {
+        // Confluence Server: no extension
+        baseName = drawio.fileName;
+      }
+
+      const candidatePngs = pngByBaseName.get(baseName) || [];
+      let matchingPng: AttachmentInfo | undefined;
+
+      // Extract the attachment ID from the Draw.io href
+      // Format: attachments/16941088/36044817.png -> ID is 36044817
+      const drawioIdMatch = drawio.href.match(/\/(\d+)\.\w+$/);
+      const drawioId = drawioIdMatch ? drawioIdMatch[1] : null;
+
+      if (drawioId) {
+        // Look for PNG with adjacent ID (usually PNG ID = Draw.io ID + small increment)
+        // In Confluence, related files often have sequential or near-sequential IDs
+        for (const png of candidatePngs) {
+          const pngIdMatch = png.href.match(/\/(\d+)\.png$/);
+          const pngId = pngIdMatch ? pngIdMatch[1] : null;
+
+          //TODO: should revisit this
+          // but seem to be the best option for now
+          // to prevent reusing the first drawio preview image if there are more with the same name
+          if (pngId && drawioId) {
+            const idDiff = Math.abs(parseInt(pngId) - parseInt(drawioId));
+            // PNG is usually within ~30 IDs of the Draw.io file
+            if (idDiff <= 30) {
+              // Verify filename match
+              if (
+                png.fileName === `${baseName}.drawio.png` ||
+                (!drawio.fileName.endsWith('.drawio') &&
+                  png.fileName === `${baseName}.png`)
+              ) {
+                matchingPng = png;
+                break;
+              }
+            }
+          }
+        }
+      }
+
+      // Fallback to name-only matching if ID-based matching fails
+      if (!matchingPng) {
+        for (const png of candidatePngs) {
+          if (png.fileName === `${baseName}.drawio.png`) {
+            matchingPng = png;
+            break;
+          }
+          if (
+            !drawio.fileName.endsWith('.drawio') &&
+            png.fileName === `${baseName}.png`
+          ) {
+            matchingPng = png;
+            break;
+          }
+        }
+      }
+
+      if (matchingPng) {
+        this.logger.debug(
+          `Found Draw.io pair: ${drawio.fileName} -> ${matchingPng.fileName}`,
+        );
+      } else {
+        this.logger.debug(`No PNG found for Draw.io file: ${drawio.fileName}`);
+      }
+
+      const pair: DrawioPair = {
+        drawioFile: drawio,
+        pngFile: matchingPng,
+        baseName,
+      };
+
+      drawioPairs.set(drawio.href, pair);
+      skipFiles.add(drawio.href);
+      if (matchingPng) {
+        skipFiles.add(matchingPng.href);
+        // Remove the matched PNG from the candidates to prevent reuse
+        const remainingPngs = pngByBaseName
+          .get(baseName)
+          ?.filter((png) => png.href !== matchingPng.href);
+        if (remainingPngs && remainingPngs.length > 0) {
+          pngByBaseName.set(baseName, remainingPngs);
+        } else {
+          pngByBaseName.delete(baseName);
+        }
+      }
+    }
+
+    return { drawioPairs, skipFiles };
+  }
+
+  private async createDrawioSvg(
+    drawioPath: string,
+    pngPath?: string,
+  ): Promise<Buffer> {
+    try {
+      const drawioContent = await fs.readFile(drawioPath, 'utf-8');
+      const drawioBase64 = Buffer.from(drawioContent).toString('base64');
+
+      let imageElement = '';
+      // If we have a PNG, include it in the SVG
+      if (pngPath) {
+        try {
+          const pngBuffer = await fs.readFile(pngPath);
+          const pngBase64 = pngBuffer.toString('base64');
+
+          imageElement = `<image href="data:image/png;base64,${pngBase64}" width="100%" height="100%"/>`;
+        } catch (error) {
+          this.logger.warn(
+            `Could not read PNG file for Draw.io diagram: ${pngPath}`,
+            error,
+          );
+        }
+      }
+
+      // Create the SVG with embedded Draw.io data and image
+      // Default dimensions for Draw.io diagrams if no image is provided
+      const svgContent = `<?xml version="1.0" encoding="UTF-8"?>
+      <svg xmlns="http://www.w3.org/2000/svg" 
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      width="600"
+      height="400"
+      viewBox="0 0 600 400"
+      content="${drawioBase64}">${imageElement}</svg>`;
+
+      return Buffer.from(svgContent, 'utf-8');
+    } catch (error) {
+      this.logger.error(`Failed to create Draw.io SVG: ${error}`);
+      throw error;
+    }
+  }
+
   private async uploadWithRetry(opts: {
     abs: string;
     storageFilePath: string;
@@ -344,11 +800,14 @@ export class ImportAttachmentService {
     } = opts;
 
     let lastError: Error;
-    
+
     for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
       try {
         const fileStream = createReadStream(abs);
-        await this.storageService.uploadStream(storageFilePath, fileStream);
+        await this.storageService.uploadStream(storageFilePath, fileStream, {
+          recreateClient: true,
+        });
+
         const stat = await fs.stat(abs);
 
         await this.db
@@ -367,35 +826,65 @@ export class ImportAttachmentService {
             spaceId: fileTask.spaceId,
           })
           .execute();
-        
+
+        // Queue PDF and DOCX files for indexing
+        const supportedExtensions = ['.pdf', '.docx'];
+        if (supportedExtensions.includes(ext.toLowerCase())) {
+          try {
+            await this.attachmentQueue.add(
+              QueueJob.ATTACHMENT_INDEX_CONTENT,
+              { attachmentId },
+              {
+                attempts: 1,
+                backoff: {
+                  type: 'exponential',
+                  delay: 3 * 60 * 1000,
+                },
+                deduplication: {
+                  id: attachmentId,
+                },
+                removeOnComplete: true,
+                removeOnFail: false,
+              },
+            );
+            this.logger.debug(
+              `Queued ${fileNameWithExt} for indexing (attachment ID: ${attachmentId})`,
+            );
+          } catch (err) {
+            this.logger.error(
+              `Failed to queue indexing for imported attachment ${attachmentId}: ${err}`,
+            );
+          }
+        }
+
         uploadStats.completed++;
-        
+
         if (uploadStats.completed % 10 === 0) {
           this.logger.debug(
-            `Upload progress: ${uploadStats.completed}/${uploadStats.total}`
+            `Upload progress: ${uploadStats.completed}/${uploadStats.total}`,
           );
         }
-        
+
         return;
       } catch (error) {
         lastError = error as Error;
         this.logger.warn(
-          `Upload attempt ${attempt}/${this.MAX_RETRIES} failed for ${fileNameWithExt}: ${error instanceof Error ? error.message : String(error)}`
+          `Upload attempt ${attempt}/${this.MAX_RETRIES} failed for ${fileNameWithExt}: ${error instanceof Error ? error.message : String(error)}`,
         );
-        
+
         if (attempt < this.MAX_RETRIES) {
-          await new Promise(resolve => 
-            setTimeout(resolve, this.RETRY_DELAY * attempt)
+          await new Promise((resolve) =>
+            setTimeout(resolve, this.RETRY_DELAY * attempt),
           );
         }
       }
     }
-    
+
     uploadStats.failed++;
     uploadStats.failedFiles.push(fileNameWithExt);
     this.logger.error(
       `Failed to upload ${fileNameWithExt} after ${this.MAX_RETRIES} attempts:`,
-      lastError
+      lastError,
     );
   }
 }

apps/server/src/integrations/import/utils/import-formatter.ts

@@ -4,6 +4,11 @@ import { v7 } from 'uuid';
 import { InsertableBacklink } from '@docmost/db/types/entity.types';
 import { Cheerio, CheerioAPI, load } from 'cheerio';
 
+// Check if text contains Unicode characters (for emojis/icons)
+function isUnicodeCharacter(text: string): boolean {
+  return text.length > 0 && text.codePointAt(0)! > 127; // Non-ASCII characters
+}
+
 export async function formatImportHtml(opts: {
   html: string;
   currentFilePath: string;
@@ -16,7 +21,11 @@ export async function formatImportHtml(opts: {
   workspaceId: string;
   pageDir?: string;
   attachmentCandidates?: string[];
-}): Promise<{ html: string; backlinks: InsertableBacklink[] }> {
+}): Promise<{
+  html: string;
+  backlinks: InsertableBacklink[];
+  pageIcon?: string;
+}> {
   const {
     html,
     currentFilePath,
@@ -28,6 +37,17 @@ export async function formatImportHtml(opts: {
   const $: CheerioAPI = load(html);
   const $root: Cheerio<any> = $.root();
 
+  let pageIcon: string | null = null;
+  // extract notion page icon
+  const headerIconSpan = $root.find('header .page-header-icon .icon');
+
+  if (headerIconSpan.length > 0) {
+    const iconText = headerIconSpan.text().trim();
+    if (iconText && isUnicodeCharacter(iconText)) {
+      pageIcon = iconText;
+    }
+  }
+
   notionFormatter($, $root);
   defaultHtmlFormatter($, $root);
 
@@ -44,6 +64,7 @@ export async function formatImportHtml(opts: {
   return {
     html: $root.html() || '',
     backlinks,
+    pageIcon: pageIcon || undefined,
   };
 }
 
@@ -69,6 +90,10 @@ export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) {
 }
 
 export function notionFormatter($: CheerioAPI, $root: Cheerio<any>) {
+  // remove page header icon and cover image
+  $root.find('.page-header-icon').remove();
+  $root.find('.page-cover-image').remove();
+
   // remove empty description paragraphs
   $root.find('p.page-description').each((_, el) => {
     if (!$(el).text().trim()) $(el).remove();
@@ -189,6 +214,9 @@ export function notionFormatter($: CheerioAPI, $root: Cheerio<any>) {
       $fig.replaceWith($newAnchor);
     });
 
+  // remove user icons
+  $root.find('span.user img.user-icon').remove();
+
   // remove toc
   $root.find('nav.table_of_contents').remove();
 }

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -9,6 +9,8 @@ export enum QueueName {
 export enum QueueJob {
   SEND_EMAIL = 'send-email',
   DELETE_SPACE_ATTACHMENTS = 'delete-space-attachments',
+  ATTACHMENT_INDEX_CONTENT = 'attachment-index-content',
+  ATTACHMENT_INDEXING = 'attachment-indexing',
   DELETE_PAGE_ATTACHMENTS = 'delete-page-attachments',
   PAGE_CONTENT_UPDATE = 'page-content-update',
 

apps/server/src/integrations/storage/drivers/local.driver.ts

@@ -28,7 +28,7 @@ export class LocalDriver implements StorageDriver {
     }
   }
 
-  async uploadStream(filePath: string, file: Readable): Promise<void> {
+  async uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise<void> {
     try {
       const fullPath = this._fullPath(filePath);
       await fs.mkdir(dirname(fullPath), { recursive: true });

apps/server/src/integrations/storage/drivers/s3.driver.ts

@@ -41,12 +41,26 @@ export class S3Driver implements StorageDriver {
     }
   }
 
-  async uploadStream(filePath: string, file: Readable): Promise<void> {
+  async uploadStream(
+    filePath: string,
+    file: Readable,
+    options?: { recreateClient?: boolean },
+  ): Promise<void> {
+    let clientToUse = this.s3Client;
+    let shouldDestroyClient = false;
+
+    // optionally recreate client to avoid socket hang errors
+    // (during multi-attachments imports)
+    if (options?.recreateClient) {
+      clientToUse = new S3Client(this.config as any);
+      shouldDestroyClient = true;
+    }
+
     try {
       const contentType = getMimeType(filePath);
 
       const upload = new Upload({
-        client: this.s3Client,
+        client: clientToUse,
         params: {
           Bucket: this.config.bucket,
           Key: filePath,
@@ -58,6 +72,10 @@ export class S3Driver implements StorageDriver {
       await upload.done();
     } catch (err) {
       throw new Error(`Failed to upload file: ${(err as Error).message}`);
+    } finally {
+      if (shouldDestroyClient && clientToUse) {
+        clientToUse.destroy();
+      }
     }
   }
 

apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts

@@ -3,7 +3,7 @@ import { Readable } from 'stream';
 export interface StorageDriver {
   upload(filePath: string, file: Buffer): Promise<void>;
 
-  uploadStream(filePath: string, file: Readable): Promise<void>;
+  uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise<void>;
 
   copy(fromFilePath: string, toFilePath: string): Promise<void>;
 

apps/server/src/integrations/storage/storage.service.ts

@@ -15,8 +15,8 @@ export class StorageService {
     this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
   }
 
-  async uploadStream(filePath: string, fileContent: Readable) {
-    await this.storageDriver.uploadStream(filePath, fileContent);
+  async uploadStream(filePath: string, fileContent: Readable, options?: { recreateClient?: boolean }) {
+    await this.storageDriver.uploadStream(filePath, fileContent, options);
     this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
   }