expand to zip imports

fix(ee): checkbox filtering

apps/server/src/core/workspace/services/workspace.service.ts

@@ -396,7 +396,10 @@ export class WorkspaceService {
       }
     }
 
-    if (updateWorkspaceDto.aiSearch) {
+    if (
+      updateWorkspaceDto.aiSearch &&
+      this.environmentService.getAiVectorDriver() !== 'turbopuffer'
+    ) {
       const tableExists = await isPageEmbeddingsTableExists(this.db);
       if (!tableExists) {
         throw new BadRequestException(

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -505,8 +505,19 @@ export class FileImportTaskService {
                 attachmentCandidates,
               });
 
+
+            const processedHTML =
+              await this.importAttachmentService.processEmbeddedAttachments({
+                html: htmlContent,
+                pageId: page.id,
+                workspaceId: fileTask.workspaceId,
+                spaceId: fileTask.spaceId,
+                creatorId: fileTask.creatorId,
+                trx,
+              });
+
             const { html, backlinks, pageIcon } = await formatImportHtml({
-              html: htmlContent,
+              html: processedHTML,
               currentFilePath: page.filePath,
               filePathToPageMetaMap: filePathToPageMetaMap,
               creatorId: fileTask.creatorId,

apps/server/src/integrations/import/services/import-attachment.service.ts

@@ -1,7 +1,7 @@
 import { Injectable, Logger } from '@nestjs/common';
 import * as path from 'path';
 import { InjectKysely } from 'nestjs-kysely';
-import { KyselyDB } from '@docmost/db/types/kysely.types';
+import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
 import { cleanUrlString } from '../utils/file.utils';
 import { StorageService } from '../../storage/storage.service';
 import { createReadStream } from 'node:fs';
@@ -20,6 +20,11 @@ import pLimit from 'p-limit';
 import { InjectQueue } from '@nestjs/bullmq';
 import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../queue/constants';
+import { isBase64 } from "class-validator";
+import { EnvironmentService } from '../../environment/environment.service';
+import * as bytes from 'bytes';
+import * as mimeTypes from 'mime-types';
+import { dbOrTx } from '@docmost/db/utils';
 
 interface AttachmentInfo {
   href: string;
@@ -33,6 +38,34 @@ interface DrawioPair {
   baseName: string;
 }
 
+interface AttachmentMeta {
+  pageId: string;
+  workspaceId: string;
+  spaceId: string;
+  creatorId: string;
+  trx: KyselyTransaction
+}
+
+interface UploadStats {
+  total: number;
+  completed: number;
+  failed: number;
+  failedFiles: string[];
+}
+
+const MIME_EXTENSION_OVERRIDES: Record<string, string> = {
+  'image/jpeg': '.jpg',
+  'audio/mpeg': '.mp3',
+};
+
+function resolveExtensionForMimeType(mimeType: string): string | null {
+  const override = MIME_EXTENSION_OVERRIDES[mimeType];
+  if (override) return override;
+
+  const ext = mimeTypes.extension(mimeType);
+  return ext ? `.${ext}` : null;
+}
+
 @Injectable()
 export class ImportAttachmentService {
   private readonly logger = new Logger(ImportAttachmentService.name);
@@ -42,10 +75,281 @@ export class ImportAttachmentService {
 
   constructor(
     private readonly storageService: StorageService,
+    private readonly environmentService: EnvironmentService,
     @InjectKysely() private readonly db: KyselyDB,
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
+  async processEmbeddedAttachments(
+    opts: AttachmentMeta & { html: string },
+  ): Promise<string> {
+    const { html, ...rest } = opts;
+    const $ = load(html);
+    const limit = pLimit(this.CONCURRENT_UPLOADS);
+
+    const uploadStats: UploadStats = {
+      total: 0,
+      completed: 0,
+      failed: 0,
+      failedFiles: [],
+    };
+
+    type UploadedEmbed = {
+      apiFilePath: string;
+      attachmentId: string;
+      fileName: string;
+    } | null;
+
+    const processed = new Map<string, Promise<UploadedEmbed>>();
+
+    const resolveUri = (uri?: string): Promise<UploadedEmbed> => {
+      const normalized = uri?.trim();
+      if (!normalized || !normalized.toLowerCase().startsWith('data:')) {
+        return Promise.resolve(null);
+      }
+
+      const existing = processed.get(normalized);
+      if (existing) {
+        return existing;
+      }
+
+      const task = limit(async () => {
+        try {
+          const result = await this.uploadDataUri({
+            uri: normalized,
+            ...rest,
+          });
+
+          if (result) {
+            uploadStats.completed++;
+          }
+
+          return result;
+        } catch (error) {
+          uploadStats.failed++;
+          uploadStats.failedFiles.push(normalized.slice(0, 80));
+
+          this.logger.error(
+            `Failed to process embedded attachment: ${
+              error instanceof Error ? error.message : String(error)
+            }`,
+          );
+
+          return null;
+        }
+      });
+
+      processed.set(normalized, task);
+      return task;
+    };
+
+    const attributeReplacements: Array<{
+      element: ReturnType<typeof $>;
+      attribute: string;
+      promise: Promise<UploadedEmbed>;
+      isImage?: boolean;
+    }> = [];
+
+    const posterReplacements: Array<{
+      element: ReturnType<typeof $>;
+      promise: Promise<UploadedEmbed>;
+    }> = [];
+
+    const embedReplacements: Array<{
+      element: ReturnType<typeof $>;
+      promise: Promise<UploadedEmbed>;
+    }> = [];
+
+    // img/video/audio/source: src attribute
+    for (const element of $(
+      'img[src], video[src], audio[src], source[src]',
+    ).toArray()) {
+      const $element = $(element);
+
+      attributeReplacements.push({
+        element: $element,
+        attribute: 'src',
+        promise: resolveUri($element.attr('src')),
+        isImage: $element.is('img'),
+      });
+    }
+
+    // video poster (thumbnail image, often a separate data URI)
+    for (const element of $('video[poster]').toArray()) {
+      const $element = $(element);
+
+      posterReplacements.push({
+        element: $element,
+        promise: resolveUri($element.attr('poster')),
+      });
+    }
+
+    // the client does not currently support srcset.
+    // so we upload only the highest resolution, or the first image
+    for (const element of $('img[srcset]').toArray()) {
+      const $element = $(element);
+      const srcset = $element.attr('srcset');
+      if (!srcset) continue;
+
+      const candidates = srcset.split(/,\s+(?=\S)/);
+
+      let firstEmbeddedCandidate: string | undefined;
+      let bestWidthCandidate: any;
+      let bestDensityCandidate: any;
+      for (const candidate of candidates) {
+        if (!candidate) continue;
+
+        const trimmed = candidate.trim();
+        const match = trimmed.match(/^(\S+)(?:\s+(\d+(?:\.\d+)?)(w|x))?$/i);
+        if (!match) continue;
+
+        const [, uri, descriptorValue, descriptorType] = match;
+        if (!uri.toLowerCase().startsWith('data:')) {
+          continue;
+        }
+
+        const value = descriptorValue ? Number(descriptorValue) : undefined;
+        const type = descriptorType?.toLowerCase();
+
+        // Keep the first embedded image as the final fallback.
+        if (!firstEmbeddedCandidate) {
+          firstEmbeddedCandidate = uri;
+          if (type !== 'w' && type !== 'x') {
+            break;
+          }
+        }
+
+        if (
+          type === 'w' &&
+          value !== undefined &&
+          (!bestWidthCandidate || value > bestWidthCandidate.width)
+        ) {
+          bestWidthCandidate = {
+            uri,
+            width: value,
+          };
+        }
+
+        if (
+          type === 'x' &&
+          value !== undefined &&
+          (!bestDensityCandidate || value > bestDensityCandidate.density)
+        ) {
+          bestDensityCandidate = {
+            uri,
+            density: value,
+          };
+        }
+      }
+
+      const selectedUri =
+        bestWidthCandidate?.uri ??
+        bestDensityCandidate?.uri ??
+        firstEmbeddedCandidate;
+
+      attributeReplacements.push({
+        element: $element,
+        attribute: 'src',
+        promise: resolveUri(selectedUri),
+        isImage: true,
+      });
+    }
+
+    // the client represents embeds as iframes.
+    // so we convert embeds and objects as iframes
+    for (const element of $('object[data], embed[src]').toArray()) {
+      const $element = $(element);
+      const sourceAttribute = $element.is('object') ? 'data' : 'src';
+
+      embedReplacements.push({
+        element: $element,
+        promise: resolveUri($element.attr(sourceAttribute)),
+      });
+    }
+
+    for (const element of $('iframe[src]').toArray()) {
+      const $iframe = $(element);
+
+      attributeReplacements.push({
+        element: $iframe,
+        attribute: 'src',
+        promise: resolveUri($iframe.attr('src')),
+      });
+    }
+
+    // anchors
+    for (const element of $('a[href]').toArray()) {
+      const $element = $(element);
+
+      attributeReplacements.push({
+        element: $element,
+        attribute: 'href',
+        promise: resolveUri($element.attr('href')),
+      });
+    }
+
+    uploadStats.total = processed.size;
+
+    await Promise.all([
+      ...attributeReplacements.map(
+        async ({ element, attribute, promise, isImage }) => {
+          const result = await promise;
+          if (!result) return;
+
+          element
+            .attr(attribute, result.apiFilePath)
+            .attr('data-attachment-id', result.attachmentId);
+
+          if (isImage) {
+            element.attr('data-align', element.attr('data-align') ?? 'center');
+          }
+        },
+      ),
+      ...posterReplacements.map(async ({ element, promise }) => {
+        const result = await promise;
+        if (!result) return;
+
+        element
+          .attr('poster', result.apiFilePath)
+          .attr('src', result.apiFilePath)
+          .attr('preload', 'metadata')
+          .attr('controls');
+      }),
+      ...embedReplacements.map(async ({ element, promise }) => {
+        const result = await promise;
+        if (!result) return;
+
+        const $iframe = $('<iframe>')
+          .attr('src', result.apiFilePath)
+          .attr('data-attachment-id', result.attachmentId);
+
+        for (const attribute of ['width', 'height', 'title']) {
+          const value = element.attr(attribute);
+          if (value) {
+            $iframe.attr(attribute, value);
+          }
+        }
+
+        element.replaceWith($iframe);
+      }),
+    ]);
+
+    if (uploadStats.total > 0) {
+      this.logger.debug(
+        `Embedded upload completed: ${uploadStats.completed}/${uploadStats.total} successful, ${uploadStats.failed} failed`,
+      );
+
+      if (uploadStats.failed > 0) {
+        this.logger.warn(
+          `Failed to upload ${uploadStats.failed} embedded attachments:`,
+          uploadStats.failedFiles,
+        );
+      }
+    }
+
+    return $.root().html() || '';
+  }
+
   async processAttachments(opts: {
     html: string;
     pageRelativePath: string;
@@ -669,6 +973,131 @@ export class ImportAttachmentService {
     return $.root().html() || '';
   }
 
+  private async uploadStorageWithRetry(
+    storageFilePath: string,
+    content: Buffer,
+  ): Promise<void> {
+    let lastError: unknown;
+
+    for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
+      try {
+        await this.storageService.upload(storageFilePath, content);
+        return;
+      } catch (error) {
+        lastError = error;
+
+        this.logger.warn(
+          `Storage upload attempt ${attempt}/${this.MAX_RETRIES} failed for ${storageFilePath}: ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
+
+        if (attempt < this.MAX_RETRIES) {
+          await new Promise((resolve) =>
+            setTimeout(resolve, this.RETRY_DELAY * attempt),
+          );
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  private uploadDataUri = async ({
+    uri,
+    creatorId,
+    workspaceId,
+    pageId,
+    spaceId,
+    trx,
+  }: AttachmentMeta & { uri: string }): Promise<{
+    apiFilePath: string;
+    attachmentId: string;
+    fileName: string;
+  } | null> => {
+    if (!uri.toLowerCase().startsWith('data:')) {
+      return null;
+    }
+
+    const commaIndex = uri.indexOf(',');
+    if (commaIndex === -1) return null;
+
+    const metadata = uri.slice(5, commaIndex);
+    const encoded = uri.slice(commaIndex + 1).replace(/\s/g, '');
+
+    const metadataParts = metadata.split(';');
+    const mimeType = metadataParts.shift()?.toLowerCase();
+
+    if (!mimeType || !metadataParts.includes('base64')) {
+      return null;
+    }
+
+    const fileExt = resolveExtensionForMimeType(mimeType);
+    if (!fileExt) {
+      this.logger.warn(`Skipping unsupported embedded MIME type: ${mimeType}`);
+      return null;
+    }
+
+    if (!isBase64(encoded)) {
+      this.logger.warn(`Skipping malformed embedded ${mimeType} payload`);
+      return null;
+    }
+
+    const maxFileSize = bytes(this.environmentService.getFileUploadSizeLimit());
+
+    // before allocation a buffer to decode
+    // we want to reject obviously oversized files
+    const estimatedSize = Math.floor(encoded.length * 0.75);
+    if (estimatedSize > maxFileSize) {
+      this.logger.warn(
+        `Skipping embedded ${mimeType} payload exceeding size limit (${estimatedSize} bytes)`,
+      );
+      return null;
+    }
+
+    const buffer = Buffer.from(encoded, 'base64');
+    if (buffer.length === 0) return null;
+    if (buffer.length > maxFileSize) {
+      this.logger.warn(
+        `Skipping embedded ${mimeType} payload exceeding size limit (${buffer.length} bytes)`,
+      );
+      return null;
+    }
+
+    const attachmentId = v7();
+    const fileName = `${attachmentId}` + fileExt;
+    const storageFilePath = `${getAttachmentFolderPath(
+      AttachmentType.File,
+      workspaceId,
+    )}/${attachmentId}/${fileName}`;
+    const apiFilePath = `/api/files/${attachmentId}/${fileName}`;
+
+    await this.uploadStorageWithRetry(storageFilePath, buffer);
+    const db = dbOrTx(this.db, trx);
+    await db
+      .insertInto('attachments')
+      .values({
+        id: attachmentId,
+        filePath: storageFilePath,
+        fileName,
+        fileSize: buffer.length,
+        mimeType,
+        type: 'file',
+        fileExt: fileExt,
+        creatorId,
+        workspaceId,
+        pageId,
+        spaceId,
+      })
+      .execute();
+
+    return {
+      apiFilePath,
+      attachmentId,
+      fileName,
+    };
+  };
+
   private analyzeAttachments(
     attachments: AttachmentInfo[],
     isConfluenceImport?: boolean,

apps/server/src/integrations/import/services/import.service.ts

@@ -31,6 +31,8 @@ import { QueueJob, QueueName } from '../../queue/constants';
 import { ModuleRef } from '@nestjs/core';
 import { load } from 'cheerio';
 import { normalizeImportHtml } from '../utils/import-formatter';
+import { ImportAttachmentService } from './import-attachment.service';
+import { executeTx } from "@docmost/db/utils";
 
 @Injectable()
 export class ImportService {
@@ -43,6 +45,7 @@ export class ImportService {
     @InjectQueue(QueueName.FILE_TASK_QUEUE)
     private readonly fileTaskQueue: Queue,
     private moduleRef: ModuleRef,
+    private readonly importAttachmentService: ImportAttachmentService,
   ) {}
 
   async importPage(
@@ -62,79 +65,86 @@ export class ImportService {
     let prosemirrorState = null;
     let createdPage = null;
 
-    // For DOCX, we need the page ID upfront so images can reference it
-    const pageId =
-      fileExtension === '.docx' || fileExtension === '.pdf'
-        ? uuid7()
-        : undefined;
+    // Generate the page ID upfront so imported attachments can reference it.
+    const pageId = uuid7();
 
     try {
-      if (fileExtension.endsWith('.md')) {
-        prosemirrorState = await this.processMarkdown(fileContent);
-      } else if (fileExtension.endsWith('.html')) {
-        prosemirrorState = await this.processHTML(fileContent);
-      } else if (fileExtension.endsWith('.docx')) {
-        prosemirrorState = await this.processDocx(
-          fileBuffer,
-          workspaceId,
-          spaceId,
-          pageId,
-          userId,
+      createdPage = await executeTx(this.db, async (trx) => {
+        if (fileExtension.endsWith('.md') || fileExtension.endsWith('.html')) {
+          const rawHtml = fileExtension.endsWith('.md')
+            ? await markdownToHtml(fileContent)
+            : fileContent;
+
+          const processedHtml =
+            await this.importAttachmentService.processEmbeddedAttachments({
+              html: rawHtml,
+              pageId,
+              workspaceId,
+              spaceId,
+              creatorId: userId,
+              trx,
+            });
+
+          prosemirrorState = await this.processHTML(processedHtml);
+        } else if (fileExtension.endsWith('.docx')) {
+          prosemirrorState = await this.processDocx(
+            fileBuffer,
+            workspaceId,
+            spaceId,
+            pageId,
+            userId,
+          );
+        } else if (fileExtension.endsWith('.pdf')) {
+          prosemirrorState = await this.processPdf(
+            fileBuffer,
+            workspaceId,
+            spaceId,
+            pageId,
+            userId,
+          );
+        }
+
+        if (!prosemirrorState) {
+          const message = 'Failed to create ProseMirror state';
+          this.logger.error(message);
+          throw new BadRequestException(message);
+        }
+
+        const { title, prosemirrorJson } = this.extractTitleAndRemoveHeading(
+          prosemirrorState,
+          { anyHeadingLevel: true },
         );
-      } else if (fileExtension.endsWith('.pdf')) {
-        prosemirrorState = await this.processPdf(
-          fileBuffer,
-          workspaceId,
-          spaceId,
-          pageId,
-          userId,
-        );
-      }
-    } catch (err) {
-      const message = 'Error processing file content';
-      this.logger.error(message, err);
-      throw new BadRequestException(message);
-    }
 
-    if (!prosemirrorState) {
-      const message = 'Failed to create ProseMirror state';
-      this.logger.error(message);
-      throw new BadRequestException(message);
-    }
-
-    const { title, prosemirrorJson } = this.extractTitleAndRemoveHeading(
-      prosemirrorState,
-      { anyHeadingLevel: true },
-    );
-
-    const pageTitle = title || fileName;
-
-    if (prosemirrorJson) {
-      try {
+        const pageTitle = title || fileName;
         const pagePosition = await this.getNewPagePosition(spaceId);
 
-        createdPage = await this.pageRepo.insertPage({
-          ...(pageId ? { id: pageId } : {}),
-          slugId: generateSlugId(),
-          title: pageTitle,
-          content: prosemirrorJson,
-          textContent: jsonToText(prosemirrorJson),
-          ydoc: await this.createYdoc(prosemirrorJson),
-          position: pagePosition,
-          spaceId: spaceId,
-          creatorId: userId,
-          workspaceId: workspaceId,
-          lastUpdatedById: userId,
-        });
+        const page = await this.pageRepo.insertPage(
+          {
+            id: pageId,
+            slugId: generateSlugId(),
+            title: pageTitle,
+            content: prosemirrorJson,
+            textContent: jsonToText(prosemirrorJson),
+            ydoc: await this.createYdoc(prosemirrorJson),
+            position: pagePosition,
+            spaceId: spaceId,
+            creatorId: userId,
+            workspaceId: workspaceId,
+            lastUpdatedById: userId,
+          },
+          trx,
+        );
 
         this.logger.debug(
-          `Successfully imported "${title}${fileExtension}. ID: ${createdPage.id} - SlugId: ${createdPage.slugId}"`,
+          `Successfully imported "${title}${fileExtension}. ID: ${page.id} - SlugId: ${page.slugId}"`,
         );
-      } catch (err) {
-        const message = 'Failed to create imported page';
-        this.logger.error(message, err);
-        throw new BadRequestException(message);
-      }
+
+        return page;
+      });
+    } catch (err) {
+      const message = 'Failed to create imported page';
+      this.logger.error(message, err);
+      throw new BadRequestException(message);
     }
 
     return createdPage;

apps/server/src/integrations/throttle/throttle.module.ts

@@ -3,7 +3,7 @@ import { ThrottlerModule } from '@nestjs/throttler';
 import { ThrottlerStorageRedisService } from '@nest-lab/throttler-storage-redis';
 import { EnvironmentService } from '../environment/environment.service';
 import { EnvironmentModule } from '../environment/environment.module';
-import { parseRedisUrl } from '../../common/helpers';
+import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
 import { AUTH_THROTTLER, AI_CHAT_THROTTLER } from './throttler-names';
 import Redis from 'ioredis';
 
@@ -27,6 +27,8 @@ import Redis from 'ioredis';
               password: redisConfig.password,
               db: redisConfig.db,
               family: redisConfig.family,
+              tls: redisConfig.tls,
+              retryStrategy: createRetryStrategy(),
               keyPrefix: 'throttle:',
             }),
           ),