From 8c7461c12532c028e276ab93d8f837d53d5e646b Mon Sep 17 00:00:00 2001 From: Salihu Date: Wed, 2 Sep 2026 18:25:45 +0100 Subject: [PATCH] base64 import init --- .../services/import-attachment.service.ts | 167 +++++++++++++++++- .../import/services/import.service.ts | 139 ++++++++------- 2 files changed, 241 insertions(+), 65 deletions(-) diff --git a/apps/server/src/integrations/import/services/import-attachment.service.ts b/apps/server/src/integrations/import/services/import-attachment.service.ts index 9100149be..b2a4985ee 100644 --- a/apps/server/src/integrations/import/services/import-attachment.service.ts +++ b/apps/server/src/integrations/import/services/import-attachment.service.ts @@ -1,7 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import * as path from 'path'; import { InjectKysely } from 'nestjs-kysely'; -import { KyselyDB } from '@docmost/db/types/kysely.types'; +import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types'; import { cleanUrlString } from '../utils/file.utils'; import { StorageService } from '../../storage/storage.service'; import { createReadStream } from 'node:fs'; @@ -20,6 +20,11 @@ import pLimit from 'p-limit'; import { InjectQueue } from '@nestjs/bullmq'; import { Queue } from 'bullmq'; import { QueueJob, QueueName } from '../../queue/constants'; +import { isBase64 } from "class-validator"; +import { EnvironmentService } from "src/integrations/environment/environment.service"; +import * as bytes from 'bytes'; +import * as mimeTypes from 'mime-types'; +import { dbOrTx } from '@docmost/db/utils'; interface AttachmentInfo { href: string; @@ -33,6 +38,27 @@ interface DrawioPair { baseName: string; } +interface AttachmentMeta { + pageId: string; + workspaceId: string; + spaceId: string; + creatorId: string; + trx: KyselyTransaction +} + +const MIME_EXTENSION_OVERRIDES: Record = { + 'image/jpeg': '.jpg', + 'audio/mpeg': '.mp3', +}; + +function resolveExtensionForMimeType(mimeType: string): string | null { + const override = MIME_EXTENSION_OVERRIDES[mimeType]; + if (override) return override; + + const ext = mimeTypes.extension(mimeType); + return ext ? `.${ext}` : null; +} + @Injectable() export class ImportAttachmentService { private readonly logger = new Logger(ImportAttachmentService.name); @@ -42,10 +68,62 @@ export class ImportAttachmentService { constructor( private readonly storageService: StorageService, + private readonly environmentService: EnvironmentService, @InjectKysely() private readonly db: KyselyDB, @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue, ) {} + /** + * Replaces Base64 data URIs in standalone page imports with stored files. + * Archive imports use processAttachments() because their files already + * exist on disk; standalone imports need to materialize these payloads. + */ + + async processEmbeddedAttachments( + opts: AttachmentMeta & { html: string }, + ): Promise { + const { html, ...rest } = opts; + const $ = load(html); + const processed = new Map(); + + for (const element of $( + 'img[src], video[src], audio[src], source[src]', + ).toArray()) { + const $element = $(element); + const src = $element.attr('src'); + const normalized = src.trim(); + + let apiFilePath = processed.get(normalized); + if (!apiFilePath) { + apiFilePath = await this.uploadDataUri({ uri: normalized, ...rest }); + processed.set(normalized, apiFilePath); + } + if (!apiFilePath) continue; + + $element + .attr('src', apiFilePath) + .attr('data-attachment-id', apiFilePath.split('/')[3]); + if ($element.is('img')) { + $element.attr('data-align', $element.attr('data-align') ?? 'center'); + } + } + + for (const element of $('a[href]').toArray()) { + const $element = $(element); + const href = $element.attr('href'); + const normalized = href.trim(); + + let apiFilePath = processed.get(normalized); + if (!apiFilePath) { + apiFilePath = await this.uploadDataUri({ uri: normalized, ...rest }); + processed.set(normalized, apiFilePath); + } + if (apiFilePath) $element.attr('href', apiFilePath); + } + + return $.root().html() || ''; + } + async processAttachments(opts: { html: string; pageRelativePath: string; @@ -669,6 +747,93 @@ export class ImportAttachmentService { return $.root().html() || ''; } + private uploadDataUri = async ({ + uri, + creatorId, + workspaceId, + pageId, + spaceId, + trx + }: AttachmentMeta & { uri: string }): Promise => { + if (!uri.toLowerCase().startsWith('data:')) { + return null; + } + + const commaIndex = uri.indexOf(','); + if (commaIndex === -1) return null; + + const metadata = uri.slice(5, commaIndex); + const encoded = uri.slice(commaIndex + 1).replace(/\s/g, ''); + + const metadataParts = metadata.split(';'); + const mimeType = metadataParts.shift()?.toLowerCase(); + + if (!mimeType || !metadataParts.includes('base64')) { + return null; + } + + const fileExt = resolveExtensionForMimeType(mimeType); + if (!fileExt) { + this.logger.warn(`Skipping unsupported embedded MIME type: ${mimeType}`); + return null; + } + + if (!isBase64(encoded)) { + this.logger.warn(`Skipping malformed embedded ${mimeType} payload`); + return null; + } + + const maxFileSize = bytes(this.environmentService.getFileUploadSizeLimit()); + + // before allocation a buffer to decode + // we want to reject obviously oversized files + const estimatedSize = Math.floor(encoded.length * 0.75); + if (estimatedSize > maxFileSize) { + this.logger.warn( + `Skipping embedded ${mimeType} payload exceeding size limit (${estimatedSize} bytes)`, + ); + return null; + } + + const buffer = Buffer.from(encoded, 'base64'); + if (buffer.length === 0) return null; + if (buffer.length > maxFileSize) { + this.logger.warn( + `Skipping embedded ${mimeType} payload exceeding size limit (${buffer.length} bytes)`, + ); + return null; + } + + const attachmentId = v7(); + const fileName = `imported-${attachmentId}` + fileExt; + const storageFilePath = `${getAttachmentFolderPath( + AttachmentType.File, + workspaceId, + )}/${attachmentId}/${fileName}`; + const apiFilePath = `/api/files/${attachmentId}/${fileName}`; + + await this.storageService.upload(storageFilePath, buffer); + const db = dbOrTx(this.db, trx) + await db + .insertInto('attachments') + .values({ + id: attachmentId, + filePath: storageFilePath, + fileName, + fileSize: buffer.length, + mimeType, + type: 'file', + fileExt: fileExt, + creatorId, + workspaceId, + pageId, + spaceId, + }) + .execute(); + + return apiFilePath; + }; + private analyzeAttachments( attachments: AttachmentInfo[], isConfluenceImport?: boolean, diff --git a/apps/server/src/integrations/import/services/import.service.ts b/apps/server/src/integrations/import/services/import.service.ts index 9d0044040..45a47cc74 100644 --- a/apps/server/src/integrations/import/services/import.service.ts +++ b/apps/server/src/integrations/import/services/import.service.ts @@ -31,6 +31,8 @@ import { QueueJob, QueueName } from '../../queue/constants'; import { ModuleRef } from '@nestjs/core'; import { load } from 'cheerio'; import { normalizeImportHtml } from '../utils/import-formatter'; +import { ImportAttachmentService } from './import-attachment.service'; +import { executeTx } from "@docmost/db/utils"; @Injectable() export class ImportService { @@ -43,6 +45,7 @@ export class ImportService { @InjectQueue(QueueName.FILE_TASK_QUEUE) private readonly fileTaskQueue: Queue, private moduleRef: ModuleRef, + private readonly importAttachmentService: ImportAttachmentService, ) {} async importPage( @@ -62,79 +65,87 @@ export class ImportService { let prosemirrorState = null; let createdPage = null; - // For DOCX, we need the page ID upfront so images can reference it - const pageId = - fileExtension === '.docx' || fileExtension === '.pdf' - ? uuid7() - : undefined; + // Generate the page ID upfront so imported attachments can reference it. + const pageId = uuid7(); try { - if (fileExtension.endsWith('.md')) { - prosemirrorState = await this.processMarkdown(fileContent); - } else if (fileExtension.endsWith('.html')) { - prosemirrorState = await this.processHTML(fileContent); - } else if (fileExtension.endsWith('.docx')) { - prosemirrorState = await this.processDocx( - fileBuffer, - workspaceId, - spaceId, - pageId, - userId, + createdPage = await executeTx(this.db, async (trx) => { + + if (fileExtension.endsWith('.md') || fileExtension.endsWith('.html')) { + const rawHtml = fileExtension.endsWith('.md') + ? await markdownToHtml(fileContent) + : fileContent; + + const processedHtml = + await this.importAttachmentService.processEmbeddedAttachments({ + html: rawHtml, + pageId, + workspaceId, + spaceId, + creatorId: userId, + trx, + }); + + prosemirrorState = await this.processHTML(processedHtml); + } else if (fileExtension.endsWith('.docx')) { + prosemirrorState = await this.processDocx( + fileBuffer, + workspaceId, + spaceId, + pageId, + userId, + ); + } else if (fileExtension.endsWith('.pdf')) { + prosemirrorState = await this.processPdf( + fileBuffer, + workspaceId, + spaceId, + pageId, + userId, + ); + } + + if (!prosemirrorState) { + const message = 'Failed to create ProseMirror state'; + this.logger.error(message); + throw new BadRequestException(message); + } + + const { title, prosemirrorJson } = this.extractTitleAndRemoveHeading( + prosemirrorState, + { anyHeadingLevel: true }, ); - } else if (fileExtension.endsWith('.pdf')) { - prosemirrorState = await this.processPdf( - fileBuffer, - workspaceId, - spaceId, - pageId, - userId, - ); - } - } catch (err) { - const message = 'Error processing file content'; - this.logger.error(message, err); - throw new BadRequestException(message); - } - if (!prosemirrorState) { - const message = 'Failed to create ProseMirror state'; - this.logger.error(message); - throw new BadRequestException(message); - } - - const { title, prosemirrorJson } = this.extractTitleAndRemoveHeading( - prosemirrorState, - { anyHeadingLevel: true }, - ); - - const pageTitle = title || fileName; - - if (prosemirrorJson) { - try { + const pageTitle = title || fileName; const pagePosition = await this.getNewPagePosition(spaceId); - createdPage = await this.pageRepo.insertPage({ - ...(pageId ? { id: pageId } : {}), - slugId: generateSlugId(), - title: pageTitle, - content: prosemirrorJson, - textContent: jsonToText(prosemirrorJson), - ydoc: await this.createYdoc(prosemirrorJson), - position: pagePosition, - spaceId: spaceId, - creatorId: userId, - workspaceId: workspaceId, - lastUpdatedById: userId, - }); + const page = await this.pageRepo.insertPage( + { + id: pageId, + slugId: generateSlugId(), + title: pageTitle, + content: prosemirrorJson, + textContent: jsonToText(prosemirrorJson), + ydoc: await this.createYdoc(prosemirrorJson), + position: pagePosition, + spaceId: spaceId, + creatorId: userId, + workspaceId: workspaceId, + lastUpdatedById: userId, + }, + trx, + ); this.logger.debug( - `Successfully imported "${title}${fileExtension}. ID: ${createdPage.id} - SlugId: ${createdPage.slugId}"`, + `Successfully imported "${title}${fileExtension}. ID: ${page.id} - SlugId: ${page.slugId}"`, ); - } catch (err) { - const message = 'Failed to create imported page'; - this.logger.error(message, err); - throw new BadRequestException(message); - } + + return page; + }); + } catch (err) { + const message = 'Failed to create imported page'; + this.logger.error(message, err); + throw new BadRequestException(message); } return createdPage;