diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index 59447b276..bc5c3a311 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -505,8 +505,19 @@ export class FileImportTaskService { attachmentCandidates, }); + + const processedHTML = + await this.importAttachmentService.processEmbeddedAttachments({ + html: htmlContent, + pageId: page.id, + workspaceId: fileTask.workspaceId, + spaceId: fileTask.spaceId, + creatorId: fileTask.creatorId, + trx, + }); + const { html, backlinks, pageIcon } = await formatImportHtml({ - html: htmlContent, + html: processedHTML, currentFilePath: page.filePath, filePathToPageMetaMap: filePathToPageMetaMap, creatorId: fileTask.creatorId, diff --git a/apps/server/src/integrations/import/services/import-attachment.service.ts b/apps/server/src/integrations/import/services/import-attachment.service.ts index e0944905f..d5a3dea6d 100644 --- a/apps/server/src/integrations/import/services/import-attachment.service.ts +++ b/apps/server/src/integrations/import/services/import-attachment.service.ts @@ -46,6 +46,13 @@ interface AttachmentMeta { trx: KyselyTransaction } +interface UploadStats { + total: number; + completed: number; + failed: number; + failedFiles: string[]; +} + const MIME_EXTENSION_OVERRIDES: Record = { 'image/jpeg': '.jpg', 'audio/mpeg': '.mp3', @@ -73,60 +80,108 @@ export class ImportAttachmentService { @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue, ) {} - /** - * Replaces Base64 data URIs in standalone page imports with stored files. - * Archive imports use processAttachments() because their files already - * exist on disk; standalone imports need to materialize these payloads. - */ async processEmbeddedAttachments( opts: AttachmentMeta & { html: string }, ): Promise { const { html, ...rest } = opts; const $ = load(html); - const processed = new Map(); + const limit = pLimit(this.CONCURRENT_UPLOADS); - const resolveUri = async (uri: string): Promise => { - const normalized = uri?.trim(); - if (!normalized) return null; - if (processed.has(normalized)) return processed.get(normalized); - - const apiFilePath = await this.uploadDataUri({ - uri: normalized, - ...rest, - }); - if (apiFilePath) processed.set(normalized, apiFilePath); - return apiFilePath; + const uploadStats: UploadStats = { + total: 0, + completed: 0, + failed: 0, + failedFiles: [], }; + type UploadedEmbed = { + apiFilePath: string; + attachmentId: string; + fileName: string; + } | null; + + const processed = new Map>(); + + const resolveUri = (uri?: string): Promise => { + const normalized = uri?.trim(); + if (!normalized || !normalized.toLowerCase().startsWith('data:')) { + return Promise.resolve(null); + } + + const existing = processed.get(normalized); + if (existing) { + return existing; + } + + const task = limit(async () => { + try { + const result = await this.uploadDataUri({ + uri: normalized, + ...rest, + }); + + if (result) { + uploadStats.completed++; + } + + return result; + } catch (error) { + uploadStats.failed++; + uploadStats.failedFiles.push(normalized.slice(0, 80)); + + this.logger.error( + `Failed to process embedded attachment: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + + return null; + } + }); + + processed.set(normalized, task); + return task; + }; + + const attributeReplacements: Array<{ + element: ReturnType; + attribute: string; + promise: Promise; + isImage?: boolean; + }> = []; + + const posterReplacements: Array<{ + element: ReturnType; + promise: Promise; + }> = []; + + const embedReplacements: Array<{ + element: ReturnType; + promise: Promise; + }> = []; + // img/video/audio/source: src attribute for (const element of $( 'img[src], video[src], audio[src], source[src]', ).toArray()) { const $element = $(element); - const apiFilePath = await resolveUri($element.attr('src')); - if (!apiFilePath) continue; - $element - .attr('src', apiFilePath) - .attr('data-attachment-id', apiFilePath.split('/')[3]); - if ($element.is('img')) { - $element.attr('data-align', $element.attr('data-align') ?? 'center'); - } + attributeReplacements.push({ + element: $element, + attribute: 'src', + promise: resolveUri($element.attr('src')), + isImage: $element.is('img'), + }); } // video poster (thumbnail image, often a separate data URI) for (const element of $('video[poster]').toArray()) { const $element = $(element); - const apiFilePath = await resolveUri($element.attr('poster')); - console.log({apiFilePath}) - if (apiFilePath) { - $element - .attr('poster', apiFilePath) - .attr('src', apiFilePath) - .attr('preload', 'metadata') - .attr('controls') - }; + posterReplacements.push({ + element: $element, + promise: resolveUri($element.attr('poster')), + }); } // the client does not currently support srcset. @@ -159,7 +214,7 @@ export class ImportAttachmentService { // Keep the first embedded image as the final fallback. if (!firstEmbeddedCandidate) { firstEmbeddedCandidate = uri; - if (type !== "w" && type !== "x"){ + if (type !== 'w' && type !== 'x') { break; } } @@ -192,13 +247,12 @@ export class ImportAttachmentService { bestDensityCandidate?.uri ?? firstEmbeddedCandidate; - const apiFilePath = await resolveUri(selectedUri); - if (!apiFilePath) continue; - - $element - .attr('src', apiFilePath) - .attr('data-attachment-id', apiFilePath.split('/')[3]) - .attr('data-align', $element.attr('data-align') ?? 'center'); + attributeReplacements.push({ + element: $element, + attribute: 'src', + promise: resolveUri(selectedUri), + isImage: true, + }); } // the client represents embeds as iframes. @@ -206,43 +260,91 @@ export class ImportAttachmentService { for (const element of $('object[data], embed[src]').toArray()) { const $element = $(element); const sourceAttribute = $element.is('object') ? 'data' : 'src'; - const uri = $element.attr(sourceAttribute); - const apiFilePath = await resolveUri(uri); - if (!apiFilePath) continue; - - const $iframe = $('