From 47c54174b3e62c51b1e1036a07205d6da0b6010f Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Thu, 11 Sep 2025 00:50:15 +0100 Subject: [PATCH 01/15] sync --- apps/server/src/ee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/ee b/apps/server/src/ee index e71f70c2..d90ce7a2 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit e71f70c29950efc02f9fe795b7d4b4c9d1060db4 +Subproject commit d90ce7a20f20575c4afa74b0373a18e9555ed0fe From 7ada3cb1f9abde14d8f952b4ee3076ac4b9d803c Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Sat, 13 Sep 2025 03:14:59 +0100 Subject: [PATCH 02/15] fix: page import task (#1551) * fix import * - fix notion importer - support notion page icon import - fix horizontal rule css - rename service file * sync * 3 mins delay --- .../src/features/editor/styles/core.css | 3 +- apps/server/src/ee | 2 +- .../src/integrations/import/import.module.ts | 4 +- .../import/processors/file-task.processor.ts | 4 +- ...service.ts => file-import-task.service.ts} | 7 +- .../services/import-attachment.service.ts | 408 ++++++++++-------- .../import/utils/import-formatter.ts | 27 +- 7 files changed, 253 insertions(+), 202 deletions(-) rename apps/server/src/integrations/import/services/{file-task.service.ts => file-import-task.service.ts} (98%) diff --git a/apps/client/src/features/editor/styles/core.css b/apps/client/src/features/editor/styles/core.css index 051921de..f08f5aa9 100644 --- a/apps/client/src/features/editor/styles/core.css +++ b/apps/client/src/features/editor/styles/core.css @@ -94,8 +94,7 @@ hr { border: none; - border-top: 2px solid #ced4da; - margin: 2rem 0; + border-top: 1px solid #ced4da; &:hover { cursor: pointer; diff --git a/apps/server/src/ee b/apps/server/src/ee index d90ce7a2..d03a6a3f 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit d90ce7a20f20575c4afa74b0373a18e9555ed0fe +Subproject commit d03a6a3f2de77df4447b56135e1600243bd67173 diff --git a/apps/server/src/integrations/import/import.module.ts b/apps/server/src/integrations/import/import.module.ts index 40a49023..8fffde54 100644 --- a/apps/server/src/integrations/import/import.module.ts +++ b/apps/server/src/integrations/import/import.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { ImportService } from './services/import.service'; import { ImportController } from './import.controller'; import { StorageModule } from '../storage/storage.module'; -import { FileTaskService } from './services/file-task.service'; +import { FileImportTaskService } from './services/file-import-task.service'; import { FileTaskProcessor } from './processors/file-task.processor'; import { ImportAttachmentService } from './services/import-attachment.service'; import { FileTaskController } from './file-task.controller'; @@ -11,7 +11,7 @@ import { PageModule } from '../../core/page/page.module'; @Module({ providers: [ ImportService, - FileTaskService, + FileImportTaskService, FileTaskProcessor, ImportAttachmentService, ], diff --git a/apps/server/src/integrations/import/processors/file-task.processor.ts b/apps/server/src/integrations/import/processors/file-task.processor.ts index 9431ccec..38ef8dec 100644 --- a/apps/server/src/integrations/import/processors/file-task.processor.ts +++ b/apps/server/src/integrations/import/processors/file-task.processor.ts @@ -2,7 +2,7 @@ import { Logger, OnModuleDestroy } from '@nestjs/common'; import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; import { Job } from 'bullmq'; import { QueueJob, QueueName } from 'src/integrations/queue/constants'; -import { FileTaskService } from '../services/file-task.service'; +import { FileImportTaskService } from '../services/file-import-task.service'; import { FileTaskStatus } from '../utils/file.utils'; import { StorageService } from '../../storage/storage.service'; @@ -11,7 +11,7 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { private readonly logger = new Logger(FileTaskProcessor.name); constructor( - private readonly fileTaskService: FileTaskService, + private readonly fileTaskService: FileImportTaskService, private readonly storageService: StorageService, ) { super(); diff --git a/apps/server/src/integrations/import/services/file-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts similarity index 98% rename from apps/server/src/integrations/import/services/file-task.service.ts rename to apps/server/src/integrations/import/services/file-import-task.service.ts index f054017d..30338568 100644 --- a/apps/server/src/integrations/import/services/file-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -33,8 +33,8 @@ import { PageService } from '../../../core/page/services/page.service'; import { ImportPageNode } from '../dto/file-task-dto'; @Injectable() -export class FileTaskService { - private readonly logger = new Logger(FileTaskService.name); +export class FileImportTaskService { + private readonly logger = new Logger(FileImportTaskService.name); constructor( private readonly storageService: StorageService, @@ -266,7 +266,7 @@ export class FileTaskService { attachmentCandidates, }); - const { html, backlinks } = await formatImportHtml({ + const { html, backlinks, pageIcon } = await formatImportHtml({ html: htmlContent, currentFilePath: page.filePath, filePathToPageMetaMap: filePathToPageMetaMap, @@ -286,6 +286,7 @@ export class FileTaskService { id: page.id, slugId: page.slugId, title: title || page.name, + icon: pageIcon || null, content: prosemirrorJson, textContent: jsonToText(prosemirrorJson), ydoc: await this.importService.createYdoc(prosemirrorJson), diff --git a/apps/server/src/integrations/import/services/import-attachment.service.ts b/apps/server/src/integrations/import/services/import-attachment.service.ts index 660239eb..92780395 100644 --- a/apps/server/src/integrations/import/services/import-attachment.service.ts +++ b/apps/server/src/integrations/import/services/import-attachment.service.ts @@ -35,7 +35,7 @@ interface DrawioPair { @Injectable() export class ImportAttachmentService { private readonly logger = new Logger(ImportAttachmentService.name); - private readonly CONCURRENT_UPLOADS = 3; + private readonly CONCURRENT_UPLOADS = 5; private readonly MAX_RETRIES = 2; private readonly RETRY_DELAY = 2000; @@ -53,6 +53,7 @@ export class ImportAttachmentService { fileTask: FileTask; attachmentCandidates: Map; pageAttachments?: AttachmentInfo[]; + isConfluenceImport?: boolean; }): Promise { const { html, @@ -62,6 +63,7 @@ export class ImportAttachmentService { fileTask, attachmentCandidates, pageAttachments = [], + isConfluenceImport, } = opts; const attachmentTasks: (() => Promise)[] = []; @@ -90,7 +92,10 @@ export class ImportAttachmentService { >(); // Analyze attachments to identify Draw.io pairs - const { drawioPairs, skipFiles } = this.analyzeAttachments(pageAttachments); + const { drawioPairs, skipFiles } = this.analyzeAttachments( + pageAttachments, + isConfluenceImport, + ); // Map to store processed Draw.io SVGs const drawioSvgMap = new Map< @@ -235,202 +240,197 @@ export class ImportAttachmentService { const pageDir = path.dirname(pageRelativePath); const $ = load(html); - // Cache for resolved paths to avoid repeated lookups - const resolvedPathCache = new Map(); + // image + for (const imgEl of $('img').toArray()) { + const $img = $(imgEl); + const src = cleanUrlString($img.attr('src') ?? '')!; + if (!src || src.startsWith('http')) continue; - const getCachedResolvedPath = (rawPath: string): string | null => { - if (resolvedPathCache.has(rawPath)) { - return resolvedPathCache.get(rawPath)!; - } - const resolved = resolveRelativeAttachmentPath( - rawPath, + const relPath = resolveRelativeAttachmentPath( + src, pageDir, attachmentCandidates, ); - resolvedPathCache.set(rawPath, resolved); - return resolved; - }; + if (!relPath) continue; - // Cache for file stats to avoid repeated file system calls - const statCache = new Map(); + // Check if this image is part of a Draw.io pair + const drawioSvg = drawioSvgMap.get(relPath); + if (drawioSvg) { + const $drawio = $('
') + .attr('data-type', 'drawio') + .attr('data-src', drawioSvg.apiFilePath) + .attr('data-title', 'diagram') + .attr('data-width', '100%') + .attr('data-align', 'center') + .attr('data-attachment-id', drawioSvg.attachmentId); - const getCachedStat = async (absPath: string) => { - if (statCache.has(absPath)) { - return statCache.get(absPath); + $img.replaceWith($drawio); + unwrapFromParagraph($, $drawio); + continue; } - const stat = await fs.stat(absPath); - statCache.set(absPath, stat); - return stat; - }; - // Single DOM traversal for all attachment elements - const selector = - 'img, video, div[data-type="attachment"], a, div[data-type="excalidraw"], div[data-type="drawio"]'; - const elements = $(selector).toArray(); + const { attachmentId, apiFilePath } = processFile(relPath); - for (const element of elements) { - const $el = $(element); - const tagName = element.tagName.toLowerCase(); + const width = $img.attr('width') ?? '100%'; + const align = $img.attr('data-align') ?? 'center'; - // Process based on element type - if (tagName === 'img') { - const src = cleanUrlString($el.attr('src') ?? ''); - if (!src || src.startsWith('http')) continue; + $img + .attr('src', apiFilePath) + .attr('data-attachment-id', attachmentId) + .attr('width', width) + .attr('data-align', align); - const relPath = getCachedResolvedPath(src); - if (!relPath) continue; + unwrapFromParagraph($, $img); + } - // Check if this image is part of a Draw.io pair - const drawioSvg = drawioSvgMap.get(relPath); - if (drawioSvg) { - const $drawio = $('
') - .attr('data-type', 'drawio') - .attr('data-src', drawioSvg.apiFilePath) - .attr('data-title', 'diagram') - .attr('data-width', '100%') - .attr('data-align', 'center') - .attr('data-attachment-id', drawioSvg.attachmentId); + // video + for (const vidEl of $('video').toArray()) { + const $vid = $(vidEl); + const src = cleanUrlString($vid.attr('src') ?? '')!; + if (!src || src.startsWith('http')) continue; - $el.replaceWith($drawio); - unwrapFromParagraph($, $drawio); - continue; - } + const relPath = resolveRelativeAttachmentPath( + src, + pageDir, + attachmentCandidates, + ); + if (!relPath) continue; - const { attachmentId, apiFilePath, abs } = processFile(relPath); - const stat = await getCachedStat(abs); + const { attachmentId, apiFilePath } = processFile(relPath); - $el + const width = $vid.attr('width') ?? '100%'; + const align = $vid.attr('data-align') ?? 'center'; + + $vid + .attr('src', apiFilePath) + .attr('data-attachment-id', attachmentId) + .attr('width', width) + .attr('data-align', align); + + unwrapFromParagraph($, $vid); + } + + //
+ for (const el of $('div[data-type="attachment"]').toArray()) { + const $oldDiv = $(el); + const rawUrl = cleanUrlString($oldDiv.attr('data-attachment-url') ?? '')!; + if (!rawUrl || rawUrl.startsWith('http')) continue; + + const relPath = resolveRelativeAttachmentPath( + rawUrl, + pageDir, + attachmentCandidates, + ); + if (!relPath) continue; + + const { attachmentId, apiFilePath, abs } = processFile(relPath); + const fileName = path.basename(abs); + const mime = getMimeType(abs); + + const $newDiv = $('
') + .attr('data-type', 'attachment') + .attr('data-attachment-url', apiFilePath) + .attr('data-attachment-name', fileName) + .attr('data-attachment-mime', mime) + .attr('data-attachment-id', attachmentId); + + $oldDiv.replaceWith($newDiv); + unwrapFromParagraph($, $newDiv); + } + + // rewrite other attachments via + for (const aEl of $('a').toArray()) { + const $a = $(aEl); + const href = cleanUrlString($a.attr('href') ?? '')!; + if (!href || href.startsWith('http')) continue; + + const relPath = resolveRelativeAttachmentPath( + href, + pageDir, + attachmentCandidates, + ); + if (!relPath) continue; + + // Check if this is a Draw.io file + const drawioSvg = drawioSvgMap.get(relPath); + if (drawioSvg) { + const $drawio = $('
') + .attr('data-type', 'drawio') + .attr('data-src', drawioSvg.apiFilePath) + .attr('data-title', 'diagram') + .attr('data-width', '100%') + .attr('data-align', 'center') + .attr('data-attachment-id', drawioSvg.attachmentId); + + $a.replaceWith($drawio); + unwrapFromParagraph($, $drawio); + continue; + } + + // Skip files that should be ignored + if (skipFiles.has(relPath)) { + $a.remove(); + continue; + } + + const { attachmentId, apiFilePath, abs } = processFile(relPath); + const ext = path.extname(relPath).toLowerCase(); + + if (ext === '.mp4') { + const $video = $('
') + .attr('data-type', 'attachment') + .attr('data-attachment-url', apiFilePath) + .attr('data-attachment-name', attachmentName) + .attr('data-attachment-mime', getMimeType(abs)) + .attr('data-attachment-id', attachmentId); - const relPath = getCachedResolvedPath(src); + $a.replaceWith($div); + unwrapFromParagraph($, $div); + } + } + + // excalidraw and drawio + for (const type of ['excalidraw', 'drawio'] as const) { + for (const el of $(`div[data-type="${type}"]`).toArray()) { + const $oldDiv = $(el); + const rawSrc = cleanUrlString($oldDiv.attr('data-src') ?? '')!; + if (!rawSrc || rawSrc.startsWith('http')) continue; + + const relPath = resolveRelativeAttachmentPath( + rawSrc, + pageDir, + attachmentCandidates, + ); if (!relPath) continue; const { attachmentId, apiFilePath, abs } = processFile(relPath); - const stat = await getCachedStat(abs); + const fileName = path.basename(abs); - $el - .attr('src', apiFilePath) - .attr('data-attachment-id', attachmentId) - .attr('data-size', stat.size.toString()) - .attr('width', $el.attr('width') ?? '100%') - .attr('data-align', $el.attr('data-align') ?? 'center'); + const width = $oldDiv.attr('data-width') || '100%'; + const align = $oldDiv.attr('data-align') || 'center'; - unwrapFromParagraph($, $el); - } else if (tagName === 'div') { - const dataType = $el.attr('data-type'); + const $newDiv = $('
') + .attr('data-type', type) + .attr('data-src', apiFilePath) + .attr('data-title', fileName) + .attr('data-width', width) + .attr('data-align', align) + .attr('data-attachment-id', attachmentId); - if (dataType === 'attachment') { - const rawUrl = cleanUrlString($el.attr('data-attachment-url') ?? ''); - if (!rawUrl || rawUrl.startsWith('http')) continue; - - const relPath = getCachedResolvedPath(rawUrl); - if (!relPath) continue; - - const { attachmentId, apiFilePath, abs } = processFile(relPath); - const stat = await getCachedStat(abs); - const fileName = path.basename(abs); - const mime = getMimeType(abs); - - const $newDiv = $('
') - .attr('data-type', 'attachment') - .attr('data-attachment-url', apiFilePath) - .attr('data-attachment-name', fileName) - .attr('data-attachment-mime', mime) - .attr('data-attachment-size', stat.size.toString()) - .attr('data-attachment-id', attachmentId); - - $el.replaceWith($newDiv); - unwrapFromParagraph($, $newDiv); - } else if (dataType === 'excalidraw' || dataType === 'drawio') { - const rawSrc = cleanUrlString($el.attr('data-src') ?? ''); - if (!rawSrc || rawSrc.startsWith('http')) continue; - - const relPath = getCachedResolvedPath(rawSrc); - if (!relPath) continue; - - const { attachmentId, apiFilePath, abs } = processFile(relPath); - const stat = await getCachedStat(abs); - const fileName = path.basename(abs); - - const $newDiv = $('
') - .attr('data-type', dataType) - .attr('data-src', apiFilePath) - .attr('data-title', fileName) - .attr('data-width', $el.attr('data-width') || '100%') - .attr('data-size', stat.size.toString()) - .attr('data-align', $el.attr('data-align') || 'center') - .attr('data-attachment-id', attachmentId); - - $el.replaceWith($newDiv); - unwrapFromParagraph($, $newDiv); - } - } else if (tagName === 'a') { - const href = cleanUrlString($el.attr('href') ?? ''); - if (!href || href.startsWith('http')) continue; - - const relPath = getCachedResolvedPath(href); - if (!relPath) continue; - - // Check if this is a Draw.io file - const drawioSvg = drawioSvgMap.get(relPath); - if (drawioSvg) { - const $drawio = $('
') - .attr('data-type', 'drawio') - .attr('data-src', drawioSvg.apiFilePath) - .attr('data-title', 'diagram') - .attr('data-width', '100%') - .attr('data-align', 'center') - .attr('data-attachment-id', drawioSvg.attachmentId); - - $el.replaceWith($drawio); - unwrapFromParagraph($, $drawio); - continue; - } - - // Skip files that should be ignored - if (skipFiles.has(relPath)) { - $el.remove(); - continue; - } - - const { attachmentId, apiFilePath, abs } = processFile(relPath); - const stat = await getCachedStat(abs); - const ext = path.extname(relPath).toLowerCase(); - - if (ext === '.mp4') { - const $video = $('
') - .attr('data-type', 'attachment') - .attr('data-attachment-url', apiFilePath) - .attr('data-attachment-name', attachmentName) - .attr('data-attachment-mime', getMimeType(abs)) - .attr('data-attachment-size', stat.size.toString()) - .attr('data-attachment-id', attachmentId); - - $el.replaceWith($div); - unwrapFromParagraph($, $div); - } + $oldDiv.replaceWith($newDiv); + unwrapFromParagraph($, $newDiv); } } @@ -492,24 +492,17 @@ export class ImportAttachmentService { // This attachment was in the list but not referenced in HTML - add it const { attachmentId, apiFilePath, abs } = processFile(href); + const mime = mimeType || getMimeType(abs); - try { - const stat = await fs.stat(abs); - const mime = mimeType || getMimeType(abs); + // Add as attachment node at the end + const $attachmentDiv = $('
') + .attr('data-type', 'attachment') + .attr('data-attachment-url', apiFilePath) + .attr('data-attachment-name', fileName) + .attr('data-attachment-mime', mime) + .attr('data-attachment-id', attachmentId); - // Add as attachment node at the end - const $attachmentDiv = $('
') - .attr('data-type', 'attachment') - .attr('data-attachment-url', apiFilePath) - .attr('data-attachment-name', fileName) - .attr('data-attachment-mime', mime) - .attr('data-attachment-size', stat.size.toString()) - .attr('data-attachment-id', attachmentId); - - $.root().append($attachmentDiv); - } catch (error) { - this.logger.error(`Failed to process attachment ${fileName}:`, error); - } + $.root().append($attachmentDiv); } // wait for all uploads & DB inserts @@ -534,16 +527,49 @@ export class ImportAttachmentService { } } + // Post-process DOM elements to add file sizes after uploads complete + // This avoids blocking file operations during initial DOM processing + const elementsNeedingSize = $('[data-attachment-id]:not([data-size])'); + for (const element of elementsNeedingSize.toArray()) { + const $el = $(element); + const attachmentId = $el.attr('data-attachment-id'); + if (!attachmentId) continue; + + // Find the corresponding processed file info + const processedEntry = Array.from(processed.values()).find( + (entry) => entry.attachmentId === attachmentId, + ); + + if (processedEntry) { + try { + const stat = await fs.stat(processedEntry.abs); + $el.attr('data-size', stat.size.toString()); + } catch (error) { + this.logger.debug( + `Could not get size for ${processedEntry.abs}:`, + error, + ); + } + } + } + return $.root().html() || ''; } - private analyzeAttachments(attachments: AttachmentInfo[]): { + private analyzeAttachments( + attachments: AttachmentInfo[], + isConfluenceImport?: boolean, + ): { drawioPairs: Map; skipFiles: Set; } { const drawioPairs = new Map(); const skipFiles = new Set(); + if (!isConfluenceImport) { + return { drawioPairs, skipFiles }; + } + // Group attachments by type const drawioFiles: AttachmentInfo[] = []; const pngByBaseName = new Map(); @@ -807,7 +833,7 @@ export class ImportAttachmentService { attempts: 1, backoff: { type: 'exponential', - delay: 30 * 1000, + delay: 3 * 60 * 1000, }, deduplication: { id: attachmentId, diff --git a/apps/server/src/integrations/import/utils/import-formatter.ts b/apps/server/src/integrations/import/utils/import-formatter.ts index 92291d39..da30a862 100644 --- a/apps/server/src/integrations/import/utils/import-formatter.ts +++ b/apps/server/src/integrations/import/utils/import-formatter.ts @@ -4,6 +4,11 @@ import { v7 } from 'uuid'; import { InsertableBacklink } from '@docmost/db/types/entity.types'; import { Cheerio, CheerioAPI, load } from 'cheerio'; +// Check if text contains Unicode characters (for emojis/icons) +function isUnicodeCharacter(text: string): boolean { + return text.length > 0 && text.codePointAt(0)! > 127; // Non-ASCII characters +} + export async function formatImportHtml(opts: { html: string; currentFilePath: string; @@ -16,7 +21,11 @@ export async function formatImportHtml(opts: { workspaceId: string; pageDir?: string; attachmentCandidates?: string[]; -}): Promise<{ html: string; backlinks: InsertableBacklink[] }> { +}): Promise<{ + html: string; + backlinks: InsertableBacklink[]; + pageIcon?: string; +}> { const { html, currentFilePath, @@ -28,6 +37,17 @@ export async function formatImportHtml(opts: { const $: CheerioAPI = load(html); const $root: Cheerio = $.root(); + let pageIcon: string | null = null; + // extract notion page icon + const headerIconSpan = $root.find('header .page-header-icon .icon'); + + if (headerIconSpan.length > 0) { + const iconText = headerIconSpan.text().trim(); + if (iconText && isUnicodeCharacter(iconText)) { + pageIcon = iconText; + } + } + notionFormatter($, $root); defaultHtmlFormatter($, $root); @@ -44,6 +64,7 @@ export async function formatImportHtml(opts: { return { html: $root.html() || '', backlinks, + pageIcon: pageIcon || undefined, }; } @@ -69,6 +90,10 @@ export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio) { } export function notionFormatter($: CheerioAPI, $root: Cheerio) { + // remove page header icon and cover image + $root.find('.page-header-icon').remove(); + $root.find('.page-cover-image').remove(); + // remove empty description paragraphs $root.find('p.page-description').each((_, el) => { if (!$(el).text().trim()) $(el).remove(); From 8e16ad952a5e02de73b4cb0836268f6802c17cb1 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Sat, 13 Sep 2025 03:15:53 +0100 Subject: [PATCH 03/15] v0.23.1 --- apps/client/package.json | 2 +- apps/server/package.json | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/client/package.json b/apps/client/package.json index 20a8539c..031646e8 100644 --- a/apps/client/package.json +++ b/apps/client/package.json @@ -1,7 +1,7 @@ { "name": "client", "private": true, - "version": "0.23.0", + "version": "0.23.1", "scripts": { "dev": "vite", "build": "tsc && vite build", diff --git a/apps/server/package.json b/apps/server/package.json index 1760ed87..414170d3 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -1,6 +1,6 @@ { "name": "server", - "version": "0.23.0", + "version": "0.23.1", "description": "", "author": "", "private": true, diff --git a/package.json b/package.json index ec2f9f9a..645cf1e1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "docmost", "homepage": "https://docmost.com", - "version": "0.23.0", + "version": "0.23.1", "private": true, "scripts": { "build": "nx run-many -t build", From f413720e152699761fe477857e2f0e6b21e95f80 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Sun, 14 Sep 2025 03:00:23 +0100 Subject: [PATCH 04/15] - sync - reinstantiate S3 client to fix file upload errors during import - delete import zip file after use --- apps/server/src/ee | 2 +- .../import/processors/file-task.processor.ts | 35 +++++++++++++++++-- .../services/import-attachment.service.ts | 11 ++++-- .../import/utils/import-formatter.ts | 3 ++ .../storage/drivers/local.driver.ts | 2 +- .../integrations/storage/drivers/s3.driver.ts | 22 ++++++++++-- .../interfaces/storage-driver.interface.ts | 2 +- .../integrations/storage/storage.service.ts | 4 +-- 8 files changed, 68 insertions(+), 13 deletions(-) diff --git a/apps/server/src/ee b/apps/server/src/ee index d03a6a3f..fd34d418 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit d03a6a3f2de77df4447b56135e1600243bd67173 +Subproject commit fd34d4183aaae765b27f95e49830c6ff2ac9aa1f diff --git a/apps/server/src/integrations/import/processors/file-task.processor.ts b/apps/server/src/integrations/import/processors/file-task.processor.ts index 38ef8dec..7b65f5ab 100644 --- a/apps/server/src/integrations/import/processors/file-task.processor.ts +++ b/apps/server/src/integrations/import/processors/file-task.processor.ts @@ -41,15 +41,32 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { @OnWorkerEvent('failed') async onFailed(job: Job) { this.logger.error( - `Error processing ${job.name} job. Reason: ${job.failedReason}`, + `Error processing ${job.name} job. Import Task ID: ${job.data.fileTaskId}. Reason: ${job.failedReason}`, ); + await this.handleFailedJob(job); + } + + @OnWorkerEvent('stalled') + async onStalled(job: Job) { + this.logger.error( + `Job ${job.name} stalled. . Import Task ID: ${job.data.fileTaskId}.. Job ID: ${job.id}`, + ); + + // Set failedReason for stalled jobs since it's not automatically set + job.failedReason = 'Job stalled and was marked as failed'; + await this.handleFailedJob(job); + } + + private async handleFailedJob(job: Job) { try { const fileTaskId = job.data.fileTaskId; + const reason = job.failedReason || 'Unknown error'; + await this.fileTaskService.updateTaskStatus( fileTaskId, FileTaskStatus.Failed, - job.failedReason, + reason, ); const fileTask = await this.fileTaskService.getFileTask(fileTaskId); @@ -62,10 +79,22 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { } @OnWorkerEvent('completed') - onCompleted(job: Job) { + async onCompleted(job: Job) { this.logger.log( `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`, ); + + try { + const fileTask = await this.fileTaskService.getFileTask( + job.data.fileTaskId, + ); + if (fileTask) { + await this.storageService.delete(fileTask.filePath); + this.logger.debug(`Deleted imported zip file: ${fileTask.filePath}`); + } + } catch (err) { + this.logger.error(`Failed to delete imported zip file:`, err); + } } async onModuleDestroy(): Promise { diff --git a/apps/server/src/integrations/import/services/import-attachment.service.ts b/apps/server/src/integrations/import/services/import-attachment.service.ts index 92780395..04a18fda 100644 --- a/apps/server/src/integrations/import/services/import-attachment.service.ts +++ b/apps/server/src/integrations/import/services/import-attachment.service.ts @@ -35,7 +35,7 @@ interface DrawioPair { @Injectable() export class ImportAttachmentService { private readonly logger = new Logger(ImportAttachmentService.name); - private readonly CONCURRENT_UPLOADS = 5; + private readonly CONCURRENT_UPLOADS = 1; private readonly MAX_RETRIES = 2; private readonly RETRY_DELAY = 2000; @@ -139,7 +139,9 @@ export class ImportAttachmentService { const stream = Readable.from(svgBuffer); // Upload to storage - await this.storageService.uploadStream(storageFilePath, stream); + await this.storageService.uploadStream(storageFilePath, stream, { + recreateClient: true, + }); // Insert into database await this.db @@ -802,7 +804,10 @@ export class ImportAttachmentService { for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) { try { const fileStream = createReadStream(abs); - await this.storageService.uploadStream(storageFilePath, fileStream); + await this.storageService.uploadStream(storageFilePath, fileStream, { + recreateClient: true, + }); + const stat = await fs.stat(abs); await this.db diff --git a/apps/server/src/integrations/import/utils/import-formatter.ts b/apps/server/src/integrations/import/utils/import-formatter.ts index da30a862..54fc50dd 100644 --- a/apps/server/src/integrations/import/utils/import-formatter.ts +++ b/apps/server/src/integrations/import/utils/import-formatter.ts @@ -214,6 +214,9 @@ export function notionFormatter($: CheerioAPI, $root: Cheerio) { $fig.replaceWith($newAnchor); }); + // remove user icons + $root.find('span.user img.user-icon').remove(); + // remove toc $root.find('nav.table_of_contents').remove(); } diff --git a/apps/server/src/integrations/storage/drivers/local.driver.ts b/apps/server/src/integrations/storage/drivers/local.driver.ts index caa6ab36..5171066c 100644 --- a/apps/server/src/integrations/storage/drivers/local.driver.ts +++ b/apps/server/src/integrations/storage/drivers/local.driver.ts @@ -28,7 +28,7 @@ export class LocalDriver implements StorageDriver { } } - async uploadStream(filePath: string, file: Readable): Promise { + async uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise { try { const fullPath = this._fullPath(filePath); await fs.mkdir(dirname(fullPath), { recursive: true }); diff --git a/apps/server/src/integrations/storage/drivers/s3.driver.ts b/apps/server/src/integrations/storage/drivers/s3.driver.ts index f9852347..f6d48677 100644 --- a/apps/server/src/integrations/storage/drivers/s3.driver.ts +++ b/apps/server/src/integrations/storage/drivers/s3.driver.ts @@ -41,12 +41,26 @@ export class S3Driver implements StorageDriver { } } - async uploadStream(filePath: string, file: Readable): Promise { + async uploadStream( + filePath: string, + file: Readable, + options?: { recreateClient?: boolean }, + ): Promise { + let clientToUse = this.s3Client; + let shouldDestroyClient = false; + + // optionally recreate client to avoid socket hang errors + // (during multi-attachments imports) + if (options?.recreateClient) { + clientToUse = new S3Client(this.config as any); + shouldDestroyClient = true; + } + try { const contentType = getMimeType(filePath); const upload = new Upload({ - client: this.s3Client, + client: clientToUse, params: { Bucket: this.config.bucket, Key: filePath, @@ -58,6 +72,10 @@ export class S3Driver implements StorageDriver { await upload.done(); } catch (err) { throw new Error(`Failed to upload file: ${(err as Error).message}`); + } finally { + if (shouldDestroyClient && clientToUse) { + clientToUse.destroy(); + } } } diff --git a/apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts b/apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts index c4511a80..22a86d2b 100644 --- a/apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts +++ b/apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts @@ -3,7 +3,7 @@ import { Readable } from 'stream'; export interface StorageDriver { upload(filePath: string, file: Buffer): Promise; - uploadStream(filePath: string, file: Readable): Promise; + uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise; copy(fromFilePath: string, toFilePath: string): Promise; diff --git a/apps/server/src/integrations/storage/storage.service.ts b/apps/server/src/integrations/storage/storage.service.ts index e9577ee9..d796351b 100644 --- a/apps/server/src/integrations/storage/storage.service.ts +++ b/apps/server/src/integrations/storage/storage.service.ts @@ -15,8 +15,8 @@ export class StorageService { this.logger.debug(`File uploaded successfully. Path: ${filePath}`); } - async uploadStream(filePath: string, fileContent: Readable) { - await this.storageDriver.uploadStream(filePath, fileContent); + async uploadStream(filePath: string, fileContent: Readable, options?: { recreateClient?: boolean }) { + await this.storageDriver.uploadStream(filePath, fileContent, options); this.logger.debug(`File uploaded successfully. Path: ${filePath}`); } From 61d1cf88a7b06589bb88ff73ae4bef190ac4324e Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Mon, 15 Sep 2025 12:52:31 +0100 Subject: [PATCH 05/15] fix: reset file inputs after import --- .../page/components/page-import-modal.tsx | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/apps/client/src/features/page/components/page-import-modal.tsx b/apps/client/src/features/page/components/page-import-modal.tsx index 0223d78f..a2df380e 100644 --- a/apps/client/src/features/page/components/page-import-modal.tsx +++ b/apps/client/src/features/page/components/page-import-modal.tsx @@ -24,7 +24,7 @@ import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts"; import { useAtom } from "jotai"; import { buildTree } from "@/features/page/tree/utils"; import { IPage } from "@/features/page/types/page.types.ts"; -import React, { useEffect, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { ConfluenceIcon } from "@/components/icons/confluence-icon.tsx"; import { getFileImportSizeLimit, isCloud } from "@/lib/config.ts"; @@ -84,6 +84,12 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { const [fileTaskId, setFileTaskId] = useState(null); const emit = useQueryEmit(); + const markdownFileRef = useRef<() => void>(null); + const htmlFileRef = useRef<() => void>(null); + const notionFileRef = useRef<() => void>(null); + const confluenceFileRef = useRef<() => void>(null); + const zipFileRef = useRef<() => void>(null); + const canUseConfluence = isCloud() || workspace?.hasLicenseKey; const handleZipUpload = async (selectedFile: File, source: string) => { @@ -116,6 +122,15 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { }); setFileTaskId(importTask.id); + + // Reset file input after successful upload + if (source === "notion" && notionFileRef.current) { + notionFileRef.current(); + } else if (source === "confluence" && confluenceFileRef.current) { + confluenceFileRef.current(); + } else if (source === "generic" && zipFileRef.current) { + zipFileRef.current(); + } } catch (err) { console.log("Failed to upload import file", err); notifications.update({ @@ -243,6 +258,10 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { setTreeData(fullTree); } + // Reset file inputs after successful upload + if (markdownFileRef.current) markdownFileRef.current(); + if (htmlFileRef.current) htmlFileRef.current(); + const pageCountText = pageCount === 1 ? `1 ${t("page")}` : `${pageCount} ${t("pages")}`; @@ -272,7 +291,7 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { return ( <> - + {(props) => ( - + + + + + - -
+ + {t("Delete space")} {t("Delete this space with all its pages and data.")} -
- - - + + + + + - {cards} - + {data?.items && data.items.length > 9 && ( - {isDescendantShared ? ( + {isCloud() && isTrial ? ( + <> + + + + + {t("Upgrade to share pages")} + + + {t( + "Page sharing is available on paid plans. Upgrade to share your pages publicly.", + )} + + + + ) : isDescendantShared ? ( <> {t("Inherits public sharing from")} Date: Wed, 17 Sep 2025 23:50:27 +0100 Subject: [PATCH 10/15] fix: enhance page import (#1570) * change import process * fix processor * fix page name in notion import * preserve confluence table bg color * sync --- .../import/processors/file-task.processor.ts | 41 ++-- .../services/file-import-task.service.ts | 219 ++++++++++++------ .../services/import-attachment.service.ts | 2 +- .../integrations/import/utils/import.utils.ts | 6 + packages/editor-ext/src/lib/table/cell.ts | 18 +- packages/editor-ext/src/lib/table/header.ts | 20 +- 6 files changed, 199 insertions(+), 107 deletions(-) diff --git a/apps/server/src/integrations/import/processors/file-task.processor.ts b/apps/server/src/integrations/import/processors/file-task.processor.ts index 7b65f5ab..20001dd7 100644 --- a/apps/server/src/integrations/import/processors/file-task.processor.ts +++ b/apps/server/src/integrations/import/processors/file-task.processor.ts @@ -47,15 +47,23 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { await this.handleFailedJob(job); } - @OnWorkerEvent('stalled') - async onStalled(job: Job) { - this.logger.error( - `Job ${job.name} stalled. . Import Task ID: ${job.data.fileTaskId}.. Job ID: ${job.id}`, + @OnWorkerEvent('completed') + async onCompleted(job: Job) { + this.logger.log( + `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`, ); - // Set failedReason for stalled jobs since it's not automatically set - job.failedReason = 'Job stalled and was marked as failed'; - await this.handleFailedJob(job); + try { + const fileTask = await this.fileTaskService.getFileTask( + job.data.fileTaskId, + ); + if (fileTask) { + await this.storageService.delete(fileTask.filePath); + this.logger.debug(`Deleted imported zip file: ${fileTask.filePath}`); + } + } catch (err) { + this.logger.error(`Failed to delete imported zip file:`, err); + } } private async handleFailedJob(job: Job) { @@ -78,25 +86,6 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { } } - @OnWorkerEvent('completed') - async onCompleted(job: Job) { - this.logger.log( - `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`, - ); - - try { - const fileTask = await this.fileTaskService.getFileTask( - job.data.fileTaskId, - ); - if (fileTask) { - await this.storageService.delete(fileTask.filePath); - this.logger.debug(`Deleted imported zip file: ${fileTask.filePath}`); - } - } catch (err) { - this.logger.error(`Failed to delete imported zip file:`, err); - } - } - async onModuleDestroy(): Promise { if (this.worker) { await this.worker.close(); diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index 30338568..f7d93ec0 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -24,6 +24,7 @@ import { formatImportHtml } from '../utils/import-formatter'; import { buildAttachmentCandidates, collectMarkdownAndHtmlFiles, + stripNotionID, } from '../utils/import.utils'; import { executeTx } from '@docmost/db/utils'; import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo'; @@ -159,17 +160,12 @@ export class FileImportTaskService { .split(path.sep) .join('/'); // normalize to forward-slashes const ext = path.extname(relPath).toLowerCase(); - let content = await fs.readFile(absPath, 'utf-8'); - - if (ext.toLowerCase() === '.md') { - content = await markdownToHtml(content); - } pagesMap.set(relPath, { id: v7(), slugId: generateSlugId(), - name: path.basename(relPath, ext), - content, + name: stripNotionID(path.basename(relPath, ext)), + content: '', parentPageId: null, fileExtension: ext, filePath: relPath, @@ -254,71 +250,160 @@ export class FileImportTaskService { }); }); - const pageResults = await Promise.all( - Array.from(pagesMap.values()).map(async (page) => { - const htmlContent = - await this.importAttachmentService.processAttachments({ - html: page.content, - pageRelativePath: page.filePath, - extractDir, - pageId: page.id, - fileTask, - attachmentCandidates, - }); + // Group pages by level (topological sort for parent-child relationships) + const pagesByLevel = new Map>(); + const pageLevel = new Map(); - const { html, backlinks, pageIcon } = await formatImportHtml({ - html: htmlContent, - currentFilePath: page.filePath, - filePathToPageMetaMap: filePathToPageMetaMap, - creatorId: fileTask.creatorId, - sourcePageId: page.id, - workspaceId: fileTask.workspaceId, - }); + // Calculate levels using BFS + const calculateLevels = () => { + const queue: Array<{ filePath: string; level: number }> = []; - const pmState = getProsemirrorContent( - await this.importService.processHTML(html), + // Start with root pages (no parent) + for (const [filePath, page] of pagesMap.entries()) { + if (!page.parentPageId) { + queue.push({ filePath, level: 0 }); + pageLevel.set(filePath, 0); + } + } + + // BFS to assign levels + while (queue.length > 0) { + const { filePath, level } = queue.shift()!; + const currentPage = pagesMap.get(filePath)!; + + // Find children of current page + for (const [childFilePath, childPage] of pagesMap.entries()) { + if ( + childPage.parentPageId === currentPage.id && + !pageLevel.has(childFilePath) + ) { + pageLevel.set(childFilePath, level + 1); + queue.push({ filePath: childFilePath, level: level + 1 }); + } + } + } + + // Group pages by level + for (const [filePath, page] of pagesMap.entries()) { + const level = pageLevel.get(filePath) || 0; + if (!pagesByLevel.has(level)) { + pagesByLevel.set(level, []); + } + pagesByLevel.get(level)!.push([filePath, page]); + } + }; + + calculateLevels(); + + if (pagesMap.size < 1) return; + + // Process pages level by level sequentially to respect foreign key constraints + const allBacklinks: any[] = []; + const validPageIds = new Set(); + let totalPagesProcessed = 0; + + // Sort levels to process in order + const sortedLevels = Array.from(pagesByLevel.keys()).sort((a, b) => a - b); + + try { + await executeTx(this.db, async (trx) => { + // Process pages level by level sequentially within the transaction + for (const level of sortedLevels) { + const levelPages = pagesByLevel.get(level)!; + + for (const [filePath, page] of levelPages) { + const absPath = path.join(extractDir, filePath); + let content = await fs.readFile(absPath, 'utf-8'); + + if (page.fileExtension.toLowerCase() === '.md') { + content = await markdownToHtml(content); + } + + const htmlContent = + await this.importAttachmentService.processAttachments({ + html: content, + pageRelativePath: page.filePath, + extractDir, + pageId: page.id, + fileTask, + attachmentCandidates, + }); + + const { html, backlinks, pageIcon } = await formatImportHtml({ + html: htmlContent, + currentFilePath: page.filePath, + filePathToPageMetaMap: filePathToPageMetaMap, + creatorId: fileTask.creatorId, + sourcePageId: page.id, + workspaceId: fileTask.workspaceId, + }); + + const pmState = getProsemirrorContent( + await this.importService.processHTML(html), + ); + + const { title, prosemirrorJson } = + this.importService.extractTitleAndRemoveHeading(pmState); + + const insertablePage: InsertablePage = { + id: page.id, + slugId: page.slugId, + title: title || page.name, + icon: pageIcon || null, + content: prosemirrorJson, + textContent: jsonToText(prosemirrorJson), + ydoc: await this.importService.createYdoc(prosemirrorJson), + position: page.position!, + spaceId: fileTask.spaceId, + workspaceId: fileTask.workspaceId, + creatorId: fileTask.creatorId, + lastUpdatedById: fileTask.creatorId, + parentPageId: page.parentPageId, + }; + + await trx.insertInto('pages').values(insertablePage).execute(); + + // Track valid page IDs and collect backlinks + validPageIds.add(insertablePage.id); + allBacklinks.push(...backlinks); + totalPagesProcessed++; + + // Log progress periodically + if (totalPagesProcessed % 50 === 0) { + this.logger.debug(`Processed ${totalPagesProcessed} pages...`); + } + } + } + + const filteredBacklinks = allBacklinks.filter( + ({ sourcePageId, targetPageId }) => + validPageIds.has(sourcePageId) && validPageIds.has(targetPageId), ); - const { title, prosemirrorJson } = - this.importService.extractTitleAndRemoveHeading(pmState); + // Insert backlinks in batches + if (filteredBacklinks.length > 0) { + const BACKLINK_BATCH_SIZE = 100; + for ( + let i = 0; + i < filteredBacklinks.length; + i += BACKLINK_BATCH_SIZE + ) { + const backlinkChunk = filteredBacklinks.slice( + i, + Math.min(i + BACKLINK_BATCH_SIZE, filteredBacklinks.length), + ); + await this.backlinkRepo.insertBacklink(backlinkChunk, trx); + } + } - const insertablePage: InsertablePage = { - id: page.id, - slugId: page.slugId, - title: title || page.name, - icon: pageIcon || null, - content: prosemirrorJson, - textContent: jsonToText(prosemirrorJson), - ydoc: await this.importService.createYdoc(prosemirrorJson), - position: page.position!, - spaceId: fileTask.spaceId, - workspaceId: fileTask.workspaceId, - creatorId: fileTask.creatorId, - lastUpdatedById: fileTask.creatorId, - parentPageId: page.parentPageId, - }; - - return { insertablePage, backlinks }; - }), - ); - - const insertablePages = pageResults.map((r) => r.insertablePage); - const insertableBacklinks = pageResults.flatMap((r) => r.backlinks); - - if (insertablePages.length < 1) return; - const validPageIds = new Set(insertablePages.map((row) => row.id)); - const filteredBacklinks = insertableBacklinks.filter( - ({ sourcePageId, targetPageId }) => - validPageIds.has(sourcePageId) && validPageIds.has(targetPageId), - ); - - await executeTx(this.db, async (trx) => { - await trx.insertInto('pages').values(insertablePages).execute(); - - if (filteredBacklinks.length > 0) { - await this.backlinkRepo.insertBacklink(filteredBacklinks, trx); - } - }); + this.logger.log( + `Successfully imported ${totalPagesProcessed} pages with ${filteredBacklinks.length} backlinks`, + ); + }); + } catch (error) { + this.logger.error('Failed to import files:', error); + throw new Error(`File import failed: ${error?.['message']}`); + } } async getFileTask(fileTaskId: string) { diff --git a/apps/server/src/integrations/import/services/import-attachment.service.ts b/apps/server/src/integrations/import/services/import-attachment.service.ts index 04a18fda..43f4d3fa 100644 --- a/apps/server/src/integrations/import/services/import-attachment.service.ts +++ b/apps/server/src/integrations/import/services/import-attachment.service.ts @@ -35,7 +35,7 @@ interface DrawioPair { @Injectable() export class ImportAttachmentService { private readonly logger = new Logger(ImportAttachmentService.name); - private readonly CONCURRENT_UPLOADS = 1; + private readonly CONCURRENT_UPLOADS = 3; private readonly MAX_RETRIES = 2; private readonly RETRY_DELAY = 2000; diff --git a/apps/server/src/integrations/import/utils/import.utils.ts b/apps/server/src/integrations/import/utils/import.utils.ts index 19cc66bc..1fa10d7a 100644 --- a/apps/server/src/integrations/import/utils/import.utils.ts +++ b/apps/server/src/integrations/import/utils/import.utils.ts @@ -64,3 +64,9 @@ export async function collectMarkdownAndHtmlFiles( await walk(dir); return results; } + +export function stripNotionID(fileName: string): string { + // Handle optional separator (space or dash) + 32 alphanumeric chars at end + const notionIdPattern = /[ -]?[a-z0-9]{32}$/i; + return fileName.replace(notionIdPattern, '').trim(); +} diff --git a/packages/editor-ext/src/lib/table/cell.ts b/packages/editor-ext/src/lib/table/cell.ts index 25a311b9..63df7dcf 100644 --- a/packages/editor-ext/src/lib/table/cell.ts +++ b/packages/editor-ext/src/lib/table/cell.ts @@ -2,33 +2,39 @@ import { TableCell as TiptapTableCell } from "@tiptap/extension-table-cell"; export const TableCell = TiptapTableCell.extend({ name: "tableCell", - content: "(paragraph | heading | bulletList | orderedList | taskList | blockquote | callout | image | video | attachment | mathBlock | details | codeBlock)+", - + content: + "(paragraph | heading | bulletList | orderedList | taskList | blockquote | callout | image | video | attachment | mathBlock | details | codeBlock)+", + addAttributes() { return { ...this.parent?.(), backgroundColor: { default: null, - parseHTML: (element) => element.style.backgroundColor || null, + parseHTML: (element) => + element.style.backgroundColor || + element.getAttribute("data-background-color") || + null, renderHTML: (attributes) => { if (!attributes.backgroundColor) { return {}; } return { style: `background-color: ${attributes.backgroundColor}`, - 'data-background-color': attributes.backgroundColor, + "data-background-color": attributes.backgroundColor, }; }, }, backgroundColorName: { default: null, - parseHTML: (element) => element.getAttribute('data-background-color-name') || null, + parseHTML: (element) => + element.getAttribute("data-background-color-name") || null, renderHTML: (attributes) => { if (!attributes.backgroundColorName) { return {}; } return { - 'data-background-color-name': attributes.backgroundColorName.toLowerCase(), + "data-background-color-name": + attributes.backgroundColorName.toLowerCase(), }; }, }, diff --git a/packages/editor-ext/src/lib/table/header.ts b/packages/editor-ext/src/lib/table/header.ts index 399a8cf0..501f089d 100644 --- a/packages/editor-ext/src/lib/table/header.ts +++ b/packages/editor-ext/src/lib/table/header.ts @@ -2,36 +2,42 @@ import { TableHeader as TiptapTableHeader } from "@tiptap/extension-table-header export const TableHeader = TiptapTableHeader.extend({ name: "tableHeader", - content: "(paragraph | heading | bulletList | orderedList | taskList | blockquote | callout | image | video | attachment | mathBlock | details | codeBlock)+", - + content: + "(paragraph | heading | bulletList | orderedList | taskList | blockquote | callout | image | video | attachment | mathBlock | details | codeBlock)+", + addAttributes() { return { ...this.parent?.(), backgroundColor: { default: null, - parseHTML: (element) => element.style.backgroundColor || null, + parseHTML: (element) => + element.style.backgroundColor || + element.getAttribute("data-background-color") || + null, renderHTML: (attributes) => { if (!attributes.backgroundColor) { return {}; } return { style: `background-color: ${attributes.backgroundColor}`, - 'data-background-color': attributes.backgroundColor, + "data-background-color": attributes.backgroundColor, }; }, }, backgroundColorName: { default: null, - parseHTML: (element) => element.getAttribute('data-background-color-name') || null, + parseHTML: (element) => + element.getAttribute("data-background-color-name") || null, renderHTML: (attributes) => { if (!attributes.backgroundColorName) { return {}; } return { - 'data-background-color-name': attributes.backgroundColorName.toLowerCase(), + "data-background-color-name": + attributes.backgroundColorName.toLowerCase(), }; }, }, }; }, -}); \ No newline at end of file +}); From ac17521717351cb56ab1b8386a9e02446a6ac796 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Thu, 18 Sep 2025 13:24:16 +0100 Subject: [PATCH 11/15] sync --- apps/server/src/ee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/ee b/apps/server/src/ee index bcc077ae..3af21def 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit bcc077ae3bf05a5fe8f1e547960163ff766bae01 +Subproject commit 3af21def15eb7ea7eb400086eeaf97991938a5fd From cf5bbb10df3cf9dcbc978cb715d9521f7c32fde6 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:34:13 +0100 Subject: [PATCH 12/15] fix import html processing --- .../import/utils/import-formatter.ts | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/apps/server/src/integrations/import/utils/import-formatter.ts b/apps/server/src/integrations/import/utils/import-formatter.ts index 54fc50dd..14a2530c 100644 --- a/apps/server/src/integrations/import/utils/import-formatter.ts +++ b/apps/server/src/integrations/import/utils/import-formatter.ts @@ -222,17 +222,40 @@ export function notionFormatter($: CheerioAPI, $root: Cheerio) { } export function unwrapFromParagraph($: CheerioAPI, $node: Cheerio) { - // find the nearest

or ancestor - let $wrapper = $node.closest('p, a'); + // Keep track of processed wrappers to avoid infinite loops + const processedWrappers = new Set(); + let $wrapper = $node.closest('p, a'); while ($wrapper.length) { - // if the wrapper has only our node inside, replace it entirely - if ($wrapper.contents().length === 1) { + const wrapperElement = $wrapper.get(0); + + // If we've already processed this wrapper, break to avoid infinite loop + if (processedWrappers.has(wrapperElement)) { + break; + } + + processedWrappers.add(wrapperElement); + + // Check if the wrapper contains only whitespace and our target node + const hasOnlyTargetNode = + $wrapper.contents().filter((_, el) => { + const $el = $(el); + // Skip whitespace-only text nodes. NodeType 3 = text node + if (el.nodeType === 3 && !$el.text().trim()) { + return false; + } + // Return true if this is not our target node + return !$el.is($node) && !$node.is($el); + }).length === 0; + + if (hasOnlyTargetNode) { + // Replace the wrapper entirely with our node $wrapper.replaceWith($node); } else { - // otherwise just move the node to before the wrapper + // Move the node to before the wrapper, preserving other content $wrapper.before($node); } + // look again for any new wrapper around $node $wrapper = $node.closest('p, a'); } From 4de25a8b94d7903314c879810eb5e622d434697c Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:52:53 +0100 Subject: [PATCH 13/15] invalidate queries on space deletion --- .../src/features/space/queries/space-query.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/apps/client/src/features/space/queries/space-query.ts b/apps/client/src/features/space/queries/space-query.ts index b51e195e..e15320d6 100644 --- a/apps/client/src/features/space/queries/space-query.ts +++ b/apps/client/src/features/space/queries/space-query.ts @@ -152,13 +152,36 @@ export function useDeleteSpaceMutation() { }); } - const spaces = queryClient.getQueryData(["spaces"]) as any; + // Remove space-specific queries + if (variables.id) { + queryClient.removeQueries({ + queryKey: ["space", variables.id], + exact: true, + }); + + // Invalidate recent changes + queryClient.invalidateQueries({ + queryKey: ["recent-changes"], + }); + + queryClient.invalidateQueries({ + queryKey: ["recent-changes", variables.id], + }); + } + + // Update spaces list cache + /* const spaces = queryClient.getQueryData(["spaces"]) as any; if (spaces) { spaces.items = spaces.items?.filter( (space: ISpace) => space.id !== variables.id, ); queryClient.setQueryData(["spaces"], spaces); - } + }*/ + + // Invalidate all spaces queries to refresh lists + queryClient.invalidateQueries({ + predicate: (item) => ["spaces"].includes(item.queryKey[0] as string), + }); }, onError: (error) => { const errorMessage = error["response"]?.data?.message; From 3c4cab0d2a8508fdeadb1e980db2af7a4a305675 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Thu, 18 Sep 2025 18:00:28 +0100 Subject: [PATCH 14/15] v0.23.2 --- apps/client/package.json | 2 +- apps/server/package.json | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/client/package.json b/apps/client/package.json index 031646e8..0cff05cc 100644 --- a/apps/client/package.json +++ b/apps/client/package.json @@ -1,7 +1,7 @@ { "name": "client", "private": true, - "version": "0.23.1", + "version": "0.23.2", "scripts": { "dev": "vite", "build": "tsc && vite build", diff --git a/apps/server/package.json b/apps/server/package.json index fb2afe8f..71865a07 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -1,6 +1,6 @@ { "name": "server", - "version": "0.23.1", + "version": "0.23.2", "description": "", "author": "", "private": true, diff --git a/package.json b/package.json index 645cf1e1..9177f81e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "docmost", "homepage": "https://docmost.com", - "version": "0.23.1", + "version": "0.23.2", "private": true, "scripts": { "build": "nx run-many -t build", From 1f3b0c72769ec9e643171b0cc0d400e8d47758e6 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Wed, 24 Sep 2025 21:25:39 +0100 Subject: [PATCH 15/15] cloud fix --- .../client/src/features/search/components/search-spotlight.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/client/src/features/search/components/search-spotlight.tsx b/apps/client/src/features/search/components/search-spotlight.tsx index e7e3c9f7..045c7477 100644 --- a/apps/client/src/features/search/components/search-spotlight.tsx +++ b/apps/client/src/features/search/components/search-spotlight.tsx @@ -8,6 +8,7 @@ import { SearchSpotlightFilters } from "./search-spotlight-filters.tsx"; import { useUnifiedSearch } from "../hooks/use-unified-search.ts"; import { SearchResultItem } from "./search-result-item.tsx"; import { useLicense } from "@/ee/hooks/use-license.tsx"; +import { isCloud } from "@/lib/config.ts"; interface SearchSpotlightProps { spaceId?: string; @@ -43,7 +44,7 @@ export function SearchSpotlight({ spaceId }: SearchSpotlightProps) { // Determine result type for rendering const isAttachmentSearch = - filters.contentType === "attachment" && hasLicenseKey; + filters.contentType === "attachment" && (hasLicenseKey || isCloud()); const resultItems = (searchResults || []).map((result) => (