From 6ccb2bb8727549239562774e886ddbef01993c8c Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:39:39 +0000 Subject: [PATCH] feat(export): add metadata file to preserve page icons and ordering on import (#1877) * feat(export): add metadata file to preserve page icons and ordering on import - Export includes `docmost-metadata.json` - Import reads metadata to restore icons and sort siblings by original position * cleanup * bonus fixes * handle unknown prosemirror nodes * add docmost app version --- .../page/components/page-import-modal.tsx | 4 ++ .../space/components/delete-space-modal.tsx | 7 ++- .../src/collaboration/collaboration.util.ts | 49 ++++++++++++++++++- .../helpers/types/export-metadata.types.ts | 14 ++++++ .../src/integrations/export/export.service.ts | 34 ++++++++++++- .../integrations/import/dto/file-task-dto.ts | 1 + .../services/file-import-task.service.ts | 44 +++++++++++++++-- .../integrations/import/utils/import.utils.ts | 34 ++++++++++++- 8 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/common/helpers/types/export-metadata.types.ts diff --git a/apps/client/src/features/page/components/page-import-modal.tsx b/apps/client/src/features/page/components/page-import-modal.tsx index a2df380e..be0264b6 100644 --- a/apps/client/src/features/page/components/page-import-modal.tsx +++ b/apps/client/src/features/page/components/page-import-modal.tsx @@ -172,6 +172,10 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { queryKey: ["root-sidebar-pages", fileTask.spaceId], }); + await queryClient.invalidateQueries({ + queryKey: ["recent-changes", fileTask.spaceId], + }); + setTimeout(() => { emit({ operation: "refetchRootTreeNodeEvent", diff --git a/apps/client/src/features/space/components/delete-space-modal.tsx b/apps/client/src/features/space/components/delete-space-modal.tsx index f697322d..8a89e720 100644 --- a/apps/client/src/features/space/components/delete-space-modal.tsx +++ b/apps/client/src/features/space/components/delete-space-modal.tsx @@ -6,6 +6,7 @@ import { ISpace } from "../types/space.types"; import { useNavigate } from "react-router-dom"; import APP_ROUTE from "@/lib/app-route"; import { Trans, useTranslation } from "react-i18next"; +import { useState } from "react"; interface DeleteSpaceModalProps { space: ISpace; @@ -14,6 +15,7 @@ interface DeleteSpaceModalProps { export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) { const { t } = useTranslation(); const [opened, { open, close }] = useDisclosure(false); + const [isDeleting, setIsDeleting] = useState(false); const deleteSpaceMutation = useDeleteSpaceMutation(); const navigate = useNavigate(); @@ -35,12 +37,15 @@ export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) { return; } + setIsDeleting(true); try { // pass slug too so we can clear the local cache await deleteSpaceMutation.mutateAsync({ id: space.id, slug: space.slug }); navigate(APP_ROUTE.HOME); } catch (error) { console.error("Failed to delete space", error); + } finally { + setIsDeleting(false); } }; @@ -79,7 +84,7 @@ export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) { - diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index 16ca5bd5..afe1be08 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -41,7 +41,8 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; // see: https://github.com/ueberdosis/tiptap/issues/5352 // see:https://github.com/ueberdosis/tiptap/issues/4089 //import { generateJSON } from '@tiptap/html'; -import { Node } from '@tiptap/pm/model'; +import { Node, Schema } from '@tiptap/pm/model'; +import { Logger } from '@nestjs/common'; export const tiptapExtensions = [ StarterKit.configure({ @@ -110,9 +111,53 @@ export function jsonToText(tiptapJson: JSONContent) { } export function jsonToNode(tiptapJson: JSONContent) { - return Node.fromJSON(getSchema(tiptapExtensions), tiptapJson); + const schema = getSchema(tiptapExtensions); + try { + return Node.fromJSON(schema, tiptapJson); + } catch (error) { + if ( + error instanceof RangeError && + error.message.includes('Unknown node type') + ) { + Logger.warn('Stripping unknown node types from document:', error.message); + const cleanedJson = stripUnknownNodes(tiptapJson, schema); + return Node.fromJSON(schema, cleanedJson); + } + throw error; + } } export function getPageId(documentName: string) { return documentName.split('.')[1]; } + +function stripUnknownNodes( + json: JSONContent, + schema: Schema, +): JSONContent | null { + if (!json || typeof json !== 'object') return json; + + // Recursively clean children first, flattening any unwrapped content + if (json.content && Array.isArray(json.content)) { + const newContent: JSONContent[] = []; + for (const child of json.content) { + const cleaned = stripUnknownNodes(child, schema); + if (Array.isArray(cleaned)) { + newContent.push(...cleaned); + } else if (cleaned) { + newContent.push(cleaned); + } + } + json.content = newContent; + } + + // Check if this node is unknown AFTER processing children + if (json.type && !schema.nodes[json.type]) { + // Unwrap: return cleaned children directly instead of wrapping + return ( + json.content && json.content.length > 0 ? json.content : null + ) as any; + } + + return json; +} diff --git a/apps/server/src/common/helpers/types/export-metadata.types.ts b/apps/server/src/common/helpers/types/export-metadata.types.ts new file mode 100644 index 00000000..42ef4c68 --- /dev/null +++ b/apps/server/src/common/helpers/types/export-metadata.types.ts @@ -0,0 +1,14 @@ +export type ExportPageMetadata = { + pageId: string; + slugId: string; + icon: string | null; + position: string; + parentPath: string | null; +}; + +export type ExportMetadata = { + exportedAt: string; + source: 'docmost'; + version: string; + pages: Record; +}; diff --git a/apps/server/src/integrations/export/export.service.ts b/apps/server/src/integrations/export/export.service.ts index e33ac11b..44047174 100644 --- a/apps/server/src/integrations/export/export.service.ts +++ b/apps/server/src/integrations/export/export.service.ts @@ -20,11 +20,17 @@ import { replaceInternalLinks, updateAttachmentUrlsToLocalPaths, } from './utils'; +import { + ExportMetadata, + ExportPageMetadata, +} from '../../common/helpers/types/export-metadata.types'; import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { Node } from '@tiptap/pm/model'; import { EditorState } from '@tiptap/pm/state'; // eslint-disable-next-line @typescript-eslint/no-require-imports import slugify = require('@sindresorhus/slugify'); +// eslint-disable-next-line @typescript-eslint/no-require-imports +const packageJson = require('../../../package.json'); import { EnvironmentService } from '../environment/environment.service'; import { getAttachmentIds, @@ -155,12 +161,15 @@ export class ExportService { 'pages.id', 'pages.slugId', 'pages.title', + 'pages.icon', + 'pages.position', 'pages.content', 'pages.parentPageId', 'pages.spaceId', 'pages.workspaceId', ]) .where('spaceId', '=', spaceId) + .where('deletedAt', 'is', null) .execute(); const tree = buildTree(pages as Page[]); @@ -189,10 +198,12 @@ export class ExportService { includeAttachments: boolean, ): Promise { const slugIdToPath: Record = {}; + const pageIdToFilePath: Record = {}; + const pagesMetadata: Record = {}; computeLocalPath(tree, format, null, '', slugIdToPath); - const stack: { folder: JSZip; parentPageId: string }[] = [ + const stack: { folder: JSZip; parentPageId: string | null }[] = [ { folder: zip, parentPageId: null }, ]; @@ -232,12 +243,33 @@ export class ExportService { `${pageTitle}${getExportExtension(format)}`, pageExportContent, ); + + pageIdToFilePath[page.id] = currentPagePath; + + const parentPath = parentPageId ? pageIdToFilePath[parentPageId] : null; + pagesMetadata[currentPagePath] = { + pageId: page.id, + slugId: page.slugId, + icon: page.icon ?? null, + position: page.position, + parentPath, + }; + if (childPages.length > 0) { const pageFolder = folder.folder(pageTitle); stack.push({ folder: pageFolder, parentPageId: page.id }); } } } + + const metadata: ExportMetadata = { + exportedAt: new Date().toISOString(), + source: 'docmost', + version: packageJson.version, + pages: pagesMetadata, + }; + + zip.file('docmost-metadata.json', JSON.stringify(metadata, null, 2)); } async zipAttachments(prosemirrorJson: any, spaceId: string, zip: JSZip) { diff --git a/apps/server/src/integrations/import/dto/file-task-dto.ts b/apps/server/src/integrations/import/dto/file-task-dto.ts index 9cdea395..84736813 100644 --- a/apps/server/src/integrations/import/dto/file-task-dto.ts +++ b/apps/server/src/integrations/import/dto/file-task-dto.ts @@ -15,4 +15,5 @@ export type ImportPageNode = { parentPageId: string | null; fileExtension: string; filePath: string; + icon?: string | null; }; \ No newline at end of file diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index 5cf39054..8ae79598 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -24,6 +24,8 @@ import { formatImportHtml } from '../utils/import-formatter'; import { buildAttachmentCandidates, collectMarkdownAndHtmlFiles, + encodeFilePath, + readDocmostMetadata, stripNotionID, } from '../utils/import.utils'; import { executeTx } from '@docmost/db/utils'; @@ -154,6 +156,7 @@ export class FileImportTaskService { const { extractDir, fileTask } = opts; const allFiles = await collectMarkdownAndHtmlFiles(extractDir); const attachmentCandidates = await buildAttachmentCandidates(extractDir); + const docmostMetadata = await readDocmostMetadata(extractDir); const pagesMap = new Map(); @@ -164,6 +167,9 @@ export class FileImportTaskService { .join('/'); // normalize to forward-slashes const ext = path.extname(relPath).toLowerCase(); + const encodedPath = encodeFilePath(relPath); + const pageMetadata = docmostMetadata?.pages[encodedPath]; + pagesMap.set(relPath, { id: v7(), slugId: generateSlugId(), @@ -172,6 +178,7 @@ export class FileImportTaskService { parentPageId: null, fileExtension: ext, filePath: relPath, + icon: pageMetadata?.icon ?? null, }); } @@ -224,6 +231,8 @@ export class FileImportTaskService { if (!pagesMap.has(mdPath) && !pagesMap.has(htmlPath)) { const folderName = path.basename(folderPath); + const encodedMdPath = encodeFilePath(mdPath); + const placeholderMetadata = docmostMetadata?.pages[encodedMdPath]; pagesMap.set(mdPath, { id: v7(), slugId: generateSlugId(), @@ -232,6 +241,7 @@ export class FileImportTaskService { parentPageId: null, fileExtension: '.md', filePath: mdPath, + icon: placeholderMetadata?.icon ?? null, }); } }); @@ -266,11 +276,39 @@ export class FileImportTaskService { siblingsMap.set(page.parentPageId, group); }); + const encodedPathsMap = new Map(); + if (docmostMetadata) { + pagesMap.forEach((_, filePath) => { + encodedPathsMap.set(filePath, encodeFilePath(filePath)); + }); + } + + // Sort siblings by metadata position if available, otherwise alphabetically + const sortSiblings = (siblings: ImportPageNode[]) => { + if (docmostMetadata) { + siblings.sort((a, b) => { + const posA = + docmostMetadata.pages[encodedPathsMap.get(a.filePath)]?.position; + const posB = + docmostMetadata.pages[encodedPathsMap.get(b.filePath)]?.position; + if (posA && posB) { + // Use direct comparison to match PostgreSQL collation 'C' (byte order) + if (posA < posB) return -1; + if (posA > posB) return 1; + return 0; + } + return a.name.localeCompare(b.name); + }); + } else { + siblings.sort((a, b) => a.name.localeCompare(b.name)); + } + }; + // get root pages const rootSibs = siblingsMap.get(null); if (rootSibs?.length) { - rootSibs.sort((a, b) => a.name.localeCompare(b.name)); + sortSiblings(rootSibs); // get first position key from the server const nextPosition = await this.pageService.nextPagePosition( @@ -292,7 +330,7 @@ export class FileImportTaskService { siblingsMap.forEach((sibs, parentId) => { if (parentId === null) return; // root already done - sibs.sort((a, b) => a.name.localeCompare(b.name)); + sortSiblings(sibs); let prevPos: string | null = null; for (const page of sibs) { @@ -426,7 +464,7 @@ export class FileImportTaskService { id: page.id, slugId: page.slugId, title: title || page.name, - icon: pageIcon || null, + icon: page.icon || pageIcon || null, content: prosemirrorJson, textContent: jsonToText(prosemirrorJson), ydoc: await this.importService.createYdoc(prosemirrorJson), diff --git a/apps/server/src/integrations/import/utils/import.utils.ts b/apps/server/src/integrations/import/utils/import.utils.ts index c8f5fe51..cd348652 100644 --- a/apps/server/src/integrations/import/utils/import.utils.ts +++ b/apps/server/src/integrations/import/utils/import.utils.ts @@ -1,6 +1,7 @@ import { Logger } from '@nestjs/common'; import { promises as fs } from 'fs'; import * as path from 'path'; +import { ExportMetadata } from '../../../common/helpers/types/export-metadata.types'; export async function buildAttachmentCandidates( extractDir: string, @@ -35,9 +36,15 @@ export function resolveRelativeAttachmentPath( try { mainRel = decodeURIComponent(mainRel); } catch (err) { - Logger.warn(`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, 'ImportUtils'); + Logger.warn( + `URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, + 'ImportUtils', + ); } - const fallback = path.normalize(path.join(pageDir, mainRel)).split(path.sep).join('/'); + const fallback = path + .normalize(path.join(pageDir, mainRel)) + .split(path.sep) + .join('/'); if (attachmentCandidates.has(mainRel)) { return mainRel; @@ -76,3 +83,26 @@ export function stripNotionID(fileName: string): string { const notionIdPattern = /[ -]?[a-z0-9]{32}$/i; return fileName.replace(notionIdPattern, '').trim(); } + +export function encodeFilePath(filePath: string): string { + return filePath + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/'); +} + +export async function readDocmostMetadata( + extractDir: string, +): Promise { + const metadataPath = path.join(extractDir, 'docmost-metadata.json'); + try { + const content = await fs.readFile(metadataPath, 'utf-8'); + const metadata = JSON.parse(content) as ExportMetadata; + if (metadata.source === 'docmost' && metadata.pages) { + return metadata; + } + return null; + } catch { + return null; + } +}