Compare commits

..

5 Commits

Author SHA1 Message Date
Philipinho 207c119792 fix collab server logging too 2026-01-29 00:41:00 +00:00
Philipinho 4683a85a8b - fix: set default Nest logger and bufferLogs to false for pino compatibility
- handle redis error event
2026-01-29 00:36:48 +00:00
Philip Okugbe 74e915546b feat: collab redis extension with server affinity (#1873)
* feat(collab): better redis extension
* move types to own file
* debug logging
* fix: graceful collab shutdown
* rename default prefix
* pass wsAdapter to gateway
* expose event handler
* unique collab serverId generation
* uninstall @hocuspocus/extension-redis package
* expose more functions
* sync with latest
* cleanup
* fastify router options
* cleanup type
2026-01-27 17:05:05 +00:00
Philipinho 3523600f40 add timestamps 2026-01-27 16:49:22 +00:00
Philip Okugbe 6ccb2bb872 feat(export): add metadata file to preserve page icons and ordering on import (#1877)
* feat(export): add metadata file to preserve page icons and ordering on import
- Export includes `docmost-metadata.json`
- Import reads metadata to restore icons and sort siblings by original position

* cleanup

* bonus fixes

* handle unknown prosemirror nodes

* add docmost app  version
2026-01-27 16:39:39 +00:00
13 changed files with 203 additions and 12 deletions
@@ -172,6 +172,10 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
queryKey: ["root-sidebar-pages", fileTask.spaceId],
});
await queryClient.invalidateQueries({
queryKey: ["recent-changes", fileTask.spaceId],
});
setTimeout(() => {
emit({
operation: "refetchRootTreeNodeEvent",
@@ -6,6 +6,7 @@ import { ISpace } from "../types/space.types";
import { useNavigate } from "react-router-dom";
import APP_ROUTE from "@/lib/app-route";
import { Trans, useTranslation } from "react-i18next";
import { useState } from "react";
interface DeleteSpaceModalProps {
space: ISpace;
@@ -14,6 +15,7 @@ interface DeleteSpaceModalProps {
export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) {
const { t } = useTranslation();
const [opened, { open, close }] = useDisclosure(false);
const [isDeleting, setIsDeleting] = useState(false);
const deleteSpaceMutation = useDeleteSpaceMutation();
const navigate = useNavigate();
@@ -35,12 +37,15 @@ export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) {
return;
}
setIsDeleting(true);
try {
// pass slug too so we can clear the local cache
await deleteSpaceMutation.mutateAsync({ id: space.id, slug: space.slug });
navigate(APP_ROUTE.HOME);
} catch (error) {
console.error("Failed to delete space", error);
} finally {
setIsDeleting(false);
}
};
@@ -79,7 +84,7 @@ export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) {
<Button onClick={close} variant="default">
{t("Cancel")}
</Button>
<Button onClick={handleDelete} color="red">
<Button onClick={handleDelete} color="red" loading={isDeleting}>
{t("Confirm")}
</Button>
</Group>
@@ -41,7 +41,8 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
// see: https://github.com/ueberdosis/tiptap/issues/5352
// see:https://github.com/ueberdosis/tiptap/issues/4089
//import { generateJSON } from '@tiptap/html';
import { Node } from '@tiptap/pm/model';
import { Node, Schema } from '@tiptap/pm/model';
import { Logger } from '@nestjs/common';
export const tiptapExtensions = [
StarterKit.configure({
@@ -110,9 +111,53 @@ export function jsonToText(tiptapJson: JSONContent) {
}
export function jsonToNode(tiptapJson: JSONContent) {
return Node.fromJSON(getSchema(tiptapExtensions), tiptapJson);
const schema = getSchema(tiptapExtensions);
try {
return Node.fromJSON(schema, tiptapJson);
} catch (error) {
if (
error instanceof RangeError &&
error.message.includes('Unknown node type')
) {
Logger.warn('Stripping unknown node types from document:', error.message);
const cleanedJson = stripUnknownNodes(tiptapJson, schema);
return Node.fromJSON(schema, cleanedJson);
}
throw error;
}
}
export function getPageId(documentName: string) {
return documentName.split('.')[1];
}
function stripUnknownNodes(
json: JSONContent,
schema: Schema,
): JSONContent | null {
if (!json || typeof json !== 'object') return json;
// Recursively clean children first, flattening any unwrapped content
if (json.content && Array.isArray(json.content)) {
const newContent: JSONContent[] = [];
for (const child of json.content) {
const cleaned = stripUnknownNodes(child, schema);
if (Array.isArray(cleaned)) {
newContent.push(...cleaned);
} else if (cleaned) {
newContent.push(cleaned);
}
}
json.content = newContent;
}
// Check if this node is unknown AFTER processing children
if (json.type && !schema.nodes[json.type]) {
// Unwrap: return cleaned children directly instead of wrapping
return (
json.content && json.content.length > 0 ? json.content : null
) as any;
}
return json;
}
@@ -79,6 +79,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents);
this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`);
this.sub.on('messageBuffer', this.handleRedisMessage);
this.pub.on('error', () => {});
this.sub.on('error', () => {});
}
private getKey(documentName: string) {
return `${this.lockPrefix}:${documentName}`;
@@ -19,7 +19,8 @@ async function bootstrap() {
},
}),
{
bufferLogs: true,
logger: false,
bufferLogs: false,
},
);
@@ -0,0 +1,16 @@
export type ExportPageMetadata = {
pageId: string;
slugId: string;
icon: string | null;
position: string;
parentPath: string | null;
createdAt: string;
updatedAt: string;
};
export type ExportMetadata = {
exportedAt: string;
source: 'docmost';
version: string;
pages: Record<string, ExportPageMetadata>;
};
@@ -422,6 +422,8 @@ export class PageRepo {
'parentPageId',
'spaceId',
'workspaceId',
'createdAt',
'updatedAt',
])
.$if(opts?.includeContent, (qb) => qb.select('content'))
.where('id', '=', parentPageId)
@@ -438,6 +440,8 @@ export class PageRepo {
'p.parentPageId',
'p.spaceId',
'p.workspaceId',
'p.createdAt',
'p.updatedAt',
])
.$if(opts?.includeContent, (qb) => qb.select('p.content'))
.innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id')
@@ -20,11 +20,17 @@ import {
replaceInternalLinks,
updateAttachmentUrlsToLocalPaths,
} from './utils';
import {
ExportMetadata,
ExportPageMetadata,
} from '../../common/helpers/types/export-metadata.types';
import { PageRepo } from '@docmost/db/repos/page/page.repo';
import { Node } from '@tiptap/pm/model';
import { EditorState } from '@tiptap/pm/state';
// eslint-disable-next-line @typescript-eslint/no-require-imports
import slugify = require('@sindresorhus/slugify');
// eslint-disable-next-line @typescript-eslint/no-require-imports
const packageJson = require('../../../package.json');
import { EnvironmentService } from '../environment/environment.service';
import {
getAttachmentIds,
@@ -155,12 +161,17 @@ export class ExportService {
'pages.id',
'pages.slugId',
'pages.title',
'pages.icon',
'pages.position',
'pages.content',
'pages.parentPageId',
'pages.spaceId',
'pages.workspaceId',
'pages.createdAt',
'pages.updatedAt',
])
.where('spaceId', '=', spaceId)
.where('deletedAt', 'is', null)
.execute();
const tree = buildTree(pages as Page[]);
@@ -189,10 +200,12 @@ export class ExportService {
includeAttachments: boolean,
): Promise<void> {
const slugIdToPath: Record<string, string> = {};
const pageIdToFilePath: Record<string, string> = {};
const pagesMetadata: Record<string, ExportPageMetadata> = {};
computeLocalPath(tree, format, null, '', slugIdToPath);
const stack: { folder: JSZip; parentPageId: string }[] = [
const stack: { folder: JSZip; parentPageId: string | null }[] = [
{ folder: zip, parentPageId: null },
];
@@ -232,12 +245,35 @@ export class ExportService {
`${pageTitle}${getExportExtension(format)}`,
pageExportContent,
);
pageIdToFilePath[page.id] = currentPagePath;
const parentPath = parentPageId ? pageIdToFilePath[parentPageId] : null;
pagesMetadata[currentPagePath] = {
pageId: page.id,
slugId: page.slugId,
icon: page.icon ?? null,
position: page.position,
parentPath,
createdAt: page.createdAt?.toISOString() ?? new Date().toISOString(),
updatedAt: page.updatedAt?.toISOString() ?? new Date().toISOString(),
};
if (childPages.length > 0) {
const pageFolder = folder.folder(pageTitle);
stack.push({ folder: pageFolder, parentPageId: page.id });
}
}
}
const metadata: ExportMetadata = {
exportedAt: new Date().toISOString(),
source: 'docmost',
version: packageJson.version,
pages: pagesMetadata,
};
zip.file('docmost-metadata.json', JSON.stringify(metadata, null, 2));
}
async zipAttachments(prosemirrorJson: any, spaceId: string, zip: JSZip) {
@@ -15,4 +15,5 @@ export type ImportPageNode = {
parentPageId: string | null;
fileExtension: string;
filePath: string;
icon?: string | null;
};
@@ -24,6 +24,8 @@ import { formatImportHtml } from '../utils/import-formatter';
import {
buildAttachmentCandidates,
collectMarkdownAndHtmlFiles,
encodeFilePath,
readDocmostMetadata,
stripNotionID,
} from '../utils/import.utils';
import { executeTx } from '@docmost/db/utils';
@@ -154,6 +156,7 @@ export class FileImportTaskService {
const { extractDir, fileTask } = opts;
const allFiles = await collectMarkdownAndHtmlFiles(extractDir);
const attachmentCandidates = await buildAttachmentCandidates(extractDir);
const docmostMetadata = await readDocmostMetadata(extractDir);
const pagesMap = new Map<string, ImportPageNode>();
@@ -164,6 +167,9 @@ export class FileImportTaskService {
.join('/'); // normalize to forward-slashes
const ext = path.extname(relPath).toLowerCase();
const encodedPath = encodeFilePath(relPath);
const pageMetadata = docmostMetadata?.pages[encodedPath];
pagesMap.set(relPath, {
id: v7(),
slugId: generateSlugId(),
@@ -172,6 +178,7 @@ export class FileImportTaskService {
parentPageId: null,
fileExtension: ext,
filePath: relPath,
icon: pageMetadata?.icon ?? null,
});
}
@@ -224,6 +231,8 @@ export class FileImportTaskService {
if (!pagesMap.has(mdPath) && !pagesMap.has(htmlPath)) {
const folderName = path.basename(folderPath);
const encodedMdPath = encodeFilePath(mdPath);
const placeholderMetadata = docmostMetadata?.pages[encodedMdPath];
pagesMap.set(mdPath, {
id: v7(),
slugId: generateSlugId(),
@@ -232,6 +241,7 @@ export class FileImportTaskService {
parentPageId: null,
fileExtension: '.md',
filePath: mdPath,
icon: placeholderMetadata?.icon ?? null,
});
}
});
@@ -266,11 +276,39 @@ export class FileImportTaskService {
siblingsMap.set(page.parentPageId, group);
});
const encodedPathsMap = new Map<string, string>();
if (docmostMetadata) {
pagesMap.forEach((_, filePath) => {
encodedPathsMap.set(filePath, encodeFilePath(filePath));
});
}
// Sort siblings by metadata position if available, otherwise alphabetically
const sortSiblings = (siblings: ImportPageNode[]) => {
if (docmostMetadata) {
siblings.sort((a, b) => {
const posA =
docmostMetadata.pages[encodedPathsMap.get(a.filePath)]?.position;
const posB =
docmostMetadata.pages[encodedPathsMap.get(b.filePath)]?.position;
if (posA && posB) {
// Use direct comparison to match PostgreSQL collation 'C' (byte order)
if (posA < posB) return -1;
if (posA > posB) return 1;
return 0;
}
return a.name.localeCompare(b.name);
});
} else {
siblings.sort((a, b) => a.name.localeCompare(b.name));
}
};
// get root pages
const rootSibs = siblingsMap.get(null);
if (rootSibs?.length) {
rootSibs.sort((a, b) => a.name.localeCompare(b.name));
sortSiblings(rootSibs);
// get first position key from the server
const nextPosition = await this.pageService.nextPagePosition(
@@ -292,7 +330,7 @@ export class FileImportTaskService {
siblingsMap.forEach((sibs, parentId) => {
if (parentId === null) return; // root already done
sibs.sort((a, b) => a.name.localeCompare(b.name));
sortSiblings(sibs);
let prevPos: string | null = null;
for (const page of sibs) {
@@ -426,7 +464,7 @@ export class FileImportTaskService {
id: page.id,
slugId: page.slugId,
title: title || page.name,
icon: pageIcon || null,
icon: page.icon || pageIcon || null,
content: prosemirrorJson,
textContent: jsonToText(prosemirrorJson),
ydoc: await this.importService.createYdoc(prosemirrorJson),
@@ -1,6 +1,7 @@
import { Logger } from '@nestjs/common';
import { promises as fs } from 'fs';
import * as path from 'path';
import { ExportMetadata } from '../../../common/helpers/types/export-metadata.types';
export async function buildAttachmentCandidates(
extractDir: string,
@@ -35,9 +36,15 @@ export function resolveRelativeAttachmentPath(
try {
mainRel = decodeURIComponent(mainRel);
} catch (err) {
Logger.warn(`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, 'ImportUtils');
Logger.warn(
`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`,
'ImportUtils',
);
}
const fallback = path.normalize(path.join(pageDir, mainRel)).split(path.sep).join('/');
const fallback = path
.normalize(path.join(pageDir, mainRel))
.split(path.sep)
.join('/');
if (attachmentCandidates.has(mainRel)) {
return mainRel;
@@ -76,3 +83,26 @@ export function stripNotionID(fileName: string): string {
const notionIdPattern = /[ -]?[a-z0-9]{32}$/i;
return fileName.replace(notionIdPattern, '').trim();
}
export function encodeFilePath(filePath: string): string {
return filePath
.split('/')
.map((segment) => encodeURIComponent(segment))
.join('/');
}
export async function readDocmostMetadata(
extractDir: string,
): Promise<ExportMetadata | null> {
const metadataPath = path.join(extractDir, 'docmost-metadata.json');
try {
const content = await fs.readFile(metadataPath, 'utf-8');
const metadata = JSON.parse(content) as ExportMetadata;
if (metadata.source === 'docmost' && metadata.pages) {
return metadata;
}
return null;
} catch {
return null;
}
}
+8 -2
View File
@@ -24,7 +24,11 @@ async function bootstrap() {
}),
{
rawBody: true,
bufferLogs: true,
// disable Nest logger so pino handles all logs
// bufferLogs must be false else pino will fail
// to log OnApplicationBootstrap logs
logger: false,
bufferLogs: false,
},
);
@@ -101,7 +105,9 @@ async function bootstrap() {
const port = process.env.PORT || 3000;
await app.listen(port, '0.0.0.0', () => {
logger.log(`Listening on http://127.0.0.1:${port} / ${process.env.APP_URL}`);
logger.log(
`Listening on http://127.0.0.1:${port} / ${process.env.APP_URL}`,
);
});
}
@@ -23,6 +23,9 @@ export class WsRedisIoAdapter extends IoAdapter {
const pubClient = new Redis(process.env.REDIS_URL, options);
const subClient = new Redis(process.env.REDIS_URL, options);
pubClient.on('error', (err) => () => {});
subClient.on('error', (err) => () => {});
this.adapterConstructor = createAdapter(pubClient, subClient);
}