From 54775f537d3aaca3df373b7528bde33e07016107 Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Sun, 25 Jan 2026 00:48:43 +0000 Subject: [PATCH 01/10] fix: handle malformed URLs gracefully during import/export (#1868) * Handling malformed URLs gracefully * Allow import of invalid URLs, but adding logging. --------- Co-authored-by: gpapp --- .../features/page/services/page-service.ts | 9 ++++++++- .../features/space/services/space-service.ts | 9 ++++++++- apps/server/src/integrations/export/utils.ts | 20 ++++++++++++++----- .../import/utils/import-formatter.ts | 13 +++++++++++- .../integrations/import/utils/import.utils.ts | 10 ++++++++-- 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/apps/client/src/features/page/services/page-service.ts b/apps/client/src/features/page/services/page-service.ts index 8d76438a..c5b6f252 100644 --- a/apps/client/src/features/page/services/page-service.ts +++ b/apps/client/src/features/page/services/page-service.ts @@ -118,7 +118,14 @@ export async function exportPage(data: IExportPageParams): Promise { .split("filename=")[1] .replace(/"/g, ""); - saveAs(req.data, decodeURIComponent(fileName)); + let decodedFileName = fileName; + try { + decodedFileName = decodeURIComponent(fileName); + } catch (err) { + // fallback to raw filename + } + + saveAs(req.data, decodedFileName); } export async function importPage(file: File, spaceId: string) { diff --git a/apps/client/src/features/space/services/space-service.ts b/apps/client/src/features/space/services/space-service.ts index f9894099..fb6987ad 100644 --- a/apps/client/src/features/space/services/space-service.ts +++ b/apps/client/src/features/space/services/space-service.ts @@ -69,5 +69,12 @@ export async function exportSpace(data: IExportSpaceParams): Promise { .split("filename=")[1] .replace(/"/g, ""); - saveAs(req.data, decodeURIComponent(fileName)); + let decodedFileName = fileName; + try { + decodedFileName = decodeURIComponent(fileName); + } catch (err) { + // fallback to raw filename + } + + saveAs(req.data, decodedFileName); } diff --git a/apps/server/src/integrations/export/utils.ts b/apps/server/src/integrations/export/utils.ts index fe1815b0..266141c2 100644 --- a/apps/server/src/integrations/export/utils.ts +++ b/apps/server/src/integrations/export/utils.ts @@ -1,4 +1,5 @@ import { jsonToNode } from 'src/collaboration/collaboration.util'; +import { Logger } from '@nestjs/common'; import { ExportFormat } from './dto/export-dto'; import { Node } from '@tiptap/pm/model'; import { validate as isValidUUID } from 'uuid'; @@ -88,7 +89,7 @@ export function replaceInternalLinks( // if link and text are same, use page title if (markLink === node.text) { //@ts-expect-error - node.text = getInternalLinkPageName(relativePath); + node.text = getInternalLinkPageName(relativePath, currentPagePath); } } } @@ -99,10 +100,19 @@ export function replaceInternalLinks( return doc.toJSON(); } -export function getInternalLinkPageName(path: string): string { - return decodeURIComponent( - path?.split('/').pop().split('.').slice(0, -1).join('.'), - ); +export function getInternalLinkPageName(path: string, currentFilePath?: string): string { + const name = path?.split('/').pop().split('.').slice(0, -1).join('.'); + try { + return decodeURIComponent(name); + } catch (err) { + if (currentFilePath) { + Logger.warn( + `URI malformed in page ${currentFilePath}: ${name}. Falling back to raw name.`, + 'ExportUtils', + ); + } + return name; + } } export function extractPageSlugId(input: string): string { diff --git a/apps/server/src/integrations/import/utils/import-formatter.ts b/apps/server/src/integrations/import/utils/import-formatter.ts index 14a2530c..59f5eeec 100644 --- a/apps/server/src/integrations/import/utils/import-formatter.ts +++ b/apps/server/src/integrations/import/utils/import-formatter.ts @@ -1,4 +1,5 @@ import { getEmbedUrlAndProvider } from '@docmost/editor-ext'; +import { Logger } from '@nestjs/common'; import * as path from 'path'; import { v7 } from 'uuid'; import { InsertableBacklink } from '@docmost/db/types/entity.types'; @@ -280,8 +281,18 @@ export async function rewriteInternalLinksToMentionHtml( const $a = $(el); const raw = $a.attr('href')!; if (raw.startsWith('http') || raw.startsWith('/api/')) return; + let decodedRaw = raw; + try { + decodedRaw = decodeURIComponent(raw); + } catch (err) { + Logger.warn( + `URI malformed in page ${currentFilePath}: ${raw}. Falling back to raw path.`, + 'ImportFormatter', + ); + } + const resolved = normalize( - path.join(path.dirname(currentFilePath), decodeURIComponent(raw)), + path.join(path.dirname(currentFilePath), decodedRaw), ); const meta = filePathToPageMetaMap.get(resolved); if (!meta) return; diff --git a/apps/server/src/integrations/import/utils/import.utils.ts b/apps/server/src/integrations/import/utils/import.utils.ts index 1fa10d7a..c8f5fe51 100644 --- a/apps/server/src/integrations/import/utils/import.utils.ts +++ b/apps/server/src/integrations/import/utils/import.utils.ts @@ -1,3 +1,4 @@ +import { Logger } from '@nestjs/common'; import { promises as fs } from 'fs'; import * as path from 'path'; @@ -30,8 +31,13 @@ export function resolveRelativeAttachmentPath( pageDir: string, attachmentCandidates: Map, ): string | null { - const mainRel = decodeURIComponent(raw.replace(/^\.?\/+/, '')); - const fallback = path.normalize(path.join(pageDir, mainRel)); + let mainRel = raw.replace(/^\.?\/+/, ''); + try { + mainRel = decodeURIComponent(mainRel); + } catch (err) { + Logger.warn(`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, 'ImportUtils'); + } + const fallback = path.normalize(path.join(pageDir, mainRel)).split(path.sep).join('/'); if (attachmentCandidates.has(mainRel)) { return mainRel; From 1e441560f67024f149dff7de90a049a5dbda4150 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Sun, 25 Jan 2026 02:15:10 +0000 Subject: [PATCH 02/10] fix production logs filter --- apps/server/src/common/logger/pino.config.ts | 23 +++++++++++++------- apps/server/src/ee | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/server/src/common/logger/pino.config.ts b/apps/server/src/common/logger/pino.config.ts index 9d9a14f7..7299a8e9 100644 --- a/apps/server/src/common/logger/pino.config.ts +++ b/apps/server/src/common/logger/pino.config.ts @@ -5,13 +5,14 @@ const CONTEXTS_TO_IGNORE = [ 'InstanceLoader', 'RoutesResolver', 'RouterExplorer', + 'LegacyRouteConverter', 'WebSocketsController', ]; export function createPinoConfig(): Params { - const isProduction = process.env.NODE_ENV === 'production'; - const isDebugMode = process.env.DEBUG_MODE === 'true'; - const logHttp = process.env.LOG_HTTP === 'true'; + const isProduction = process.env.NODE_ENV?.toLowerCase() === 'production'; + const isDebugMode = process.env.DEBUG_MODE?.toLowerCase() === 'true'; + const logHttp = process.env.LOG_HTTP?.toLowerCase() === 'true'; const level = isProduction && !isDebugMode ? 'info' : 'debug'; @@ -32,14 +33,20 @@ export function createPinoConfig(): Params { : undefined, formatters: { level: (label) => ({ level: label }), - log: (object: Record) => { + }, + hooks: { + logMethod(inputArgs, method) { if (isProduction && !isDebugMode) { - const context = object['context'] as string | undefined; - if (context && CONTEXTS_TO_IGNORE.includes(context)) { - return { filtered: true }; + for (const arg of inputArgs) { + if (typeof arg === 'object' && arg !== null && 'context' in arg) { + const context = (arg as Record)['context']; + if (typeof context === 'string' && CONTEXTS_TO_IGNORE.includes(context)) { + return; + } + } } } - return object; + return method.apply(this, inputArgs); }, }, serializers: { diff --git a/apps/server/src/ee b/apps/server/src/ee index b6844b01..88e3d01f 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit b6844b019c3778d51ff1bb236f30284a0bf8f403 +Subproject commit 88e3d01f8135c2dbc628b9636ba91bb9ffd2f0eb From 1ca7d422034458bc30960d87706abfbe525f5b37 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Sun, 25 Jan 2026 02:49:25 +0000 Subject: [PATCH 03/10] fix switch space toggle --- .../features/space/components/sidebar/switch-space.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/client/src/features/space/components/sidebar/switch-space.tsx b/apps/client/src/features/space/components/sidebar/switch-space.tsx index 60349ecc..89e0f64e 100644 --- a/apps/client/src/features/space/components/sidebar/switch-space.tsx +++ b/apps/client/src/features/space/components/sidebar/switch-space.tsx @@ -3,7 +3,7 @@ import { useNavigate } from "react-router-dom"; import { SpaceSelect } from "./space-select"; import { getSpaceUrl } from "@/lib/config"; import { Button, Popover, Text } from "@mantine/core"; -import { IconChevronDown } from "@tabler/icons-react"; +import { IconChevronDown, IconChevronUp } from "@tabler/icons-react"; import { useDisclosure } from "@mantine/hooks"; import { CustomAvatar } from "@/components/ui/custom-avatar.tsx"; import { AvatarIconType } from "@/features/attachments/types/attachment.types.ts"; @@ -21,7 +21,7 @@ export function SwitchSpace({ spaceIcon, }: SwitchSpaceProps) { const navigate = useNavigate(); - const [opened, { close, open, toggle }] = useDisclosure(false); + const [opened, { close, toggle }] = useDisclosure(false); const handleSelect = (value: string) => { if (value) { @@ -44,9 +44,9 @@ export function SwitchSpace({ variant="subtle" fullWidth justify="space-between" - rightSection={} + rightSection={opened ? : } color="gray" - onClick={open} + onClick={toggle} > Date: Sun, 25 Jan 2026 12:38:44 +0000 Subject: [PATCH 04/10] fix(tree): update sidebar-pages cache directly instead of refetching on page move (#1870) --- .../src/features/page/queries/page-query.ts | 135 ++++++++++++++++-- .../page/tree/hooks/use-tree-mutation.ts | 19 +++ .../src/features/websocket/types/types.ts | 2 + .../websocket/use-query-subscription.ts | 10 +- 4 files changed, 150 insertions(+), 16 deletions(-) diff --git a/apps/client/src/features/page/queries/page-query.ts b/apps/client/src/features/page/queries/page-query.ts index 64d03ddd..e7388fe4 100644 --- a/apps/client/src/features/page/queries/page-query.ts +++ b/apps/client/src/features/page/queries/page-query.ts @@ -163,9 +163,6 @@ export function useDeletePageMutation() { export function useMovePageMutation() { return useMutation({ mutationFn: (data) => movePage(data), - onSuccess: () => { - invalidateOnMovePage(); - }, }); } @@ -458,17 +455,127 @@ export function invalidateOnUpdatePage( }); } -export function invalidateOnMovePage() { - //for move invalidate all sidebars for now (how to do???) - //invalidate all root sidebar pages - queryClient.invalidateQueries({ - queryKey: ["root-sidebar-pages"], - }); - //invalidate all sub sidebar pages - queryClient.invalidateQueries({ - queryKey: ["sidebar-pages"], - }); - // --- +export function updateCacheOnMovePage( + spaceId: string, + pageId: string, + oldParentId: string | null, + newParentId: string | null, + pageData: Partial, +) { + // Remove page from old parent's cache + const oldQueryKey = + oldParentId === null + ? ["root-sidebar-pages", spaceId] + : ["sidebar-pages", { pageId: oldParentId, spaceId }]; + + queryClient.setQueryData>>( + oldQueryKey, + (old) => { + if (!old) return old; + return { + ...old, + pages: old.pages.map((page) => ({ + ...page, + items: page.items.filter((item) => item.id !== pageId), + })), + }; + }, + ); + + // Update old parent's hasChildren flag if it has no more children + if (oldParentId !== null) { + const oldParentCache = queryClient.getQueryData< + InfiniteData> + >(["sidebar-pages", { pageId: oldParentId, spaceId }]); + + const remainingChildren = + oldParentCache?.pages.flatMap((p) => p.items).length ?? 0; + + if (remainingChildren === 0) { + // Update hasChildren in all caches where old parent appears + const allSideBarMatches = queryClient.getQueriesData({ + predicate: (query) => + query.queryKey[0] === "root-sidebar-pages" || + query.queryKey[0] === "sidebar-pages", + }); + + allSideBarMatches.forEach(([key]) => { + queryClient.setQueryData>>( + key, + (old) => { + if (!old) return old; + return { + ...old, + pages: old.pages.map((page) => ({ + ...page, + items: page.items.map((item) => + item.id === oldParentId + ? { ...item, hasChildren: false } + : item, + ), + })), + }; + }, + ); + }); + } + } + + // Add page to new parent's cache + const newQueryKey = + newParentId === null + ? ["root-sidebar-pages", spaceId] + : ["sidebar-pages", { pageId: newParentId, spaceId }]; + + queryClient.setQueryData>>>( + newQueryKey, + (old) => { + if (!old) return old; + + // Check if page already exists in new location + const exists = old.pages.some((page) => + page.items.some((item) => item.id === pageId), + ); + if (exists) return old; + + return { + ...old, + pages: old.pages.map((page, index) => { + if (index === old.pages.length - 1) { + return { + ...page, + items: [...page.items, pageData], + }; + } + return page; + }), + }; + }, + ); + + // Update new parent's hasChildren flag + if (newParentId !== null) { + const allSideBarMatches = queryClient.getQueriesData({ + predicate: (query) => + query.queryKey[0] === "root-sidebar-pages" || + query.queryKey[0] === "sidebar-pages", + }); + + allSideBarMatches.forEach(([key]) => { + queryClient.setQueryData>>(key, (old) => { + if (!old) return old; + return { + ...old, + pages: old.pages.map((page) => ({ + ...page, + items: page.items.map((item) => + item.id === newParentId ? { ...item, hasChildren: true } : item, + ), + })), + }; + }); + }); + } } export function invalidateOnDeletePage(pageId: string) { diff --git a/apps/client/src/features/page/tree/hooks/use-tree-mutation.ts b/apps/client/src/features/page/tree/hooks/use-tree-mutation.ts index b2a58f30..162992dd 100644 --- a/apps/client/src/features/page/tree/hooks/use-tree-mutation.ts +++ b/apps/client/src/features/page/tree/hooks/use-tree-mutation.ts @@ -16,6 +16,7 @@ import { useRemovePageMutation, useMovePageMutation, useUpdatePageMutation, + updateCacheOnMovePage, } from "@/features/page/queries/page-query.ts"; import { generateJitteredKeyBetween } from "fractional-indexing-jittered"; import { SpaceTreeNode } from "@/features/page/tree/types.ts"; @@ -175,9 +176,25 @@ export function useTreeMutation(spaceId: string) { parentPageId: args.parentId, }; + const draggedNode = args.dragNodes[0]; + const nodeData = draggedNode.data as SpaceTreeNode; + const oldParentId = nodeData.parentPageId ?? null; + const pageData = { + id: nodeData.id, + slugId: nodeData.slugId, + title: nodeData.name, + icon: nodeData.icon, + position: newPosition, + spaceId: nodeData.spaceId, + parentPageId: args.parentId, + hasChildren: nodeData.hasChildren, + }; + try { await movePageMutation.mutateAsync(payload); + updateCacheOnMovePage(spaceId, draggedNodeId, oldParentId, args.parentId, pageData); + setTimeout(() => { emit({ operation: "moveTreeNode", @@ -185,8 +202,10 @@ export function useTreeMutation(spaceId: string) { payload: { id: draggedNodeId, parentId: args.parentId, + oldParentId, index: args.index, position: newPosition, + pageData, }, }); }, 50); diff --git a/apps/client/src/features/websocket/types/types.ts b/apps/client/src/features/websocket/types/types.ts index 21561038..3bc5b941 100644 --- a/apps/client/src/features/websocket/types/types.ts +++ b/apps/client/src/features/websocket/types/types.ts @@ -45,8 +45,10 @@ export type MoveTreeNodeEvent = { payload: { id: string; parentId: string; + oldParentId: string | null; index: number; position: string; + pageData: Partial; }; }; diff --git a/apps/client/src/features/websocket/use-query-subscription.ts b/apps/client/src/features/websocket/use-query-subscription.ts index 3aa95417..faa7139f 100644 --- a/apps/client/src/features/websocket/use-query-subscription.ts +++ b/apps/client/src/features/websocket/use-query-subscription.ts @@ -8,7 +8,7 @@ import { IPagination } from "@/lib/types"; import { invalidateOnCreatePage, invalidateOnDeletePage, - invalidateOnMovePage, + updateCacheOnMovePage, invalidateOnUpdatePage, } from "../page/queries/page-query"; import { RQ_KEY } from "../comment/queries/comment-query"; @@ -41,7 +41,13 @@ export const useQuerySubscription = () => { invalidateOnCreatePage(data.payload.data); break; case "moveTreeNode": - invalidateOnMovePage(); + updateCacheOnMovePage( + data.spaceId, + data.payload.id, + data.payload.oldParentId, + data.payload.parentId, + data.payload.pageData, + ); break; case "deleteTreeNode": invalidateOnDeletePage(data.payload.node.id); From de5f71894a7994a225cc740565746b9e261a7141 Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:39:19 +0000 Subject: [PATCH 05/10] New Crowdin updates (#1869) * New translations translation.json (Japanese) * New translations translation.json (French) * New translations translation.json (Spanish) * New translations translation.json (German) * New translations translation.json (Italian) * New translations translation.json (Korean) * New translations translation.json (Dutch) * New translations translation.json (Russian) * New translations translation.json (Ukrainian) * New translations translation.json (Chinese Simplified) * New translations translation.json (Portuguese, Brazilian) --- apps/client/public/locales/de-DE/translation.json | 8 ++++---- apps/client/public/locales/es-ES/translation.json | 8 ++++---- apps/client/public/locales/fr-FR/translation.json | 8 ++++---- apps/client/public/locales/it-IT/translation.json | 8 ++++---- apps/client/public/locales/ja-JP/translation.json | 8 ++++---- apps/client/public/locales/ko-KR/translation.json | 8 ++++---- apps/client/public/locales/nl-NL/translation.json | 8 ++++---- apps/client/public/locales/pt-BR/translation.json | 8 ++++---- apps/client/public/locales/ru-RU/translation.json | 8 ++++---- apps/client/public/locales/uk-UA/translation.json | 8 ++++---- apps/client/public/locales/zh-CN/translation.json | 8 ++++---- 11 files changed, 44 insertions(+), 44 deletions(-) diff --git a/apps/client/public/locales/de-DE/translation.json b/apps/client/public/locales/de-DE/translation.json index 34fb5bb4..93c6f265 100644 --- a/apps/client/public/locales/de-DE/translation.json +++ b/apps/client/public/locales/de-DE/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Wählen Sie Ihre bevorzugte Benutzersprache.", "Choose your preferred page width.": "Wählen Sie Ihre bevorzugte Seitenbreite.", "Confirm": "Bestätigen", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Als Markdown kopieren", "Copy link": "Link kopieren", "Create": "Erstellen", "Create group": "Gruppe erstellen", @@ -254,7 +254,7 @@ "Export failed:": "Export fehlgeschlagen:", "export error": "Exportfehler", "Export page": "Seite exportieren", - "Export successful": "Export successful", + "Export successful": "Export erfolgreich", "Export space": "Bereich exportieren", "Export {{type}}": "Exportiere {{type}}", "File exceeds the {{limit}} attachment limit": "Datei überschreitet das Anhängelimit von {{limit}}", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Laden Sie ein beliebiges Bild von Ihrem Gerät hoch.", "Upload any video from your device.": "Laden Sie ein beliebiges Video von Ihrem Gerät hoch.", "Upload any file from your device.": "Laden Sie eine beliebige Datei von Ihrem Gerät hoch.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Lade {{name}} hoch", + "Uploading file": "Datei wird hochgeladen", "Table": "Tabelle", "Insert a table.": "Tabelle einfügen.", "Insert collapsible block.": "Einklappbaren Block einfügen.", diff --git a/apps/client/public/locales/es-ES/translation.json b/apps/client/public/locales/es-ES/translation.json index d68f64a7..af02c493 100644 --- a/apps/client/public/locales/es-ES/translation.json +++ b/apps/client/public/locales/es-ES/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Elige tu idioma de interfaz preferido.", "Choose your preferred page width.": "Elige el ancho de página que prefieras.", "Confirm": "Confirmar", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Copiar como Markdown", "Copy link": "Copiar enlace", "Create": "Crear", "Create group": "Crear grupo", @@ -254,7 +254,7 @@ "Export failed:": "Exportación fallida:", "export error": "error de exportación", "Export page": "Exportar página", - "Export successful": "Export successful", + "Export successful": "Exportación exitosa", "Export space": "Exportar espacio", "Export {{type}}": "Exportar {{type}}", "File exceeds the {{limit}} attachment limit": "El archivo supera el límite de {{limit}} adjuntos", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Sube cualquier imagen desde tu dispositivo.", "Upload any video from your device.": "Sube cualquier video desde tu dispositivo.", "Upload any file from your device.": "Sube cualquier archivo desde tu dispositivo.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Subiendo {{name}}", + "Uploading file": "Subiendo archivo", "Table": "Tabla", "Insert a table.": "Insertar una tabla.", "Insert collapsible block.": "Insertar bloque desplegable.", diff --git a/apps/client/public/locales/fr-FR/translation.json b/apps/client/public/locales/fr-FR/translation.json index 4a70b735..40a1e68a 100644 --- a/apps/client/public/locales/fr-FR/translation.json +++ b/apps/client/public/locales/fr-FR/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Choisissez votre langue d'interface préférée.", "Choose your preferred page width.": "Choisissez votre largeur de page préférée.", "Confirm": "Confirmer", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Copier comme Markdown", "Copy link": "Copier le lien", "Create": "Créer", "Create group": "Créer groupe", @@ -254,7 +254,7 @@ "Export failed:": "Échec de l'exportation :", "export error": "exporter l'erreur", "Export page": "Exporter la page", - "Export successful": "Export successful", + "Export successful": "Exportation réussie", "Export space": "Exporter l'espace", "Export {{type}}": "Exporter {{type}}", "File exceeds the {{limit}} attachment limit": "Le fichier dépasse la limite de {{limit}} pièces jointes", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Téléchargez n'importe quelle image depuis votre appareil.", "Upload any video from your device.": "Téléchargez n'importe quelle vidéo depuis votre appareil.", "Upload any file from your device.": "Téléchargez n'importe quel fichier depuis votre appareil.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Téléchargement de {{name}}", + "Uploading file": "Téléchargement du fichier", "Table": "Tableau", "Insert a table.": "Insérez un tableau.", "Insert collapsible block.": "Insérer un bloc repliable.", diff --git a/apps/client/public/locales/it-IT/translation.json b/apps/client/public/locales/it-IT/translation.json index a716a86a..ff80df0f 100644 --- a/apps/client/public/locales/it-IT/translation.json +++ b/apps/client/public/locales/it-IT/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Scegli la lingua da utilizzare per l'interfaccia.", "Choose your preferred page width.": "Scegli la larghezza della pagina che preferisci.", "Confirm": "Conferma", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Copia come Markdown", "Copy link": "Copia link", "Create": "Crea", "Create group": "Crea gruppo", @@ -254,7 +254,7 @@ "Export failed:": "Esportazione fallita:", "export error": "errore di esportazione", "Export page": "Esporta pagina", - "Export successful": "Export successful", + "Export successful": "Esportazione riuscita", "Export space": "Esporta spazio", "Export {{type}}": "Esporta {{type}}", "File exceeds the {{limit}} attachment limit": "Il file supera il limite per gli allegati di {{limit}}", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Carica un'immagine dal tuo dispositivo.", "Upload any video from your device.": "Carica qualsiasi video dal tuo dispositivo.", "Upload any file from your device.": "Carica qualsiasi file dal tuo dispositivo.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Caricamento di {{name}}", + "Uploading file": "Caricamento file", "Table": "Tabella", "Insert a table.": "Inserisci una tabella.", "Insert collapsible block.": "Inserisci blocco comprimibile.", diff --git a/apps/client/public/locales/ja-JP/translation.json b/apps/client/public/locales/ja-JP/translation.json index be341728..4d18e074 100644 --- a/apps/client/public/locales/ja-JP/translation.json +++ b/apps/client/public/locales/ja-JP/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "お好みの言語を選択してください", "Choose your preferred page width.": "お好みのページ幅を選択してください", "Confirm": "確認", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Markdownとしてコピー", "Copy link": "リンクをコピー", "Create": "新規作成", "Create group": "グループを作成", @@ -254,7 +254,7 @@ "Export failed:": "エクスポートに失敗しました:", "export error": "エクスポートエラー", "Export page": "エクスポートページ", - "Export successful": "Export successful", + "Export successful": "エクスポート成功", "Export space": "エクスポートスペース", "Export {{type}}": "{{type}}をエクスポート", "File exceeds the {{limit}} attachment limit": "ファイルが{{limit}}の添付制限を超えています", @@ -330,8 +330,8 @@ "Upload any image from your device.": "デバイスから画像をアップロードします", "Upload any video from your device.": "デバイスから動画をアップロードします", "Upload any file from your device.": "デバイスからファイルをアップロードします", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "{{name}} をアップロード中", + "Uploading file": "ファイルをアップロード中", "Table": "テーブル", "Insert a table.": "テーブルを挿入します", "Insert collapsible block.": "折りたたみブロックを挿入します", diff --git a/apps/client/public/locales/ko-KR/translation.json b/apps/client/public/locales/ko-KR/translation.json index 6e83db5f..d9b48b04 100644 --- a/apps/client/public/locales/ko-KR/translation.json +++ b/apps/client/public/locales/ko-KR/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "선호하는 인터페이스 언어를 선택하세요.", "Choose your preferred page width.": "선호하는 페이지 너비를 선택하세요.", "Confirm": "확인", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Markdown으로 복사", "Copy link": "링크 복사", "Create": "생성", "Create group": "팀 생성", @@ -254,7 +254,7 @@ "Export failed:": "내보내기 실패:", "export error": "내보내기 오류", "Export page": "페이지 내보내기", - "Export successful": "Export successful", + "Export successful": "내보내기 성공", "Export space": "Space 내보내기", "Export {{type}}": "{{type}} 내보내기", "File exceeds the {{limit}} attachment limit": "첨부 파일 크기 제한 {{limit}}을 초과했습니다", @@ -330,8 +330,8 @@ "Upload any image from your device.": "기기에서 이미지를 업로드하세요.", "Upload any video from your device.": "기기에서 비디오를 업로드하세요.", "Upload any file from your device.": "기기에서 파일을 업로드하세요.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "{{name}} 업로드 중", + "Uploading file": "파일 업로드 중", "Table": "테이블", "Insert a table.": "테이블 삽입.", "Insert collapsible block.": "접을 수 있는 블록 삽입.", diff --git a/apps/client/public/locales/nl-NL/translation.json b/apps/client/public/locales/nl-NL/translation.json index 9c16efe3..a7923b98 100644 --- a/apps/client/public/locales/nl-NL/translation.json +++ b/apps/client/public/locales/nl-NL/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Kies uw gewenste interfacetaal.", "Choose your preferred page width.": "Kies uw gewenste paginabreedte.", "Confirm": "Bevestig", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Kopiëren als Markdown", "Copy link": "Link kopiëren", "Create": "Aanmaken", "Create group": "Groep aanmaken", @@ -254,7 +254,7 @@ "Export failed:": "Exporteren mislukt:", "export error": "Exporteer fout", "Export page": "Exporteer pagina", - "Export successful": "Export successful", + "Export successful": "Export succesvol", "Export space": "Exporteer ruimte", "Export {{type}}": "Exporteer {{type}}", "File exceeds the {{limit}} attachment limit": "Bestand overschrijdt de bijlagelimiet van {{limit}}", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Upload een afbeelding vanaf uw apparaat.", "Upload any video from your device.": "Upload een video vanaf uw apparaat.", "Upload any file from your device.": "Upload een bestand vanaf uw apparaat.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Uploaden {{name}}", + "Uploading file": "Bestand uploaden", "Table": "Tabel", "Insert a table.": "Voeg een tabel in.", "Insert collapsible block.": "Inklapbaar blok invoegen.", diff --git a/apps/client/public/locales/pt-BR/translation.json b/apps/client/public/locales/pt-BR/translation.json index eb1442b2..30cc0b21 100644 --- a/apps/client/public/locales/pt-BR/translation.json +++ b/apps/client/public/locales/pt-BR/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Escolha o idioma da interface.", "Choose your preferred page width.": "Escolha a largura preferida da página.", "Confirm": "Confirmar", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Copiar como Markdown", "Copy link": "Copiar link", "Create": "Criar", "Create group": "Criar grupo", @@ -254,7 +254,7 @@ "Export failed:": "Falha ao exportar:", "export error": "erro de exportação", "Export page": "Exportar página", - "Export successful": "Export successful", + "Export successful": "Exportação bem-sucedida", "Export space": "Exportar espaço", "Export {{type}}": "Exportar para {{type}}", "File exceeds the {{limit}} attachment limit": "O arquivo excede o limite de anexos {{limit}}", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Envie qualquer imagem do seu dispositivo.", "Upload any video from your device.": "Envie qualquer vídeo do seu dispositivo.", "Upload any file from your device.": "Envie qualquer arquivo do seu dispositivo.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Enviando {{name}}", + "Uploading file": "Enviando arquivo", "Table": "Tabela", "Insert a table.": "Insira uma tabela.", "Insert collapsible block.": "Insira um bloco colapsável.", diff --git a/apps/client/public/locales/ru-RU/translation.json b/apps/client/public/locales/ru-RU/translation.json index b39d13a4..88e1f701 100644 --- a/apps/client/public/locales/ru-RU/translation.json +++ b/apps/client/public/locales/ru-RU/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Выберите предпочитаемый язык интерфейса.", "Choose your preferred page width.": "Выберите предпочитаемую ширину страницы.", "Confirm": "Подтвердить", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Копировать как Markdown", "Copy link": "Копировать ссылку", "Create": "Создать", "Create group": "Создать группу", @@ -254,7 +254,7 @@ "Export failed:": "Экспортирование не удалось:", "export error": "ошибка экспорта", "Export page": "Экспорт страницы", - "Export successful": "Export successful", + "Export successful": "Экспорт выполнен успешно", "Export space": "Экспорт пространства", "Export {{type}}": "Экспорт {{type}}", "File exceeds the {{limit}} attachment limit": "Файл превышает лимит вложений {{limit}}", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Загрузить любое изображение с вашего устройства.", "Upload any video from your device.": "Загрузить любое видео с вашего устройства.", "Upload any file from your device.": "Загрузить любой файл с вашего устройства.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Загрузка {{name}}", + "Uploading file": "Загрузка файла", "Table": "Таблица", "Insert a table.": "Вставить таблицу.", "Insert collapsible block.": "Вставить сворачиваемый блок.", diff --git a/apps/client/public/locales/uk-UA/translation.json b/apps/client/public/locales/uk-UA/translation.json index 2460f38f..e5cdaa40 100644 --- a/apps/client/public/locales/uk-UA/translation.json +++ b/apps/client/public/locales/uk-UA/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "Оберіть бажану мову інтерфейсу.", "Choose your preferred page width.": "Оберіть бажану ширину сторінки.", "Confirm": "Підтвердити", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "Скопіювати як Markdown", "Copy link": "Копіювати посилання", "Create": "Створити", "Create group": "Створити групу", @@ -254,7 +254,7 @@ "Export failed:": "Експортування не вдалося:", "export error": "помилка експорту", "Export page": "Експорт сторінки", - "Export successful": "Export successful", + "Export successful": "Експорт виконано успішно", "Export space": "Експорт простору", "Export {{type}}": "Експорт {{type}}", "File exceeds the {{limit}} attachment limit": "Файл перевищує ліміт вкладень {{limit}}", @@ -330,8 +330,8 @@ "Upload any image from your device.": "Завантажити будь-яке зображення з вашого пристрою.", "Upload any video from your device.": "Завантажити будь-яке відео з вашого пристрою.", "Upload any file from your device.": "Завантажити будь-який файл з вашого пристрою.", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "Завантаження {{name}}", + "Uploading file": "Завантаження файлу", "Table": "Таблиця", "Insert a table.": "Вставити таблицю.", "Insert collapsible block.": "Вставити блок, що згортається.", diff --git a/apps/client/public/locales/zh-CN/translation.json b/apps/client/public/locales/zh-CN/translation.json index ed26024f..a5eb84f1 100644 --- a/apps/client/public/locales/zh-CN/translation.json +++ b/apps/client/public/locales/zh-CN/translation.json @@ -29,7 +29,7 @@ "Choose your preferred interface language.": "选择您喜欢的界面语言。", "Choose your preferred page width.": "选择您喜欢的页面宽度。", "Confirm": "确认", - "Copy as Markdown": "Copy as Markdown", + "Copy as Markdown": "复制为Markdown", "Copy link": "复制链接", "Create": "创建", "Create group": "创建群组", @@ -254,7 +254,7 @@ "Export failed:": "导出失败：", "export error": "导出出错", "Export page": "导出页面", - "Export successful": "Export successful", + "Export successful": "导出成功", "Export space": "导出空间", "Export {{type}}": "导出为 {{type}}", "File exceeds the {{limit}} attachment limit": "文件超出了 {{limit}} 类型附件限制", @@ -330,8 +330,8 @@ "Upload any image from your device.": "从设备上传任何图像", "Upload any video from your device.": "从设备上传任何视频", "Upload any file from your device.": "从设备上传任何文件", - "Uploading {{name}}": "Uploading {{name}}", - "Uploading file": "Uploading file", + "Uploading {{name}}": "正在上传{{name}}", + "Uploading file": "正在上传文件", "Table": "表格", "Insert a table.": "插入一个表格", "Insert collapsible block.": "插入一个折叠块", From 0245a183e1f58b77d05eeb8794b5fe6d30ff7c42 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Mon, 26 Jan 2026 02:08:54 +0000 Subject: [PATCH 06/10] sync --- apps/server/src/ee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/ee b/apps/server/src/ee index 88e3d01f..f858f127 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit 88e3d01f8135c2dbc628b9636ba91bb9ffd2f0eb +Subproject commit f858f127b55e32d34e5a2b9867bf512dd152ba09 From 6ccb2bb8727549239562774e886ddbef01993c8c Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:39:39 +0000 Subject: [PATCH 07/10] feat(export): add metadata file to preserve page icons and ordering on import (#1877) * feat(export): add metadata file to preserve page icons and ordering on import - Export includes `docmost-metadata.json` - Import reads metadata to restore icons and sort siblings by original position * cleanup * bonus fixes * handle unknown prosemirror nodes * add docmost app version --- .../page/components/page-import-modal.tsx | 4 ++ .../space/components/delete-space-modal.tsx | 7 ++- .../src/collaboration/collaboration.util.ts | 49 ++++++++++++++++++- .../helpers/types/export-metadata.types.ts | 14 ++++++ .../src/integrations/export/export.service.ts | 34 ++++++++++++- .../integrations/import/dto/file-task-dto.ts | 1 + .../services/file-import-task.service.ts | 44 +++++++++++++++-- .../integrations/import/utils/import.utils.ts | 34 ++++++++++++- 8 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/common/helpers/types/export-metadata.types.ts diff --git a/apps/client/src/features/page/components/page-import-modal.tsx b/apps/client/src/features/page/components/page-import-modal.tsx index a2df380e..be0264b6 100644 --- a/apps/client/src/features/page/components/page-import-modal.tsx +++ b/apps/client/src/features/page/components/page-import-modal.tsx @@ -172,6 +172,10 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { queryKey: ["root-sidebar-pages", fileTask.spaceId], }); + await queryClient.invalidateQueries({ + queryKey: ["recent-changes", fileTask.spaceId], + }); + setTimeout(() => { emit({ operation: "refetchRootTreeNodeEvent", diff --git a/apps/client/src/features/space/components/delete-space-modal.tsx b/apps/client/src/features/space/components/delete-space-modal.tsx index f697322d..8a89e720 100644 --- a/apps/client/src/features/space/components/delete-space-modal.tsx +++ b/apps/client/src/features/space/components/delete-space-modal.tsx @@ -6,6 +6,7 @@ import { ISpace } from "../types/space.types"; import { useNavigate } from "react-router-dom"; import APP_ROUTE from "@/lib/app-route"; import { Trans, useTranslation } from "react-i18next"; +import { useState } from "react"; interface DeleteSpaceModalProps { space: ISpace; @@ -14,6 +15,7 @@ interface DeleteSpaceModalProps { export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) { const { t } = useTranslation(); const [opened, { open, close }] = useDisclosure(false); + const [isDeleting, setIsDeleting] = useState(false); const deleteSpaceMutation = useDeleteSpaceMutation(); const navigate = useNavigate(); @@ -35,12 +37,15 @@ export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) { return; } + setIsDeleting(true); try { // pass slug too so we can clear the local cache await deleteSpaceMutation.mutateAsync({ id: space.id, slug: space.slug }); navigate(APP_ROUTE.HOME); } catch (error) { console.error("Failed to delete space", error); + } finally { + setIsDeleting(false); } }; @@ -79,7 +84,7 @@ export default function DeleteSpaceModal({ space }: DeleteSpaceModalProps) { - diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index 16ca5bd5..afe1be08 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -41,7 +41,8 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; // see: https://github.com/ueberdosis/tiptap/issues/5352 // see:https://github.com/ueberdosis/tiptap/issues/4089 //import { generateJSON } from '@tiptap/html'; -import { Node } from '@tiptap/pm/model'; +import { Node, Schema } from '@tiptap/pm/model'; +import { Logger } from '@nestjs/common'; export const tiptapExtensions = [ StarterKit.configure({ @@ -110,9 +111,53 @@ export function jsonToText(tiptapJson: JSONContent) { } export function jsonToNode(tiptapJson: JSONContent) { - return Node.fromJSON(getSchema(tiptapExtensions), tiptapJson); + const schema = getSchema(tiptapExtensions); + try { + return Node.fromJSON(schema, tiptapJson); + } catch (error) { + if ( + error instanceof RangeError && + error.message.includes('Unknown node type') + ) { + Logger.warn('Stripping unknown node types from document:', error.message); + const cleanedJson = stripUnknownNodes(tiptapJson, schema); + return Node.fromJSON(schema, cleanedJson); + } + throw error; + } } export function getPageId(documentName: string) { return documentName.split('.')[1]; } + +function stripUnknownNodes( + json: JSONContent, + schema: Schema, +): JSONContent | null { + if (!json || typeof json !== 'object') return json; + + // Recursively clean children first, flattening any unwrapped content + if (json.content && Array.isArray(json.content)) { + const newContent: JSONContent[] = []; + for (const child of json.content) { + const cleaned = stripUnknownNodes(child, schema); + if (Array.isArray(cleaned)) { + newContent.push(...cleaned); + } else if (cleaned) { + newContent.push(cleaned); + } + } + json.content = newContent; + } + + // Check if this node is unknown AFTER processing children + if (json.type && !schema.nodes[json.type]) { + // Unwrap: return cleaned children directly instead of wrapping + return ( + json.content && json.content.length > 0 ? json.content : null + ) as any; + } + + return json; +} diff --git a/apps/server/src/common/helpers/types/export-metadata.types.ts b/apps/server/src/common/helpers/types/export-metadata.types.ts new file mode 100644 index 00000000..42ef4c68 --- /dev/null +++ b/apps/server/src/common/helpers/types/export-metadata.types.ts @@ -0,0 +1,14 @@ +export type ExportPageMetadata = { + pageId: string; + slugId: string; + icon: string | null; + position: string; + parentPath: string | null; +}; + +export type ExportMetadata = { + exportedAt: string; + source: 'docmost'; + version: string; + pages: Record; +}; diff --git a/apps/server/src/integrations/export/export.service.ts b/apps/server/src/integrations/export/export.service.ts index e33ac11b..44047174 100644 --- a/apps/server/src/integrations/export/export.service.ts +++ b/apps/server/src/integrations/export/export.service.ts @@ -20,11 +20,17 @@ import { replaceInternalLinks, updateAttachmentUrlsToLocalPaths, } from './utils'; +import { + ExportMetadata, + ExportPageMetadata, +} from '../../common/helpers/types/export-metadata.types'; import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { Node } from '@tiptap/pm/model'; import { EditorState } from '@tiptap/pm/state'; // eslint-disable-next-line @typescript-eslint/no-require-imports import slugify = require('@sindresorhus/slugify'); +// eslint-disable-next-line @typescript-eslint/no-require-imports +const packageJson = require('../../../package.json'); import { EnvironmentService } from '../environment/environment.service'; import { getAttachmentIds, @@ -155,12 +161,15 @@ export class ExportService { 'pages.id', 'pages.slugId', 'pages.title', + 'pages.icon', + 'pages.position', 'pages.content', 'pages.parentPageId', 'pages.spaceId', 'pages.workspaceId', ]) .where('spaceId', '=', spaceId) + .where('deletedAt', 'is', null) .execute(); const tree = buildTree(pages as Page[]); @@ -189,10 +198,12 @@ export class ExportService { includeAttachments: boolean, ): Promise { const slugIdToPath: Record = {}; + const pageIdToFilePath: Record = {}; + const pagesMetadata: Record = {}; computeLocalPath(tree, format, null, '', slugIdToPath); - const stack: { folder: JSZip; parentPageId: string }[] = [ + const stack: { folder: JSZip; parentPageId: string | null }[] = [ { folder: zip, parentPageId: null }, ]; @@ -232,12 +243,33 @@ export class ExportService { `${pageTitle}${getExportExtension(format)}`, pageExportContent, ); + + pageIdToFilePath[page.id] = currentPagePath; + + const parentPath = parentPageId ? pageIdToFilePath[parentPageId] : null; + pagesMetadata[currentPagePath] = { + pageId: page.id, + slugId: page.slugId, + icon: page.icon ?? null, + position: page.position, + parentPath, + }; + if (childPages.length > 0) { const pageFolder = folder.folder(pageTitle); stack.push({ folder: pageFolder, parentPageId: page.id }); } } } + + const metadata: ExportMetadata = { + exportedAt: new Date().toISOString(), + source: 'docmost', + version: packageJson.version, + pages: pagesMetadata, + }; + + zip.file('docmost-metadata.json', JSON.stringify(metadata, null, 2)); } async zipAttachments(prosemirrorJson: any, spaceId: string, zip: JSZip) { diff --git a/apps/server/src/integrations/import/dto/file-task-dto.ts b/apps/server/src/integrations/import/dto/file-task-dto.ts index 9cdea395..84736813 100644 --- a/apps/server/src/integrations/import/dto/file-task-dto.ts +++ b/apps/server/src/integrations/import/dto/file-task-dto.ts @@ -15,4 +15,5 @@ export type ImportPageNode = { parentPageId: string | null; fileExtension: string; filePath: string; + icon?: string | null; }; \ No newline at end of file diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index 5cf39054..8ae79598 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -24,6 +24,8 @@ import { formatImportHtml } from '../utils/import-formatter'; import { buildAttachmentCandidates, collectMarkdownAndHtmlFiles, + encodeFilePath, + readDocmostMetadata, stripNotionID, } from '../utils/import.utils'; import { executeTx } from '@docmost/db/utils'; @@ -154,6 +156,7 @@ export class FileImportTaskService { const { extractDir, fileTask } = opts; const allFiles = await collectMarkdownAndHtmlFiles(extractDir); const attachmentCandidates = await buildAttachmentCandidates(extractDir); + const docmostMetadata = await readDocmostMetadata(extractDir); const pagesMap = new Map(); @@ -164,6 +167,9 @@ export class FileImportTaskService { .join('/'); // normalize to forward-slashes const ext = path.extname(relPath).toLowerCase(); + const encodedPath = encodeFilePath(relPath); + const pageMetadata = docmostMetadata?.pages[encodedPath]; + pagesMap.set(relPath, { id: v7(), slugId: generateSlugId(), @@ -172,6 +178,7 @@ export class FileImportTaskService { parentPageId: null, fileExtension: ext, filePath: relPath, + icon: pageMetadata?.icon ?? null, }); } @@ -224,6 +231,8 @@ export class FileImportTaskService { if (!pagesMap.has(mdPath) && !pagesMap.has(htmlPath)) { const folderName = path.basename(folderPath); + const encodedMdPath = encodeFilePath(mdPath); + const placeholderMetadata = docmostMetadata?.pages[encodedMdPath]; pagesMap.set(mdPath, { id: v7(), slugId: generateSlugId(), @@ -232,6 +241,7 @@ export class FileImportTaskService { parentPageId: null, fileExtension: '.md', filePath: mdPath, + icon: placeholderMetadata?.icon ?? null, }); } }); @@ -266,11 +276,39 @@ export class FileImportTaskService { siblingsMap.set(page.parentPageId, group); }); + const encodedPathsMap = new Map(); + if (docmostMetadata) { + pagesMap.forEach((_, filePath) => { + encodedPathsMap.set(filePath, encodeFilePath(filePath)); + }); + } + + // Sort siblings by metadata position if available, otherwise alphabetically + const sortSiblings = (siblings: ImportPageNode[]) => { + if (docmostMetadata) { + siblings.sort((a, b) => { + const posA = + docmostMetadata.pages[encodedPathsMap.get(a.filePath)]?.position; + const posB = + docmostMetadata.pages[encodedPathsMap.get(b.filePath)]?.position; + if (posA && posB) { + // Use direct comparison to match PostgreSQL collation 'C' (byte order) + if (posA < posB) return -1; + if (posA > posB) return 1; + return 0; + } + return a.name.localeCompare(b.name); + }); + } else { + siblings.sort((a, b) => a.name.localeCompare(b.name)); + } + }; + // get root pages const rootSibs = siblingsMap.get(null); if (rootSibs?.length) { - rootSibs.sort((a, b) => a.name.localeCompare(b.name)); + sortSiblings(rootSibs); // get first position key from the server const nextPosition = await this.pageService.nextPagePosition( @@ -292,7 +330,7 @@ export class FileImportTaskService { siblingsMap.forEach((sibs, parentId) => { if (parentId === null) return; // root already done - sibs.sort((a, b) => a.name.localeCompare(b.name)); + sortSiblings(sibs); let prevPos: string | null = null; for (const page of sibs) { @@ -426,7 +464,7 @@ export class FileImportTaskService { id: page.id, slugId: page.slugId, title: title || page.name, - icon: pageIcon || null, + icon: page.icon || pageIcon || null, content: prosemirrorJson, textContent: jsonToText(prosemirrorJson), ydoc: await this.importService.createYdoc(prosemirrorJson), diff --git a/apps/server/src/integrations/import/utils/import.utils.ts b/apps/server/src/integrations/import/utils/import.utils.ts index c8f5fe51..cd348652 100644 --- a/apps/server/src/integrations/import/utils/import.utils.ts +++ b/apps/server/src/integrations/import/utils/import.utils.ts @@ -1,6 +1,7 @@ import { Logger } from '@nestjs/common'; import { promises as fs } from 'fs'; import * as path from 'path'; +import { ExportMetadata } from '../../../common/helpers/types/export-metadata.types'; export async function buildAttachmentCandidates( extractDir: string, @@ -35,9 +36,15 @@ export function resolveRelativeAttachmentPath( try { mainRel = decodeURIComponent(mainRel); } catch (err) { - Logger.warn(`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, 'ImportUtils'); + Logger.warn( + `URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, + 'ImportUtils', + ); } - const fallback = path.normalize(path.join(pageDir, mainRel)).split(path.sep).join('/'); + const fallback = path + .normalize(path.join(pageDir, mainRel)) + .split(path.sep) + .join('/'); if (attachmentCandidates.has(mainRel)) { return mainRel; @@ -76,3 +83,26 @@ export function stripNotionID(fileName: string): string { const notionIdPattern = /[ -]?[a-z0-9]{32}$/i; return fileName.replace(notionIdPattern, '').trim(); } + +export function encodeFilePath(filePath: string): string { + return filePath + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/'); +} + +export async function readDocmostMetadata( + extractDir: string, +): Promise { + const metadataPath = path.join(extractDir, 'docmost-metadata.json'); + try { + const content = await fs.readFile(metadataPath, 'utf-8'); + const metadata = JSON.parse(content) as ExportMetadata; + if (metadata.source === 'docmost' && metadata.pages) { + return metadata; + } + return null; + } catch { + return null; + } +} From 3523600f40f281407253b71b365e410f2d2f2d46 Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:49:22 +0000 Subject: [PATCH 08/10] add timestamps --- apps/server/src/common/helpers/types/export-metadata.types.ts | 2 ++ apps/server/src/database/repos/page/page.repo.ts | 4 ++++ apps/server/src/integrations/export/export.service.ts | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/apps/server/src/common/helpers/types/export-metadata.types.ts b/apps/server/src/common/helpers/types/export-metadata.types.ts index 42ef4c68..f901c0e2 100644 --- a/apps/server/src/common/helpers/types/export-metadata.types.ts +++ b/apps/server/src/common/helpers/types/export-metadata.types.ts @@ -4,6 +4,8 @@ export type ExportPageMetadata = { icon: string | null; position: string; parentPath: string | null; + createdAt: string; + updatedAt: string; }; export type ExportMetadata = { diff --git a/apps/server/src/database/repos/page/page.repo.ts b/apps/server/src/database/repos/page/page.repo.ts index f2b27abb..52337bb1 100644 --- a/apps/server/src/database/repos/page/page.repo.ts +++ b/apps/server/src/database/repos/page/page.repo.ts @@ -422,6 +422,8 @@ export class PageRepo { 'parentPageId', 'spaceId', 'workspaceId', + 'createdAt', + 'updatedAt', ]) .$if(opts?.includeContent, (qb) => qb.select('content')) .where('id', '=', parentPageId) @@ -438,6 +440,8 @@ export class PageRepo { 'p.parentPageId', 'p.spaceId', 'p.workspaceId', + 'p.createdAt', + 'p.updatedAt', ]) .$if(opts?.includeContent, (qb) => qb.select('p.content')) .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id') diff --git a/apps/server/src/integrations/export/export.service.ts b/apps/server/src/integrations/export/export.service.ts index 44047174..655e31d3 100644 --- a/apps/server/src/integrations/export/export.service.ts +++ b/apps/server/src/integrations/export/export.service.ts @@ -167,6 +167,8 @@ export class ExportService { 'pages.parentPageId', 'pages.spaceId', 'pages.workspaceId', + 'pages.createdAt', + 'pages.updatedAt', ]) .where('spaceId', '=', spaceId) .where('deletedAt', 'is', null) @@ -253,6 +255,8 @@ export class ExportService { icon: page.icon ?? null, position: page.position, parentPath, + createdAt: page.createdAt?.toISOString() ?? new Date().toISOString(), + updatedAt: page.updatedAt?.toISOString() ?? new Date().toISOString(), }; if (childPages.length > 0) { From 74e915546b537ea4134e2315e9d5ae175fba5f64 Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:05:05 +0000 Subject: [PATCH 09/10] feat: collab redis extension with server affinity (#1873) * feat(collab): better redis extension * move types to own file * debug logging * fix: graceful collab shutdown * rename default prefix * pass wsAdapter to gateway * expose event handler * unique collab serverId generation * uninstall @hocuspocus/extension-redis package * expose more functions * sync with latest * cleanup * fastify router options * cleanup type --- apps/server/package.json | 3 + .../adapter/collab-ws.adapter.ts | 16 +- .../collaboration/collaboration.gateway.ts | 158 +++++++- .../collaboration/collaboration.handler.ts | 42 ++ .../src/collaboration/collaboration.module.ts | 12 +- .../extensions/logger.extension.ts | 8 +- .../redis-sync/collab-proxy-socket.ts | 70 ++++ .../extensions/redis-sync/index.ts | 2 + .../redis-sync/redis-sync.extension.ts | 376 ++++++++++++++++++ .../extensions/redis-sync/redis-sync.types.ts | 121 ++++++ .../redis-sync/ws-socket-wrapper.ts | 47 +++ .../src/collaboration/server/collab-main.ts | 8 +- package.json | 1 - .../src/lib/attachment/attachment.ts | 2 +- packages/editor-ext/src/lib/image/image.ts | 2 +- packages/editor-ext/src/lib/video/video.ts | 2 +- pnpm-lock.yaml | 86 ++-- 17 files changed, 857 insertions(+), 99 deletions(-) create mode 100644 apps/server/src/collaboration/collaboration.handler.ts create mode 100644 apps/server/src/collaboration/extensions/redis-sync/collab-proxy-socket.ts create mode 100644 apps/server/src/collaboration/extensions/redis-sync/index.ts create mode 100644 apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts create mode 100644 apps/server/src/collaboration/extensions/redis-sync/redis-sync.types.ts create mode 100644 apps/server/src/collaboration/extensions/redis-sync/ws-socket-wrapper.ts diff --git a/apps/server/package.json b/apps/server/package.json index 71e68679..edecf07a 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -76,8 +76,10 @@ "kysely-migration-cli": "^0.4.2", "kysely-postgres-js": "^3.0.0", "ldapts": "^7.4.0", + "lib0": "^0.2.117", "mammoth": "^1.11.0", "mime-types": "^2.1.35", + "msgpackr": "^1.11.8", "nanoid": "3.3.11", "nestjs-kysely": "^1.2.0", "nestjs-pino": "^4.5.0", @@ -102,6 +104,7 @@ "socket.io": "^4.8.3", "stripe": "^17.5.0", "tmp-promise": "^3.0.3", + "tseep": "^1.3.1", "typesense": "^2.1.0", "ws": "^8.19.0", "yauzl": "^3.2.0" diff --git a/apps/server/src/collaboration/adapter/collab-ws.adapter.ts b/apps/server/src/collaboration/adapter/collab-ws.adapter.ts index 352fe01f..18685bf0 100644 --- a/apps/server/src/collaboration/adapter/collab-ws.adapter.ts +++ b/apps/server/src/collaboration/adapter/collab-ws.adapter.ts @@ -30,14 +30,22 @@ export class CollabWsAdapter { return this.wss; } - public destroy() { + public close() { try { - this.wss.clients.forEach((client) => { - client.terminate(); - }); this.wss.close(); } catch (err) { console.error(err); } } + + public destroy() { + try { + this.wss.close(); + this.wss.clients.forEach((client) => { + client.terminate(); + }); + } catch (err) { + console.error(err); + } + } } diff --git a/apps/server/src/collaboration/collaboration.gateway.ts b/apps/server/src/collaboration/collaboration.gateway.ts index f1d50671..b296c520 100644 --- a/apps/server/src/collaboration/collaboration.gateway.ts +++ b/apps/server/src/collaboration/collaboration.gateway.ts @@ -1,10 +1,9 @@ -import { Hocuspocus, Server as HocuspocusServer } from '@hocuspocus/server'; +import { Hocuspocus } from '@hocuspocus/server'; import { IncomingMessage } from 'http'; import WebSocket from 'ws'; import { AuthenticationExtension } from './extensions/authentication.extension'; import { PersistenceExtension } from './extensions/persistence.extension'; import { Injectable } from '@nestjs/common'; -import { Redis } from '@hocuspocus/extension-redis'; import { EnvironmentService } from '../integrations/environment/environment.service'; import { createRetryStrategy, @@ -12,19 +11,39 @@ import { RedisConfig, } from '../common/helpers'; import { LoggerExtension } from './extensions/logger.extension'; +import { + RedisSyncExtension, + SerializedHTTPRequest, +} from './extensions/redis-sync'; +import { WsSocketWrapper } from './extensions/redis-sync/ws-socket-wrapper'; +import RedisClient from 'ioredis'; +import { pack, unpack } from 'msgpackr'; +import { nanoid } from 'nanoid'; +import * as os from 'node:os'; +import { CollabWsAdapter } from './adapter/collab-ws.adapter'; +import { + CollaborationHandler, + CollabEventHandlers, +} from './collaboration.handler'; @Injectable() export class CollaborationGateway { - private hocuspocus: Hocuspocus; + private readonly hocuspocus: Hocuspocus; private redisConfig: RedisConfig; + // @ts-ignore + private readonly redisSync: RedisSyncExtension | null = + null; + private readonly withRedis: boolean; constructor( private authenticationExtension: AuthenticationExtension, private persistenceExtension: PersistenceExtension, private loggerExtension: LoggerExtension, private environmentService: EnvironmentService, + private collabEventsService: CollaborationHandler, ) { this.redisConfig = parseRedisUrl(this.environmentService.getRedisUrl()); + this.withRedis = !this.environmentService.isCollabDisableRedis(); this.hocuspocus = new Hocuspocus({ debounce: 10000, @@ -34,26 +53,80 @@ export class CollaborationGateway { this.authenticationExtension, this.persistenceExtension, this.loggerExtension, - ...(this.environmentService.isCollabDisableRedis() - ? [] - : [ - new Redis({ - host: this.redisConfig.host, - port: this.redisConfig.port, - options: { - password: this.redisConfig.password, - db: this.redisConfig.db, - family: this.redisConfig.family, - retryStrategy: createRetryStrategy(), - }, - }), - ]), ], }); + + if (this.withRedis) { + // @ts-ignore + this.redisSync = new RedisSyncExtension({ + redis: new RedisClient({ + host: this.redisConfig.host, + port: this.redisConfig.port, + password: this.redisConfig.password, + db: this.redisConfig.db, + family: this.redisConfig.family, + retryStrategy: createRetryStrategy(), + }), + serverId: `collab-${os?.hostname()}-${nanoid(10)}`, + prefix: 'collab', + pack, + unpack, + // @ts-ignore + customEvents: this.collabEventsService.getHandlers(this.hocuspocus), + }); + this.hocuspocus.configuration.extensions.push(this.redisSync); + // @ts-ignore + this.redisSync.onConfigure({ instance: this.hocuspocus }); + } + } + + private serializeRequest(request: IncomingMessage): SerializedHTTPRequest { + return { + method: request.method ?? 'GET', + url: request.url ?? '/', + headers: { + 'sec-websocket-key': request.headers['sec-websocket-key'] ?? '', + 'sec-websocket-protocol': + request.headers['sec-websocket-protocol'] ?? '', + }, + socket: { remoteAddress: request.socket?.remoteAddress ?? '' }, + }; } handleConnection(client: WebSocket, request: IncomingMessage): any { - this.hocuspocus.handleConnection(client, request); + if (this.redisSync) { + const serializedHTTPRequest = this.serializeRequest(request); + const socketId = serializedHTTPRequest.headers['sec-websocket-key']; + + // Create wrapper socket that only receives events via emit() + // This prevents double-handling since Hocuspocus won't listen to raw WebSocket events + const wrappedSocket = new WsSocketWrapper(client); + + // Route through RedisSync extension (this calls handleConnection internally) + this.redisSync.onSocketOpen(wrappedSocket as any, serializedHTTPRequest); + + // Forward raw WebSocket messages to the extension + client.on('message', (data: ArrayBuffer) => { + this.redisSync!.onSocketMessage( + wrappedSocket as any, + serializedHTTPRequest, + data, + ); + }); + + // Forward close events + client.on('close', (code: number, reason: Buffer) => { + this.redisSync!.onSocketClose(socketId, code, reason); + }); + + // Forward pong events for keepalive + client.on('pong', (data: Buffer) => { + wrappedSocket.emit('pong', data); + }); + } else { + // Fallback to direct Hocuspocus connection + this.hocuspocus.handleConnection(client, request); + } } getConnectionCount() { @@ -64,7 +137,52 @@ export class CollaborationGateway { return this.hocuspocus.getDocumentsCount(); } - async destroy(): Promise { - //await this.hocuspocus.destroy(); + handleYjsEvent( + eventName: TName, + documentName: string, + payload: Parameters[1], + ) { + return this.redisSync?.handleEvent(eventName, documentName, payload); + } + + openDirectConnection(documentName: string, context?: any) { + return this.hocuspocus.openDirectConnection(documentName, context); + } + + /* + *Can be used before calling openDirectConnection directly + */ + async lockDocument(documentName: string) { + return this.redisSync.lockDocument(documentName); + } + + /* + *Releases a document lock and stops the interval that maintains it. + */ + async releaseLock(documentName: string) { + return this.redisSync.releaseLock(documentName); + } + + async destroy(collabWsAdapter: CollabWsAdapter): Promise { + // eslint-disable-next-line no-async-promise-executor + await new Promise(async (resolve) => { + try { + // Wait for all documents to unload + this.hocuspocus.configuration.extensions.push({ + async afterUnloadDocument({ instance }) { + if (instance.getDocumentsCount() === 0) resolve(''); + }, + }); + + collabWsAdapter?.close(); + + if (this.hocuspocus.getDocumentsCount() === 0) resolve(''); + this.hocuspocus.closeConnections(); + } catch (error) { + console.error(error); + } + }); + + await this.hocuspocus.hooks('onDestroy', { instance: this.hocuspocus }); } } diff --git a/apps/server/src/collaboration/collaboration.handler.ts b/apps/server/src/collaboration/collaboration.handler.ts new file mode 100644 index 00000000..ec746550 --- /dev/null +++ b/apps/server/src/collaboration/collaboration.handler.ts @@ -0,0 +1,42 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Hocuspocus, Document } from '@hocuspocus/server'; + +export type CollabEventHandlers = ReturnType< + CollaborationHandler['getHandlers'] +>; + +@Injectable() +export class CollaborationHandler { + private readonly logger = new Logger(CollaborationHandler.name); + + constructor() {} + + getHandlers(hocuspocus: Hocuspocus) { + return { + alterState: async (documentName: string, payload: { pageId: string }) => { + // dummy + // this.logger.log('Processing', documentName, payload); + // await this.withYdocConnection(hocuspocus, documentName, {}, (doc) => { + // const fragment = doc.getXmlFragment('default'); + //}); + }, + }; + } + + async withYdocConnection( + hocuspocus: Hocuspocus, + documentName: string, + context: any = {}, + fn: (doc: Document) => void, + ): Promise { + const connection = await hocuspocus.openDirectConnection( + documentName, + context, + ); + try { + await connection.transact(fn); + } finally { + await connection.disconnect(); + } + } +} diff --git a/apps/server/src/collaboration/collaboration.module.ts b/apps/server/src/collaboration/collaboration.module.ts index 30cb0ccf..e9374c53 100644 --- a/apps/server/src/collaboration/collaboration.module.ts +++ b/apps/server/src/collaboration/collaboration.module.ts @@ -9,6 +9,7 @@ import { WebSocket } from 'ws'; import { TokenModule } from '../core/auth/token.module'; import { HistoryListener } from './listeners/history.listener'; import { LoggerExtension } from './extensions/logger.extension'; +import { CollaborationHandler } from './collaboration.handler'; @Module({ providers: [ @@ -17,6 +18,7 @@ import { LoggerExtension } from './extensions/logger.extension'; PersistenceExtension, LoggerExtension, HistoryListener, + CollaborationHandler, ], exports: [CollaborationGateway], imports: [TokenModule], @@ -46,16 +48,12 @@ export class CollaborationModule implements OnModuleInit, OnModuleDestroy { }); wss.on('error', (error) => - this.logger.log('WebSocket server error:', error), + this.logger.error('WebSocket server error:', error), ); } async onModuleDestroy(): Promise { - if (this.collaborationGateway) { - await this.collaborationGateway.destroy(); - } - if (this.collabWsAdapter) { - this.collabWsAdapter.destroy(); - } + await this.collaborationGateway?.destroy(this.collabWsAdapter); + this.collabWsAdapter?.destroy(); } } diff --git a/apps/server/src/collaboration/extensions/logger.extension.ts b/apps/server/src/collaboration/extensions/logger.extension.ts index 969fa712..bbca47bd 100644 --- a/apps/server/src/collaboration/extensions/logger.extension.ts +++ b/apps/server/src/collaboration/extensions/logger.extension.ts @@ -9,11 +9,11 @@ import { Injectable, Logger } from '@nestjs/common'; export class LoggerExtension implements Extension { private readonly logger = new Logger('Collab' + LoggerExtension.name); - async onDisconnect(data: onDisconnectPayload) { - this.logger.debug(`User disconnected from "${data.documentName}".`); - } - async afterUnloadDocument(data: onLoadDocumentPayload) { this.logger.debug('Unloaded ' + data.documentName + ' from memory'); } + + async onDisconnect(data: onDisconnectPayload) { + this.logger.debug('User disconnected from ' + data.documentName); + } } diff --git a/apps/server/src/collaboration/extensions/redis-sync/collab-proxy-socket.ts b/apps/server/src/collaboration/extensions/redis-sync/collab-proxy-socket.ts new file mode 100644 index 00000000..8ecfa00a --- /dev/null +++ b/apps/server/src/collaboration/extensions/redis-sync/collab-proxy-socket.ts @@ -0,0 +1,70 @@ +import type RedisClient from 'ioredis'; +import { EventEmitter } from 'tseep'; +import type { + Pack, + RSAMessageClose, + RSAMessagePing, + RSAMessageSend, +} from './redis-sync.types'; + +export class CollabProxySocket extends EventEmitter { + private readonly replyTo: string; + private readonly serverChannel: string; + private readonly socketId: string; + private pub: RedisClient; + private readonly pack: Pack; + readyState = 1; + + constructor( + pub: RedisClient, + pack: Pack, + replyTo: string, + serverChannel: string, + socketId: string, + ) { + super(); + this.replyTo = replyTo; + this.socketId = socketId; + this.serverChannel = serverChannel; + this.pub = pub; + this.pack = pack; + this.once('close', () => { + this.readyState = 3; + }); + } + + private publish(msg: RSAMessageClose | RSAMessagePing | RSAMessageSend) { + this.pub.publish(this.replyTo, this.pack(msg)); + } + + close(code?: number, reason?: string) { + if (this.readyState !== 1) return; + const msg: RSAMessageClose = { + type: 'close', + code, + reason, + socketId: this.socketId, + }; + this.publish(msg); + } + + ping() { + if (this.readyState !== 1) return; + const msg: RSAMessagePing = { + type: 'ping', + socketId: this.socketId, + replyTo: this.serverChannel, + }; + this.publish(msg); + } + + send(message: Uint8Array) { + if (this.readyState !== 1) return; + const msg: RSAMessageSend = { + type: 'send', + socketId: this.socketId, + message, + }; + this.publish(msg); + } +} diff --git a/apps/server/src/collaboration/extensions/redis-sync/index.ts b/apps/server/src/collaboration/extensions/redis-sync/index.ts new file mode 100644 index 00000000..a5847477 --- /dev/null +++ b/apps/server/src/collaboration/extensions/redis-sync/index.ts @@ -0,0 +1,2 @@ +export * from './redis-sync.extension'; +export type { SerializedHTTPRequest } from './redis-sync.extension'; diff --git a/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts b/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts new file mode 100644 index 00000000..5b78b9c0 --- /dev/null +++ b/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts @@ -0,0 +1,376 @@ +// Source https://github.com/ueberdosis/hocuspocus/pull/1008 - MIT +import { + Extension, + Hocuspocus, + IncomingMessage, + afterUnloadDocumentPayload, + onConfigurePayload, + onLoadDocumentPayload, +} from '@hocuspocus/server'; +import RedisClient from 'ioredis'; +import { readVarString } from 'lib0/decoding.js'; +import { CollabProxySocket } from './collab-proxy-socket'; +import { + BaseWebSocket, + Configuration, + CustomEvents, + Pack, + RSAMessage, + RSAMessageCloseProxy, + RSAMessageCustomEventComplete, + RSAMessageCustomEventStart, + RSAMessagePong, + RSAMessageProxy, + RSAMessageUnload, + SerializedHTTPRequest, + Unpack, +} from './redis-sync.types'; + +export type { Pack, SerializedHTTPRequest } from './redis-sync.types'; + +type ServerId = string; +type DocumentName = string; +type SocketId = string; + +export class RedisSyncExtension implements Extension { + priority = 1000; + private readonly pub: RedisClient; + private sub: RedisClient; + private readonly pack: Pack; + private readonly unpack: Unpack; + private originSockets: Record = {}; + private locks: Record = {}; + private lockPromises: Record> = {}; + private proxySockets: Record = {}; + private readonly prefix: string; + private readonly lockPrefix: string; + private readonly msgChannel: string; + private readonly serverId: ServerId; + private readonly customEventTTL: number; + private readonly lockTTL: number; + private instance!: Hocuspocus; + private readonly customEvents: TCE; + private replyIdCounter: number = 0; + // @ts-ignore + private pendingReplies: Record['resolve']> = + {}; + + constructor(configuration: Configuration) { + const { + redis, + pack, + unpack, + serverId, + lockTTL, + prefix, + customEvents, + customEventTTL, + } = configuration; + this.pub = redis.duplicate(); + this.sub = redis.duplicate(); + this.pack = pack; + this.unpack = unpack; + this.serverId = serverId; + this.lockTTL = lockTTL ?? 10_000; + this.customEventTTL = customEventTTL ?? 30_000; + this.prefix = prefix ?? 'collab'; + this.lockPrefix = `${this.prefix}Lock`; + this.msgChannel = `${this.prefix}Msg`; + this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents); + this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`); + this.sub.on('messageBuffer', this.handleRedisMessage); + } + private getKey(documentName: string) { + return `${this.lockPrefix}:${documentName}`; + } + + private closeProxy(socketId: string) { + const proxySocket = this.proxySockets[socketId]; + if (proxySocket) { + proxySocket.emit( + 'close', + 1000, + Buffer.from('provider_initiated', 'utf-8'), + ); + delete this.proxySockets[socketId]; + } + } + + private pongProxy(socketId: string) { + this.proxySockets[socketId]?.emit('pong'); + } + + private handleProxyMessage( + msg: Pick, + ) { + const { replyTo, message, serializedHTTPRequest } = msg; + const { headers } = serializedHTTPRequest; + const socketId = headers['sec-websocket-key']!; + let socket = this.proxySockets[socketId]; + if (!socket) { + socket = new CollabProxySocket( + this.pub, + this.pack, + replyTo, + `${this.msgChannel}:${this.serverId}`, + socketId, + ); + this.proxySockets[socketId] = socket; + this.instance.handleConnection( + socket as any, + serializedHTTPRequest as any, + {}, + ); + } + socket.emit('message', message); + } + + private getOrClaimLock(documentName: string) { + const lockPromise = this.pub.set( + this.getKey(documentName), + this.serverId, + 'PX', + this.lockTTL, + 'NX', + 'GET', + ); + this.lockPromises[documentName] = lockPromise; + // Briefly cache the serverId that claimed the doc to reduce load on redis + // When the claimant unloads the doc, it will send an unload message to immediately clear this + // a lockTTL / 2 guarantees stale reads < lockTTL upon server crash + setTimeout(() => { + delete this.lockPromises[documentName]; + }, this.lockTTL / 2); + return lockPromise; + } + + private getOrClaimLockThrottled(documentName: string) { + const existingWorkerIdPromise = this.lockPromises[documentName]; + if (existingWorkerIdPromise) return existingWorkerIdPromise; + return this.getOrClaimLock(documentName); + } + + private handleRedisMessage = async ( + _channel: Buffer, + packedMessage: Buffer, + ) => { + const msg = this.unpack(packedMessage) as RSAMessage; + const { type } = msg; + if (type === 'proxy') { + this.handleProxyMessage(msg); + return; + } + if (type === 'closeProxy') { + this.closeProxy(msg.socketId); + return; + } + if (type === 'pong') { + this.pongProxy(msg.socketId); + return; + } + if (type === 'unload') { + delete this.lockPromises[msg.documentName]; + return; + } + if (type === 'customEventStart') { + const { documentName, eventName, payload, replyTo, replyId } = msg; + const res = await this.handleEventLocally( + eventName as Extract, + documentName, + payload, + ); + const reply: RSAMessageCustomEventComplete = { + type: 'customEventComplete', + replyId, + payload: res, + }; + this.pub.publish(`${replyTo}`, this.pack(reply)); + return; + } + if (type === 'customEventComplete') { + const { replyId, payload } = msg; + const resolveFn = this.pendingReplies[replyId]; + if (!resolveFn) return; + delete this.pendingReplies[replyId]; + resolveFn(payload); + return; + } + const { socketId } = msg; + const socket = this.originSockets[socketId]; + if (!socket) { + // origin socket already cleaned up + return; + } + if (type === 'close') { + socket.close(msg.code, msg.reason); + } else if (type === 'ping') { + // Reply instantly to the proxy socket, without forwarding to client + // The origin socket handles heartbeat for itself + const { replyTo, socketId } = msg; + const reply: RSAMessagePong = { + type: 'pong', + socketId, + }; + this.pub.publish(`${replyTo}`, this.pack(reply)); + } else if (type === 'send') { + socket.send(msg.message); + } + }; + + async maintainLock(documentName: string) { + this.locks[documentName] = setInterval(() => { + this.pub.set( + this.getKey(documentName), + this.serverId, + 'PX', + this.lockTTL, + ); + }, this.lockTTL / 2); + } + + async releaseLock(documentName: string) { + clearInterval(this.locks[documentName]); + delete this.locks[documentName]; + return this.pub.del(this.getKey(documentName)); + } + + private async handleEventLocally>( + eventName: TName, + documentName: string, + payload: any, + ) { + const handler = this.customEvents[eventName]; + if (!handler) throw new Error(`Invalid eventName: ${eventName}`); + const result = await handler(documentName, payload); + return result as Promise>; + } + + async handleEvent>( + eventName: TName, + documentName: string, + payload: any, + ) { + const isDocLoadedOnInstance = this.instance.documents.has(documentName); + + if (isDocLoadedOnInstance) { + return this.handleEventLocally(eventName, documentName, payload); + } + + const proxyTo = await this.getOrClaimLockThrottled(documentName); + if (proxyTo && proxyTo !== this.serverId) { + ++this.replyIdCounter; // bug in biome thinks this.replyIdCounter is not used if written on the line below + const replyId = this.replyIdCounter; + // another server owns the doc + const proxyMessage: RSAMessageCustomEventStart = { + eventName, + documentName, + payload, + replyTo: `${this.msgChannel}:${this.serverId}`, + replyId, + type: 'customEventStart', + }; + const msg = this.pack(proxyMessage); + this.pub.publish(`${this.msgChannel}:${proxyTo}`, msg); + // @ts-ignore + const { promise, resolve, reject } = Promise.withResolvers(); + this.pendingReplies[replyId] = resolve; + setTimeout(() => { + reject('TIMEOUT'); + }, this.customEventTTL); + return promise as Promise>; + } + // This server owns the document, but hocuspocus hasn't loaded it yet + return this.handleEventLocally(eventName, documentName, payload); + } + + async lockDocument(documentName: string) { + const proxyTo = await this.getOrClaimLockThrottled(documentName); + if (proxyTo && proxyTo !== this.serverId) { + throw new Error(`Could not lock document: ${documentName}`); + } + this.maintainLock(documentName); + return () => this.releaseLock(documentName); + } + + /* WebSocket Server Hooks */ + onSocketOpen( + ws: BaseWebSocket, + serializedHTTPRequest: SerializedHTTPRequest, + context = {}, + ) { + const socketId = serializedHTTPRequest.headers['sec-websocket-key']!; + this.originSockets[socketId] = ws; + this.instance.handleConnection( + ws as any, + serializedHTTPRequest as any, + context, + ); + } + + async onSocketMessage( + ws: BaseWebSocket, + serializedHTTPRequest: SerializedHTTPRequest, + detachableMsg: ArrayBuffer, + ) { + const message = new Uint8Array(detachableMsg.slice()); + const tmpMsg = new IncomingMessage(detachableMsg); + const documentName = readVarString(tmpMsg.decoder); + const isDocLoadedOnInstance = this.instance.documents.has(documentName); + + if (isDocLoadedOnInstance) { + ws.emit('message', message); + return; + } + + const proxyTo = await this.getOrClaimLockThrottled(documentName); + if (proxyTo && proxyTo !== this.serverId) { + // another server owns the doc + const proxyMessage: RSAMessageProxy = { + serializedHTTPRequest: serializedHTTPRequest, + replyTo: `${this.msgChannel}:${this.serverId}`, + message, + type: 'proxy', + }; + const msg = this.pack(proxyMessage); + this.pub.publish(`${this.msgChannel}:${proxyTo}`, msg); + return; + } + // This server owns the document, but hocuspocus hasn't loaded it yet + ws.emit('message', message); + } + + onSocketClose(socketId: string, code?: number, reason?: ArrayBuffer) { + const socket = this.originSockets[socketId]; + if (!socket) return; + // at this point the socket is considered GC'd and we cannot call close + // The origin socket did not set up any connections for the proxy, so none of the hooks will work if we just emit + socket?.emit('close', code, reason); + delete this.originSockets[socketId]; + const msg: RSAMessageCloseProxy = { type: 'closeProxy', socketId }; + this.pub.publish(this.msgChannel, this.pack(msg)).catch(() => {}); + } + + /* Hocuspocus hooks */ + async onConfigure({ instance }: onConfigurePayload) { + this.instance = instance; + } + + async onLoadDocument(data: onLoadDocumentPayload) { + const { documentName } = data; + // Refresh the lock TTL + this.maintainLock(documentName); + } + + async afterUnloadDocument(data: afterUnloadDocumentPayload) { + const { documentName } = data; + this.releaseLock(documentName); + // Broadcast to cluster to immediately remove the cached redis value + const msg: RSAMessageUnload = { type: 'unload', documentName }; + this.pub.publish(this.msgChannel, this.pack(msg)); + } + + async onDestroy() { + this.pub.disconnect(false); + this.sub.disconnect(false); + } +} diff --git a/apps/server/src/collaboration/extensions/redis-sync/redis-sync.types.ts b/apps/server/src/collaboration/extensions/redis-sync/redis-sync.types.ts new file mode 100644 index 00000000..1bbab80a --- /dev/null +++ b/apps/server/src/collaboration/extensions/redis-sync/redis-sync.types.ts @@ -0,0 +1,121 @@ +import EventEmitter from 'node:events'; +import { IncomingHttpHeaders } from 'node:http2'; +import RedisClient from 'ioredis'; + +export type SecondParam = T extends ( + arg1: unknown, + arg2: infer A, + ...args: unknown[] +) => unknown + ? A + : never; + +export type SerializedHTTPRequest = { + method: string; + url: string; + headers: IncomingHttpHeaders; + socket: { remoteAddress: string }; +}; + +export type RSAMessageProxy = { + type: 'proxy'; + replyTo: string; + message: Uint8Array; + serializedHTTPRequest: SerializedHTTPRequest; +}; + +export type RSAMessageCloseProxy = { + type: 'closeProxy'; + socketId: string; +}; + +export type RSAMessageUnload = { + type: 'unload'; + documentName: string; +}; + +export type RSAMessageClose = { + type: 'close'; + code?: number; + reason?: string; + socketId: string; +}; + +export type RSAMessagePing = { + type: 'ping'; + socketId: string; + replyTo: string; +}; + +export type RSAMessagePong = { + type: 'pong'; + socketId: string; +}; + +export type RSAMessageSend = { + type: 'send'; + // @ts-ignore + message: Uint8Array; + socketId: string; +}; + +export type RSAMessageCustomEventStart = { + type: 'customEventStart'; + documentName: string; + eventName: TName; + payload: TPayload; + replyTo: string; + replyId: number; +}; + +export type RSAMessageCustomEventComplete = { + type: 'customEventComplete'; + replyId: number; + payload: unknown; +}; + +export type RSAMessage = + | RSAMessageProxy + | RSAMessageCloseProxy + | RSAMessageUnload + | RSAMessageClose + | RSAMessagePing + | RSAMessagePong + | RSAMessageSend + | RSAMessageCustomEventStart + | RSAMessageCustomEventComplete; + +// @ts-ignore +export type Pack = (msg: RSAMessage) => string | Buffer; + +export type Unpack = ( + // @ts-ignore + packedMessage: Uint8Array | Buffer, +) => RSAMessage; + +type ServerId = string; +type DocumentName = string; +type CustomEventName = string; + +export type CustomEvents = Record< + CustomEventName, + (documentName: string, payload: unknown) => Promise +>; + +export interface Configuration { + redis: RedisClient; + pack: Pack; + unpack: Unpack; + serverId: ServerId; + lockTTL?: number; + customEventTTL?: number; + prefix?: string; + customEvents?: TCE; +} + +export type BaseWebSocket = EventEmitter & { + readyState: number; + close(code?: number, reason?: string): void; + ping(): void; + send(message: Uint8Array): void; +}; diff --git a/apps/server/src/collaboration/extensions/redis-sync/ws-socket-wrapper.ts b/apps/server/src/collaboration/extensions/redis-sync/ws-socket-wrapper.ts new file mode 100644 index 00000000..258e6e12 --- /dev/null +++ b/apps/server/src/collaboration/extensions/redis-sync/ws-socket-wrapper.ts @@ -0,0 +1,47 @@ +import { EventEmitter } from 'events'; +import type WebSocket from 'ws'; + +/** + * Wrapper around ws WebSocket that only receives events via emit(). + * This prevents double-handling when used with RedisSyncExtension. + */ +export class WsSocketWrapper extends EventEmitter { + private ws: WebSocket; + readyState = 1; + + constructor(ws: WebSocket) { + super(); + this.ws = ws; + this.once('close', () => { + this.readyState = 3; + }); + } + + close(code?: number, reason?: string) { + if (this.readyState !== 1) return; + this.readyState = 3; + try { + this.ws.close(code, reason); + } catch (e) { + // Socket already closed + } + } + + ping() { + if (this.readyState !== 1) return; + try { + this.ws.ping(); + } catch (e) { + // Socket already closed + } + } + + send(message: Uint8Array) { + if (this.readyState !== 1) return; + try { + this.ws.send(message); + } catch (e) { + // Socket already closed + } + } +} diff --git a/apps/server/src/collaboration/server/collab-main.ts b/apps/server/src/collaboration/server/collab-main.ts index 1a10167f..4a86a71b 100644 --- a/apps/server/src/collaboration/server/collab-main.ts +++ b/apps/server/src/collaboration/server/collab-main.ts @@ -12,9 +12,11 @@ async function bootstrap() { const app = await NestFactory.create( CollabAppModule, new FastifyAdapter({ - ignoreTrailingSlash: true, - ignoreDuplicateSlashes: true, - maxParamLength: 500, + routerOptions: { + maxParamLength: 1000, + ignoreTrailingSlash: true, + ignoreDuplicateSlashes: true, + }, }), { bufferLogs: true, diff --git a/package.json b/package.json index 2b5096ef..9ffe3d83 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "@casl/ability": "6.8.0", "@docmost/editor-ext": "workspace:*", "@floating-ui/dom": "^1.7.3", - "@hocuspocus/extension-redis": "3.4.3", "@hocuspocus/provider": "3.4.3", "@hocuspocus/server": "3.4.3", "@hocuspocus/transformer": "3.4.3", diff --git a/packages/editor-ext/src/lib/attachment/attachment.ts b/packages/editor-ext/src/lib/attachment/attachment.ts index 0e37e014..a1e851a4 100644 --- a/packages/editor-ext/src/lib/attachment/attachment.ts +++ b/packages/editor-ext/src/lib/attachment/attachment.ts @@ -96,7 +96,7 @@ export const Attachment = Node.create({ mergeAttributes( { "data-type": this.name }, this.options.HTMLAttributes, - HTMLAttributes, + HTMLAttributes ), [ "a", diff --git a/packages/editor-ext/src/lib/image/image.ts b/packages/editor-ext/src/lib/image/image.ts index e6426f23..e0f5053d 100644 --- a/packages/editor-ext/src/lib/image/image.ts +++ b/packages/editor-ext/src/lib/image/image.ts @@ -25,7 +25,7 @@ declare module "@tiptap/core" { imageBlock: { setImage: (attributes: ImageAttributes) => ReturnType; setImageAt: ( - attributes: ImageAttributes & { pos: number | Range }, + attributes: ImageAttributes & { pos: number | Range } ) => ReturnType; setImageAlign: (align: "left" | "center" | "right") => ReturnType; setImageWidth: (width: number) => ReturnType; diff --git a/packages/editor-ext/src/lib/video/video.ts b/packages/editor-ext/src/lib/video/video.ts index 31c68f89..c3c6ab3e 100644 --- a/packages/editor-ext/src/lib/video/video.ts +++ b/packages/editor-ext/src/lib/video/video.ts @@ -23,7 +23,7 @@ declare module "@tiptap/core" { videoBlock: { setVideo: (attributes: VideoAttributes) => ReturnType; setVideoAt: ( - attributes: VideoAttributes & { pos: number | Range }, + attributes: VideoAttributes & { pos: number | Range } ) => ReturnType; setVideoAlign: (align: "left" | "center" | "right") => ReturnType; setVideoWidth: (width: number) => ReturnType; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e3904280..7af8c424 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,9 +30,6 @@ importers: '@floating-ui/dom': specifier: ^1.7.3 version: 1.7.3 - '@hocuspocus/extension-redis': - specifier: 3.4.3 - version: 3.4.3(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29) '@hocuspocus/provider': specifier: 3.4.3 version: 3.4.3(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29) @@ -554,12 +551,18 @@ importers: ldapts: specifier: ^7.4.0 version: 7.4.0 + lib0: + specifier: ^0.2.117 + version: 0.2.117 mammoth: specifier: ^1.11.0 version: 1.11.0 mime-types: specifier: ^2.1.35 version: 2.1.35 + msgpackr: + specifier: ^1.11.8 + version: 1.11.8 nanoid: specifier: 3.3.11 version: 3.3.11 @@ -632,6 +635,9 @@ importers: tmp-promise: specifier: ^3.0.3 version: 3.0.3 + tseep: + specifier: ^1.3.1 + version: 1.3.1 typesense: specifier: ^2.1.0 version: 2.1.0(@babel/runtime@7.25.6) @@ -2383,12 +2389,6 @@ packages: '@hocuspocus/common@3.4.3': resolution: {integrity: sha512-wnBBO9sWcVAoUPEXN1qO+zk3HaEF9VTemxB6kjuuH6e1dHnD0v12m4P4X1wiZVhmMIX/PMl/fu3MGtYWQJz8gA==} - '@hocuspocus/extension-redis@3.4.3': - resolution: {integrity: sha512-r64Vpgk6tt0VZaQPEo1dQuyur2ozr243ncDcDM+4gFPuV8ZRUjL1rvaJTidb2HCcAW2zjfwshNxw4+OixeksBA==} - peerDependencies: - y-protocols: ^1.0.6 - yjs: ^13.6.8 - '@hocuspocus/provider@3.4.3': resolution: {integrity: sha512-zt+UgVXGsEQrqnDZgavc2PT9yKJjmVjV+5YxvhlmFVFLVORqawT4l601aKmLPhvyK97un4ZApZ5rso8iO6crWg==} peerDependencies: @@ -3884,12 +3884,6 @@ packages: '@selderee/plugin-htmlparser2@0.11.0': resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==} - '@sesamecare-oss/redlock@1.4.0': - resolution: {integrity: sha512-2z589R+yxKLN4CgKxP1oN4dsg6Y548SE4bVYam/R0kHk7Q9VrQ9l66q+k1ehhSLLY4or9hcchuF9/MhuuZdjJg==} - engines: {node: '>=16'} - peerDependencies: - ioredis: '>=5' - '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} @@ -7593,13 +7587,8 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} - lib0@0.2.114: - resolution: {integrity: sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ==} - engines: {node: '>=16'} - hasBin: true - - lib0@0.2.88: - resolution: {integrity: sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==} + lib0@0.2.117: + resolution: {integrity: sha512-DeXj9X5xDCjgKLU/7RR+/HQEVzuuEUiwldwOGsHK/sfAfELGWEyTcf0x+uOvCvK3O2zPmZePXWL85vtia6GyZw==} engines: {node: '>=16'} hasBin: true @@ -7975,8 +7964,8 @@ packages: resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==} hasBin: true - msgpackr@1.11.2: - resolution: {integrity: sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==} + msgpackr@1.11.8: + resolution: {integrity: sha512-bC4UGzHhVvgDNS7kn9tV8fAucIYUBuGojcaLiz7v+P63Lmtm0Xeji8B/8tYKddALXxJLpwIeBmUN3u64C4YkRA==} multimath@2.0.0: resolution: {integrity: sha512-toRx66cAMJ+Ccz7pMIg38xSIrtnbozk0dchXezwQDMgQmbGpfxjtv68H+L00iFL8hxDaVjrmwAFSb3I6bg8Q2g==} @@ -9671,6 +9660,9 @@ packages: resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} engines: {node: '>=6'} + tseep@1.3.1: + resolution: {integrity: sha512-ZPtfk1tQnZVyr7BPtbJ93qaAh2lZuIOpTMjhrYa4XctT8xe7t4SAW9LIxrySDuYMsfNNayE51E/WNGrNVgVicQ==} + tslib@2.8.0: resolution: {integrity: sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==} @@ -12659,27 +12651,13 @@ snapshots: '@hocuspocus/common@3.4.3': dependencies: - lib0: 0.2.114 - - '@hocuspocus/extension-redis@3.4.3(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29)': - dependencies: - '@hocuspocus/server': 3.4.3(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29) - '@sesamecare-oss/redlock': 1.4.0(ioredis@5.8.2) - ioredis: 5.8.2 - kleur: 4.1.5 - lodash.debounce: 4.0.8 - y-protocols: 1.0.6(yjs@13.6.29) - yjs: 13.6.29 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate + lib0: 0.2.117 '@hocuspocus/provider@3.4.3(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29)': dependencies: '@hocuspocus/common': 3.4.3 '@lifeomic/attempt': 3.0.3 - lib0: 0.2.114 + lib0: 0.2.117 ws: 8.19.0 y-protocols: 1.0.6(yjs@13.6.29) yjs: 13.6.29 @@ -12693,7 +12671,7 @@ snapshots: async-lock: 1.4.1 async-mutex: 0.5.0 kleur: 4.1.5 - lib0: 0.2.114 + lib0: 0.2.117 ws: 8.19.0 y-protocols: 1.0.6(yjs@13.6.29) yjs: 13.6.29 @@ -14153,10 +14131,6 @@ snapshots: domhandler: 5.0.3 selderee: 0.11.0 - '@sesamecare-oss/redlock@1.4.0(ioredis@5.8.2)': - dependencies: - ioredis: 5.8.2 - '@sinclair/typebox@0.27.8': {} '@sindresorhus/slugify@1.1.0': @@ -14867,7 +14841,7 @@ snapshots: '@tiptap/y-tiptap@3.0.1(prosemirror-model@1.25.1)(prosemirror-state@1.4.3)(prosemirror-view@1.40.0)(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29)': dependencies: - lib0: 0.2.114 + lib0: 0.2.117 prosemirror-model: 1.25.1 prosemirror-state: 1.4.3 prosemirror-view: 1.40.0 @@ -16065,7 +16039,7 @@ snapshots: dependencies: cron-parser: 4.9.0 ioredis: 5.8.2 - msgpackr: 1.11.2 + msgpackr: 1.11.8 node-abort-controller: 3.1.1 semver: 7.7.2 tslib: 2.8.1 @@ -18687,11 +18661,7 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 - lib0@0.2.114: - dependencies: - isomorphic.js: 0.2.5 - - lib0@0.2.88: + lib0@0.2.117: dependencies: isomorphic.js: 0.2.5 @@ -19166,7 +19136,7 @@ snapshots: '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2 optional: true - msgpackr@1.11.2: + msgpackr@1.11.8: optionalDependencies: msgpackr-extract: 3.0.2 @@ -21046,6 +21016,8 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 + tseep@1.3.1: {} + tslib@2.8.0: {} tslib@2.8.1: {} @@ -21519,12 +21491,12 @@ snapshots: y-indexeddb@9.0.12(yjs@13.6.29): dependencies: - lib0: 0.2.88 + lib0: 0.2.117 yjs: 13.6.29 y-prosemirror@1.3.7(prosemirror-model@1.25.1)(prosemirror-state@1.4.3)(prosemirror-view@1.40.0)(y-protocols@1.0.6(yjs@13.6.29))(yjs@13.6.29): dependencies: - lib0: 0.2.114 + lib0: 0.2.117 prosemirror-model: 1.25.1 prosemirror-state: 1.4.3 prosemirror-view: 1.40.0 @@ -21533,7 +21505,7 @@ snapshots: y-protocols@1.0.6(yjs@13.6.29): dependencies: - lib0: 0.2.114 + lib0: 0.2.117 yjs: 13.6.29 y18n@4.0.3: {} @@ -21586,7 +21558,7 @@ snapshots: yjs@13.6.29: dependencies: - lib0: 0.2.114 + lib0: 0.2.117 yn@3.1.1: {} From 60501de992d7c0bfcedbe2f91eca55ced5717a89 Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:25:23 +0000 Subject: [PATCH 10/10] fix: missing logs on `OnApplicationBootstrap` hook (#1882) * - fix: set default Nest logger and bufferLogs to false for pino compatibility - handle redis error event * fix collab server logging too --- .../extensions/redis-sync/redis-sync.extension.ts | 2 ++ apps/server/src/collaboration/server/collab-main.ts | 3 ++- apps/server/src/main.ts | 10 ++++++++-- apps/server/src/ws/adapter/ws-redis.adapter.ts | 3 +++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts b/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts index 5b78b9c0..38747465 100644 --- a/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts +++ b/apps/server/src/collaboration/extensions/redis-sync/redis-sync.extension.ts @@ -79,6 +79,8 @@ export class RedisSyncExtension implements Extension { this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents); this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`); this.sub.on('messageBuffer', this.handleRedisMessage); + this.pub.on('error', () => {}); + this.sub.on('error', () => {}); } private getKey(documentName: string) { return `${this.lockPrefix}:${documentName}`; diff --git a/apps/server/src/collaboration/server/collab-main.ts b/apps/server/src/collaboration/server/collab-main.ts index 4a86a71b..4cac6878 100644 --- a/apps/server/src/collaboration/server/collab-main.ts +++ b/apps/server/src/collaboration/server/collab-main.ts @@ -19,7 +19,8 @@ async function bootstrap() { }, }), { - bufferLogs: true, + logger: false, + bufferLogs: false, }, ); diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 406921a0..e8634a09 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -24,7 +24,11 @@ async function bootstrap() { }), { rawBody: true, - bufferLogs: true, + // disable Nest logger so pino handles all logs + // bufferLogs must be false else pino will fail + // to log OnApplicationBootstrap logs + logger: false, + bufferLogs: false, }, ); @@ -101,7 +105,9 @@ async function bootstrap() { const port = process.env.PORT || 3000; await app.listen(port, '0.0.0.0', () => { - logger.log(`Listening on http://127.0.0.1:${port} / ${process.env.APP_URL}`); + logger.log( + `Listening on http://127.0.0.1:${port} / ${process.env.APP_URL}`, + ); }); } diff --git a/apps/server/src/ws/adapter/ws-redis.adapter.ts b/apps/server/src/ws/adapter/ws-redis.adapter.ts index 5aae1c7c..a221c84a 100644 --- a/apps/server/src/ws/adapter/ws-redis.adapter.ts +++ b/apps/server/src/ws/adapter/ws-redis.adapter.ts @@ -23,6 +23,9 @@ export class WsRedisIoAdapter extends IoAdapter { const pubClient = new Redis(process.env.REDIS_URL, options); const subClient = new Redis(process.env.REDIS_URL, options); + pubClient.on('error', (err) => () => {}); + subClient.on('error', (err) => () => {}); + this.adapterConstructor = createAdapter(pubClient, subClient); }