feat: page content update and output

- Set shouldRerenderOnTransaction to false

apps/server/src/collaboration/collaboration.handler.ts

@@ -1,5 +1,12 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { Hocuspocus, Document } from '@hocuspocus/server';
+import { TiptapTransformer } from '@hocuspocus/transformer';
+import {
+  prosemirrorNodeToYElement,
+  tiptapExtensions,
+} from './collaboration.util';
+import * as Y from 'yjs';
+import { User } from '@docmost/db/types/entity.types';
 
 export type CollabEventHandlers = ReturnType<
   CollaborationHandler['getHandlers']
@@ -20,6 +27,47 @@ export class CollaborationHandler {
         //   const fragment = doc.getXmlFragment('default');
         //});
       },
+      updatePageContent: async (
+        documentName: string,
+        payload: {
+          pageId: string;
+          prosemirrorJson: any;
+          operation: string;
+          user: User;
+        },
+      ) => {
+        const { pageId, prosemirrorJson, operation, user } = payload;
+        this.logger.debug(
+          'Updating page content via yjs',
+          documentName,
+          payload,
+        );
+        await this.withYdocConnection(
+          hocuspocus,
+          documentName,
+          { user },
+          (doc) => {
+            const fragment = doc.getXmlFragment('default');
+
+            if (operation === 'replace') {
+              if (fragment.length > 0) {
+                fragment.delete(0, fragment.length);
+              }
+
+              const newDoc = TiptapTransformer.toYdoc(
+                prosemirrorJson,
+                'default',
+                tiptapExtensions,
+              );
+              Y.applyUpdate(doc, Y.encodeStateAsUpdate(newDoc));
+            } else {
+              const newContent = prosemirrorJson.content || [];
+              const yElements = newContent.map(prosemirrorNodeToYElement);
+              fragment.insert(fragment.length, yElements);
+            }
+          },
+        );
+      },
     };
   }
 

apps/server/src/collaboration/collaboration.util.ts

@@ -34,6 +34,7 @@ import {
   Highlight,
   UniqueID,
   addUniqueIdsToDoc,
+  htmlToMarkdown,
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
 import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
@@ -42,6 +43,7 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
 // see:https://github.com/ueberdosis/tiptap/issues/4089
 //import { generateJSON } from '@tiptap/html';
 import { Node, Schema } from '@tiptap/pm/model';
+import * as Y from 'yjs';
 import { Logger } from '@nestjs/common';
 
 export const tiptapExtensions = [
@@ -161,3 +163,37 @@ function stripUnknownNodes(
 
   return json;
 }
+
+export function prosemirrorNodeToYElement(node: any): Y.XmlElement | Y.XmlText {
+  if (node.type === 'text') {
+    const ytext = new Y.XmlText();
+    ytext.insert(0, node.text || '');
+    if (node.marks?.length > 0) {
+      const attrs: Record<string, any> = {};
+      for (const mark of node.marks) {
+        attrs[mark.type] = mark.attrs || true;
+      }
+      ytext.format(0, node.text?.length || 0, attrs);
+    }
+    return ytext;
+  }
+
+  const element = new Y.XmlElement(node.type);
+  if (node.attrs) {
+    for (const [key, value] of Object.entries(node.attrs)) {
+      if (value !== null && value !== undefined) {
+        element.setAttribute(key, value as any);
+      }
+    }
+  }
+  if (node.content?.length > 0) {
+    const children = node.content.map(prosemirrorNodeToYElement);
+    element.insert(0, children);
+  }
+  return element;
+}
+
+export function jsonToMarkdown(tiptapJson: any): string {
+  const html = jsonToHtml(tiptapJson);
+  return htmlToMarkdown(html);
+}

apps/server/src/collaboration/extensions/persistence.extension.ts

@@ -181,7 +181,7 @@ export class PersistenceExtension implements Extension {
 
   async onChange(data: onChangePayload) {
     const documentName = data.documentName;
-    const userId = data.context?.user.id;
+    const userId = data.context?.user?.id;
     if (!userId) return;
 
     if (!this.contributors.has(documentName)) {

apps/server/src/core/page/dto/create-page.dto.ts

@@ -1,4 +1,12 @@
-import { IsOptional, IsString, IsUUID } from 'class-validator';
+import {
+  IsIn,
+  IsOptional,
+  IsString,
+  IsUUID,
+  ValidateIf,
+} from 'class-validator';
+
+export type InputFormat = 'json' | 'markdown' | 'html';
 
 export class CreatePageDto {
   @IsOptional()
@@ -15,4 +23,11 @@ export class CreatePageDto {
 
   @IsUUID()
   spaceId: string;
+
+  @IsOptional()
+  content?: string | object;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @IsIn(['json', 'markdown', 'html'])
+  input?: InputFormat;
 }

apps/server/src/core/page/dto/page.dto.ts

@@ -1,5 +1,6 @@
 import {
   IsBoolean,
+  IsIn,
   IsNotEmpty,
   IsOptional,
   IsString,
@@ -22,14 +23,20 @@ export class PageHistoryIdDto {
   historyId: string;
 }
 
+export type OutputFormat = 'json' | 'markdown' | 'html';
+
 export class PageInfoDto extends PageIdDto {
   @IsOptional()
   @IsBoolean()
-  includeSpace: boolean;
+  includeSpace?: boolean;
 
   @IsOptional()
   @IsBoolean()
-  includeContent: boolean;
+  includeContent?: boolean;
+
+  @IsOptional()
+  @IsIn(['json', 'markdown', 'html'])
+  output?: OutputFormat;
 }
 
 export class DeletePageDto extends PageIdDto {

apps/server/src/core/page/dto/update-page.dto.ts

@@ -1,8 +1,21 @@
 import { PartialType } from '@nestjs/mapped-types';
-import { CreatePageDto } from './create-page.dto';
-import { IsString } from 'class-validator';
+import { CreatePageDto, InputFormat } from './create-page.dto';
+import { IsIn, IsOptional, IsString, ValidateIf } from 'class-validator';
+
+export type ContentOperation = 'append' | 'replace';
 
 export class UpdatePageDto extends PartialType(CreatePageDto) {
   @IsString()
   pageId: string;
+
+  @IsOptional()
+  content?: string | object;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @IsIn(['append', 'replace'])
+  operation?: ContentOperation;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @IsIn(['json', 'markdown', 'html'])
+  input?: InputFormat;
 }

apps/server/src/core/page/page.controller.ts

@@ -35,6 +35,10 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { RecentPageDto } from './dto/recent-page.dto';
 import { DuplicatePageDto } from './dto/duplicate-page.dto';
 import { DeletedPageDto } from './dto/deleted-page.dto';
+import {
+  jsonToHtml,
+  jsonToMarkdown,
+} from '../../collaboration/collaboration.util';
 
 @UseGuards(JwtAuthGuard)
 @Controller('pages')
@@ -66,6 +70,17 @@ export class PageController {
       throw new ForbiddenException();
     }
 
+    if (dto.output && dto.output !== 'json' && page.content) {
+      const contentOutput =
+        dto.output === 'markdown'
+          ? jsonToMarkdown(page.content)
+          : jsonToHtml(page.content);
+      return {
+        ...page,
+        content: contentOutput,
+      };
+    }
+
     return page;
   }
 
@@ -101,7 +116,7 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    return this.pageService.update(page, updatePageDto, user.id);
+    return this.pageService.update(page, updatePageDto, user);
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/page/page.module.ts

@@ -4,11 +4,12 @@ import { PageController } from './page.controller';
 import { PageHistoryService } from './services/page-history.service';
 import { TrashCleanupService } from './services/trash-cleanup.service';
 import { StorageModule } from '../../integrations/storage/storage.module';
+import { CollaborationModule } from '../../collaboration/collaboration.module';
 
 @Module({
   controllers: [PageController],
   providers: [PageService, PageHistoryService, TrashCleanupService],
   exports: [PageService, PageHistoryService],
-  imports: [StorageModule],
+  imports: [StorageModule, CollaborationModule],
 })
 export class PageModule {}

apps/server/src/core/page/services/page.service.ts

@@ -4,8 +4,8 @@ import {
   Logger,
   NotFoundException,
 } from '@nestjs/common';
-import { CreatePageDto } from '../dto/create-page.dto';
-import { UpdatePageDto } from '../dto/update-page.dto';
+import { CreatePageDto, InputFormat } from '../dto/create-page.dto';
+import { ContentOperation, UpdatePageDto } from '../dto/update-page.dto';
 import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { InsertablePage, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
@@ -28,7 +28,11 @@ import {
   isAttachmentNode,
   removeMarkTypeFromDoc,
 } from '../../../common/helpers/prosemirror/utils';
-import { jsonToNode, jsonToText } from 'src/collaboration/collaboration.util';
+import {
+  htmlToJson,
+  jsonToNode,
+  jsonToText,
+} from 'src/collaboration/collaboration.util';
 import {
   CopyPageMapEntry,
   ICopyPageAttachment,
@@ -40,6 +44,8 @@ import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { EventName } from '../../../common/events/event.contants';
 import { EventEmitter2 } from '@nestjs/event-emitter';
+import { CollaborationGateway } from '../../../collaboration/collaboration.gateway';
+import { markdownToHtml } from '@docmost/editor-ext';
 
 @Injectable()
 export class PageService {
@@ -53,6 +59,7 @@ export class PageService {
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
     @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
     private eventEmitter: EventEmitter2,
+    private collaborationGateway: CollaborationGateway,
   ) {}
 
   async findById(
@@ -88,7 +95,42 @@ export class PageService {
       parentPageId = parentPage.id;
     }
 
-    const createdPage = await this.pageRepo.insertPage({
+    let content = undefined;
+    let textContent = undefined;
+    let ydoc = undefined;
+
+    if (createPageDto?.content && createPageDto?.input) {
+      let prosemirrorJson: any;
+
+      switch (createPageDto.input) {
+        case 'markdown': {
+          const html = await markdownToHtml(createPageDto.content as string);
+          prosemirrorJson = htmlToJson(html as string);
+          break;
+        }
+        case 'html': {
+          prosemirrorJson = htmlToJson(createPageDto.content as string);
+          break;
+        }
+        case 'json':
+        default: {
+          prosemirrorJson = createPageDto.content;
+          break;
+        }
+      }
+
+      try {
+        jsonToNode(prosemirrorJson);
+      } catch (err) {
+        throw new BadRequestException('Invalid content format');
+      }
+
+      content = prosemirrorJson;
+      textContent = jsonToText(prosemirrorJson);
+      ydoc = createYdocFromJson(prosemirrorJson);
+    }
+
+    return this.pageRepo.insertPage({
       slugId: generateSlugId(),
       title: createPageDto.title,
       position: await this.nextPagePosition(
@@ -101,9 +143,10 @@ export class PageService {
       creatorId: userId,
       workspaceId: workspaceId,
       lastUpdatedById: userId,
+      content,
+      textContent,
+      ydoc,
     });
-
-    return createdPage;
   }
 
   async nextPagePosition(spaceId: string, parentPageId?: string) {
@@ -150,23 +193,37 @@ export class PageService {
   async update(
     page: Page,
     updatePageDto: UpdatePageDto,
-    userId: string,
+    user: User,
   ): Promise<Page> {
     const contributors = new Set<string>(page.contributorIds);
-    contributors.add(userId);
+    contributors.add(user.id);
     const contributorIds = Array.from(contributors);
 
     await this.pageRepo.updatePage(
       {
         title: updatePageDto.title,
         icon: updatePageDto.icon,
-        lastUpdatedById: userId,
+        lastUpdatedById: user.id,
         updatedAt: new Date(),
         contributorIds: contributorIds,
       },
       page.id,
     );
 
+    if (
+      updatePageDto.content &&
+      updatePageDto.operation &&
+      updatePageDto.input
+    ) {
+      await this.updatePageContent(
+        page.id,
+        updatePageDto.content,
+        updatePageDto.operation,
+        updatePageDto.input,
+        user,
+      );
+    }
+
     return await this.pageRepo.findById(page.id, {
       includeSpace: true,
       includeContent: true,
@@ -176,6 +233,46 @@ export class PageService {
     });
   }
 
+  async updatePageContent(
+    pageId: string,
+    content: string | object,
+    operation: ContentOperation,
+    input: InputFormat,
+    user: User,
+  ): Promise<void> {
+    let prosemirrorJson: any;
+
+    switch (input) {
+      case 'markdown': {
+        const html = await markdownToHtml(content as string);
+        prosemirrorJson = htmlToJson(html as string);
+        break;
+      }
+      case 'html': {
+        prosemirrorJson = htmlToJson(content as string);
+        break;
+      }
+      case 'json':
+      default: {
+        prosemirrorJson = content;
+        break;
+      }
+    }
+
+    try {
+      jsonToNode(prosemirrorJson);
+    } catch (err) {
+      throw new BadRequestException('Invalid content format');
+    }
+
+    const documentName = `page.${pageId}`;
+    await this.collaborationGateway.handleYjsEvent(
+      'updatePageContent',
+      documentName,
+      { pageId, operation, prosemirrorJson, user },
+    );
+  }
+
   async getSidebarPages(
     spaceId: string,
     pagination: PaginationOptions,
@@ -209,7 +306,11 @@ export class PageService {
       cursor: pagination.cursor,
       beforeCursor: pagination.beforeCursor,
       fields: [
-        { expression: 'position', direction: 'asc', orderModifier: (ob) => ob.collate('C').asc() },
+        {
+          expression: 'position',
+          direction: 'asc',
+          orderModifier: (ob) => ob.collate('C').asc(),
+        },
         { expression: 'id', direction: 'asc' },
       ],
       parseCursor: (cursor) => ({