feat(API): page content update and retrieval (#1937)

* feat: page content update and retrieval output

* import module

* refactor naming
* support prepend

* rename contentOperation -> operation

* dry

* add yjs utils

apps/server/src/collaboration/collaboration.handler.ts

@@ -1,5 +1,12 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { Hocuspocus, Document } from '@hocuspocus/server';
+import { TiptapTransformer } from '@hocuspocus/transformer';
+import {
+  prosemirrorNodeToYElement,
+  tiptapExtensions,
+} from './collaboration.util';
+import * as Y from 'yjs';
+import { User } from '@docmost/db/types/entity.types';
 
 export type CollabEventHandlers = ReturnType<
   CollaborationHandler['getHandlers']
@@ -20,6 +27,44 @@ export class CollaborationHandler {
         //   const fragment = doc.getXmlFragment('default');
         //});
       },
+      updatePageContent: async (
+        documentName: string,
+        payload: {
+          prosemirrorJson: any;
+          operation: string;
+          user: User;
+        },
+      ) => {
+        const { prosemirrorJson, operation, user } = payload;
+        this.logger.debug('Updating page content via yjs', documentName);
+        await this.withYdocConnection(
+          hocuspocus,
+          documentName,
+          { user },
+          (doc) => {
+            const fragment = doc.getXmlFragment('default');
+
+            if (operation === 'replace') {
+              if (fragment.length > 0) {
+                fragment.delete(0, fragment.length);
+              }
+
+              const newDoc = TiptapTransformer.toYdoc(
+                prosemirrorJson,
+                'default',
+                tiptapExtensions,
+              );
+              Y.applyUpdate(doc, Y.encodeStateAsUpdate(newDoc));
+            } else {
+              const newContent = prosemirrorJson.content || [];
+              const yElements = newContent.map(prosemirrorNodeToYElement);
+              const position =
+                operation === 'prepend' ? 0 : fragment.length;
+              fragment.insert(position, yElements);
+            }
+          },
+        );
+      },
     };
   }
 

apps/server/src/collaboration/collaboration.module.ts

@@ -1,4 +1,10 @@
-import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import {
+  Global,
+  Logger,
+  Module,
+  OnModuleDestroy,
+  OnModuleInit,
+} from '@nestjs/common';
 import { AuthenticationExtension } from './extensions/authentication.extension';
 import { PersistenceExtension } from './extensions/persistence.extension';
 import { CollaborationGateway } from './collaboration.gateway';

apps/server/src/collaboration/collaboration.util.ts

@@ -34,6 +34,7 @@ import {
   Highlight,
   UniqueID,
   addUniqueIdsToDoc,
+  htmlToMarkdown,
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
 import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
@@ -42,6 +43,7 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
 // see:https://github.com/ueberdosis/tiptap/issues/4089
 //import { generateJSON } from '@tiptap/html';
 import { Node, Schema } from '@tiptap/pm/model';
+import * as Y from 'yjs';
 import { Logger } from '@nestjs/common';
 
 export const tiptapExtensions = [
@@ -161,3 +163,37 @@ function stripUnknownNodes(
 
   return json;
 }
+
+export function prosemirrorNodeToYElement(node: any): Y.XmlElement | Y.XmlText {
+  if (node.type === 'text') {
+    const ytext = new Y.XmlText();
+    ytext.insert(0, node.text || '');
+    if (node.marks?.length > 0) {
+      const attrs: Record<string, any> = {};
+      for (const mark of node.marks) {
+        attrs[mark.type] = mark.attrs || true;
+      }
+      ytext.format(0, node.text?.length || 0, attrs);
+    }
+    return ytext;
+  }
+
+  const element = new Y.XmlElement(node.type);
+  if (node.attrs) {
+    for (const [key, value] of Object.entries(node.attrs)) {
+      if (value !== null && value !== undefined) {
+        element.setAttribute(key, value as any);
+      }
+    }
+  }
+  if (node.content?.length > 0) {
+    const children = node.content.map(prosemirrorNodeToYElement);
+    element.insert(0, children);
+  }
+  return element;
+}
+
+export function jsonToMarkdown(tiptapJson: any): string {
+  const html = jsonToHtml(tiptapJson);
+  return htmlToMarkdown(html);
+}

apps/server/src/collaboration/extensions/persistence.extension.ts

@@ -181,7 +181,8 @@ export class PersistenceExtension implements Extension {
 
   async onChange(data: onChangePayload) {
     const documentName = data.documentName;
-    const userId = data.context?.user.id;
+    const userId = data.context?.user?.id;
+
     if (!userId) return;
 
     if (!this.contributors.has(documentName)) {

apps/server/src/collaboration/yjs.util.ts

@@ -0,0 +1,177 @@
+import {
+  initProseMirrorDoc,
+  relativePositionToAbsolutePosition,
+} from 'y-prosemirror';
+import * as Y from 'yjs';
+import { Document } from '@hocuspocus/server';
+import { getSchema } from '@tiptap/core';
+import { tiptapExtensions } from './collaboration.util';
+
+export type YjsSelection = {
+  anchor: any;
+  head: any;
+};
+
+export function setYjsMark(
+  doc: Document,
+  fragment: Y.XmlFragment,
+  yjsSelection: YjsSelection,
+  markName: string,
+  markAttributes: Record<string, any>,
+) {
+  const schema = getSchema(tiptapExtensions);
+  const { mapping } = initProseMirrorDoc(fragment, schema);
+
+  // Convert JSON positions to Y.js RelativePosition objects
+  const anchorRelPos = Y.createRelativePositionFromJSON(yjsSelection.anchor);
+  const headRelPos = Y.createRelativePositionFromJSON(yjsSelection.head);
+
+  const anchor = relativePositionToAbsolutePosition(
+    doc,
+    fragment,
+    anchorRelPos,
+    mapping,
+  );
+  const head = relativePositionToAbsolutePosition(
+    doc,
+    fragment,
+    headRelPos,
+    mapping,
+  );
+
+  if (anchor === null || head === null) {
+    throw new Error(
+      'Could not resolve Y.js relative positions to absolute positions',
+    );
+  }
+
+  const from = Math.min(anchor, head);
+  const to = Math.max(anchor, head);
+
+  // Apply mark directly to Y.js XmlText nodes
+  // This bypasses updateYFragment which has compatibility issues
+  applyMarkToYFragment(fragment, from, to, markName, markAttributes);
+}
+
+function applyMarkToYFragment(
+  fragment: Y.XmlFragment,
+  from: number,
+  to: number,
+  markName: string,
+  markAttributes: Record<string, any>,
+) {
+  let pos = 0;
+
+  const processItem = (item: any): boolean => {
+    if (pos >= to) return false;
+
+    if (item instanceof Y.XmlText) {
+      const textLength = item.length;
+      const itemEnd = pos + textLength;
+
+      if (itemEnd > from && pos < to) {
+        const formatFrom = Math.max(0, from - pos);
+        const formatTo = Math.min(textLength, to - pos);
+        const formatLength = formatTo - formatFrom;
+
+        if (formatLength > 0) {
+          item.format(formatFrom, formatLength, { [markName]: markAttributes });
+        }
+      }
+      pos = itemEnd;
+    } else if (item instanceof Y.XmlElement) {
+      pos++; // Opening tag
+      for (let i = 0; i < item.length; i++) {
+        if (!processItem(item.get(i))) return false;
+      }
+      pos++; // Closing tag
+    }
+    return true;
+  };
+
+  for (let i = 0; i < fragment.length; i++) {
+    if (!processItem(fragment.get(i))) break;
+  }
+}
+
+/**
+ * Removes a mark from all text in the fragment that has the specified attribute value.
+ * Useful for deleting comments by commentId.
+ */
+export function removeYjsMarkByAttribute(
+  fragment: Y.XmlFragment,
+  markName: string,
+  attributeName: string,
+  attributeValue: string,
+) {
+  const processItem = (item: any) => {
+    if (item instanceof Y.XmlText) {
+      // Get all formatting deltas to find ranges with this mark
+      const deltas = item.toDelta();
+      let offset = 0;
+
+      for (const delta of deltas) {
+        const length = delta.insert?.length ?? 0;
+        const attributes = delta.attributes ?? {};
+        const markAttr = attributes[markName];
+
+        if (markAttr && markAttr[attributeName] === attributeValue) {
+          // Remove the mark by setting it to null
+          item.format(offset, length, { [markName]: null });
+        }
+        offset += length;
+      }
+    } else if (item instanceof Y.XmlElement) {
+      for (let i = 0; i < item.length; i++) {
+        processItem(item.get(i));
+      }
+    }
+  };
+
+  for (let i = 0; i < fragment.length; i++) {
+    processItem(fragment.get(i));
+  }
+}
+
+/**
+ * Updates a mark's attributes for all text that has the specified attribute value.
+ * Useful for resolving/unresolving comments by commentId.
+ */
+export function updateYjsMarkAttribute(
+  fragment: Y.XmlFragment,
+  markName: string,
+  findByAttribute: { name: string; value: string },
+  newAttributes: Record<string, any>,
+) {
+  const processItem = (item: any) => {
+    if (item instanceof Y.XmlText) {
+      const deltas = item.toDelta();
+      let offset = 0;
+
+      for (const delta of deltas) {
+        const length = delta.insert?.length ?? 0;
+        const attributes = delta.attributes ?? {};
+        const markAttr = attributes[markName];
+
+        if (
+          markAttr &&
+          markAttr[findByAttribute.name] === findByAttribute.value
+        ) {
+          // Update the mark with new attributes (merge with existing)
+          item.format(offset, length, {
+            [markName]: { ...markAttr, ...newAttributes },
+          });
+        }
+        offset += length;
+      }
+    } else if (item instanceof Y.XmlElement) {
+      for (let i = 0; i < item.length; i++) {
+        processItem(item.get(i));
+      }
+    }
+  };
+
+  for (let i = 0; i < fragment.length; i++) {
+    processItem(fragment.get(i));
+  }
+}

apps/server/src/core/page/dto/create-page.dto.ts

@@ -1,4 +1,13 @@
-import { IsOptional, IsString, IsUUID } from 'class-validator';
+import {
+  IsIn,
+  IsOptional,
+  IsString,
+  IsUUID,
+  ValidateIf,
+} from 'class-validator';
+import { Transform } from 'class-transformer';
+
+export type ContentFormat = 'json' | 'markdown' | 'html';
 
 export class CreatePageDto {
   @IsOptional()
@@ -15,4 +24,12 @@ export class CreatePageDto {
 
   @IsUUID()
   spaceId: string;
+
+  @IsOptional()
+  content?: string | object;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @Transform(({ value }) => value?.toLowerCase() ?? 'json')
+  @IsIn(['json', 'markdown', 'html'])
+  format?: ContentFormat;
 }

apps/server/src/core/page/dto/page.dto.ts

@@ -1,10 +1,14 @@
 import {
   IsBoolean,
+  IsIn,
   IsNotEmpty,
   IsOptional,
   IsString,
   IsUUID,
 } from 'class-validator';
+import { Transform } from 'class-transformer';
+
+import { ContentFormat } from './create-page.dto';
 
 export class PageIdDto {
   @IsString()
@@ -30,6 +34,11 @@ export class PageInfoDto extends PageIdDto {
   @IsOptional()
   @IsBoolean()
   includeContent: boolean;
+
+  @IsOptional()
+  @Transform(({ value }) => value?.toLowerCase())
+  @IsIn(['json', 'markdown', 'html'])
+  format?: ContentFormat;
 }
 
 export class DeletePageDto extends PageIdDto {

apps/server/src/core/page/dto/update-page.dto.ts

@@ -1,8 +1,24 @@
 import { PartialType } from '@nestjs/mapped-types';
-import { CreatePageDto } from './create-page.dto';
+import { CreatePageDto, ContentFormat } from './create-page.dto';
-import { IsString } from 'class-validator';
+import { IsIn, IsOptional, IsString, ValidateIf } from 'class-validator';
+import { Transform } from 'class-transformer';
+
+export type ContentOperation = 'append' | 'prepend' | 'replace';
 
 export class UpdatePageDto extends PartialType(CreatePageDto) {
   @IsString()
   pageId: string;
+
+  @IsOptional()
+  content?: string | object;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @Transform(({ value }) => value?.toLowerCase())
+  @IsIn(['append', 'prepend', 'replace'])
+  operation?: ContentOperation;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @Transform(({ value }) => value?.toLowerCase() ?? 'json')
+  @IsIn(['json', 'markdown', 'html'])
+  format?: ContentFormat;
 }

apps/server/src/core/page/page.controller.ts

@@ -35,6 +35,10 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { RecentPageDto } from './dto/recent-page.dto';
 import { DuplicatePageDto } from './dto/duplicate-page.dto';
 import { DeletedPageDto } from './dto/deleted-page.dto';
+import {
+  jsonToHtml,
+  jsonToMarkdown,
+} from '../../collaboration/collaboration.util';
 
 @UseGuards(JwtAuthGuard)
 @Controller('pages')
@@ -66,6 +70,17 @@ export class PageController {
       throw new ForbiddenException();
     }
 
+    if (dto.format && dto.format !== 'json' && page.content) {
+      const contentOutput =
+        dto.format === 'markdown'
+          ? jsonToMarkdown(page.content)
+          : jsonToHtml(page.content);
+      return {
+        ...page,
+        content: contentOutput,
+      };
+    }
+
     return page;
   }
 
@@ -84,7 +99,25 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    return this.pageService.create(user.id, workspace.id, createPageDto);
+    const page = await this.pageService.create(
+      user.id,
+      workspace.id,
+      createPageDto,
+    );
+
+    if (
+      createPageDto.format &&
+      createPageDto.format !== 'json' &&
+      page.content
+    ) {
+      const contentOutput =
+        createPageDto.format === 'markdown'
+          ? jsonToMarkdown(page.content)
+          : jsonToHtml(page.content);
+      return { ...page, content: contentOutput };
+    }
+
+    return page;
   }
 
   @HttpCode(HttpStatus.OK)
@@ -101,7 +134,25 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    return this.pageService.update(page, updatePageDto, user.id);
+    const updatedPage = await this.pageService.update(
+      page,
+      updatePageDto,
+      user,
+    );
+
+    if (
+      updatePageDto.format &&
+      updatePageDto.format !== 'json' &&
+      updatedPage.content
+    ) {
+      const contentOutput =
+        updatePageDto.format === 'markdown'
+          ? jsonToMarkdown(updatedPage.content)
+          : jsonToHtml(updatedPage.content);
+      return { ...updatedPage, content: contentOutput };
+    }
+
+    return updatedPage;
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/page/page.module.ts

@@ -4,11 +4,12 @@ import { PageController } from './page.controller';
 import { PageHistoryService } from './services/page-history.service';
 import { TrashCleanupService } from './services/trash-cleanup.service';
 import { StorageModule } from '../../integrations/storage/storage.module';
+import { CollaborationModule } from '../../collaboration/collaboration.module';
 
 @Module({
   controllers: [PageController],
   providers: [PageService, PageHistoryService, TrashCleanupService],
   exports: [PageService, PageHistoryService],
-  imports: [StorageModule],
+  imports: [StorageModule, CollaborationModule],
 })
 export class PageModule {}

apps/server/src/core/page/services/page.service.ts

@@ -4,8 +4,8 @@ import {
   Logger,
   NotFoundException,
 } from '@nestjs/common';
-import { CreatePageDto } from '../dto/create-page.dto';
+import { CreatePageDto, ContentFormat } from '../dto/create-page.dto';
-import { UpdatePageDto } from '../dto/update-page.dto';
+import { ContentOperation, UpdatePageDto } from '../dto/update-page.dto';
 import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { InsertablePage, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
@@ -28,7 +28,11 @@ import {
   isAttachmentNode,
   removeMarkTypeFromDoc,
 } from '../../../common/helpers/prosemirror/utils';
-import { jsonToNode, jsonToText } from 'src/collaboration/collaboration.util';
+import {
+  htmlToJson,
+  jsonToNode,
+  jsonToText,
+} from 'src/collaboration/collaboration.util';
 import {
   CopyPageMapEntry,
   ICopyPageAttachment,
@@ -40,6 +44,8 @@ import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { EventName } from '../../../common/events/event.contants';
 import { EventEmitter2 } from '@nestjs/event-emitter';
+import { CollaborationGateway } from '../../../collaboration/collaboration.gateway';
+import { markdownToHtml } from '@docmost/editor-ext';
 
 @Injectable()
 export class PageService {
@@ -53,6 +59,7 @@ export class PageService {
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
     @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
     private eventEmitter: EventEmitter2,
+    private collaborationGateway: CollaborationGateway,
   ) {}
 
   async findById(
@@ -88,7 +95,22 @@ export class PageService {
       parentPageId = parentPage.id;
     }
 
-    const createdPage = await this.pageRepo.insertPage({
+    let content = undefined;
+    let textContent = undefined;
+    let ydoc = undefined;
+
+    if (createPageDto?.content && createPageDto?.format) {
+      const prosemirrorJson = await this.parseProsemirrorContent(
+        createPageDto.content,
+        createPageDto.format,
+      );
+
+      content = prosemirrorJson;
+      textContent = jsonToText(prosemirrorJson);
+      ydoc = createYdocFromJson(prosemirrorJson);
+    }
+
+    return this.pageRepo.insertPage({
       slugId: generateSlugId(),
       title: createPageDto.title,
       position: await this.nextPagePosition(
@@ -101,9 +123,10 @@ export class PageService {
       creatorId: userId,
       workspaceId: workspaceId,
       lastUpdatedById: userId,
+      content,
+      textContent,
+      ydoc,
     });
-
-    return createdPage;
   }
 
   async nextPagePosition(spaceId: string, parentPageId?: string) {
@@ -150,23 +173,37 @@ export class PageService {
   async update(
     page: Page,
     updatePageDto: UpdatePageDto,
-    userId: string,
+    user: User,
   ): Promise<Page> {
     const contributors = new Set<string>(page.contributorIds);
-    contributors.add(userId);
+    contributors.add(user.id);
     const contributorIds = Array.from(contributors);
 
     await this.pageRepo.updatePage(
       {
         title: updatePageDto.title,
         icon: updatePageDto.icon,
-        lastUpdatedById: userId,
+        lastUpdatedById: user.id,
         updatedAt: new Date(),
         contributorIds: contributorIds,
       },
       page.id,
     );
 
+    if (
+      updatePageDto.content &&
+      updatePageDto.operation &&
+      updatePageDto.format
+    ) {
+      await this.updatePageContent(
+        page.id,
+        updatePageDto.content,
+        updatePageDto.operation,
+        updatePageDto.format,
+        user,
+      );
+    }
+
     return await this.pageRepo.findById(page.id, {
       includeSpace: true,
       includeContent: true,
@@ -176,6 +213,23 @@ export class PageService {
     });
   }
 
+  async updatePageContent(
+    pageId: string,
+    content: string | object,
+    operation: ContentOperation,
+    format: ContentFormat,
+    user: User,
+  ): Promise<void> {
+    const prosemirrorJson = await this.parseProsemirrorContent(content, format);
+
+    const documentName = `page.${pageId}`;
+    await this.collaborationGateway.handleYjsEvent(
+      'updatePageContent',
+      documentName,
+      { operation, prosemirrorJson, user },
+    );
+  }
+
   async getSidebarPages(
     spaceId: string,
     pagination: PaginationOptions,
@@ -209,7 +263,11 @@ export class PageService {
       cursor: pagination.cursor,
       beforeCursor: pagination.beforeCursor,
       fields: [
-        { expression: 'position', direction: 'asc', orderModifier: (ob) => ob.collate('C').asc() },
+        {
+          expression: 'position',
+          direction: 'asc',
+          orderModifier: (ob) => ob.collate('C').asc(),
+        },
         { expression: 'id', direction: 'asc' },
       ],
       parseCursor: (cursor) => ({
@@ -653,4 +711,36 @@ export class PageService {
   ): Promise<void> {
     await this.pageRepo.removePage(pageId, userId, workspaceId);
   }
+
+  private async parseProsemirrorContent(
+    content: string | object,
+    format: ContentFormat,
+  ): Promise<any> {
+    let prosemirrorJson: any;
+
+    switch (format) {
+      case 'markdown': {
+        const html = await markdownToHtml(content as string);
+        prosemirrorJson = htmlToJson(html as string);
+        break;
+      }
+      case 'html': {
+        prosemirrorJson = htmlToJson(content as string);
+        break;
+      }
+      case 'json':
+      default: {
+        prosemirrorJson = content;
+        break;
+      }
+    }
+
+    try {
+      jsonToNode(prosemirrorJson);
+    } catch (err) {
+      throw new BadRequestException('Invalid content format');
+    }
+
+    return prosemirrorJson;
+  }
 }