feat(API): page content update and retrieval (#1937)

* feat: page content update and retrieval output

* import module

* refactor naming
* support prepend

* rename contentOperation -> operation

* dry

* add yjs utils

apps/server/src/core/page/dto/create-page.dto.ts

@@ -1,4 +1,13 @@
-import { IsOptional, IsString, IsUUID } from 'class-validator';
+import {
+  IsIn,
+  IsOptional,
+  IsString,
+  IsUUID,
+  ValidateIf,
+} from 'class-validator';
+import { Transform } from 'class-transformer';
+
+export type ContentFormat = 'json' | 'markdown' | 'html';
 
 export class CreatePageDto {
   @IsOptional()
@@ -15,4 +24,12 @@ export class CreatePageDto {
 
   @IsUUID()
   spaceId: string;
+
+  @IsOptional()
+  content?: string | object;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @Transform(({ value }) => value?.toLowerCase() ?? 'json')
+  @IsIn(['json', 'markdown', 'html'])
+  format?: ContentFormat;
 }

apps/server/src/core/page/dto/page.dto.ts

@@ -1,10 +1,14 @@
 import {
   IsBoolean,
+  IsIn,
   IsNotEmpty,
   IsOptional,
   IsString,
   IsUUID,
 } from 'class-validator';
+import { Transform } from 'class-transformer';
+
+import { ContentFormat } from './create-page.dto';
 
 export class PageIdDto {
   @IsString()
@@ -30,6 +34,11 @@ export class PageInfoDto extends PageIdDto {
   @IsOptional()
   @IsBoolean()
   includeContent: boolean;
+
+  @IsOptional()
+  @Transform(({ value }) => value?.toLowerCase())
+  @IsIn(['json', 'markdown', 'html'])
+  format?: ContentFormat;
 }
 
 export class DeletePageDto extends PageIdDto {

apps/server/src/core/page/dto/update-page.dto.ts

@@ -1,8 +1,24 @@
 import { PartialType } from '@nestjs/mapped-types';
-import { CreatePageDto } from './create-page.dto';
-import { IsString } from 'class-validator';
+import { CreatePageDto, ContentFormat } from './create-page.dto';
+import { IsIn, IsOptional, IsString, ValidateIf } from 'class-validator';
+import { Transform } from 'class-transformer';
+
+export type ContentOperation = 'append' | 'prepend' | 'replace';
 
 export class UpdatePageDto extends PartialType(CreatePageDto) {
   @IsString()
   pageId: string;
+
+  @IsOptional()
+  content?: string | object;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @Transform(({ value }) => value?.toLowerCase())
+  @IsIn(['append', 'prepend', 'replace'])
+  operation?: ContentOperation;
+
+  @ValidateIf((o) => o.content !== undefined)
+  @Transform(({ value }) => value?.toLowerCase() ?? 'json')
+  @IsIn(['json', 'markdown', 'html'])
+  format?: ContentFormat;
 }

apps/server/src/core/page/page.controller.ts

@@ -35,6 +35,10 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { RecentPageDto } from './dto/recent-page.dto';
 import { DuplicatePageDto } from './dto/duplicate-page.dto';
 import { DeletedPageDto } from './dto/deleted-page.dto';
+import {
+  jsonToHtml,
+  jsonToMarkdown,
+} from '../../collaboration/collaboration.util';
 
 @UseGuards(JwtAuthGuard)
 @Controller('pages')
@@ -66,6 +70,17 @@ export class PageController {
       throw new ForbiddenException();
     }
 
+    if (dto.format && dto.format !== 'json' && page.content) {
+      const contentOutput =
+        dto.format === 'markdown'
+          ? jsonToMarkdown(page.content)
+          : jsonToHtml(page.content);
+      return {
+        ...page,
+        content: contentOutput,
+      };
+    }
+
     return page;
   }
 
@@ -84,7 +99,25 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    return this.pageService.create(user.id, workspace.id, createPageDto);
+    const page = await this.pageService.create(
+      user.id,
+      workspace.id,
+      createPageDto,
+    );
+
+    if (
+      createPageDto.format &&
+      createPageDto.format !== 'json' &&
+      page.content
+    ) {
+      const contentOutput =
+        createPageDto.format === 'markdown'
+          ? jsonToMarkdown(page.content)
+          : jsonToHtml(page.content);
+      return { ...page, content: contentOutput };
+    }
+
+    return page;
   }
 
   @HttpCode(HttpStatus.OK)
@@ -101,7 +134,25 @@ export class PageController {
       throw new ForbiddenException();
     }
 
-    return this.pageService.update(page, updatePageDto, user.id);
+    const updatedPage = await this.pageService.update(
+      page,
+      updatePageDto,
+      user,
+    );
+
+    if (
+      updatePageDto.format &&
+      updatePageDto.format !== 'json' &&
+      updatedPage.content
+    ) {
+      const contentOutput =
+        updatePageDto.format === 'markdown'
+          ? jsonToMarkdown(updatedPage.content)
+          : jsonToHtml(updatedPage.content);
+      return { ...updatedPage, content: contentOutput };
+    }
+
+    return updatedPage;
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/page/page.module.ts

@@ -4,11 +4,12 @@ import { PageController } from './page.controller';
 import { PageHistoryService } from './services/page-history.service';
 import { TrashCleanupService } from './services/trash-cleanup.service';
 import { StorageModule } from '../../integrations/storage/storage.module';
+import { CollaborationModule } from '../../collaboration/collaboration.module';
 
 @Module({
   controllers: [PageController],
   providers: [PageService, PageHistoryService, TrashCleanupService],
   exports: [PageService, PageHistoryService],
-  imports: [StorageModule],
+  imports: [StorageModule, CollaborationModule],
 })
 export class PageModule {}

apps/server/src/core/page/services/page.service.ts

@@ -4,8 +4,8 @@ import {
   Logger,
   NotFoundException,
 } from '@nestjs/common';
-import { CreatePageDto } from '../dto/create-page.dto';
-import { UpdatePageDto } from '../dto/update-page.dto';
+import { CreatePageDto, ContentFormat } from '../dto/create-page.dto';
+import { ContentOperation, UpdatePageDto } from '../dto/update-page.dto';
 import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { InsertablePage, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
@@ -28,7 +28,11 @@ import {
   isAttachmentNode,
   removeMarkTypeFromDoc,
 } from '../../../common/helpers/prosemirror/utils';
-import { jsonToNode, jsonToText } from 'src/collaboration/collaboration.util';
+import {
+  htmlToJson,
+  jsonToNode,
+  jsonToText,
+} from 'src/collaboration/collaboration.util';
 import {
   CopyPageMapEntry,
   ICopyPageAttachment,
@@ -40,6 +44,8 @@ import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { EventName } from '../../../common/events/event.contants';
 import { EventEmitter2 } from '@nestjs/event-emitter';
+import { CollaborationGateway } from '../../../collaboration/collaboration.gateway';
+import { markdownToHtml } from '@docmost/editor-ext';
 
 @Injectable()
 export class PageService {
@@ -53,6 +59,7 @@ export class PageService {
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
     @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
     private eventEmitter: EventEmitter2,
+    private collaborationGateway: CollaborationGateway,
   ) {}
 
   async findById(
@@ -88,7 +95,22 @@ export class PageService {
       parentPageId = parentPage.id;
     }
 
-    const createdPage = await this.pageRepo.insertPage({
+    let content = undefined;
+    let textContent = undefined;
+    let ydoc = undefined;
+
+    if (createPageDto?.content && createPageDto?.format) {
+      const prosemirrorJson = await this.parseProsemirrorContent(
+        createPageDto.content,
+        createPageDto.format,
+      );
+
+      content = prosemirrorJson;
+      textContent = jsonToText(prosemirrorJson);
+      ydoc = createYdocFromJson(prosemirrorJson);
+    }
+
+    return this.pageRepo.insertPage({
       slugId: generateSlugId(),
       title: createPageDto.title,
       position: await this.nextPagePosition(
@@ -101,9 +123,10 @@ export class PageService {
       creatorId: userId,
       workspaceId: workspaceId,
       lastUpdatedById: userId,
+      content,
+      textContent,
+      ydoc,
     });
-
-    return createdPage;
   }
 
   async nextPagePosition(spaceId: string, parentPageId?: string) {
@@ -150,23 +173,37 @@ export class PageService {
   async update(
     page: Page,
     updatePageDto: UpdatePageDto,
-    userId: string,
+    user: User,
   ): Promise<Page> {
     const contributors = new Set<string>(page.contributorIds);
-    contributors.add(userId);
+    contributors.add(user.id);
     const contributorIds = Array.from(contributors);
 
     await this.pageRepo.updatePage(
       {
         title: updatePageDto.title,
         icon: updatePageDto.icon,
-        lastUpdatedById: userId,
+        lastUpdatedById: user.id,
         updatedAt: new Date(),
         contributorIds: contributorIds,
       },
       page.id,
     );
 
+    if (
+      updatePageDto.content &&
+      updatePageDto.operation &&
+      updatePageDto.format
+    ) {
+      await this.updatePageContent(
+        page.id,
+        updatePageDto.content,
+        updatePageDto.operation,
+        updatePageDto.format,
+        user,
+      );
+    }
+
     return await this.pageRepo.findById(page.id, {
       includeSpace: true,
       includeContent: true,
@@ -176,6 +213,23 @@ export class PageService {
     });
   }
 
+  async updatePageContent(
+    pageId: string,
+    content: string | object,
+    operation: ContentOperation,
+    format: ContentFormat,
+    user: User,
+  ): Promise<void> {
+    const prosemirrorJson = await this.parseProsemirrorContent(content, format);
+
+    const documentName = `page.${pageId}`;
+    await this.collaborationGateway.handleYjsEvent(
+      'updatePageContent',
+      documentName,
+      { operation, prosemirrorJson, user },
+    );
+  }
+
   async getSidebarPages(
     spaceId: string,
     pagination: PaginationOptions,
@@ -209,7 +263,11 @@ export class PageService {
       cursor: pagination.cursor,
       beforeCursor: pagination.beforeCursor,
       fields: [
-        { expression: 'position', direction: 'asc', orderModifier: (ob) => ob.collate('C').asc() },
+        {
+          expression: 'position',
+          direction: 'asc',
+          orderModifier: (ob) => ob.collate('C').asc(),
+        },
         { expression: 'id', direction: 'asc' },
       ],
       parseCursor: (cursor) => ({
@@ -653,4 +711,36 @@ export class PageService {
   ): Promise<void> {
     await this.pageRepo.removePage(pageId, userId, workspaceId);
   }
+
+  private async parseProsemirrorContent(
+    content: string | object,
+    format: ContentFormat,
+  ): Promise<any> {
+    let prosemirrorJson: any;
+
+    switch (format) {
+      case 'markdown': {
+        const html = await markdownToHtml(content as string);
+        prosemirrorJson = htmlToJson(html as string);
+        break;
+      }
+      case 'html': {
+        prosemirrorJson = htmlToJson(content as string);
+        break;
+      }
+      case 'json':
+      default: {
+        prosemirrorJson = content;
+        break;
+      }
+    }
+
+    try {
+      jsonToNode(prosemirrorJson);
+    } catch (err) {
+      throw new BadRequestException('Invalid content format');
+    }
+
+    return prosemirrorJson;
+  }
 }