diff --git a/apps/server/src/collaboration/collaboration.handler.ts b/apps/server/src/collaboration/collaboration.handler.ts index ec746550..87dc5010 100644 --- a/apps/server/src/collaboration/collaboration.handler.ts +++ b/apps/server/src/collaboration/collaboration.handler.ts @@ -1,5 +1,12 @@ import { Injectable, Logger } from '@nestjs/common'; import { Hocuspocus, Document } from '@hocuspocus/server'; +import { TiptapTransformer } from '@hocuspocus/transformer'; +import { + prosemirrorNodeToYElement, + tiptapExtensions, +} from './collaboration.util'; +import * as Y from 'yjs'; +import { User } from '@docmost/db/types/entity.types'; export type CollabEventHandlers = ReturnType< CollaborationHandler['getHandlers'] @@ -20,6 +27,44 @@ export class CollaborationHandler { // const fragment = doc.getXmlFragment('default'); //}); }, + updatePageContent: async ( + documentName: string, + payload: { + prosemirrorJson: any; + operation: string; + user: User; + }, + ) => { + const { prosemirrorJson, operation, user } = payload; + this.logger.debug('Updating page content via yjs', documentName); + await this.withYdocConnection( + hocuspocus, + documentName, + { user }, + (doc) => { + const fragment = doc.getXmlFragment('default'); + + if (operation === 'replace') { + if (fragment.length > 0) { + fragment.delete(0, fragment.length); + } + + const newDoc = TiptapTransformer.toYdoc( + prosemirrorJson, + 'default', + tiptapExtensions, + ); + Y.applyUpdate(doc, Y.encodeStateAsUpdate(newDoc)); + } else { + const newContent = prosemirrorJson.content || []; + const yElements = newContent.map(prosemirrorNodeToYElement); + const position = + operation === 'prepend' ? 0 : fragment.length; + fragment.insert(position, yElements); + } + }, + ); + }, }; } diff --git a/apps/server/src/collaboration/collaboration.module.ts b/apps/server/src/collaboration/collaboration.module.ts index c4444d0b..c2dffd22 100644 --- a/apps/server/src/collaboration/collaboration.module.ts +++ b/apps/server/src/collaboration/collaboration.module.ts @@ -1,4 +1,10 @@ -import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { + Global, + Logger, + Module, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { AuthenticationExtension } from './extensions/authentication.extension'; import { PersistenceExtension } from './extensions/persistence.extension'; import { CollaborationGateway } from './collaboration.gateway'; diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index afe1be08..a29bb22a 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -34,6 +34,7 @@ import { Highlight, UniqueID, addUniqueIdsToDoc, + htmlToMarkdown, } from '@docmost/editor-ext'; import { generateText, getSchema, JSONContent } from '@tiptap/core'; import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; @@ -42,6 +43,7 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; // see:https://github.com/ueberdosis/tiptap/issues/4089 //import { generateJSON } from '@tiptap/html'; import { Node, Schema } from '@tiptap/pm/model'; +import * as Y from 'yjs'; import { Logger } from '@nestjs/common'; export const tiptapExtensions = [ @@ -161,3 +163,37 @@ function stripUnknownNodes( return json; } + +export function prosemirrorNodeToYElement(node: any): Y.XmlElement | Y.XmlText { + if (node.type === 'text') { + const ytext = new Y.XmlText(); + ytext.insert(0, node.text || ''); + if (node.marks?.length > 0) { + const attrs: Record = {}; + for (const mark of node.marks) { + attrs[mark.type] = mark.attrs || true; + } + ytext.format(0, node.text?.length || 0, attrs); + } + return ytext; + } + + const element = new Y.XmlElement(node.type); + if (node.attrs) { + for (const [key, value] of Object.entries(node.attrs)) { + if (value !== null && value !== undefined) { + element.setAttribute(key, value as any); + } + } + } + if (node.content?.length > 0) { + const children = node.content.map(prosemirrorNodeToYElement); + element.insert(0, children); + } + return element; +} + +export function jsonToMarkdown(tiptapJson: any): string { + const html = jsonToHtml(tiptapJson); + return htmlToMarkdown(html); +} diff --git a/apps/server/src/collaboration/extensions/persistence.extension.ts b/apps/server/src/collaboration/extensions/persistence.extension.ts index 4548b40c..5a358d99 100644 --- a/apps/server/src/collaboration/extensions/persistence.extension.ts +++ b/apps/server/src/collaboration/extensions/persistence.extension.ts @@ -181,7 +181,8 @@ export class PersistenceExtension implements Extension { async onChange(data: onChangePayload) { const documentName = data.documentName; - const userId = data.context?.user.id; + const userId = data.context?.user?.id; + if (!userId) return; if (!this.contributors.has(documentName)) { diff --git a/apps/server/src/collaboration/yjs.util.ts b/apps/server/src/collaboration/yjs.util.ts new file mode 100644 index 00000000..3e494bbc --- /dev/null +++ b/apps/server/src/collaboration/yjs.util.ts @@ -0,0 +1,177 @@ +import { + initProseMirrorDoc, + relativePositionToAbsolutePosition, +} from 'y-prosemirror'; +import * as Y from 'yjs'; +import { Document } from '@hocuspocus/server'; +import { getSchema } from '@tiptap/core'; +import { tiptapExtensions } from './collaboration.util'; + +export type YjsSelection = { + anchor: any; + head: any; +}; + +export function setYjsMark( + doc: Document, + fragment: Y.XmlFragment, + yjsSelection: YjsSelection, + markName: string, + markAttributes: Record, +) { + const schema = getSchema(tiptapExtensions); + const { mapping } = initProseMirrorDoc(fragment, schema); + + // Convert JSON positions to Y.js RelativePosition objects + const anchorRelPos = Y.createRelativePositionFromJSON(yjsSelection.anchor); + const headRelPos = Y.createRelativePositionFromJSON(yjsSelection.head); + + const anchor = relativePositionToAbsolutePosition( + doc, + fragment, + anchorRelPos, + mapping, + ); + const head = relativePositionToAbsolutePosition( + doc, + fragment, + headRelPos, + mapping, + ); + + if (anchor === null || head === null) { + throw new Error( + 'Could not resolve Y.js relative positions to absolute positions', + ); + } + + const from = Math.min(anchor, head); + const to = Math.max(anchor, head); + + // Apply mark directly to Y.js XmlText nodes + // This bypasses updateYFragment which has compatibility issues + applyMarkToYFragment(fragment, from, to, markName, markAttributes); +} + +function applyMarkToYFragment( + fragment: Y.XmlFragment, + from: number, + to: number, + markName: string, + markAttributes: Record, +) { + let pos = 0; + + const processItem = (item: any): boolean => { + if (pos >= to) return false; + + if (item instanceof Y.XmlText) { + const textLength = item.length; + const itemEnd = pos + textLength; + + if (itemEnd > from && pos < to) { + const formatFrom = Math.max(0, from - pos); + const formatTo = Math.min(textLength, to - pos); + const formatLength = formatTo - formatFrom; + + if (formatLength > 0) { + item.format(formatFrom, formatLength, { [markName]: markAttributes }); + } + } + pos = itemEnd; + } else if (item instanceof Y.XmlElement) { + pos++; // Opening tag + for (let i = 0; i < item.length; i++) { + if (!processItem(item.get(i))) return false; + } + pos++; // Closing tag + } + return true; + }; + + for (let i = 0; i < fragment.length; i++) { + if (!processItem(fragment.get(i))) break; + } +} + +/** + * Removes a mark from all text in the fragment that has the specified attribute value. + * Useful for deleting comments by commentId. + */ +export function removeYjsMarkByAttribute( + fragment: Y.XmlFragment, + markName: string, + attributeName: string, + attributeValue: string, +) { + const processItem = (item: any) => { + if (item instanceof Y.XmlText) { + // Get all formatting deltas to find ranges with this mark + const deltas = item.toDelta(); + let offset = 0; + + for (const delta of deltas) { + const length = delta.insert?.length ?? 0; + const attributes = delta.attributes ?? {}; + const markAttr = attributes[markName]; + + if (markAttr && markAttr[attributeName] === attributeValue) { + // Remove the mark by setting it to null + item.format(offset, length, { [markName]: null }); + } + offset += length; + } + } else if (item instanceof Y.XmlElement) { + for (let i = 0; i < item.length; i++) { + processItem(item.get(i)); + } + } + }; + + for (let i = 0; i < fragment.length; i++) { + processItem(fragment.get(i)); + } +} + +/** + * Updates a mark's attributes for all text that has the specified attribute value. + * Useful for resolving/unresolving comments by commentId. + */ +export function updateYjsMarkAttribute( + fragment: Y.XmlFragment, + markName: string, + findByAttribute: { name: string; value: string }, + newAttributes: Record, +) { + const processItem = (item: any) => { + if (item instanceof Y.XmlText) { + const deltas = item.toDelta(); + let offset = 0; + + for (const delta of deltas) { + const length = delta.insert?.length ?? 0; + const attributes = delta.attributes ?? {}; + const markAttr = attributes[markName]; + + if ( + markAttr && + markAttr[findByAttribute.name] === findByAttribute.value + ) { + // Update the mark with new attributes (merge with existing) + item.format(offset, length, { + [markName]: { ...markAttr, ...newAttributes }, + }); + } + offset += length; + } + } else if (item instanceof Y.XmlElement) { + for (let i = 0; i < item.length; i++) { + processItem(item.get(i)); + } + } + }; + + for (let i = 0; i < fragment.length; i++) { + processItem(fragment.get(i)); + } +} diff --git a/apps/server/src/core/page/dto/create-page.dto.ts b/apps/server/src/core/page/dto/create-page.dto.ts index 397e4a42..5cf71e5a 100644 --- a/apps/server/src/core/page/dto/create-page.dto.ts +++ b/apps/server/src/core/page/dto/create-page.dto.ts @@ -1,4 +1,13 @@ -import { IsOptional, IsString, IsUUID } from 'class-validator'; +import { + IsIn, + IsOptional, + IsString, + IsUUID, + ValidateIf, +} from 'class-validator'; +import { Transform } from 'class-transformer'; + +export type ContentFormat = 'json' | 'markdown' | 'html'; export class CreatePageDto { @IsOptional() @@ -15,4 +24,12 @@ export class CreatePageDto { @IsUUID() spaceId: string; + + @IsOptional() + content?: string | object; + + @ValidateIf((o) => o.content !== undefined) + @Transform(({ value }) => value?.toLowerCase() ?? 'json') + @IsIn(['json', 'markdown', 'html']) + format?: ContentFormat; } diff --git a/apps/server/src/core/page/dto/page.dto.ts b/apps/server/src/core/page/dto/page.dto.ts index e897d3a5..c53f8ade 100644 --- a/apps/server/src/core/page/dto/page.dto.ts +++ b/apps/server/src/core/page/dto/page.dto.ts @@ -1,10 +1,14 @@ import { IsBoolean, + IsIn, IsNotEmpty, IsOptional, IsString, IsUUID, } from 'class-validator'; +import { Transform } from 'class-transformer'; + +import { ContentFormat } from './create-page.dto'; export class PageIdDto { @IsString() @@ -30,6 +34,11 @@ export class PageInfoDto extends PageIdDto { @IsOptional() @IsBoolean() includeContent: boolean; + + @IsOptional() + @Transform(({ value }) => value?.toLowerCase()) + @IsIn(['json', 'markdown', 'html']) + format?: ContentFormat; } export class DeletePageDto extends PageIdDto { diff --git a/apps/server/src/core/page/dto/update-page.dto.ts b/apps/server/src/core/page/dto/update-page.dto.ts index 7bd2e2a6..026b3076 100644 --- a/apps/server/src/core/page/dto/update-page.dto.ts +++ b/apps/server/src/core/page/dto/update-page.dto.ts @@ -1,8 +1,24 @@ import { PartialType } from '@nestjs/mapped-types'; -import { CreatePageDto } from './create-page.dto'; -import { IsString } from 'class-validator'; +import { CreatePageDto, ContentFormat } from './create-page.dto'; +import { IsIn, IsOptional, IsString, ValidateIf } from 'class-validator'; +import { Transform } from 'class-transformer'; + +export type ContentOperation = 'append' | 'prepend' | 'replace'; export class UpdatePageDto extends PartialType(CreatePageDto) { @IsString() pageId: string; + + @IsOptional() + content?: string | object; + + @ValidateIf((o) => o.content !== undefined) + @Transform(({ value }) => value?.toLowerCase()) + @IsIn(['append', 'prepend', 'replace']) + operation?: ContentOperation; + + @ValidateIf((o) => o.content !== undefined) + @Transform(({ value }) => value?.toLowerCase() ?? 'json') + @IsIn(['json', 'markdown', 'html']) + format?: ContentFormat; } diff --git a/apps/server/src/core/page/page.controller.ts b/apps/server/src/core/page/page.controller.ts index 08daecf8..e1227f28 100644 --- a/apps/server/src/core/page/page.controller.ts +++ b/apps/server/src/core/page/page.controller.ts @@ -35,6 +35,10 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { RecentPageDto } from './dto/recent-page.dto'; import { DuplicatePageDto } from './dto/duplicate-page.dto'; import { DeletedPageDto } from './dto/deleted-page.dto'; +import { + jsonToHtml, + jsonToMarkdown, +} from '../../collaboration/collaboration.util'; @UseGuards(JwtAuthGuard) @Controller('pages') @@ -66,6 +70,17 @@ export class PageController { throw new ForbiddenException(); } + if (dto.format && dto.format !== 'json' && page.content) { + const contentOutput = + dto.format === 'markdown' + ? jsonToMarkdown(page.content) + : jsonToHtml(page.content); + return { + ...page, + content: contentOutput, + }; + } + return page; } @@ -84,7 +99,25 @@ export class PageController { throw new ForbiddenException(); } - return this.pageService.create(user.id, workspace.id, createPageDto); + const page = await this.pageService.create( + user.id, + workspace.id, + createPageDto, + ); + + if ( + createPageDto.format && + createPageDto.format !== 'json' && + page.content + ) { + const contentOutput = + createPageDto.format === 'markdown' + ? jsonToMarkdown(page.content) + : jsonToHtml(page.content); + return { ...page, content: contentOutput }; + } + + return page; } @HttpCode(HttpStatus.OK) @@ -101,7 +134,25 @@ export class PageController { throw new ForbiddenException(); } - return this.pageService.update(page, updatePageDto, user.id); + const updatedPage = await this.pageService.update( + page, + updatePageDto, + user, + ); + + if ( + updatePageDto.format && + updatePageDto.format !== 'json' && + updatedPage.content + ) { + const contentOutput = + updatePageDto.format === 'markdown' + ? jsonToMarkdown(updatedPage.content) + : jsonToHtml(updatedPage.content); + return { ...updatedPage, content: contentOutput }; + } + + return updatedPage; } @HttpCode(HttpStatus.OK) diff --git a/apps/server/src/core/page/page.module.ts b/apps/server/src/core/page/page.module.ts index 9dfba84a..1d70d9fa 100644 --- a/apps/server/src/core/page/page.module.ts +++ b/apps/server/src/core/page/page.module.ts @@ -4,11 +4,12 @@ import { PageController } from './page.controller'; import { PageHistoryService } from './services/page-history.service'; import { TrashCleanupService } from './services/trash-cleanup.service'; import { StorageModule } from '../../integrations/storage/storage.module'; +import { CollaborationModule } from '../../collaboration/collaboration.module'; @Module({ controllers: [PageController], providers: [PageService, PageHistoryService, TrashCleanupService], exports: [PageService, PageHistoryService], - imports: [StorageModule], + imports: [StorageModule, CollaborationModule], }) export class PageModule {} diff --git a/apps/server/src/core/page/services/page.service.ts b/apps/server/src/core/page/services/page.service.ts index 3b02e14e..964ea42e 100644 --- a/apps/server/src/core/page/services/page.service.ts +++ b/apps/server/src/core/page/services/page.service.ts @@ -4,8 +4,8 @@ import { Logger, NotFoundException, } from '@nestjs/common'; -import { CreatePageDto } from '../dto/create-page.dto'; -import { UpdatePageDto } from '../dto/update-page.dto'; +import { CreatePageDto, ContentFormat } from '../dto/create-page.dto'; +import { ContentOperation, UpdatePageDto } from '../dto/update-page.dto'; import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { InsertablePage, Page, User } from '@docmost/db/types/entity.types'; import { PaginationOptions } from '@docmost/db/pagination/pagination-options'; @@ -28,7 +28,11 @@ import { isAttachmentNode, removeMarkTypeFromDoc, } from '../../../common/helpers/prosemirror/utils'; -import { jsonToNode, jsonToText } from 'src/collaboration/collaboration.util'; +import { + htmlToJson, + jsonToNode, + jsonToText, +} from 'src/collaboration/collaboration.util'; import { CopyPageMapEntry, ICopyPageAttachment, @@ -40,6 +44,8 @@ import { Queue } from 'bullmq'; import { QueueJob, QueueName } from '../../../integrations/queue/constants'; import { EventName } from '../../../common/events/event.contants'; import { EventEmitter2 } from '@nestjs/event-emitter'; +import { CollaborationGateway } from '../../../collaboration/collaboration.gateway'; +import { markdownToHtml } from '@docmost/editor-ext'; @Injectable() export class PageService { @@ -53,6 +59,7 @@ export class PageService { @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue, @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue, private eventEmitter: EventEmitter2, + private collaborationGateway: CollaborationGateway, ) {} async findById( @@ -88,7 +95,22 @@ export class PageService { parentPageId = parentPage.id; } - const createdPage = await this.pageRepo.insertPage({ + let content = undefined; + let textContent = undefined; + let ydoc = undefined; + + if (createPageDto?.content && createPageDto?.format) { + const prosemirrorJson = await this.parseProsemirrorContent( + createPageDto.content, + createPageDto.format, + ); + + content = prosemirrorJson; + textContent = jsonToText(prosemirrorJson); + ydoc = createYdocFromJson(prosemirrorJson); + } + + return this.pageRepo.insertPage({ slugId: generateSlugId(), title: createPageDto.title, position: await this.nextPagePosition( @@ -101,9 +123,10 @@ export class PageService { creatorId: userId, workspaceId: workspaceId, lastUpdatedById: userId, + content, + textContent, + ydoc, }); - - return createdPage; } async nextPagePosition(spaceId: string, parentPageId?: string) { @@ -150,23 +173,37 @@ export class PageService { async update( page: Page, updatePageDto: UpdatePageDto, - userId: string, + user: User, ): Promise { const contributors = new Set(page.contributorIds); - contributors.add(userId); + contributors.add(user.id); const contributorIds = Array.from(contributors); await this.pageRepo.updatePage( { title: updatePageDto.title, icon: updatePageDto.icon, - lastUpdatedById: userId, + lastUpdatedById: user.id, updatedAt: new Date(), contributorIds: contributorIds, }, page.id, ); + if ( + updatePageDto.content && + updatePageDto.operation && + updatePageDto.format + ) { + await this.updatePageContent( + page.id, + updatePageDto.content, + updatePageDto.operation, + updatePageDto.format, + user, + ); + } + return await this.pageRepo.findById(page.id, { includeSpace: true, includeContent: true, @@ -176,6 +213,23 @@ export class PageService { }); } + async updatePageContent( + pageId: string, + content: string | object, + operation: ContentOperation, + format: ContentFormat, + user: User, + ): Promise { + const prosemirrorJson = await this.parseProsemirrorContent(content, format); + + const documentName = `page.${pageId}`; + await this.collaborationGateway.handleYjsEvent( + 'updatePageContent', + documentName, + { operation, prosemirrorJson, user }, + ); + } + async getSidebarPages( spaceId: string, pagination: PaginationOptions, @@ -209,7 +263,11 @@ export class PageService { cursor: pagination.cursor, beforeCursor: pagination.beforeCursor, fields: [ - { expression: 'position', direction: 'asc', orderModifier: (ob) => ob.collate('C').asc() }, + { + expression: 'position', + direction: 'asc', + orderModifier: (ob) => ob.collate('C').asc(), + }, { expression: 'id', direction: 'asc' }, ], parseCursor: (cursor) => ({ @@ -653,4 +711,36 @@ export class PageService { ): Promise { await this.pageRepo.removePage(pageId, userId, workspaceId); } + + private async parseProsemirrorContent( + content: string | object, + format: ContentFormat, + ): Promise { + let prosemirrorJson: any; + + switch (format) { + case 'markdown': { + const html = await markdownToHtml(content as string); + prosemirrorJson = htmlToJson(html as string); + break; + } + case 'html': { + prosemirrorJson = htmlToJson(content as string); + break; + } + case 'json': + default: { + prosemirrorJson = content; + break; + } + } + + try { + jsonToNode(prosemirrorJson); + } catch (err) { + throw new BadRequestException('Invalid content format'); + } + + return prosemirrorJson; + } }