diff --git a/apps/client/src/features/page/components/page-import-modal.tsx b/apps/client/src/features/page/components/page-import-modal.tsx index baa9536e..de0cddcb 100644 --- a/apps/client/src/features/page/components/page-import-modal.tsx +++ b/apps/client/src/features/page/components/page-import-modal.tsx @@ -84,14 +84,14 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { } } - const newTreeNodes = buildTree(pages); - const fullTree = treeData.concat(newTreeNodes); + if (pages?.length > 0 && pageCount > 0) { + const newTreeNodes = buildTree(pages); + const fullTree = treeData.concat(newTreeNodes); - if (newTreeNodes?.length && fullTree?.length > 0) { - setTreeData(fullTree); - } + if (newTreeNodes?.length && fullTree?.length > 0) { + setTreeData(fullTree); + } - if (pageCount > 0) { const pageCountText = pageCount === 1 ? "1 page" : `${pageCount} pages`; notifications.update({ diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index d1e55f9d..8f8c98ab 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -28,7 +28,9 @@ import { TrailingNode, } from '@docmost/editor-ext'; import { generateText, JSONContent } from '@tiptap/core'; -import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; +import { generateHTML } from '../common/helpers/prosemirror/html'; +// default tiptap library works best generating prosemirror json state +import { generateJSON } from '@tiptap/html'; export const tiptapExtensions = [ StarterKit, diff --git a/apps/server/src/integrations/import/import.controller.ts b/apps/server/src/integrations/import/import.controller.ts index 461b9daa..902376d5 100644 --- a/apps/server/src/integrations/import/import.controller.ts +++ b/apps/server/src/integrations/import/import.controller.ts @@ -20,6 +20,7 @@ import { } from '../../core/casl/interfaces/space-ability.type'; import { FileInterceptor } from '../../common/interceptors/file.interceptor'; import * as bytes from 'bytes'; +import * as path from 'path'; import { MAX_FILE_SIZE } from '../../core/attachment/attachment.constants'; import { ImportService } from './import.service'; import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator'; @@ -42,6 +43,8 @@ export class ImportController { @AuthUser() user: User, @AuthWorkspace() workspace: Workspace, ) { + const validFileExtensions = ['.md', '.html']; + const maxFileSize = bytes(MAX_FILE_SIZE); let file = null; @@ -62,6 +65,12 @@ export class ImportController { throw new BadRequestException('Failed to upload file'); } + if ( + !validFileExtensions.includes(path.extname(file.filename).toLowerCase()) + ) { + throw new BadRequestException('Invalid import file type.'); + } + const spaceId = file.fields?.spaceId?.value; if (!spaceId) { diff --git a/apps/server/src/integrations/import/import.service.ts b/apps/server/src/integrations/import/import.service.ts index be84dda6..175855f5 100644 --- a/apps/server/src/integrations/import/import.service.ts +++ b/apps/server/src/integrations/import/import.service.ts @@ -4,12 +4,11 @@ import { MultipartFile } from '@fastify/multipart'; import { sanitize } from 'sanitize-filename-ts'; import * as path from 'path'; import { htmlToJson } from '../../collaboration/collaboration.util'; -import { marked } from 'marked'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; import { generateSlugId } from '../../common/helpers'; import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; -import { transformHTML } from './utils/html.utils'; +import { markdownToHtml } from './utils/marked.utils'; @Injectable() export class ImportService { @@ -36,16 +35,23 @@ export class ImportService { let prosemirrorState = null; let createdPage = null; - if (fileExtension.endsWith('.md') && fileMimeType === 'text/markdown') { - prosemirrorState = await this.processMarkdown(fileContent); - } - - if (fileExtension.endsWith('.html') && fileMimeType === 'text/html') { - prosemirrorState = await this.processHTML(fileContent); + try { + if (fileExtension.endsWith('.md') && fileMimeType === 'text/markdown') { + prosemirrorState = await this.processMarkdown(fileContent); + } else if ( + fileExtension.endsWith('.html') && + fileMimeType === 'text/html' + ) { + prosemirrorState = await this.processHTML(fileContent); + } + } catch (err) { + const message = 'Error processing file content'; + this.logger.error(message, err); + throw new BadRequestException(message); } if (!prosemirrorState) { - const message = 'Unsupported file format or mime type'; + const message = 'Failed to create ProseMirror state'; this.logger.error(message); throw new BadRequestException(message); } @@ -69,8 +75,12 @@ export class ImportService { workspaceId: workspaceId, lastUpdatedById: userId, }); + + this.logger.debug( + `Successfully imported "${title}${fileExtension}. ID: ${createdPage.id} - SlugId: ${createdPage.slugId}"`, + ); } catch (err) { - const message = 'Failed to create page'; + const message = 'Failed to create imported page'; this.logger.error(message, err); throw new BadRequestException(message); } @@ -80,14 +90,20 @@ export class ImportService { } async processMarkdown(markdownInput: string): Promise { - // turn markdown to html - const html = await marked.parse(markdownInput); - return await this.processHTML(html); + try { + const html = await markdownToHtml(markdownInput); + return this.processHTML(html); + } catch (err) { + throw err; + } } async processHTML(htmlInput: string): Promise { - // turn html to prosemirror state - return htmlToJson(transformHTML(htmlInput)); + try { + return htmlToJson(htmlInput); + } catch (err) { + throw err; + } } extractTitleAndRemoveHeading(prosemirrorState: any) { diff --git a/apps/server/src/integrations/import/utils/html.utils.ts b/apps/server/src/integrations/import/utils/html.utils.ts deleted file mode 100644 index 02c3d778..00000000 --- a/apps/server/src/integrations/import/utils/html.utils.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { Window, DOMParser } from 'happy-dom'; - -function transformTaskList(html: string): string { - const window = new Window(); - const doc = new DOMParser(window).parseFromString(html, 'text/html'); - - const ulElements = doc.querySelectorAll('ul'); - ulElements.forEach((ul) => { - let isTaskList = false; - - const liElements = ul.querySelectorAll('li'); - liElements.forEach((li) => { - const checkbox = li.querySelector('input[type="checkbox"]'); - - if (checkbox) { - isTaskList = true; - // Add taskItem data type - li.setAttribute('data-type', 'taskItem'); - // Set data-checked attribute based on the checkbox state - // @ts-ignore - li.setAttribute('data-checked', checkbox.checked ? 'true' : 'false'); - // Remove the checkbox from the li - checkbox.remove(); - - // Move the content of

out of the

and remove

- const pElements = li.querySelectorAll('p'); - pElements.forEach((p) => { - // Append the content of the

element to its parent (the

  • element) - while (p.firstChild) { - li.appendChild(p.firstChild); - } - // Remove the now empty

    element - p.remove(); - }); - } - }); - - // If any

  • contains a checkbox, mark the
      as a task list - if (isTaskList) { - ul.setAttribute('data-type', 'taskList'); - } - }); - - return doc.body.innerHTML; -} - -function transformCallouts(html: string): string { - const window = new Window(); - const doc = new DOMParser(window).parseFromString(html, 'text/html'); - - const calloutRegex = /:::(\w+)\s*([\s\S]*?)\s*:::/g; - - const createCalloutDiv = (type: string, content: string): HTMLElement => { - const div = doc.createElement('div'); - div.setAttribute('data-type', 'callout'); - div.setAttribute('data-callout-type', type); - const p = doc.createElement('p'); - p.textContent = content.trim(); - div.appendChild(p); - return div as unknown as HTMLElement; - }; - - const pElements = doc.querySelectorAll('p'); - - pElements.forEach((p) => { - if (calloutRegex.test(p.innerHTML) && !p.closest('ul, ol')) { - calloutRegex.lastIndex = 0; - const [, type, content] = calloutRegex.exec(p.innerHTML) || []; - const calloutDiv = createCalloutDiv(type, content); - // @ts-ignore - p.replaceWith(calloutDiv); - } - }); - - return doc.body.innerHTML; -} - -export function transformHTML(html: string): string { - return transformTaskList(transformCallouts(html)); -} diff --git a/apps/server/src/integrations/import/utils/marked.utils.ts b/apps/server/src/integrations/import/utils/marked.utils.ts new file mode 100644 index 00000000..c0283a01 --- /dev/null +++ b/apps/server/src/integrations/import/utils/marked.utils.ts @@ -0,0 +1,36 @@ +import { marked } from 'marked'; + +marked.use({ + renderer: { + // @ts-ignore + list(body: string, isOrdered: boolean, start: number) { + if (isOrdered) { + const startAttr = start !== 1 ? ` start="${start}"` : ''; + return `
        \n${body}
      \n`; + } + + const dataType = body.includes(`\n${body}
    \n`; + }, + // @ts-ignore + listitem({ text, raw, task: isTask, checked: isChecked }): string { + if (!isTask) { + return `
  • ${text}
    • \n`; + } + const checkedAttr = isChecked + ? 'data-checked="true"' + : 'data-checked="false"'; + return `
  • ${text}
    • \n`; + }, + }, +}); + +export async function markdownToHtml(markdownInput: string): Promise { + const YAML_FONT_MATTER_REGEX = /^\s*---[\s\S]*?---\s*/; + + const markdown = markdownInput + .replace(YAML_FONT_MATTER_REGEX, '') + .trimStart(); + + return marked.parse(markdown); +} diff --git a/package.json b/package.json index 2c250439..ca36f395 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@tiptap/extension-typography": "^2.5.4", "@tiptap/extension-underline": "^2.5.4", "@tiptap/extension-youtube": "^2.5.4", + "@tiptap/html": "^2.5.4", "@tiptap/pm": "^2.5.4", "@tiptap/react": "^2.5.4", "@tiptap/starter-kit": "^2.5.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25ee7b0a..06a26ddb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,9 @@ importers: '@tiptap/extension-youtube': specifier: ^2.5.4 version: 2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4)) + '@tiptap/html': + specifier: ^2.5.4 + version: 2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))(@tiptap/pm@2.5.4) '@tiptap/pm': specifier: ^2.5.4 version: 2.5.4 @@ -3780,6 +3783,12 @@ packages: peerDependencies: '@tiptap/core': ^2.5.4 + '@tiptap/html@2.5.4': + resolution: {integrity: sha512-Fcvsa7kkO+Id7WBFimDN5zdHksVGVnyHnffaN/PaAgbKmzP53BC38Pd0XuHS+KL6btqQIFE2GlqNYnyIos7i+g==} + peerDependencies: + '@tiptap/core': ^2.5.4 + '@tiptap/pm': ^2.5.4 + '@tiptap/pm@2.5.4': resolution: {integrity: sha512-oFIsuniptdUXn93x4aM2sVN3hYKo9Fj55zAkYrWhwxFYUYcPxd5ibra2we+wRK5TaiPu098wpC+yMSTZ/KKMpA==} @@ -4757,6 +4766,10 @@ packages: css-to-mat@1.1.1: resolution: {integrity: sha512-kvpxFYZb27jRd2vium35G7q5XZ2WJ9rWjDUMNT36M3Hc41qCrLXFM5iEKMGXcrPsKfXEN+8l/riB4QzwwwiEyQ==} + css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -6178,8 +6191,8 @@ packages: makeerror@1.0.12: resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==} - markdown-it@14.0.0: - resolution: {integrity: sha512-seFjF0FIcPt4P9U39Bq1JYblX0KZCjDLFFQPHpL5AzHpqPEKtosxmdq/LTVZnjfH7tjt9BxStm+wXcDBNuYmzw==} + markdown-it@14.1.0: + resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} hasBin: true marked@13.0.2: @@ -7790,8 +7803,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - uc.micro@2.0.0: - resolution: {integrity: sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==} + uc.micro@2.1.0: + resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} uid2@1.0.0: resolution: {integrity: sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==} @@ -8160,6 +8173,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zeed-dom@0.10.11: + resolution: {integrity: sha512-7ukbu6aQKx34OQ7PfUIxOuAhk2MvyZY/t4/IJsVzy76zuMzfhE74+Dbyp8SHiUJPTPkF0FflP1KVrGJ7gk9IHw==} + engines: {node: '>=14.13.1'} + zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -12059,6 +12076,12 @@ snapshots: dependencies: '@tiptap/core': 2.5.4(@tiptap/pm@2.5.4) + '@tiptap/html@2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))(@tiptap/pm@2.5.4)': + dependencies: + '@tiptap/core': 2.5.4(@tiptap/pm@2.5.4) + '@tiptap/pm': 2.5.4 + zeed-dom: 0.10.11 + '@tiptap/pm@2.5.4': dependencies: prosemirror-changeset: 2.2.1 @@ -13272,6 +13295,8 @@ snapshots: '@daybrush/utils': 1.13.0 '@scena/matrix': 1.1.1 + css-what@6.1.0: {} + cssesc@3.0.0: {} cssstyle@3.0.0: @@ -14899,7 +14924,7 @@ snapshots: linkify-it@5.0.0: dependencies: - uc.micro: 2.0.0 + uc.micro: 2.1.0 linkifyjs@4.1.3: {} @@ -14994,14 +15019,14 @@ snapshots: dependencies: tmpl: 1.0.5 - markdown-it@14.0.0: + markdown-it@14.1.0: dependencies: argparse: 2.0.1 entities: 4.5.0 linkify-it: 5.0.0 mdurl: 2.0.0 punycode.js: 2.3.1 - uc.micro: 2.0.0 + uc.micro: 2.1.0 marked@13.0.2: {} @@ -15690,7 +15715,7 @@ snapshots: prosemirror-markdown@1.13.0: dependencies: - markdown-it: 14.0.0 + markdown-it: 14.1.0 prosemirror-model: 1.22.1 prosemirror-menu@1.2.4: @@ -16778,7 +16803,7 @@ snapshots: typescript@5.5.2: {} - uc.micro@2.0.0: {} + uc.micro@2.1.0: {} uid2@1.0.0: {} @@ -17101,4 +17126,8 @@ snapshots: yocto-queue@0.1.0: {} + zeed-dom@0.10.11: + dependencies: + css-what: 6.1.0 + zod@3.23.8: {}