feat: aduio

apps/client/src/features/editor/components/audio/audio-menu.tsx

@@ -0,0 +1,123 @@
+import { BubbleMenu as BaseBubbleMenu } from "@tiptap/react/menus";
+import { findParentNode, posToDOMRect, useEditorState } from "@tiptap/react";
+import { useCallback } from "react";
+import { Node as PMNode } from "@tiptap/pm/model";
+import {
+  EditorMenuProps,
+  ShouldShowProps,
+} from "@/features/editor/components/table/types/types.ts";
+import { ActionIcon, Tooltip } from "@mantine/core";
+import {
+  IconDownload,
+  IconTrash,
+} from "@tabler/icons-react";
+import { useTranslation } from "react-i18next";
+import { getFileUrl } from "@/lib/config.ts";
+import classes from "../common/toolbar-menu.module.css";
+
+export function AudioMenu({ editor }: EditorMenuProps) {
+  const { t } = useTranslation();
+
+  const editorState = useEditorState({
+    editor,
+    selector: (ctx) => {
+      if (!ctx.editor) {
+        return null;
+      }
+
+      const audioAttrs = ctx.editor.getAttributes("audio");
+
+      return {
+        isAudio: ctx.editor.isActive("audio"),
+        src: audioAttrs?.src || null,
+      };
+    },
+  });
+
+  const shouldShow = useCallback(
+    ({ state }: ShouldShowProps) => {
+      if (!state) {
+        return false;
+      }
+
+      return editor.isActive("audio") && editor.getAttributes("audio").src;
+    },
+    [editor],
+  );
+
+  const getReferencedVirtualElement = useCallback(() => {
+    if (!editor) return;
+    const { selection } = editor.state;
+    const predicate = (node: PMNode) => node.type.name === "audio";
+    const parent = findParentNode(predicate)(selection);
+
+    if (parent) {
+      const dom = editor.view.nodeDOM(parent?.pos) as HTMLElement;
+      const domRect = dom.getBoundingClientRect();
+      return {
+        getBoundingClientRect: () => domRect,
+        getClientRects: () => [domRect],
+      };
+    }
+
+    const domRect = posToDOMRect(editor.view, selection.from, selection.to);
+    return {
+      getBoundingClientRect: () => domRect,
+      getClientRects: () => [domRect],
+    };
+  }, [editor]);
+
+  const handleDownload = useCallback(() => {
+    if (!editorState?.src) return;
+    const url = getFileUrl(editorState.src);
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = "";
+    a.click();
+  }, [editorState?.src]);
+
+  const handleDelete = useCallback(() => {
+    editor.commands.deleteSelection();
+  }, [editor]);
+
+  return (
+    <BaseBubbleMenu
+      editor={editor}
+      pluginKey={`audio-menu`}
+      updateDelay={0}
+      getReferencedVirtualElement={getReferencedVirtualElement}
+      options={{
+        placement: "top",
+        offset: 8,
+        flip: false,
+      }}
+      shouldShow={shouldShow}
+    >
+      <div className={classes.toolbar}>
+        <Tooltip position="top" label={t("Download")} withinPortal={false}>
+          <ActionIcon
+            onClick={handleDownload}
+            size="lg"
+            aria-label={t("Download")}
+            variant="subtle"
+          >
+            <IconDownload size={18} />
+          </ActionIcon>
+        </Tooltip>
+
+        <Tooltip position="top" label={t("Delete")} withinPortal={false}>
+          <ActionIcon
+            onClick={handleDelete}
+            size="lg"
+            aria-label={t("Delete")}
+            variant="subtle"
+          >
+            <IconTrash size={18} />
+          </ActionIcon>
+        </Tooltip>
+      </div>
+    </BaseBubbleMenu>
+  );
+}
+
+export default AudioMenu;

apps/client/src/features/editor/components/audio/audio-view.module.css

@@ -0,0 +1,33 @@
+.audioWrapper {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  max-width: 100%;
+  border-radius: 8px;
+  overflow: hidden;
+  animation: pulse 1.2s ease-in-out infinite;
+
+  @mixin light {
+    background: linear-gradient(-90deg, var(--mantine-color-gray-3) 0%, var(--mantine-color-gray-1) 50%, var(--mantine-color-gray-3) 100%);
+    background-size: 400% 400%;
+  }
+
+  @mixin dark {
+    background: linear-gradient(-90deg, var(--mantine-color-dark-6) 0%, var(--mantine-color-dark-5) 50%, var(--mantine-color-dark-6) 100%);
+    background-size: 400% 400%;
+  }
+
+  @keyframes pulse {
+    0% {
+      background-position: 0% 0%;
+    }
+    100% {
+      background-position: -135% 0%;
+    }
+  }
+}
+.audio {
+  display: block;
+  width: 100%;
+  border-radius: 8px;
+}

apps/client/src/features/editor/components/audio/audio-view.tsx

@@ -0,0 +1,59 @@
+import { NodeViewProps, NodeViewWrapper } from "@tiptap/react";
+import { Group, Loader, Text } from "@mantine/core";
+import { useMemo } from "react";
+import { getFileUrl } from "@/lib/config.ts";
+import classes from "./audio-view.module.css";
+import { useTranslation } from "react-i18next";
+
+export default function AudioView(props: NodeViewProps) {
+  const { t } = useTranslation();
+  const { editor, node } = props;
+  const { src, placeholder } = node.attrs;
+
+  const previewSrc = useMemo(() => {
+    editor.storage.shared.audioPreviews =
+      editor.storage.shared.audioPreviews || {};
+
+    if (placeholder?.id) {
+      return editor.storage.shared.audioPreviews[placeholder.id];
+    }
+
+    return null;
+  }, [placeholder, editor]);
+
+  return (
+    <NodeViewWrapper data-drag-handle>
+      <div className={classes.audioWrapper}>
+        {src && (
+          <audio
+            className={classes.audio}
+            preload="metadata"
+            controls
+            src={getFileUrl(src)}
+          />
+        )}
+        {!src && previewSrc && (
+          <Group pos="relative" w="100%">
+            <audio
+              className={classes.audio}
+              preload="metadata"
+              controls
+              src={previewSrc}
+            />
+            <Loader size={20} pos="absolute" top={6} right={6} />
+          </Group>
+        )}
+        {!src && !previewSrc && (
+          <Group justify="center" wrap="nowrap" gap="xs" maw="100%" px="md" h={54}>
+            <Loader size={20} style={{ flexShrink: 0 }} />
+            <Text component="span" size="sm" truncate="end">
+              {placeholder?.name
+                ? t("Uploading {{name}}", { name: placeholder.name })
+                : t("Uploading file")}
+            </Text>
+          </Group>
+        )}
+      </div>
+    </NodeViewWrapper>
+  );
+}

apps/client/src/features/editor/components/audio/upload-audio-action.tsx

@@ -0,0 +1,36 @@
+import { handleAudioUpload } from "@docmost/editor-ext";
+import { uploadFile } from "@/features/page/services/page-service.ts";
+import { notifications } from "@mantine/notifications";
+import { getFileUploadSizeLimit } from "@/lib/config.ts";
+import { formatBytes } from "@/lib";
+import i18n from "@/i18n.ts";
+
+export const uploadAudioAction = handleAudioUpload({
+  onUpload: async (file: File, pageId: string): Promise<any> => {
+    try {
+      return await uploadFile(file, pageId);
+    } catch (err) {
+      notifications.show({
+        color: "red",
+        message: err?.response.data.message,
+      });
+      throw err;
+    }
+  },
+  validateFn: (file) => {
+    if (!file.type.includes("audio/")) {
+      return false;
+    }
+
+    if (file.size > getFileUploadSizeLimit()) {
+      notifications.show({
+        color: "red",
+        message: i18n.t("File exceeds the {{limit}} attachment limit", {
+          limit: formatBytes(getFileUploadSizeLimit()),
+        }),
+      });
+      return false;
+    }
+    return true;
+  },
+});

apps/client/src/features/editor/components/slash-menu/menu-items.ts

@@ -12,6 +12,7 @@ import {
   IconMath,
   IconMathFunction,
   IconMovie,
+  IconMusic,
   IconPaperclip,
   IconPhoto,
   IconTable,
@@ -30,6 +31,7 @@ import {
 } from "@/features/editor/components/slash-menu/types";
 import { uploadImageAction } from "@/features/editor/components/image/upload-image-action.tsx";
 import { uploadVideoAction } from "@/features/editor/components/video/upload-video-action.tsx";
+import { uploadAudioAction } from "@/features/editor/components/audio/upload-audio-action.tsx";
 import { uploadAttachmentAction } from "@/features/editor/components/attachment/upload-attachment-action.tsx";
 import IconExcalidraw from "@/components/icons/icon-excalidraw";
 import IconMermaid from "@/components/icons/icon-mermaid";
@@ -224,6 +226,39 @@ const CommandGroups: SlashMenuGroupedItemsType = {
         input.click();
       },
     },
+    {
+      title: "Audio",
+      description: "Upload any audio from your device.",
+      searchTerms: ["audio", "music", "sound", "mp3"],
+      icon: IconMusic,
+      command: ({ editor, range }) => {
+        editor.chain().focus().deleteRange(range).run();
+
+        // @ts-ignore
+        const pageId = editor.storage?.pageId;
+        if (!pageId) return;
+
+        // upload audio
+        const input = document.createElement("input");
+        input.type = "file";
+        input.accept = "audio/*";
+        input.multiple = true;
+        input.style.display = "none";
+        document.body.appendChild(input);
+        input.onchange = async () => {
+          if (input.files?.length) {
+            for (const file of input.files) {
+              const pos = editor.view.state.selection.from;
+
+              uploadAudioAction(file, editor, pos, pageId);
+            }
+          }
+
+          input.remove();
+        };
+        input.click();
+      },
+    },
     {
       title: "File attachment",
       description: "Upload any file from your device.",

apps/client/src/features/editor/extensions/extensions.ts

@@ -30,6 +30,7 @@ import {
   TiptapImage,
   Callout,
   TiptapVideo,
+  TiptapAudio,
   LinkExtension,
   Selection,
   Attachment,
@@ -68,6 +69,7 @@ import ImageView from "@/features/editor/components/image/image-view.tsx";
 import CalloutView from "@/features/editor/components/callout/callout-view.tsx";
 import StatusView from "@/features/editor/components/status/status-view.tsx";
 import VideoView from "@/features/editor/components/video/video-view.tsx";
+import AudioView from "@/features/editor/components/audio/audio-view.tsx";
 import AttachmentView from "@/features/editor/components/attachment/attachment-view.tsx";
 import CodeBlockView from "@/features/editor/components/code-block/code-block-view.tsx";
 import DrawioView from "../components/drawio/drawio-view";
@@ -269,6 +271,9 @@ export const mainExtensions = [
       className: buildResizeClasses("node-video"),
     },
   }),
+  TiptapAudio.configure({
+    view: AudioView,
+  }),
   Callout.configure({
     view: CalloutView,
   }),

apps/client/src/features/editor/page-editor.tsx

@@ -45,6 +45,7 @@ import TableMenu from "@/features/editor/components/table/table-menu.tsx";
 import ImageMenu from "@/features/editor/components/image/image-menu.tsx";
 import CalloutMenu from "@/features/editor/components/callout/callout-menu.tsx";
 import VideoMenu from "@/features/editor/components/video/video-menu.tsx";
+import AudioMenu from "@/features/editor/components/audio/audio-menu.tsx";
 import SubpagesMenu from "@/features/editor/components/subpages/subpages-menu.tsx";
 import {
   handleFileDrop,
@@ -414,6 +415,7 @@ export default function PageEditor({
             <TableCellMenu editor={editor} appendTo={menuContainerRef} />
             <ImageMenu editor={editor} />
             <VideoMenu editor={editor} />
+            <AudioMenu editor={editor} />
             <CalloutMenu editor={editor} />
             <SubpagesMenu editor={editor} />
             <ExcalidrawMenu editor={editor} />

apps/server/src/collaboration/collaboration.util.ts

@@ -24,6 +24,7 @@ import {
   CustomTable,
   TiptapImage,
   TiptapVideo,
+  TiptapAudio,
   TrailingNode,
   Attachment,
   Drawio,
@@ -86,6 +87,7 @@ export const tiptapExtensions = [
   Youtube,
   TiptapImage,
   TiptapVideo,
+  TiptapAudio,
   Callout,
   Attachment,
   CustomCodeBlock,

apps/server/src/core/attachment/attachment.constants.ts

@@ -15,4 +15,9 @@ export const inlineFileExtensions = [
   '.pdf',
   '.mp4',
   '.mov',
+  '.mp3',
+  '.wav',
+  '.ogg',
+  '.m4a',
+  '.webm',
 ];

package.json

@@ -30,6 +30,7 @@
     "@joplin/turndown-plugin-gfm": "^1.0.64",
     "@sindresorhus/slugify": "3.0.0",
     "@tiptap/core": "3.20.4",
+    "@tiptap/extension-audio": "3.20.4",
     "@tiptap/extension-code-block": "3.20.4",
     "@tiptap/extension-collaboration": "3.20.4",
     "@tiptap/extension-collaboration-caret": "3.20.4",

packages/editor-ext/src/index.ts

@@ -11,6 +11,7 @@ export * from "./lib/media-utils";
 export * from "./lib/link";
 export * from "./lib/selection";
 export * from "./lib/attachment";
+export * from "./lib/audio";
 export * from "./lib/custom-code-block";
 export * from "./lib/drawio";
 export * from "./lib/excalidraw";

packages/editor-ext/src/lib/audio/audio-upload.ts

@@ -0,0 +1,139 @@
+import { MediaUploadOptions, UploadFn } from "../media-utils";
+import { IAttachment } from "../types";
+import { generateNodeId } from "../utils";
+import { Node } from "@tiptap/pm/model";
+import { Command } from "@tiptap/core";
+
+const findAudioNodeByPlaceholderId = (
+  doc: Node,
+  placeholderId: string,
+): { node: Node; pos: number } | null => {
+  let result: { node: Node; pos: number } | null = null;
+
+  doc.descendants((node, pos) => {
+    if (result) return false;
+
+    if (
+      node.type.name === "audio" &&
+      node.attrs.placeholder?.id === placeholderId
+    ) {
+      result = { node, pos };
+      return false;
+    }
+
+    return true;
+  });
+
+  return result;
+};
+
+const handleAudioUpload =
+  ({ validateFn, onUpload }: MediaUploadOptions): UploadFn =>
+  async (file, editor, pos, pageId) => {
+    const validated = validateFn?.(file);
+    // @ts-ignore
+    if (!validated) return;
+
+    const objectUrl = URL.createObjectURL(file);
+    const placeholderId = generateNodeId();
+
+    let placeholderInserted = false;
+
+    editor.storage.shared.audioPreviews =
+      editor.storage.shared.audioPreviews || {};
+    editor.storage.shared.audioPreviews[placeholderId] = objectUrl;
+
+    const insertPlaceholder = (): Command => {
+      return ({ tr, state }) => {
+        const initialPlaceholderNode = state.schema.nodes.audio?.create({
+          placeholder: {
+            id: placeholderId,
+            name: file.name,
+          },
+        });
+
+        if (!initialPlaceholderNode) return false;
+
+        const { parent } = tr.doc.resolve(pos);
+        const isEmptyTextBlock = parent.isTextblock && !parent.childCount;
+
+        if (isEmptyTextBlock) {
+          tr.replaceRangeWith(pos - 1, pos + 1, initialPlaceholderNode);
+        } else {
+          tr.insert(pos, initialPlaceholderNode);
+        }
+
+        return true;
+      };
+    };
+
+    const replacePlaceholderWithAudio = (attachment: IAttachment): Command => {
+      return ({ tr }) => {
+        const { pos: currentPos = null } =
+          findAudioNodeByPlaceholderId(tr.doc, placeholderId) || {};
+
+        if (currentPos === null || !attachment) return;
+
+        tr.setNodeMarkup(currentPos, undefined, {
+          src: `/api/files/${attachment.id}/${attachment.fileName}`,
+          attachmentId: attachment.id,
+          size: attachment.fileSize,
+        });
+
+        return true;
+      };
+    };
+
+    const removePlaceholder = (): Command => {
+      return ({ tr }) => {
+        const { pos: currentPos = null } =
+          findAudioNodeByPlaceholderId(tr.doc, placeholderId) || {};
+
+        if (currentPos === null) return false;
+
+        tr.delete(currentPos, currentPos + 2);
+
+        return true;
+      };
+    };
+
+    const insertPlaceholderTimeout = setTimeout(() => {
+      editor.commands.command(insertPlaceholder());
+      placeholderInserted = true;
+    }, 250);
+
+    const disposePreviewFile = () => {
+      URL.revokeObjectURL(objectUrl);
+
+      if (editor.storage.shared.audioPreviews) {
+        delete editor.storage.shared.audioPreviews[placeholderId];
+      }
+    };
+
+    try {
+      const attachment: IAttachment = await onUpload(file, pageId);
+
+      clearTimeout(insertPlaceholderTimeout);
+
+      if (placeholderInserted) {
+        setTimeout(() => {
+          editor.commands.command(replacePlaceholderWithAudio(attachment));
+          disposePreviewFile();
+        }, 100);
+      } else {
+        editor
+          .chain()
+          .command(insertPlaceholder())
+          .command(replacePlaceholderWithAudio(attachment))
+          .run();
+        disposePreviewFile();
+      }
+    } catch (error) {
+      clearTimeout(insertPlaceholderTimeout);
+
+      editor.commands.command(removePlaceholder());
+      disposePreviewFile();
+    }
+  };
+
+export { handleAudioUpload };

packages/editor-ext/src/lib/audio/audio.ts

@@ -0,0 +1,124 @@
+import { Node, mergeAttributes } from "@tiptap/core";
+import { ReactNodeViewRenderer } from "@tiptap/react";
+import { normalizeFileUrl } from "../media-utils";
+
+export interface AudioOptions {
+  view: any;
+  HTMLAttributes: Record<string, any>;
+}
+
+export interface AudioAttributes {
+  src?: string;
+  attachmentId?: string;
+  size?: number;
+  placeholder?: {
+    id: string;
+    name: string;
+  };
+}
+
+declare module "@tiptap/core" {
+  interface Commands<ReturnType> {
+    audioBlock: {
+      setAudio: (attributes: AudioAttributes) => ReturnType;
+    };
+  }
+}
+
+export const TiptapAudio = Node.create<AudioOptions>({
+  name: "audio",
+
+  group: "block",
+  isolating: true,
+  atom: true,
+  defining: true,
+  draggable: true,
+
+  addOptions() {
+    return {
+      view: null,
+      HTMLAttributes: {},
+    };
+  },
+
+  addAttributes() {
+    return {
+      src: {
+        default: "",
+        parseHTML: (element) => element.getAttribute("src"),
+        renderHTML: (attributes) => ({
+          src: attributes.src,
+        }),
+      },
+      attachmentId: {
+        default: undefined,
+        parseHTML: (element) => element.getAttribute("data-attachment-id"),
+        renderHTML: (attributes: AudioAttributes) => ({
+          "data-attachment-id": attributes.attachmentId,
+        }),
+      },
+      size: {
+        default: null,
+        parseHTML: (element) => element.getAttribute("data-size"),
+        renderHTML: (attributes: AudioAttributes) => ({
+          "data-size": attributes.size,
+        }),
+      },
+      placeholder: {
+        default: null,
+        rendered: false,
+      },
+    };
+  },
+
+  parseHTML() {
+    return [
+      {
+        tag: "audio",
+      },
+    ];
+  },
+
+  renderHTML({ HTMLAttributes }) {
+    return [
+      "audio",
+      mergeAttributes(
+        { controls: "true", preload: "metadata" },
+        this.options.HTMLAttributes,
+        HTMLAttributes,
+      ),
+      ["source", { src: HTMLAttributes.src }],
+    ];
+  },
+
+  addCommands() {
+    return {
+      setAudio:
+        (attrs: AudioAttributes) =>
+        ({ commands }) => {
+          return commands.insertContent({
+            type: "audio",
+            attrs: attrs,
+          });
+        },
+    };
+  },
+
+  addNodeView() {
+    if (this.options.view) {
+      this.editor.isInitialized = true;
+      return ReactNodeViewRenderer(this.options.view);
+    }
+
+    return ({ node, HTMLAttributes }) => {
+      const dom = document.createElement("div");
+      const audio = document.createElement("audio");
+      audio.src = normalizeFileUrl(node.attrs.src);
+      audio.controls = true;
+      audio.preload = "metadata";
+      audio.style.width = "100%";
+      dom.append(audio);
+      return { dom };
+    };
+  },
+});

packages/editor-ext/src/lib/audio/index.ts

@@ -0,0 +1,2 @@
+export { TiptapAudio } from "./audio";
+export * from "./audio-upload";

pnpm-lock.yaml

@@ -75,6 +75,9 @@ importers:
       '@tiptap/core':
         specifier: 3.20.4
         version: 3.20.4(@tiptap/pm@3.20.4)
+      '@tiptap/extension-audio':
+        specifier: 3.20.4
+        version: 3.20.4(@tiptap/core@3.20.4(@tiptap/pm@3.20.4))
       '@tiptap/extension-code-block':
         specifier: 3.20.4
         version: 3.20.4(@tiptap/core@3.20.4(@tiptap/pm@3.20.4))(@tiptap/pm@3.20.4)
@@ -4659,6 +4662,11 @@ packages:
     peerDependencies:
       '@tiptap/pm': ^3.20.4
 
+  '@tiptap/extension-audio@3.20.4':
+    resolution: {integrity: sha512-zX90pxpEYpV5jSrwtQw8Nmh2uK4WC+xwSG5MXVh4VLG8SnSE/vg/vCCqFiSHjXNfw68dctd6HJ0MJigwnuS0lw==}
+    peerDependencies:
+      '@tiptap/core': ^3.20.4
+
   '@tiptap/extension-blockquote@3.20.4':
     resolution: {integrity: sha512-9sskyyhYj2oKat//lyZVXCp9YrPt4oJAZnGHYWXS0xlskjsLElrfKKlM4vpbhGss3VrhQRoEGqWLnIaJYPF1zw==}
     peerDependencies:
@@ -15282,6 +15290,10 @@ snapshots:
     dependencies:
       '@tiptap/pm': 3.20.4
 
+  '@tiptap/extension-audio@3.20.4(@tiptap/core@3.20.4(@tiptap/pm@3.20.4))':
+    dependencies:
+      '@tiptap/core': 3.20.4(@tiptap/pm@3.20.4)
+
   '@tiptap/extension-blockquote@3.20.4(@tiptap/core@3.20.4(@tiptap/pm@3.20.4))':
     dependencies:
       '@tiptap/core': 3.20.4(@tiptap/pm@3.20.4)