feat: page history diff (#1891)

* Show actual history changes
* V2 - WIP
* feat: page history diff
* fix: exclude content from history listing

---------

Co-authored-by: Jason Norwood-Young <jason@10layer.com>

packages/editor-ext/src/index.ts

@@ -24,3 +24,4 @@ export * from "./lib/highlight";
 export * from "./lib/heading/heading";
 export * from "./lib/unique-id";
 export * from "./lib/shared-storage";
+export * from "./lib/recreate-transform";

packages/editor-ext/src/lib/recreate-transform/copy.ts

@@ -0,0 +1,3 @@
+export function copy<T>(value: T): T {
+  return JSON.parse(JSON.stringify(value));
+}

packages/editor-ext/src/lib/recreate-transform/getFromPath.ts

@@ -0,0 +1,17 @@
+import { AnyObject } from "./types";
+
+/**
+ * get target value from json-pointer (e.g. /content/0/content)
+ * @param  {AnyObject} obj  object to resolve path into
+ * @param  {string}    path json-pointer
+ * @return {any} target value
+ */
+export function getFromPath(obj: AnyObject, path: string): any {
+  const pathParts = path.split("/");
+  pathParts.shift(); // remove root-entry
+  while (pathParts.length) {
+    const property = pathParts.shift();
+    obj = obj[property];
+  }
+  return obj;
+}

packages/editor-ext/src/lib/recreate-transform/getReplaceStep.ts

@@ -0,0 +1,29 @@
+import { ReplaceStep } from "@tiptap/pm/transform";
+import { Node } from "@tiptap/pm/model";
+
+export function getReplaceStep(fromDoc: Node, toDoc: Node) {
+  let start = toDoc.content.findDiffStart(fromDoc.content);
+  if (start === null) {
+    return false;
+  }
+
+  // @ts-ignore property access to content
+  let { a: endA, b: endB } = toDoc.content.findDiffEnd(fromDoc.content);
+  const overlap = start - Math.min(endA, endB);
+  if (overlap > 0) {
+    // If there is an overlap, there is some freedom of choice in how to calculate the
+    // start/end boundary. for an inserted/removed slice. We choose the extreme with
+    // the lowest depth value.
+    if (
+      fromDoc.resolve(start - overlap).depth <
+      toDoc.resolve(endA + overlap).depth
+    ) {
+      start -= overlap;
+    } else {
+      endA += overlap;
+      endB += overlap;
+    }
+  }
+
+  return new ReplaceStep(start, endB, toDoc.slice(start, endA));
+}

packages/editor-ext/src/lib/recreate-transform/index.ts

@@ -0,0 +1,4 @@
+// https://gitlab.com/mpapp-public/prosemirror-recreate-steps - MIT
+// https://github.com/sueddeutsche/prosemirror-recreate-transform - MIT
+export { recreateTransform, RecreateTransform } from "./recreateTransform";
+export type { Options } from "./recreateTransform";

packages/editor-ext/src/lib/recreate-transform/recreateTransform.ts

@@ -0,0 +1,279 @@
+import { Transform } from "@tiptap/pm/transform";
+import { Node, Schema } from "@tiptap/pm/model";
+import { applyPatch, createPatch, Operation } from "rfc6902";
+import { diffWordsWithSpace, diffChars } from "diff";
+import { AnyObject } from "./types";
+import { getReplaceStep } from "./getReplaceStep";
+import { simplifyTransform } from "./simplifyTransform";
+import { removeMarks } from "./removeMarks";
+import { getFromPath } from "./getFromPath";
+import { copy } from "./copy";
+
+export interface Options {
+  complexSteps?: boolean;
+  wordDiffs?: boolean;
+  simplifyDiff?: boolean;
+}
+
+export class RecreateTransform {
+  fromDoc: Node;
+  toDoc: Node;
+  complexSteps: boolean;
+  wordDiffs: boolean;
+  simplifyDiff: boolean;
+  schema: Schema;
+  tr: Transform;
+  /* current working document data, may get updated while recalculating node steps */
+  currentJSON: AnyObject;
+  /* final document as json data */
+  finalJSON: AnyObject;
+  ops: Array<Operation>;
+
+  constructor(fromDoc: Node, toDoc: Node, options: Options = {}) {
+    const o = {
+      complexSteps: true,
+      wordDiffs: false,
+      simplifyDiff: true,
+      ...options,
+    };
+
+    this.fromDoc = fromDoc;
+    this.toDoc = toDoc;
+    this.complexSteps = o.complexSteps; // Whether to return steps other than ReplaceSteps
+    this.wordDiffs = o.wordDiffs; // Whether to make text diffs cover entire words
+    this.simplifyDiff = o.simplifyDiff;
+    this.schema = fromDoc.type.schema;
+    this.tr = new Transform(fromDoc);
+  }
+
+  init() {
+    if (this.complexSteps) {
+      // For First steps: we create versions of the documents without marks as
+      // these will only confuse the diffing mechanism and marks won't cause
+      // any mapping changes anyway.
+      this.currentJSON = removeMarks(this.fromDoc).toJSON();
+      this.finalJSON = removeMarks(this.toDoc).toJSON();
+      this.ops = createPatch(this.currentJSON, this.finalJSON);
+      this.recreateChangeContentSteps();
+      this.recreateChangeMarkSteps();
+    } else {
+      // We don't differentiate between mark changes and other changes.
+      this.currentJSON = this.fromDoc.toJSON();
+      this.finalJSON = this.toDoc.toJSON();
+      this.ops = createPatch(this.currentJSON, this.finalJSON);
+      this.recreateChangeContentSteps();
+    }
+
+    if (this.simplifyDiff) {
+      this.tr = simplifyTransform(this.tr) || this.tr;
+    }
+
+    return this.tr;
+  }
+
+  /** convert json-diff to prosemirror steps */
+  recreateChangeContentSteps() {
+    // First step: find content changing steps.
+    let ops = [];
+    while (this.ops.length) {
+      // get next
+      let op = this.ops.shift();
+      ops.push(op);
+
+      let toDoc;
+      const afterStepJSON = copy(this.currentJSON); // working document receiving patches
+      const pathParts = op.path.split("/");
+
+      // collect operations until we receive a valid document:
+      // apply ops-patches until a valid prosemirror document is retrieved,
+      // then try to create a transformation step or retry with next operation
+      while (toDoc == null) {
+        applyPatch(afterStepJSON, [op]);
+
+        try {
+          toDoc = this.schema.nodeFromJSON(afterStepJSON);
+          toDoc.check();
+        } catch (error) {
+          toDoc = null;
+          if (this.ops.length > 0) {
+            op = this.ops.shift();
+            ops.push(op);
+          } else {
+            throw new Error(`No valid diff possible applying ${op.path}`);
+          }
+        }
+      }
+
+      // apply operation (ignoring afterStepJSON)
+      if (
+        this.complexSteps &&
+        ops.length === 1 &&
+        (pathParts.includes("attrs") || pathParts.includes("type"))
+      ) {
+        // Node markup is changing
+        this.addSetNodeMarkup(); // a lost update is ignored
+        ops = [];
+        // console.log("%cop", logStyle, "- update node", ops);
+      } else if (
+        ops.length === 1 &&
+        op.op === "replace" &&
+        pathParts[pathParts.length - 1] === "text"
+      ) {
+        // Text is being replaced, we apply text diffing to find the smallest possible diffs.
+        this.addReplaceTextSteps(op, afterStepJSON);
+        ops = [];
+        // console.log("%cop", logStyle, "- replace", ops);
+      } else if (this.addReplaceStep(toDoc, afterStepJSON)) {
+        // operations have been applied
+        ops = [];
+        // console.log("%cop", logStyle, "- other", ops);
+      }
+    }
+  }
+
+  /** update node with attrs and marks, may also change type */
+  addSetNodeMarkup() {
+    // first diff in document is supposed to be a node-change (in type and/or attributes)
+    // thus simply find the first change and apply a node change step, then recalculate the diff
+    // after updating the document
+    const fromDoc = this.schema.nodeFromJSON(this.currentJSON);
+    const toDoc = this.schema.nodeFromJSON(this.finalJSON);
+    const start = toDoc.content.findDiffStart(fromDoc.content);
+    // @note start is the same (first) position for current and target document
+    const fromNode = fromDoc.nodeAt(start);
+    const toNode = toDoc.nodeAt(start);
+
+    if (start != null) {
+      // @note this completly updates all attributes in one step, by completely replacing node
+      const nodeType = fromNode.type === toNode.type ? null : toNode.type;
+      try {
+        this.tr.setNodeMarkup(start, nodeType, toNode.attrs, toNode.marks);
+      } catch (e) {
+        // if nodetypes differ, the updated node-type and contents might not be compatible
+        // with schema and requires a replace
+        if (nodeType && e.message.includes("Invalid content")) {
+          // @todo add test-case for this scenario
+          this.tr.replaceWith(start, start + fromNode.nodeSize, toNode);
+        } else {
+          throw e;
+        }
+      }
+      this.currentJSON = removeMarks(this.tr.doc).toJSON();
+      // setting the node markup may have invalidated the following ops, so we calculate them again.
+      this.ops = createPatch(this.currentJSON, this.finalJSON);
+      return true;
+    }
+    return false;
+  }
+
+  recreateChangeMarkSteps() {
+    // Now the documents should be the same, except their marks, so everything should map 1:1.
+    // Second step: Iterate through the toDoc and make sure all marks are the same in tr.doc
+    this.toDoc.descendants((tNode, tPos) => {
+      if (!tNode.isInline) {
+        return true;
+      }
+
+      this.tr.doc.nodesBetween(tPos, tPos + tNode.nodeSize, (fNode, fPos) => {
+        if (!fNode.isInline) {
+          return true;
+        }
+        const from = Math.max(tPos, fPos);
+        const to = Math.min(tPos + tNode.nodeSize, fPos + fNode.nodeSize);
+        fNode.marks.forEach((nodeMark) => {
+          if (!nodeMark.isInSet(tNode.marks)) {
+            this.tr.removeMark(from, to, nodeMark);
+          }
+        });
+        tNode.marks.forEach((nodeMark) => {
+          if (!nodeMark.isInSet(fNode.marks)) {
+            this.tr.addMark(from, to, nodeMark);
+          }
+        });
+      });
+    });
+  }
+
+  /**
+   * retrieve and possibly apply replace-step based from doc changes
+   * From http://prosemirror.net/examples/footnote/
+   */
+  addReplaceStep(toDoc: Node, afterStepJSON: AnyObject) {
+    const fromDoc = this.schema.nodeFromJSON(this.currentJSON);
+    const step = getReplaceStep(fromDoc, toDoc);
+
+    if (!step) {
+      return false;
+    } else if (!this.tr.maybeStep(step).failed) {
+      this.currentJSON = afterStepJSON;
+      return true; // @change previously null
+    }
+
+    throw new Error("No valid step found.");
+  }
+
+  /** retrieve and possibly apply text replace-steps based from doc changes */
+  addReplaceTextSteps(op, afterStepJSON) {
+    // We find the position number of the first character in the string
+    const op1 = { ...op, value: "xx" };
+    const op2 = { ...op, value: "yy" };
+    const afterOP1JSON = copy(this.currentJSON);
+    const afterOP2JSON = copy(this.currentJSON);
+    applyPatch(afterOP1JSON, [op1]);
+    applyPatch(afterOP2JSON, [op2]);
+    const op1Doc = this.schema.nodeFromJSON(afterOP1JSON);
+    const op2Doc = this.schema.nodeFromJSON(afterOP2JSON);
+
+    // get text diffs
+    const finalText = op.value;
+    const currentText = getFromPath(this.currentJSON, op.path);
+    const textDiffs = this.wordDiffs
+      ? diffWordsWithSpace(currentText, finalText)
+      : diffChars(currentText, finalText);
+
+    let offset = op1Doc.content.findDiffStart(op2Doc.content);
+    const marks = op1Doc.resolve(offset + 1).marks();
+
+    while (textDiffs.length) {
+      const diff = textDiffs.shift();
+
+      if (diff.added) {
+        const textNode = this.schema
+          .nodeFromJSON({ type: "text", text: diff.value })
+          .mark(marks);
+
+        if (textDiffs.length && textDiffs[0].removed) {
+          const nextDiff = textDiffs.shift();
+          this.tr.replaceWith(offset, offset + nextDiff.value.length, textNode);
+        } else {
+          this.tr.insert(offset, textNode);
+        }
+        offset += diff.value.length;
+      } else if (diff.removed) {
+        if (textDiffs.length && textDiffs[0].added) {
+          const nextDiff = textDiffs.shift();
+          const textNode = this.schema
+            .nodeFromJSON({ type: "text", text: nextDiff.value })
+            .mark(marks);
+          this.tr.replaceWith(offset, offset + diff.value.length, textNode);
+          offset += nextDiff.value.length;
+        } else {
+          this.tr.delete(offset, offset + diff.value.length);
+        }
+      } else {
+        offset += diff.value.length;
+      }
+    }
+
+    this.currentJSON = afterStepJSON;
+  }
+}
+
+export function recreateTransform(
+  fromDoc: Node,
+  toDoc: Node,
+  options: Options = {},
+): Transform {
+  const recreator = new RecreateTransform(fromDoc, toDoc, options);
+  return recreator.init();
+}

packages/editor-ext/src/lib/recreate-transform/removeMarks.ts

@@ -0,0 +1,8 @@
+import { Transform } from "@tiptap/pm/transform";
+import { Node } from "@tiptap/pm/model";
+
+export function removeMarks(doc: Node) {
+  const tr = new Transform(doc);
+  tr.removeMark(0, doc.nodeSize - 2);
+  return tr.doc;
+}

packages/editor-ext/src/lib/recreate-transform/simplifyTransform.ts

@@ -0,0 +1,30 @@
+import { Transform, ReplaceStep, Step } from "@tiptap/pm/transform";
+import { getReplaceStep } from "./getReplaceStep";
+
+// join adjacent ReplaceSteps
+export function simplifyTransform(tr: Transform) {
+  if (!tr.steps.length) {
+    return undefined;
+  }
+
+  const newTr = new Transform(tr.docs[0]);
+  const oldSteps = tr.steps.slice();
+
+  while (oldSteps.length) {
+    let step = oldSteps.shift();
+    while (oldSteps.length && step.merge(oldSteps[0])) {
+      const addedStep = oldSteps.shift();
+      if (step instanceof ReplaceStep && addedStep instanceof ReplaceStep) {
+        step = getReplaceStep(
+          newTr.doc,
+          addedStep.apply(step.apply(newTr.doc).doc).doc,
+          // @ts-ignore
+        ) as Step<any>;
+      } else {
+        step = step.merge(addedStep);
+      }
+    }
+    newTr.step(step);
+  }
+  return newTr;
+}

packages/editor-ext/src/lib/recreate-transform/types.ts

@@ -0,0 +1,3 @@
+export interface AnyObject {
+    [p: string]: any;
+}