feat: switch to cursor pagination (#1884)

* add cursor pagination function

* support custom order modifier
* refactor returned object

* feat(db): migrate paginated endpoints to cursor-based pagination

* sync

* support hasPrevPage boolean

* feat(client): migrate pagination from offset to cursor-based

* support beforeCursor/prevCursor

* wrap search results in items array for API consistency

apps/server/src/core/comment/comment.service.ts

@@ -9,16 +9,14 @@ import { UpdateCommentDto } from './dto/update-comment.dto';
 import { CommentRepo } from '@docmost/db/repos/comment/comment.repo';
 import { Comment, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
 import { PageRepo } from '@docmost/db/repos/page/page.repo';
-import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class CommentService {
   constructor(
     private commentRepo: CommentRepo,
     private pageRepo: PageRepo,
-    private spaceMemberRepo: SpaceMemberRepo,
   ) {}
 
   async findById(commentId: string) {
@@ -68,14 +66,14 @@ export class CommentService {
   async findByPageId(
     pageId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Comment>> {
+  ): Promise<CursorPaginationResult<Comment>> {
     const page = await this.pageRepo.findById(pageId);
 
     if (!page) {
       throw new BadRequestException('Page not found');
     }
 
-    return await this.commentRepo.findPageComments(pageId, pagination);
+    return this.commentRepo.findPageComments(pageId, pagination);
   }
 
   async update(

apps/server/src/core/group/services/group.service.ts

@@ -11,7 +11,7 @@ import { UpdateGroupDto } from '../dto/update-group.dto';
 import { KyselyTransaction } from '@docmost/db/types/kysely.types';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { Group, InsertableGroup, User } from '@docmost/db/types/entity.types';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 import { GroupUserService } from './group-user.service';
 
 @Injectable()
@@ -132,12 +132,8 @@ export class GroupService {
   async getWorkspaceGroups(
     workspaceId: string,
     paginationOptions: PaginationOptions,
-  ): Promise<PaginationResult<Group>> {
-    const groups = await this.groupRepo.getGroupsPaginated(
-      workspaceId,
-      paginationOptions,
-    );
-    return groups;
+  ): Promise<CursorPaginationResult<Group>> {
+    return this.groupRepo.getGroupsPaginated(workspaceId, paginationOptions);
   }
 
   async deleteGroup(groupId: string, workspaceId: string): Promise<void> {

apps/server/src/core/page/services/page-history.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { PageHistoryRepo } from '@docmost/db/repos/page/page-history.repo';
 import { PageHistory } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class PageHistoryService {
@@ -15,12 +15,10 @@ export class PageHistoryService {
   async findHistoryByPageId(
     pageId: string,
     paginationOptions: PaginationOptions,
-  ): Promise<PaginationResult<any>> {
-    const pageHistory = await this.pageHistoryRepo.findPageHistoryByPageId(
+  ): Promise<CursorPaginationResult<PageHistory>> {
+    return this.pageHistoryRepo.findPageHistoryByPageId(
       pageId,
       paginationOptions,
     );
-
-    return pageHistory;
   }
 }

apps/server/src/core/page/services/page.service.ts

@@ -10,9 +10,9 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { InsertablePage, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import {
-  executeWithPagination,
-  PaginationResult,
-} from '@docmost/db/pagination/pagination';
+  CursorPaginationResult,
+  executeWithCursorPagination,
+} from '@docmost/db/pagination/cursor-pagination';
 import { InjectKysely } from 'nestjs-kysely';
 import { KyselyDB } from '@docmost/db/types/kysely.types';
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
@@ -180,7 +180,7 @@ export class PageService {
     spaceId: string,
     pagination: PaginationOptions,
     pageId?: string,
-  ): Promise<any> {
+  ): Promise<CursorPaginationResult<Partial<Page> & { hasChildren: boolean }>> {
     let query = this.db
       .selectFrom('pages')
       .select([
@@ -195,7 +195,6 @@ export class PageService {
         'deletedAt',
       ])
       .select((eb) => this.pageRepo.withHasChildren(eb))
-      .orderBy('position', (ob) => ob.collate('C').asc())
       .where('deletedAt', 'is', null)
       .where('spaceId', '=', spaceId);
 
@@ -205,12 +204,19 @@ export class PageService {
       query = query.where('parentPageId', 'is', null);
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: 250,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'position', direction: 'asc', orderModifier: (ob) => ob.collate('C').asc() },
+        { expression: 'id', direction: 'asc' },
+      ],
+      parseCursor: (cursor) => ({
+        position: cursor.position,
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   async movePageToSpace(rootPage: Page, spaceId: string) {
@@ -259,7 +265,7 @@ export class PageService {
 
         await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
           pageId: pageIds,
-          workspaceId: rootPage.workspaceId
+          workspaceId: rootPage.workspaceId,
         });
       }
     });
@@ -387,9 +393,14 @@ export class PageService {
           workspaceId: page.workspaceId,
           creatorId: authUser.id,
           lastUpdatedById: authUser.id,
-          parentPageId: page.id === rootPage.id
-            ? (isDuplicateInSameSpace ? rootPage.parentPageId : null)
-            : (page.parentPageId ? pageMap.get(page.parentPageId)?.newPageId : null),
+          parentPageId:
+            page.id === rootPage.id
+              ? isDuplicateInSameSpace
+                ? rootPage.parentPageId
+                : null
+              : page.parentPageId
+                ? pageMap.get(page.parentPageId)?.newPageId
+                : null,
         };
       }),
     );
@@ -569,22 +580,22 @@ export class PageService {
   async getRecentSpacePages(
     spaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Page>> {
-    return await this.pageRepo.getRecentPagesInSpace(spaceId, pagination);
+  ): Promise<CursorPaginationResult<Page>> {
+    return this.pageRepo.getRecentPagesInSpace(spaceId, pagination);
   }
 
   async getRecentPages(
     userId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Page>> {
-    return await this.pageRepo.getRecentPages(userId, pagination);
+  ): Promise<CursorPaginationResult<Page>> {
+    return this.pageRepo.getRecentPages(userId, pagination);
   }
 
   async getDeletedSpacePages(
     spaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Page>> {
-    return await this.pageRepo.getDeletedPagesInSpace(spaceId, pagination);
+  ): Promise<CursorPaginationResult<Page>> {
+    return this.pageRepo.getDeletedPagesInSpace(spaceId, pagination);
   }
 
   async forceDelete(pageId: string, workspaceId: string): Promise<void> {

apps/server/src/core/search/search.service.ts

@@ -26,11 +26,11 @@ export class SearchService {
       userId?: string;
       workspaceId: string;
     },
-  ): Promise<SearchResponseDto[]> {
+  ): Promise<{ items: SearchResponseDto[] }> {
     const { query } = searchParams;
 
     if (query.length < 1) {
-      return;
+      return { items: [] };
     }
     const searchQuery = tsquery(query.trim() + '*');
 
@@ -62,7 +62,7 @@ export class SearchService {
       )
       .where('deletedAt', 'is', null)
       .orderBy('rank', 'desc')
-      .limit(searchParams.limit | 25)
+      .limit(searchParams.limit || 25)
       .offset(searchParams.offset || 0);
 
     if (!searchParams.shareId) {
@@ -86,7 +86,7 @@ export class SearchService {
       const shareId = searchParams.shareId;
       const share = await this.shareRepo.findById(shareId);
       if (!share || share.workspaceId !== opts.workspaceId) {
-        return [];
+        return { items: [] };
       }
 
       const pageIdsToSearch = [];
@@ -108,10 +108,10 @@ export class SearchService {
           .where('id', 'in', pageIdsToSearch)
           .where('workspaceId', '=', opts.workspaceId);
       } else {
-        return [];
+        return { items: [] };
       }
     } else {
-      return [];
+      return { items: [] };
     }
 
     //@ts-ignore
@@ -127,7 +127,7 @@ export class SearchService {
       return result;
     });
 
-    return searchResults;
+    return { items: searchResults };
   }
 
   async searchSuggestions(

apps/server/src/core/space/services/space-member.service.ts

@@ -13,7 +13,7 @@ import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
 import { RemoveSpaceMemberDto } from '../dto/remove-space-member.dto';
 import { UpdateSpaceMemberRoleDto } from '../dto/update-space-member-role.dto';
 import { SpaceRole } from '../../../common/helpers/types/permission';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class SpaceMemberService {
@@ -68,18 +68,16 @@ export class SpaceMemberService {
     spaceId: string,
     workspaceId: string,
     pagination: PaginationOptions,
-  ) {
+  ): Promise<CursorPaginationResult<any>> {
     const space = await this.spaceRepo.findById(spaceId, workspaceId);
     if (!space) {
       throw new NotFoundException('Space not found');
     }
 
-    const members = await this.spaceMemberRepo.getSpaceMembersPaginated(
+    return await this.spaceMemberRepo.getSpaceMembersPaginated(
       spaceId,
       pagination,
     );
-
-    return members;
   }
 
   async addMembersToSpaceBatch(
@@ -276,7 +274,7 @@ export class SpaceMemberService {
   async getUserSpaces(
     userId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Space>> {
-    return await this.spaceMemberRepo.getUserSpaces(userId, pagination);
+  ): Promise<CursorPaginationResult<Space>> {
+    return this.spaceMemberRepo.getUserSpaces(userId, pagination);
   }
 }

apps/server/src/core/space/services/space.service.ts

@@ -8,7 +8,6 @@ import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
 import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
 import { Space, User } from '@docmost/db/types/entity.types';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
 import { UpdateSpaceDto } from '../dto/update-space.dto';
 import { executeTx } from '@docmost/db/utils';
 import { InjectKysely } from 'nestjs-kysely';
@@ -17,6 +16,7 @@ import { SpaceRole } from '../../../common/helpers/types/permission';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
 import { Queue } from 'bullmq';
 import { InjectQueue } from '@nestjs/bullmq';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class SpaceService {
@@ -130,13 +130,8 @@ export class SpaceService {
   async getWorkspaceSpaces(
     workspaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Space>> {
-    const spaces = await this.spaceRepo.getSpacesInWorkspace(
-      workspaceId,
-      pagination,
-    );
-
-    return spaces;
+  ): Promise<CursorPaginationResult<Space>> {
+    return this.spaceRepo.getSpacesInWorkspace(workspaceId, pagination);
   }
 
   async deleteSpace(spaceId: string, workspaceId: string): Promise<void> {

apps/server/src/core/workspace/services/workspace-invitation.service.ts

@@ -23,7 +23,7 @@ import InvitationAcceptedEmail from '@docmost/transactional/emails/invitation-ac
 import { TokenService } from '../../auth/services/token.service';
 import { nanoIdGen } from '../../../common/helpers';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { DomainService } from 'src/integrations/environment/domain.service';
 import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
@@ -64,12 +64,13 @@ export class WorkspaceInvitationService {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async getInvitationById(invitationId: string, workspace: Workspace) {

apps/server/src/core/workspace/services/workspace.service.ts

@@ -19,7 +19,6 @@ import { User } from '@docmost/db/types/entity.types';
 import { GroupUserRepo } from '@docmost/db/repos/group/group-user.repo';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
 import { UpdateWorkspaceUserRoleDto } from '../dto/update-workspace-user-role.dto';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { EnvironmentService } from '../../../integrations/environment/environment.service';
@@ -28,12 +27,12 @@ import { jsonArrayFrom } from 'kysely/helpers/postgres';
 import { addDays } from 'date-fns';
 import { DISALLOWED_HOSTNAMES, WorkspaceStatus } from '../workspace.constants';
 import { v4 } from 'uuid';
-import { AttachmentType } from 'src/core/attachment/attachment.constants';
 import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { Queue } from 'bullmq';
 import { generateRandomSuffixNumbers } from '../../../common/helpers';
 import { isPageEmbeddingsTableExists } from '@docmost/db/helpers/helpers';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class WorkspaceService {
@@ -376,13 +375,8 @@ export class WorkspaceService {
   async getWorkspaceUsers(
     workspaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<User>> {
-    const users = await this.userRepo.getUsersPaginated(
-      workspaceId,
-      pagination,
-    );
-
-    return users;
+  ): Promise<CursorPaginationResult<User>> {
+    return this.userRepo.getUsersPaginated(workspaceId, pagination);
   }
 
   async updateWorkspaceUserRole(

apps/server/src/database/pagination/cursor-pagination.ts

@@ -0,0 +1,348 @@
+// adapted from https://github.com/charlie-hadden/kysely-paginate/blob/main/src/cursor.ts - MIT
+import {
+  OrderByDirection,
+  OrderByModifiers,
+  ReferenceExpression,
+  SelectQueryBuilder,
+  StringReference,
+} from 'kysely';
+
+type SortField<DB, TB extends keyof DB, O> =
+  | {
+      expression:
+        | (StringReference<DB, TB> & keyof O & string)
+        | (StringReference<DB, TB> & `${string}.${keyof O & string}`);
+      direction: OrderByDirection;
+      orderModifier?: OrderByModifiers;
+      key?: keyof O & string;
+    }
+  | {
+      expression: ReferenceExpression<DB, TB>;
+      direction: OrderByDirection;
+      orderModifier?: OrderByModifiers;
+      key: keyof O & string;
+    };
+
+type ExtractSortFieldKey<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends SortField<DB, TB, O>,
+> = T['key'] extends keyof O & string
+  ? T['key']
+  : T['expression'] extends keyof O & string
+    ? T['expression']
+    : T['expression'] extends `${string}.${infer K}`
+      ? K extends keyof O & string
+        ? K
+        : never
+      : never;
+
+type Fields<DB, TB extends keyof DB, O> = ReadonlyArray<
+  Readonly<SortField<DB, TB, O>>
+>;
+
+type FieldNames<DB, TB extends keyof DB, O, T extends Fields<DB, TB, O>> = {
+  [TIndex in keyof T]: ExtractSortFieldKey<DB, TB, O, T[TIndex]>;
+};
+
+type EncodeCursorValues<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+> = {
+  [TIndex in keyof T]: [
+    ExtractSortFieldKey<DB, TB, O, T[TIndex]>,
+    O[ExtractSortFieldKey<DB, TB, O, T[TIndex]>],
+  ];
+};
+
+export type CursorEncoder<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+> = (values: EncodeCursorValues<DB, TB, O, T>) => string;
+
+type DecodedCursor<DB, TB extends keyof DB, O, T extends Fields<DB, TB, O>> = {
+  [TField in ExtractSortFieldKey<DB, TB, O, T[number]>]: string;
+};
+
+export type CursorDecoder<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+> = (
+  cursor: string,
+  fields: FieldNames<DB, TB, O, T>,
+) => DecodedCursor<DB, TB, O, T>;
+
+type ParsedCursorValues<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+> = {
+  [TField in ExtractSortFieldKey<DB, TB, O, T[number]>]: O[TField];
+};
+
+export type CursorParser<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+> = (cursor: DecodedCursor<DB, TB, O, T>) => ParsedCursorValues<DB, TB, O, T>;
+
+type CursorPaginationResultRow<
+  TRow,
+  TCursorKey extends string | boolean | undefined,
+> = TRow & {
+  [K in TCursorKey extends undefined
+    ? never
+    : TCursorKey extends false
+      ? never
+      : TCursorKey extends true
+        ? '$cursor'
+        : TCursorKey]: string;
+};
+
+type CursorPaginationMeta = {
+  limit: number;
+  hasNextPage: boolean;
+  hasPrevPage: boolean;
+  nextCursor: string | null;
+  prevCursor: string | null;
+};
+
+export type CursorPaginationResult<
+  TRow,
+  TCursorKey extends string | boolean | undefined = undefined,
+> = {
+  meta: CursorPaginationMeta;
+  items: CursorPaginationResultRow<TRow, TCursorKey>[];
+};
+
+export async function executeWithCursorPagination<
+  DB,
+  TB extends keyof DB,
+  O,
+  const TFields extends Fields<DB, TB, O>,
+  TCursorKey extends string | boolean | undefined = undefined,
+>(
+  qb: SelectQueryBuilder<DB, TB, O>,
+  opts: {
+    perPage: number;
+    cursor?: string;
+    beforeCursor?: string;
+    cursorPerRow?: TCursorKey;
+    fields: TFields;
+    encodeCursor?: CursorEncoder<DB, TB, O, TFields>;
+    decodeCursor?: CursorDecoder<DB, TB, O, TFields>;
+    parseCursor:
+      | CursorParser<DB, TB, O, TFields>
+      | { parse: CursorParser<DB, TB, O, TFields> };
+  },
+): Promise<CursorPaginationResult<O, TCursorKey>> {
+  const encodeCursor = opts.encodeCursor ?? defaultEncodeCursor;
+  const decodeCursor = opts.decodeCursor ?? defaultDecodeCursor;
+
+  const parseCursor =
+    typeof opts.parseCursor === 'function'
+      ? opts.parseCursor
+      : opts.parseCursor.parse;
+
+  const fields = opts.fields.map((field) => {
+    let key = field.key;
+
+    if (!key && typeof field.expression === 'string') {
+      const expressionParts = field.expression.split('.');
+
+      key = (expressionParts[1] ?? expressionParts[0]) as
+        | (keyof O & string)
+        | undefined;
+    }
+
+    if (!key) throw new Error('missing key');
+
+    return { ...field, key };
+  });
+
+  function generateCursor(row: O): string {
+    const cursorFieldValues = fields.map(({ key }) => [
+      key,
+      row[key],
+    ]) as EncodeCursorValues<DB, TB, O, TFields>;
+
+    return encodeCursor(cursorFieldValues);
+  }
+
+  const fieldNames = fields.map((field) => field.key) as FieldNames<
+    DB,
+    TB,
+    O,
+    TFields
+  >;
+
+  function applyCursor(
+    qb: SelectQueryBuilder<DB, TB, O>,
+    encoded: string,
+    defaultDirection: OrderByDirection,
+  ) {
+    const decoded = decodeCursor(encoded, fieldNames);
+    const cursor = parseCursor(decoded);
+
+    return qb.where(({ and, or, eb }) => {
+      let expression;
+
+      for (let i = fields.length - 1; i >= 0; --i) {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const field = fields[i]!;
+
+        const comparison = field.direction === defaultDirection ? '>' : '<';
+        const value = cursor[field.key as keyof typeof cursor];
+
+        const conditions = [eb(field.expression, comparison, value)];
+
+        if (expression) {
+          conditions.push(and([eb(field.expression, '=', value), expression]));
+        }
+
+        expression = or(conditions);
+      }
+
+      if (!expression) {
+        throw new Error('Error building cursor expression');
+      }
+
+      return expression;
+    });
+  }
+
+  if (opts.cursor) qb = applyCursor(qb, opts.cursor, 'asc');
+  if (opts.beforeCursor) qb = applyCursor(qb, opts.beforeCursor, 'desc');
+
+  const reversed = !!opts.beforeCursor && !opts.cursor;
+
+  for (const { expression, direction, orderModifier } of fields) {
+    qb = qb.orderBy(
+      expression,
+      orderModifier ??
+        (reversed ? (direction === 'asc' ? 'desc' : 'asc') : direction),
+    );
+  }
+
+  const rows = await qb.limit(opts.perPage + 1).execute();
+
+  const hasNextPage = rows.length > opts.perPage;
+
+  // If we fetched an extra row to determine if we have a next page, that
+  // shouldn't be in the returned results
+  if (rows.length > opts.perPage) rows.pop();
+
+  if (reversed) rows.reverse();
+
+  const startRow = rows[0];
+  const endRow = rows[rows.length - 1];
+
+  const hasPrevPage = !!opts.cursor;
+  const prevCursor = hasPrevPage && startRow ? generateCursor(startRow) : null;
+  const nextCursor = hasNextPage && endRow ? generateCursor(endRow) : null;
+
+  return {
+    items: rows.map((row) => {
+      if (opts.cursorPerRow) {
+        const cursorKey =
+          typeof opts.cursorPerRow === 'string' ? opts.cursorPerRow : '$cursor';
+
+        (row as any)[cursorKey] = generateCursor(row);
+      }
+
+      return row as CursorPaginationResultRow<O, TCursorKey>;
+    }),
+    meta: {
+      limit: opts.perPage,
+      hasNextPage,
+      hasPrevPage,
+      nextCursor,
+      prevCursor,
+    },
+  };
+}
+
+export function defaultEncodeCursor<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+>(values: EncodeCursorValues<DB, TB, O, T>) {
+  const cursor = new URLSearchParams();
+
+  for (const [key, value] of values) {
+    switch (typeof value) {
+      case 'string':
+        cursor.set(key, value);
+        break;
+
+      case 'number':
+      case 'bigint':
+        cursor.set(key, value.toString(10));
+        break;
+
+      case 'object': {
+        if (value instanceof Date) {
+          cursor.set(key, value.toISOString());
+          break;
+        }
+      }
+
+      // eslint-disable-next-line no-fallthrough
+      default:
+        throw new Error(`Unable to encode '${key.toString()}'`);
+    }
+  }
+
+  return Buffer.from(cursor.toString(), 'utf8').toString('base64url');
+}
+
+export function defaultDecodeCursor<
+  DB,
+  TB extends keyof DB,
+  O,
+  T extends Fields<DB, TB, O>,
+>(
+  cursor: string,
+  fields: FieldNames<DB, TB, O, T>,
+): DecodedCursor<DB, TB, O, T> {
+  let parsed;
+
+  try {
+    parsed = [
+      ...new URLSearchParams(
+        Buffer.from(cursor, 'base64url').toString('utf8'),
+      ).entries(),
+    ];
+  } catch {
+    throw new Error('Unparsable cursor');
+  }
+
+  if (parsed.length !== fields.length) {
+    throw new Error('Unexpected number of fields');
+  }
+
+  for (let i = 0; i < fields.length; i++) {
+    const field = parsed[i];
+    const expectedName = fields[i];
+
+    if (!field) {
+      throw new Error('Unable to find field');
+    }
+
+    if (field[0] !== expectedName) {
+      throw new Error('Unexpected field name');
+    }
+  }
+
+  return Object.fromEntries(parsed) as DecodedCursor<DB, TB, O, T>;
+}

apps/server/src/database/pagination/pagination-options.ts

@@ -9,11 +9,6 @@ import {
 } from 'class-validator';
 
 export class PaginationOptions {
-  @IsOptional()
-  @IsNumber()
-  @Min(1)
-  page = 1;
-
   @IsOptional()
   @IsNumber()
   @IsPositive()
@@ -21,6 +16,14 @@ export class PaginationOptions {
   @Max(100)
   limit = 20;
 
+  @IsOptional()
+  @IsString()
+  cursor?: string;
+
+  @IsOptional()
+  @IsString()
+  beforeCursor?: string;
+
   @IsOptional()
   @IsString()
   query: string;

apps/server/src/database/repos/comment/comment.repo.ts

@@ -8,7 +8,7 @@ import {
   UpdatableComment,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { ExpressionBuilder } from 'kysely';
 import { DB } from '@docmost/db/types/db';
 import { jsonObjectFrom } from 'kysely/helpers/postgres';
@@ -37,15 +37,15 @@ export class CommentRepo {
       .selectAll('comments')
       .select((eb) => this.withCreator(eb))
       .select((eb) => this.withResolvedBy(eb))
-      .where('pageId', '=', pageId)
-      .orderBy('createdAt', 'asc');
+      .where('pageId', '=', pageId);
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async updateComment(

apps/server/src/database/repos/group/group-user.repo.ts

@@ -9,7 +9,7 @@ import { dbOrTx, executeTx } from '@docmost/db/utils';
 import { sql } from 'kysely';
 import { GroupUser, InsertableGroupUser } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '../../pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 
@@ -52,8 +52,7 @@ export class GroupUserRepo {
       .selectFrom('groupUsers')
       .innerJoin('users', 'users.id', 'groupUsers.userId')
       .selectAll('users')
-      .where('groupId', '=', groupId)
-      .orderBy('createdAt', 'asc');
+      .where('groupId', '=', groupId);
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -61,9 +60,12 @@ export class GroupUserRepo {
       );
     }
 
-    const result = await executeWithPagination(query, {
-      page: pagination.page,
+    const result = await executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'users.id', direction: 'asc', key: 'id' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
 
     result.items.map((user) => {

apps/server/src/database/repos/group/group.repo.ts

@@ -10,8 +10,8 @@ import {
 import { ExpressionBuilder, sql } from 'kysely';
 import { PaginationOptions } from '../../pagination/pagination-options';
 import { DB } from '@docmost/db/types/db';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
 import { DefaultGroup } from '../../../core/group/dto/create-group.dto';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class GroupRepo {
@@ -104,17 +104,19 @@ export class GroupRepo {
   }
 
   async getGroupsPaginated(workspaceId: string, pagination: PaginationOptions) {
-    let query = this.db
+    let baseQuery = this.db
       .selectFrom('groups')
       .selectAll('groups')
       .select((eb) => this.withMemberCount(eb))
-      .where('workspaceId', '=', workspaceId)
-      .orderBy('memberCount', 'desc')
-      .orderBy('createdAt', 'asc');
+      .where('workspaceId', '=', workspaceId);
 
     if (pagination.query) {
-      query = query.where((eb) =>
-        eb(sql`f_unaccent(name)`, 'ilike', sql`f_unaccent(${'%' + pagination.query + '%'})`).or(
+      baseQuery = baseQuery.where((eb) =>
+        eb(
+          sql`f_unaccent(name)`,
+          'ilike',
+          sql`f_unaccent(${'%' + pagination.query + '%'})`,
+        ).or(
           sql`f_unaccent(description)`,
           'ilike',
           sql`f_unaccent(${'%' + pagination.query + '%'})`,
@@ -122,12 +124,24 @@ export class GroupRepo {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    const query = this.db.selectFrom(baseQuery.as('sub')).selectAll('sub');
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        {
+          expression: 'sub.memberCount',
+          direction: 'desc',
+          key: 'memberCount',
+        },
+        { expression: 'sub.id', direction: 'asc', key: 'id' },
+      ],
+      parseCursor: (cursor) => ({
+        memberCount: parseInt(cursor.memberCount, 10),
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   withMemberCount(eb: ExpressionBuilder<DB, 'groups'>) {

apps/server/src/database/repos/page/page-history.repo.ts

@@ -8,7 +8,7 @@ import {
   PageHistory,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { jsonObjectFrom } from 'kysely/helpers/postgres';
 import { ExpressionBuilder } from 'kysely';
 import { DB } from '@docmost/db/types/db';
@@ -65,15 +65,15 @@ export class PageHistoryRepo {
       .selectFrom('pageHistory')
       .selectAll()
       .select((eb) => this.withLastUpdatedBy(eb))
-      .where('pageId', '=', pageId)
-      .orderBy('createdAt', 'desc');
+      .where('pageId', '=', pageId);
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'desc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async findPageLastHistory(pageId: string, trx?: KyselyTransaction) {

apps/server/src/database/repos/page/page.repo.ts

@@ -8,7 +8,7 @@ import {
   UpdatablePage,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { validate as isValidUUID } from 'uuid';
 import { ExpressionBuilder, sql } from 'kysely';
 import { DB } from '@docmost/db/types/db';
@@ -281,15 +281,21 @@ export class PageRepo {
       .select(this.baseFields)
       .select((eb) => this.withSpace(eb))
       .where('spaceId', '=', spaceId)
-      .where('deletedAt', 'is', null)
-      .orderBy('updatedAt', 'desc');
+      .where('deletedAt', 'is', null);
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'updatedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        updatedAt: new Date(cursor.updatedAt),
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   async getRecentPages(userId: string, pagination: PaginationOptions) {
@@ -298,12 +304,20 @@ export class PageRepo {
       .select(this.baseFields)
       .select((eb) => this.withSpace(eb))
       .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(userId))
-      .where('deletedAt', 'is', null)
-      .orderBy('updatedAt', 'desc');
+      .where('deletedAt', 'is', null);
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'updatedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        updatedAt: new Date(cursor.updatedAt),
+        id: cursor.id,
+      }),
     });
   }
 
@@ -331,15 +345,21 @@ export class PageRepo {
             ),
           ),
         ]),
-      )
-      .orderBy('deletedAt', 'desc');
+      );
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'deletedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        deletedAt: new Date(cursor.deletedAt),
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   withSpace(eb: ExpressionBuilder<DB, 'pages'>) {

apps/server/src/database/repos/share/share.repo.ts

@@ -8,7 +8,7 @@ import {
   UpdatableShare,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { validate as isValidUUID } from 'uuid';
 import { ExpressionBuilder, sql } from 'kysely';
 import { DB } from '@docmost/db/types/db';
@@ -143,12 +143,20 @@ export class ShareRepo {
       .select((eb) => this.withPage(eb))
       .select((eb) => this.withSpace(eb, userId))
       .select((eb) => this.withCreator(eb))
-      .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(userId))
-      .orderBy('updatedAt', 'desc');
+      .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(userId));
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'updatedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        updatedAt: new Date(cursor.updatedAt),
+        id: cursor.id,
+      }),
     });
   }
 

apps/server/src/database/repos/space/space-member.repo.ts

@@ -10,7 +10,7 @@ import {
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '../../pagination/pagination-options';
 import { MemberInfo, UserSpaceRole } from './types';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
 
@@ -98,7 +98,7 @@ export class SpaceMemberRepo {
     spaceId: string,
     pagination: PaginationOptions,
   ) {
-    let query = this.db
+    let baseQuery = this.db
       .selectFrom('spaceMembers')
       .leftJoin('users', 'users.id', 'spaceMembers.userId')
       .leftJoin('groups', 'groups.id', 'spaceMembers.groupId')
@@ -114,12 +114,11 @@ export class SpaceMemberRepo {
         'spaceMembers.createdAt',
       ])
       .select((eb) => this.groupRepo.withMemberCount(eb))
-      .where('spaceId', '=', spaceId)
-      .orderBy((eb) => eb('groups.id', 'is not', null), 'desc')
-      .orderBy('spaceMembers.createdAt', 'asc');
+      .select(sql<number>`case when groups.id is not null then 1 else 0 end`.as('isGroup'))
+      .where('spaceId', '=', spaceId);
 
     if (pagination.query) {
-      query = query.where((eb) =>
+      baseQuery = baseQuery.where((eb) =>
         eb(
           sql`f_unaccent(users.name)`,
           'ilike',
@@ -138,9 +137,20 @@ export class SpaceMemberRepo {
       );
     }
 
-    const result = await executeWithPagination(query, {
-      page: pagination.page,
+    const query = this.db.selectFrom(baseQuery.as('sub')).selectAll('sub');
+
+    const result = await executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'sub.isGroup', direction: 'desc', key: 'isGroup' },
+        { expression: 'sub.createdAt', direction: 'asc', key: 'createdAt' },
+      ],
+      parseCursor: (cursor) => ({
+        isGroup: parseInt(cursor.isGroup, 10),
+        createdAt: new Date(cursor.createdAt),
+      }),
     });
 
     let memberInfo: MemberInfo;
@@ -235,8 +245,7 @@ export class SpaceMemberRepo {
       .selectFrom('spaces')
       .selectAll()
       .select((eb) => [this.spaceRepo.withMemberCount(eb)])
-      .where('id', 'in', this.getUserSpaceIdsQuery(userId))
-      .orderBy('createdAt', 'asc');
+      .where('id', 'in', this.getUserSpaceIdsQuery(userId));
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -252,9 +261,12 @@ export class SpaceMemberRepo {
       );
     }
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
   }
 }

apps/server/src/database/repos/space/space.repo.ts

@@ -9,7 +9,7 @@ import {
 } from '@docmost/db/types/entity.types';
 import { ExpressionBuilder, sql } from 'kysely';
 import { PaginationOptions } from '../../pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { DB } from '@docmost/db/types/db';
 import { validate as isValidUUID } from 'uuid';
 import { EventEmitter2 } from '@nestjs/event-emitter';
@@ -110,8 +110,7 @@ export class SpaceRepo {
       .selectFrom('spaces')
       .selectAll('spaces')
       .select((eb) => [this.withMemberCount(eb)])
-      .where('workspaceId', '=', workspaceId)
-      .orderBy('createdAt', 'asc');
+      .where('workspaceId', '=', workspaceId);
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -127,12 +126,13 @@ export class SpaceRepo {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   withMemberCount(eb: ExpressionBuilder<DB, 'spaces'>) {

apps/server/src/database/repos/user/user.repo.ts

@@ -10,7 +10,7 @@ import {
   User,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '../../pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { ExpressionBuilder, sql } from 'kysely';
 import { jsonObjectFrom } from 'kysely/helpers/postgres';
 
@@ -145,8 +145,7 @@ export class UserRepo {
       .selectFrom('users')
       .select(this.baseFields)
       .where('workspaceId', '=', workspaceId)
-      .where('deletedAt', 'is', null)
-      .orderBy('createdAt', 'asc');
+      .where('deletedAt', 'is', null);
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -162,12 +161,13 @@ export class UserRepo {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async updatePreference(

apps/server/src/integrations/import/file-task.controller.ts

@@ -27,7 +27,7 @@ import { AuthUser } from '../../common/decorators/auth-user.decorator';
 import { FileTaskIdDto } from './dto/file-task-dto';
 import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 
 @Controller('file-tasks')
 export class FileTaskController {
@@ -56,12 +56,14 @@ export class FileTaskController {
     const query = this.db
       .selectFrom('fileTasks')
       .selectAll()
-      .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(user.id))
-      .orderBy('createdAt', 'desc');
+      .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(user.id));
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'desc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
   }