feat: switch to cursor pagination (#1884)

* add cursor pagination function

* support custom order modifier
* refactor returned object

* feat(db): migrate paginated endpoints to cursor-based pagination

* sync

* support hasPrevPage boolean

* feat(client): migrate pagination from offset to cursor-based

* support beforeCursor/prevCursor

* wrap search results in items array for API consistency

apps/server/src/core/comment/comment.service.ts

@@ -9,16 +9,14 @@ import { UpdateCommentDto } from './dto/update-comment.dto';
 import { CommentRepo } from '@docmost/db/repos/comment/comment.repo';
 import { Comment, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
 import { PageRepo } from '@docmost/db/repos/page/page.repo';
-import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class CommentService {
   constructor(
     private commentRepo: CommentRepo,
     private pageRepo: PageRepo,
-    private spaceMemberRepo: SpaceMemberRepo,
   ) {}
 
   async findById(commentId: string) {
@@ -68,14 +66,14 @@ export class CommentService {
   async findByPageId(
     pageId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Comment>> {
+  ): Promise<CursorPaginationResult<Comment>> {
     const page = await this.pageRepo.findById(pageId);
 
     if (!page) {
       throw new BadRequestException('Page not found');
     }
 
-    return await this.commentRepo.findPageComments(pageId, pagination);
+    return this.commentRepo.findPageComments(pageId, pagination);
   }
 
   async update(

apps/server/src/core/group/services/group.service.ts

@@ -11,7 +11,7 @@ import { UpdateGroupDto } from '../dto/update-group.dto';
 import { KyselyTransaction } from '@docmost/db/types/kysely.types';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { Group, InsertableGroup, User } from '@docmost/db/types/entity.types';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 import { GroupUserService } from './group-user.service';
 
 @Injectable()
@@ -132,12 +132,8 @@ export class GroupService {
   async getWorkspaceGroups(
     workspaceId: string,
     paginationOptions: PaginationOptions,
-  ): Promise<PaginationResult<Group>> {
-    const groups = await this.groupRepo.getGroupsPaginated(
-      workspaceId,
-      paginationOptions,
-    );
-    return groups;
+  ): Promise<CursorPaginationResult<Group>> {
+    return this.groupRepo.getGroupsPaginated(workspaceId, paginationOptions);
   }
 
   async deleteGroup(groupId: string, workspaceId: string): Promise<void> {

apps/server/src/core/page/services/page-history.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { PageHistoryRepo } from '@docmost/db/repos/page/page-history.repo';
 import { PageHistory } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class PageHistoryService {
@@ -15,12 +15,10 @@ export class PageHistoryService {
   async findHistoryByPageId(
     pageId: string,
     paginationOptions: PaginationOptions,
-  ): Promise<PaginationResult<any>> {
-    const pageHistory = await this.pageHistoryRepo.findPageHistoryByPageId(
+  ): Promise<CursorPaginationResult<PageHistory>> {
+    return this.pageHistoryRepo.findPageHistoryByPageId(
       pageId,
       paginationOptions,
     );
-
-    return pageHistory;
   }
 }

apps/server/src/core/page/services/page.service.ts

@@ -10,9 +10,9 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo';
 import { InsertablePage, Page, User } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import {
-  executeWithPagination,
-  PaginationResult,
-} from '@docmost/db/pagination/pagination';
+  CursorPaginationResult,
+  executeWithCursorPagination,
+} from '@docmost/db/pagination/cursor-pagination';
 import { InjectKysely } from 'nestjs-kysely';
 import { KyselyDB } from '@docmost/db/types/kysely.types';
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
@@ -180,7 +180,7 @@ export class PageService {
     spaceId: string,
     pagination: PaginationOptions,
     pageId?: string,
-  ): Promise<any> {
+  ): Promise<CursorPaginationResult<Partial<Page> & { hasChildren: boolean }>> {
     let query = this.db
       .selectFrom('pages')
       .select([
@@ -195,7 +195,6 @@ export class PageService {
         'deletedAt',
       ])
       .select((eb) => this.pageRepo.withHasChildren(eb))
-      .orderBy('position', (ob) => ob.collate('C').asc())
       .where('deletedAt', 'is', null)
       .where('spaceId', '=', spaceId);
 
@@ -205,12 +204,19 @@ export class PageService {
       query = query.where('parentPageId', 'is', null);
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: 250,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'position', direction: 'asc', orderModifier: (ob) => ob.collate('C').asc() },
+        { expression: 'id', direction: 'asc' },
+      ],
+      parseCursor: (cursor) => ({
+        position: cursor.position,
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   async movePageToSpace(rootPage: Page, spaceId: string) {
@@ -259,7 +265,7 @@ export class PageService {
 
         await this.aiQueue.add(QueueJob.PAGE_MOVED_TO_SPACE, {
           pageId: pageIds,
-          workspaceId: rootPage.workspaceId
+          workspaceId: rootPage.workspaceId,
         });
       }
     });
@@ -387,9 +393,14 @@ export class PageService {
           workspaceId: page.workspaceId,
           creatorId: authUser.id,
           lastUpdatedById: authUser.id,
-          parentPageId: page.id === rootPage.id
-            ? (isDuplicateInSameSpace ? rootPage.parentPageId : null)
-            : (page.parentPageId ? pageMap.get(page.parentPageId)?.newPageId : null),
+          parentPageId:
+            page.id === rootPage.id
+              ? isDuplicateInSameSpace
+                ? rootPage.parentPageId
+                : null
+              : page.parentPageId
+                ? pageMap.get(page.parentPageId)?.newPageId
+                : null,
         };
       }),
     );
@@ -569,22 +580,22 @@ export class PageService {
   async getRecentSpacePages(
     spaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Page>> {
-    return await this.pageRepo.getRecentPagesInSpace(spaceId, pagination);
+  ): Promise<CursorPaginationResult<Page>> {
+    return this.pageRepo.getRecentPagesInSpace(spaceId, pagination);
   }
 
   async getRecentPages(
     userId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Page>> {
-    return await this.pageRepo.getRecentPages(userId, pagination);
+  ): Promise<CursorPaginationResult<Page>> {
+    return this.pageRepo.getRecentPages(userId, pagination);
   }
 
   async getDeletedSpacePages(
     spaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Page>> {
-    return await this.pageRepo.getDeletedPagesInSpace(spaceId, pagination);
+  ): Promise<CursorPaginationResult<Page>> {
+    return this.pageRepo.getDeletedPagesInSpace(spaceId, pagination);
   }
 
   async forceDelete(pageId: string, workspaceId: string): Promise<void> {

apps/server/src/core/search/search.service.ts

@@ -26,11 +26,11 @@ export class SearchService {
       userId?: string;
       workspaceId: string;
     },
-  ): Promise<SearchResponseDto[]> {
+  ): Promise<{ items: SearchResponseDto[] }> {
     const { query } = searchParams;
 
     if (query.length < 1) {
-      return;
+      return { items: [] };
     }
     const searchQuery = tsquery(query.trim() + '*');
 
@@ -62,7 +62,7 @@ export class SearchService {
       )
       .where('deletedAt', 'is', null)
       .orderBy('rank', 'desc')
-      .limit(searchParams.limit | 25)
+      .limit(searchParams.limit || 25)
       .offset(searchParams.offset || 0);
 
     if (!searchParams.shareId) {
@@ -86,7 +86,7 @@ export class SearchService {
       const shareId = searchParams.shareId;
       const share = await this.shareRepo.findById(shareId);
       if (!share || share.workspaceId !== opts.workspaceId) {
-        return [];
+        return { items: [] };
       }
 
       const pageIdsToSearch = [];
@@ -108,10 +108,10 @@ export class SearchService {
           .where('id', 'in', pageIdsToSearch)
           .where('workspaceId', '=', opts.workspaceId);
       } else {
-        return [];
+        return { items: [] };
       }
     } else {
-      return [];
+      return { items: [] };
     }
 
     //@ts-ignore
@@ -127,7 +127,7 @@ export class SearchService {
       return result;
     });
 
-    return searchResults;
+    return { items: searchResults };
   }
 
   async searchSuggestions(

apps/server/src/core/space/services/space-member.service.ts

@@ -13,7 +13,7 @@ import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
 import { RemoveSpaceMemberDto } from '../dto/remove-space-member.dto';
 import { UpdateSpaceMemberRoleDto } from '../dto/update-space-member-role.dto';
 import { SpaceRole } from '../../../common/helpers/types/permission';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class SpaceMemberService {
@@ -68,18 +68,16 @@ export class SpaceMemberService {
     spaceId: string,
     workspaceId: string,
     pagination: PaginationOptions,
-  ) {
+  ): Promise<CursorPaginationResult<any>> {
     const space = await this.spaceRepo.findById(spaceId, workspaceId);
     if (!space) {
       throw new NotFoundException('Space not found');
     }
 
-    const members = await this.spaceMemberRepo.getSpaceMembersPaginated(
+    return await this.spaceMemberRepo.getSpaceMembersPaginated(
       spaceId,
       pagination,
     );
-
-    return members;
   }
 
   async addMembersToSpaceBatch(
@@ -276,7 +274,7 @@ export class SpaceMemberService {
   async getUserSpaces(
     userId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Space>> {
-    return await this.spaceMemberRepo.getUserSpaces(userId, pagination);
+  ): Promise<CursorPaginationResult<Space>> {
+    return this.spaceMemberRepo.getUserSpaces(userId, pagination);
   }
 }

apps/server/src/core/space/services/space.service.ts

@@ -8,7 +8,6 @@ import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
 import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
 import { Space, User } from '@docmost/db/types/entity.types';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
 import { UpdateSpaceDto } from '../dto/update-space.dto';
 import { executeTx } from '@docmost/db/utils';
 import { InjectKysely } from 'nestjs-kysely';
@@ -17,6 +16,7 @@ import { SpaceRole } from '../../../common/helpers/types/permission';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
 import { Queue } from 'bullmq';
 import { InjectQueue } from '@nestjs/bullmq';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class SpaceService {
@@ -130,13 +130,8 @@ export class SpaceService {
   async getWorkspaceSpaces(
     workspaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<Space>> {
-    const spaces = await this.spaceRepo.getSpacesInWorkspace(
-      workspaceId,
-      pagination,
-    );
-
-    return spaces;
+  ): Promise<CursorPaginationResult<Space>> {
+    return this.spaceRepo.getSpacesInWorkspace(workspaceId, pagination);
   }
 
   async deleteSpace(spaceId: string, workspaceId: string): Promise<void> {

apps/server/src/core/workspace/services/workspace-invitation.service.ts

@@ -23,7 +23,7 @@ import InvitationAcceptedEmail from '@docmost/transactional/emails/invitation-ac
 import { TokenService } from '../../auth/services/token.service';
 import { nanoIdGen } from '../../../common/helpers';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { DomainService } from 'src/integrations/environment/domain.service';
 import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
@@ -64,12 +64,13 @@ export class WorkspaceInvitationService {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async getInvitationById(invitationId: string, workspace: Workspace) {

apps/server/src/core/workspace/services/workspace.service.ts

@@ -19,7 +19,6 @@ import { User } from '@docmost/db/types/entity.types';
 import { GroupUserRepo } from '@docmost/db/repos/group/group-user.repo';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { PaginationResult } from '@docmost/db/pagination/pagination';
 import { UpdateWorkspaceUserRoleDto } from '../dto/update-workspace-user-role.dto';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { EnvironmentService } from '../../../integrations/environment/environment.service';
@@ -28,12 +27,12 @@ import { jsonArrayFrom } from 'kysely/helpers/postgres';
 import { addDays } from 'date-fns';
 import { DISALLOWED_HOSTNAMES, WorkspaceStatus } from '../workspace.constants';
 import { v4 } from 'uuid';
-import { AttachmentType } from 'src/core/attachment/attachment.constants';
 import { InjectQueue } from '@nestjs/bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
 import { Queue } from 'bullmq';
 import { generateRandomSuffixNumbers } from '../../../common/helpers';
 import { isPageEmbeddingsTableExists } from '@docmost/db/helpers/helpers';
+import { CursorPaginationResult } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class WorkspaceService {
@@ -376,13 +375,8 @@ export class WorkspaceService {
   async getWorkspaceUsers(
     workspaceId: string,
     pagination: PaginationOptions,
-  ): Promise<PaginationResult<User>> {
-    const users = await this.userRepo.getUsersPaginated(
-      workspaceId,
-      pagination,
-    );
-
-    return users;
+  ): Promise<CursorPaginationResult<User>> {
+    return this.userRepo.getUsersPaginated(workspaceId, pagination);
   }
 
   async updateWorkspaceUserRole(