feat: switch to cursor pagination (#1884)

* add cursor pagination function

* support custom order modifier
* refactor returned object

* feat(db): migrate paginated endpoints to cursor-based pagination

* sync

* support hasPrevPage boolean

* feat(client): migrate pagination from offset to cursor-based

* support beforeCursor/prevCursor

* wrap search results in items array for API consistency

apps/server/src/database/repos/comment/comment.repo.ts

@@ -8,7 +8,7 @@ import {
   UpdatableComment,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { ExpressionBuilder } from 'kysely';
 import { DB } from '@docmost/db/types/db';
 import { jsonObjectFrom } from 'kysely/helpers/postgres';
@@ -37,15 +37,15 @@ export class CommentRepo {
       .selectAll('comments')
       .select((eb) => this.withCreator(eb))
       .select((eb) => this.withResolvedBy(eb))
-      .where('pageId', '=', pageId)
-      .orderBy('createdAt', 'asc');
+      .where('pageId', '=', pageId);
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async updateComment(

apps/server/src/database/repos/group/group-user.repo.ts

@@ -9,7 +9,7 @@ import { dbOrTx, executeTx } from '@docmost/db/utils';
 import { sql } from 'kysely';
 import { GroupUser, InsertableGroupUser } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '../../pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 
@@ -52,8 +52,7 @@ export class GroupUserRepo {
       .selectFrom('groupUsers')
       .innerJoin('users', 'users.id', 'groupUsers.userId')
       .selectAll('users')
-      .where('groupId', '=', groupId)
-      .orderBy('createdAt', 'asc');
+      .where('groupId', '=', groupId);
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -61,9 +60,12 @@ export class GroupUserRepo {
       );
     }
 
-    const result = await executeWithPagination(query, {
-      page: pagination.page,
+    const result = await executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'users.id', direction: 'asc', key: 'id' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
 
     result.items.map((user) => {

apps/server/src/database/repos/group/group.repo.ts

@@ -10,8 +10,8 @@ import {
 import { ExpressionBuilder, sql } from 'kysely';
 import { PaginationOptions } from '../../pagination/pagination-options';
 import { DB } from '@docmost/db/types/db';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
 import { DefaultGroup } from '../../../core/group/dto/create-group.dto';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 
 @Injectable()
 export class GroupRepo {
@@ -104,17 +104,19 @@ export class GroupRepo {
   }
 
   async getGroupsPaginated(workspaceId: string, pagination: PaginationOptions) {
-    let query = this.db
+    let baseQuery = this.db
       .selectFrom('groups')
       .selectAll('groups')
       .select((eb) => this.withMemberCount(eb))
-      .where('workspaceId', '=', workspaceId)
-      .orderBy('memberCount', 'desc')
-      .orderBy('createdAt', 'asc');
+      .where('workspaceId', '=', workspaceId);
 
     if (pagination.query) {
-      query = query.where((eb) =>
-        eb(sql`f_unaccent(name)`, 'ilike', sql`f_unaccent(${'%' + pagination.query + '%'})`).or(
+      baseQuery = baseQuery.where((eb) =>
+        eb(
+          sql`f_unaccent(name)`,
+          'ilike',
+          sql`f_unaccent(${'%' + pagination.query + '%'})`,
+        ).or(
           sql`f_unaccent(description)`,
           'ilike',
           sql`f_unaccent(${'%' + pagination.query + '%'})`,
@@ -122,12 +124,24 @@ export class GroupRepo {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    const query = this.db.selectFrom(baseQuery.as('sub')).selectAll('sub');
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        {
+          expression: 'sub.memberCount',
+          direction: 'desc',
+          key: 'memberCount',
+        },
+        { expression: 'sub.id', direction: 'asc', key: 'id' },
+      ],
+      parseCursor: (cursor) => ({
+        memberCount: parseInt(cursor.memberCount, 10),
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   withMemberCount(eb: ExpressionBuilder<DB, 'groups'>) {

apps/server/src/database/repos/page/page-history.repo.ts

@@ -8,7 +8,7 @@ import {
   PageHistory,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { jsonObjectFrom } from 'kysely/helpers/postgres';
 import { ExpressionBuilder } from 'kysely';
 import { DB } from '@docmost/db/types/db';
@@ -65,15 +65,15 @@ export class PageHistoryRepo {
       .selectFrom('pageHistory')
       .selectAll()
       .select((eb) => this.withLastUpdatedBy(eb))
-      .where('pageId', '=', pageId)
-      .orderBy('createdAt', 'desc');
+      .where('pageId', '=', pageId);
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'desc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async findPageLastHistory(pageId: string, trx?: KyselyTransaction) {

apps/server/src/database/repos/page/page.repo.ts

@@ -8,7 +8,7 @@ import {
   UpdatablePage,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { validate as isValidUUID } from 'uuid';
 import { ExpressionBuilder, sql } from 'kysely';
 import { DB } from '@docmost/db/types/db';
@@ -281,15 +281,21 @@ export class PageRepo {
       .select(this.baseFields)
       .select((eb) => this.withSpace(eb))
       .where('spaceId', '=', spaceId)
-      .where('deletedAt', 'is', null)
-      .orderBy('updatedAt', 'desc');
+      .where('deletedAt', 'is', null);
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'updatedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        updatedAt: new Date(cursor.updatedAt),
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   async getRecentPages(userId: string, pagination: PaginationOptions) {
@@ -298,12 +304,20 @@ export class PageRepo {
       .select(this.baseFields)
       .select((eb) => this.withSpace(eb))
       .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(userId))
-      .where('deletedAt', 'is', null)
-      .orderBy('updatedAt', 'desc');
+      .where('deletedAt', 'is', null);
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'updatedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        updatedAt: new Date(cursor.updatedAt),
+        id: cursor.id,
+      }),
     });
   }
 
@@ -331,15 +345,21 @@ export class PageRepo {
             ),
           ),
         ]),
-      )
-      .orderBy('deletedAt', 'desc');
+      );
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'deletedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        deletedAt: new Date(cursor.deletedAt),
+        id: cursor.id,
+      }),
     });
-
-    return result;
   }
 
   withSpace(eb: ExpressionBuilder<DB, 'pages'>) {

apps/server/src/database/repos/share/share.repo.ts

@@ -8,7 +8,7 @@ import {
   UpdatableShare,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { validate as isValidUUID } from 'uuid';
 import { ExpressionBuilder, sql } from 'kysely';
 import { DB } from '@docmost/db/types/db';
@@ -143,12 +143,20 @@ export class ShareRepo {
       .select((eb) => this.withPage(eb))
       .select((eb) => this.withSpace(eb, userId))
       .select((eb) => this.withCreator(eb))
-      .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(userId))
-      .orderBy('updatedAt', 'desc');
+      .where('spaceId', 'in', this.spaceMemberRepo.getUserSpaceIdsQuery(userId));
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'updatedAt', direction: 'desc' },
+        { expression: 'id', direction: 'desc' },
+      ],
+      parseCursor: (cursor) => ({
+        updatedAt: new Date(cursor.updatedAt),
+        id: cursor.id,
+      }),
     });
   }
 

apps/server/src/database/repos/space/space-member.repo.ts

@@ -10,7 +10,7 @@ import {
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '../../pagination/pagination-options';
 import { MemberInfo, UserSpaceRole } from './types';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { GroupRepo } from '@docmost/db/repos/group/group.repo';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
 
@@ -98,7 +98,7 @@ export class SpaceMemberRepo {
     spaceId: string,
     pagination: PaginationOptions,
   ) {
-    let query = this.db
+    let baseQuery = this.db
       .selectFrom('spaceMembers')
       .leftJoin('users', 'users.id', 'spaceMembers.userId')
       .leftJoin('groups', 'groups.id', 'spaceMembers.groupId')
@@ -114,12 +114,11 @@ export class SpaceMemberRepo {
         'spaceMembers.createdAt',
       ])
       .select((eb) => this.groupRepo.withMemberCount(eb))
-      .where('spaceId', '=', spaceId)
-      .orderBy((eb) => eb('groups.id', 'is not', null), 'desc')
-      .orderBy('spaceMembers.createdAt', 'asc');
+      .select(sql<number>`case when groups.id is not null then 1 else 0 end`.as('isGroup'))
+      .where('spaceId', '=', spaceId);
 
     if (pagination.query) {
-      query = query.where((eb) =>
+      baseQuery = baseQuery.where((eb) =>
         eb(
           sql`f_unaccent(users.name)`,
           'ilike',
@@ -138,9 +137,20 @@ export class SpaceMemberRepo {
       );
     }
 
-    const result = await executeWithPagination(query, {
-      page: pagination.page,
+    const query = this.db.selectFrom(baseQuery.as('sub')).selectAll('sub');
+
+    const result = await executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [
+        { expression: 'sub.isGroup', direction: 'desc', key: 'isGroup' },
+        { expression: 'sub.createdAt', direction: 'asc', key: 'createdAt' },
+      ],
+      parseCursor: (cursor) => ({
+        isGroup: parseInt(cursor.isGroup, 10),
+        createdAt: new Date(cursor.createdAt),
+      }),
     });
 
     let memberInfo: MemberInfo;
@@ -235,8 +245,7 @@ export class SpaceMemberRepo {
       .selectFrom('spaces')
       .selectAll()
       .select((eb) => [this.spaceRepo.withMemberCount(eb)])
-      .where('id', 'in', this.getUserSpaceIdsQuery(userId))
-      .orderBy('createdAt', 'asc');
+      .where('id', 'in', this.getUserSpaceIdsQuery(userId));
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -252,9 +261,12 @@ export class SpaceMemberRepo {
       );
     }
 
-    return executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
   }
 }

apps/server/src/database/repos/space/space.repo.ts

@@ -9,7 +9,7 @@ import {
 } from '@docmost/db/types/entity.types';
 import { ExpressionBuilder, sql } from 'kysely';
 import { PaginationOptions } from '../../pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { DB } from '@docmost/db/types/db';
 import { validate as isValidUUID } from 'uuid';
 import { EventEmitter2 } from '@nestjs/event-emitter';
@@ -110,8 +110,7 @@ export class SpaceRepo {
       .selectFrom('spaces')
       .selectAll('spaces')
       .select((eb) => [this.withMemberCount(eb)])
-      .where('workspaceId', '=', workspaceId)
-      .orderBy('createdAt', 'asc');
+      .where('workspaceId', '=', workspaceId);
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -127,12 +126,13 @@ export class SpaceRepo {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   withMemberCount(eb: ExpressionBuilder<DB, 'spaces'>) {

apps/server/src/database/repos/user/user.repo.ts

@@ -10,7 +10,7 @@ import {
   User,
 } from '@docmost/db/types/entity.types';
 import { PaginationOptions } from '../../pagination/pagination-options';
-import { executeWithPagination } from '@docmost/db/pagination/pagination';
+import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
 import { ExpressionBuilder, sql } from 'kysely';
 import { jsonObjectFrom } from 'kysely/helpers/postgres';
 
@@ -145,8 +145,7 @@ export class UserRepo {
       .selectFrom('users')
       .select(this.baseFields)
       .where('workspaceId', '=', workspaceId)
-      .where('deletedAt', 'is', null)
-      .orderBy('createdAt', 'asc');
+      .where('deletedAt', 'is', null);
 
     if (pagination.query) {
       query = query.where((eb) =>
@@ -162,12 +161,13 @@ export class UserRepo {
       );
     }
 
-    const result = executeWithPagination(query, {
-      page: pagination.page,
+    return executeWithCursorPagination(query, {
       perPage: pagination.limit,
+      cursor: pagination.cursor,
+      beforeCursor: pagination.beforeCursor,
+      fields: [{ expression: 'id', direction: 'asc' }],
+      parseCursor: (cursor) => ({ id: cursor.id }),
     });
-
-    return result;
   }
 
   async updatePreference(