mirror of
https://github.com/docmost/docmost.git
synced 2026-06-10 01:52:43 +08:00
feat(base): add /bases/rows/count with estimate and capped-exact modes
Row-count display on a filtered view shouldn't force a full COUNT(*) on
every list fetch. New endpoint returns either an EXPLAIN-plan estimate
(default, ~1ms, no execution) or a LIMIT-capped exact count that short-
circuits to `{ capped: true }` once the match set passes EXACT_COUNT_CAP.
Clients call it in parallel with the rows query so the grid still paints
at its own pace.
- DTO + repo.countEstimate/countExact reusing the list predicate shape
- service picks the mode; controller mirrors the list Read ability check
- client hook keyed by filter/search/exact so a "show exact" toggle
doesn't clobber the estimate cache
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
useInfiniteQuery,
|
||||
useMutation,
|
||||
useQuery,
|
||||
InfiniteData,
|
||||
} from "@tanstack/react-query";
|
||||
import {
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
deleteRows,
|
||||
listRows,
|
||||
reorderRow,
|
||||
countRows,
|
||||
} from "@/features/base/services/base-service";
|
||||
import {
|
||||
IBaseRow,
|
||||
@@ -21,6 +23,7 @@ import {
|
||||
FilterNode,
|
||||
SearchSpec,
|
||||
ViewSortConfig,
|
||||
CountRowsResult,
|
||||
} from "@/features/base/types/base.types";
|
||||
import { notifications } from "@mantine/notifications";
|
||||
import { queryClient } from "@/main";
|
||||
@@ -277,6 +280,35 @@ export function useDeleteRowsMutation() {
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
* Row count for the current view. Fires in parallel with `useBaseRowsQuery`
|
||||
* — doesn't block first paint. Keyed by filter + search so an independent
|
||||
* view with a different filter gets its own cached count; `exact` is part
|
||||
* of the key so a "show exact" toggle doesn't clobber the estimate cache.
|
||||
*/
|
||||
export function useBaseRowsCountQuery(
|
||||
baseId: string | undefined,
|
||||
filter?: FilterNode,
|
||||
search?: SearchSpec,
|
||||
exact = false,
|
||||
) {
|
||||
const activeFilter = normalizeFilter(filter);
|
||||
const activeSearch = search?.query ? search : undefined;
|
||||
|
||||
return useQuery<CountRowsResult>({
|
||||
queryKey: ["base-rows-count", baseId, activeFilter, activeSearch, exact],
|
||||
queryFn: () =>
|
||||
countRows({
|
||||
baseId: baseId!,
|
||||
filter: activeFilter,
|
||||
search: activeSearch,
|
||||
exact,
|
||||
}),
|
||||
enabled: !!baseId,
|
||||
staleTime: 30 * 1000,
|
||||
});
|
||||
}
|
||||
|
||||
export function useReorderRowMutation() {
|
||||
const { t } = useTranslation();
|
||||
return useMutation<void, Error, ReorderRowInput, RowCacheContext>({
|
||||
|
||||
@@ -23,6 +23,8 @@ import {
|
||||
FilterNode,
|
||||
SearchSpec,
|
||||
ViewSortConfig,
|
||||
CountRowsInput,
|
||||
CountRowsResult,
|
||||
} from "@/features/base/types/base.types";
|
||||
import { IPagination } from "@/lib/types";
|
||||
|
||||
@@ -151,6 +153,13 @@ export async function reorderRow(data: ReorderRowInput): Promise<void> {
|
||||
await api.post("/bases/rows/reorder", data);
|
||||
}
|
||||
|
||||
export async function countRows(
|
||||
data: CountRowsInput,
|
||||
): Promise<CountRowsResult> {
|
||||
const req = await api.post<CountRowsResult>("/bases/rows/count", data);
|
||||
return req.data;
|
||||
}
|
||||
|
||||
// --- Views ---
|
||||
|
||||
export async function createView(data: CreateViewInput): Promise<IBaseView> {
|
||||
|
||||
@@ -239,6 +239,21 @@ export type ReorderPropertyInput = {
|
||||
requestId?: string;
|
||||
};
|
||||
|
||||
export type CountRowsInput = {
|
||||
baseId: string;
|
||||
filter?: FilterNode;
|
||||
search?: SearchSpec;
|
||||
// When true the server returns an exact (but capped) count. Otherwise
|
||||
// a cheap EXPLAIN-plan estimate. Paired with `capped` on the result.
|
||||
exact?: boolean;
|
||||
};
|
||||
|
||||
export type CountRowsResult = {
|
||||
value: number;
|
||||
exact: boolean;
|
||||
capped: boolean;
|
||||
};
|
||||
|
||||
export type CreateRowInput = {
|
||||
baseId: string;
|
||||
cells?: Record<string, unknown>;
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
RowIdDto,
|
||||
ListRowsDto,
|
||||
ReorderRowDto,
|
||||
CountRowsDto,
|
||||
} from '../dto/update-row.dto';
|
||||
import { AuthUser } from '../../../common/decorators/auth-user.decorator';
|
||||
import { AuthWorkspace } from '../../../common/decorators/auth-workspace.decorator';
|
||||
@@ -160,6 +161,26 @@ export class BaseRowController {
|
||||
return this.baseRowService.list(dto, pagination, workspace.id);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('count')
|
||||
async count(
|
||||
@Body() dto: CountRowsDto,
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
const base = await this.baseRepo.findById(dto.baseId);
|
||||
if (!base) {
|
||||
throw new NotFoundException('Base not found');
|
||||
}
|
||||
|
||||
const ability = await this.spaceAbility.createForUser(user, base.spaceId);
|
||||
if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Base)) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
|
||||
return this.baseRowService.count(dto, workspace.id);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('reorder')
|
||||
async reorder(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
IsBoolean,
|
||||
IsIn,
|
||||
IsNotEmpty,
|
||||
IsObject,
|
||||
@@ -83,6 +84,25 @@ export class ListRowsDto {
|
||||
search?: unknown;
|
||||
}
|
||||
|
||||
export class CountRowsDto {
|
||||
@IsUUID()
|
||||
baseId: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsObject()
|
||||
filter?: unknown;
|
||||
|
||||
@IsOptional()
|
||||
@IsObject()
|
||||
search?: unknown;
|
||||
|
||||
// Opt-in for an exact (but capped) count. Default is an EXPLAIN-plan
|
||||
// estimate — cheap and good enough for "≈ N rows" headers.
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
exact?: boolean;
|
||||
}
|
||||
|
||||
export class ReorderRowDto {
|
||||
@IsUUID()
|
||||
rowId: string;
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
DeleteRowsDto,
|
||||
ListRowsDto,
|
||||
ReorderRowDto,
|
||||
CountRowsDto,
|
||||
} from '../dto/update-row.dto';
|
||||
import {
|
||||
BasePropertyTypeValue,
|
||||
@@ -46,6 +47,12 @@ import {
|
||||
} from '../events/base-events';
|
||||
import { FormulaService } from '../formula/formula.service';
|
||||
|
||||
// Cap for `count({ exact: true })`. Beyond this we return `capped: true` and
|
||||
// the UI shows "N+". Chosen so a cell-wise scan stays well under 100ms on
|
||||
// the existing `idx_base_rows_cells_gin_path_ops` index; callers that need
|
||||
// true totals should fall back to `exact: false` (planner estimate).
|
||||
const EXACT_COUNT_CAP = 10_000;
|
||||
|
||||
@Injectable()
|
||||
export class BaseRowService {
|
||||
constructor(
|
||||
@@ -244,6 +251,39 @@ export class BaseRowService {
|
||||
});
|
||||
}
|
||||
|
||||
async count(dto: CountRowsDto, workspaceId: string) {
|
||||
const properties = await this.basePropertyRepo.findByBaseId(dto.baseId);
|
||||
const schema: PropertySchema = new Map(properties.map((p) => [p.id, p]));
|
||||
|
||||
const filter = this.normaliseFilter({ filter: dto.filter });
|
||||
const search = this.normaliseSearch(dto.search);
|
||||
|
||||
if (dto.exact) {
|
||||
const { value, capped } = await this.baseRowRepo.countExact({
|
||||
baseId: dto.baseId,
|
||||
workspaceId,
|
||||
filter,
|
||||
search,
|
||||
schema,
|
||||
cap: EXACT_COUNT_CAP,
|
||||
});
|
||||
return { value, exact: true as const, capped };
|
||||
}
|
||||
|
||||
const estimate = await this.baseRowRepo.countEstimate({
|
||||
baseId: dto.baseId,
|
||||
workspaceId,
|
||||
filter,
|
||||
search,
|
||||
schema,
|
||||
});
|
||||
return {
|
||||
value: estimate ?? 0,
|
||||
exact: false as const,
|
||||
capped: false,
|
||||
};
|
||||
}
|
||||
|
||||
async reorder(dto: ReorderRowDto, workspaceId: string, userId?: string) {
|
||||
const row = await this.baseRowRepo.findById(dto.rowId, { workspaceId });
|
||||
if (!row || row.baseId !== dto.baseId) {
|
||||
@@ -274,7 +314,7 @@ export class BaseRowService {
|
||||
|
||||
// --- private helpers ------------------------------------------------
|
||||
|
||||
private normaliseFilter(dto: ListRowsDto): FilterNode | undefined {
|
||||
private normaliseFilter(dto: { filter?: unknown }): FilterNode | undefined {
|
||||
if (!dto.filter) return undefined;
|
||||
|
||||
const parsed = filterGroupSchema.safeParse(dto.filter);
|
||||
|
||||
@@ -11,12 +11,14 @@ import {
|
||||
CursorPaginationResult,
|
||||
executeWithCursorPagination,
|
||||
} from '@docmost/db/pagination/cursor-pagination';
|
||||
import { sql, SqlBool } from 'kysely';
|
||||
import { CompiledQuery, sql, SqlBool } from 'kysely';
|
||||
import {
|
||||
FilterNode,
|
||||
PropertySchema,
|
||||
SearchSpec,
|
||||
SortSpec,
|
||||
buildSearch,
|
||||
buildWhere,
|
||||
runListQuery,
|
||||
} from '../../../core/base/engine';
|
||||
|
||||
@@ -128,6 +130,109 @@ export class BaseRowRepo {
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
* EXPLAIN-based row estimate for the same predicate shape `list` uses.
|
||||
* Doesn't execute the query — ~1ms even on large bases. Returns `null`
|
||||
* when the plan JSON is unexpected. Caller treats the number as
|
||||
* approximate; planner accuracy depends on fresh stats (ANALYZE).
|
||||
*/
|
||||
async countEstimate(opts: {
|
||||
baseId: string;
|
||||
workspaceId: string;
|
||||
filter?: FilterNode;
|
||||
search?: SearchSpec;
|
||||
schema: PropertySchema;
|
||||
trx?: KyselyTransaction;
|
||||
}): Promise<number | null> {
|
||||
const db = dbOrTx(this.db, opts.trx);
|
||||
|
||||
let qb = db
|
||||
.selectFrom('baseRows')
|
||||
.select(sql<number>`1`.as('x'))
|
||||
.where('baseId', '=', opts.baseId)
|
||||
.where('workspaceId', '=', opts.workspaceId)
|
||||
.where('deletedAt', 'is', null);
|
||||
|
||||
if (opts.search) {
|
||||
const spec = opts.search;
|
||||
qb = qb.where((eb) => buildSearch(eb, spec));
|
||||
}
|
||||
if (opts.filter) {
|
||||
const filter = opts.filter;
|
||||
const schema = opts.schema;
|
||||
qb = qb.where((eb) => buildWhere(eb, filter, schema));
|
||||
}
|
||||
|
||||
// Embedding `qb` inside a `sql` template would wrap it in parens as a
|
||||
// subquery, which PG rejects after EXPLAIN. Splice the raw SQL + reuse
|
||||
// the compiled parameters via a hand-built CompiledQuery instead.
|
||||
const compiled = qb.compile();
|
||||
const explain: CompiledQuery = {
|
||||
...compiled,
|
||||
sql: `EXPLAIN (FORMAT JSON) ${compiled.sql}`,
|
||||
};
|
||||
const res = await db.executeQuery<{ 'QUERY PLAN': unknown }>(explain);
|
||||
|
||||
// `QUERY PLAN` is a json column — postgres.js auto-parses it into an
|
||||
// array. Fall back to JSON.parse for drivers that return it as text.
|
||||
let plan = (res.rows[0] as any)?.['QUERY PLAN'];
|
||||
if (typeof plan === 'string') {
|
||||
try {
|
||||
plan = JSON.parse(plan);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
const n = Array.isArray(plan) ? plan[0]?.Plan?.['Plan Rows'] : undefined;
|
||||
return typeof n === 'number' ? Math.max(0, Math.round(n)) : null;
|
||||
}
|
||||
|
||||
/*
|
||||
* Capped exact count. Wraps the filtered scan in a `LIMIT cap + 1`
|
||||
* subquery so PG stops touching rows once we've proven the filter
|
||||
* matches more than `cap` — callers render "cap+" in the UI instead.
|
||||
*/
|
||||
async countExact(opts: {
|
||||
baseId: string;
|
||||
workspaceId: string;
|
||||
filter?: FilterNode;
|
||||
search?: SearchSpec;
|
||||
schema: PropertySchema;
|
||||
cap: number;
|
||||
trx?: KyselyTransaction;
|
||||
}): Promise<{ value: number; capped: boolean }> {
|
||||
const db = dbOrTx(this.db, opts.trx);
|
||||
|
||||
let inner = db
|
||||
.selectFrom('baseRows')
|
||||
.select(sql<number>`1`.as('x'))
|
||||
.where('baseId', '=', opts.baseId)
|
||||
.where('workspaceId', '=', opts.workspaceId)
|
||||
.where('deletedAt', 'is', null);
|
||||
|
||||
if (opts.search) {
|
||||
const spec = opts.search;
|
||||
inner = inner.where((eb) => buildSearch(eb, spec));
|
||||
}
|
||||
if (opts.filter) {
|
||||
const filter = opts.filter;
|
||||
const schema = opts.schema;
|
||||
inner = inner.where((eb) => buildWhere(eb, filter, schema));
|
||||
}
|
||||
|
||||
inner = inner.limit(opts.cap + 1);
|
||||
const compiled = inner.compile();
|
||||
const wrapped: CompiledQuery = {
|
||||
...compiled,
|
||||
sql: `SELECT count(*)::text AS n FROM (${compiled.sql}) AS t`,
|
||||
};
|
||||
const res = await db.executeQuery<{ n: string }>(wrapped);
|
||||
|
||||
const n = Number(res.rows[0]?.n ?? 0);
|
||||
if (n > opts.cap) return { value: opts.cap, capped: true };
|
||||
return { value: n, capped: false };
|
||||
}
|
||||
|
||||
async getLastPosition(
|
||||
baseId: string,
|
||||
opts: WorkspaceOpts,
|
||||
|
||||
Reference in New Issue
Block a user