feat(base): log duckdb heap + spill per base on cold load

This commit is contained in:
Philipinho
2026-04-23 14:07:36 +01:00
parent 17db634029
commit 5b96dfe6c9
3 changed files with 53 additions and 1 deletions
@@ -298,6 +298,30 @@ export class BaseQueryCacheService
return this.collections.get(baseId);
}
// Returns the memory footprint of every currently resident collection.
residencySnapshot(): Array<{
baseId: string;
rows: number;
heapMb: number;
spilledMb: number;
}> {
const out: Array<{
baseId: string;
rows: number;
heapMb: number;
spilledMb: number;
}> = [];
for (const [baseId, c] of this.collections) {
out.push({
baseId,
rows: c.rowCount,
heapMb: +(c.heapBytes / (1024 * 1024)).toFixed(1),
spilledMb: +(c.spilledBytes / (1024 * 1024)).toFixed(1),
});
}
return out;
}
/*
* Apply a change envelope received from Redis pub/sub to the local
* collection (if any). Rows that target bases not resident on this node
@@ -507,6 +531,9 @@ export class BaseQueryCacheService
baseId: baseId.slice(0, 8),
findMs,
loadMs,
rows: loaded.rowCount,
heapMb: +(loaded.heapBytes / (1024 * 1024)).toFixed(1),
spilledMb: +(loaded.spilledBytes / (1024 * 1024)).toFixed(1),
}),
);
}
@@ -113,8 +113,22 @@ export class CollectionLoader {
(countResult.getRowObjects()[0] as { c: bigint | number }).c,
);
const memoryResult = await connection.runAndReadAll(
`SELECT
COALESCE(sum(memory_usage_bytes), 0)::BIGINT AS used_bytes,
COALESCE(sum(temporary_storage_bytes), 0)::BIGINT AS spilled_bytes
FROM duckdb_memory()`,
);
const mem = memoryResult.getRowObjects()[0] as {
used_bytes: bigint | number;
spilled_bytes: bigint | number;
};
const heapBytes = Number(mem.used_bytes);
const spilledBytes = Number(mem.spilled_bytes);
this.logger.debug(
`Loaded ${rowCount} rows for base ${baseId} (schemaVersion=${schemaVersion})`,
`Loaded ${rowCount} rows for base ${baseId} ` +
`(schemaVersion=${schemaVersion}, heap=${fmtMb(heapBytes)}MB, spilled=${fmtMb(spilledBytes)}MB)`,
);
return {
@@ -125,6 +139,8 @@ export class CollectionLoader {
connection,
lastAccessedAt: Date.now(),
rowCount,
heapBytes,
spilledBytes,
};
} catch (err) {
try {
@@ -150,3 +166,7 @@ export class CollectionLoader {
function quoteIdent(name: string): string {
return `"${name.replace(/"/g, '""')}"`;
}
function fmtMb(bytes: number): string {
return (bytes / (1024 * 1024)).toFixed(1);
}
@@ -29,6 +29,11 @@ export type LoadedCollection = {
lastAccessedAt: number;
// cached; set by loader, maintained by applyChange
rowCount: number;
// Memory stats captured immediately after load. Static until next
// explicit refresh — see `BaseQueryCacheService.refreshMemoryStats` if you
// need up-to-date figures after many applyChange() mutations.
heapBytes: number;
spilledBytes: number;
};
export type ChangeEnvelope =