mirror of
https://github.com/docmost/docmost.git
synced 2026-05-23 10:42:42 +08:00
feat(base): log duckdb heap + spill per base on cold load
This commit is contained in:
@@ -298,6 +298,30 @@ export class BaseQueryCacheService
|
|||||||
return this.collections.get(baseId);
|
return this.collections.get(baseId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Returns the memory footprint of every currently resident collection.
|
||||||
|
residencySnapshot(): Array<{
|
||||||
|
baseId: string;
|
||||||
|
rows: number;
|
||||||
|
heapMb: number;
|
||||||
|
spilledMb: number;
|
||||||
|
}> {
|
||||||
|
const out: Array<{
|
||||||
|
baseId: string;
|
||||||
|
rows: number;
|
||||||
|
heapMb: number;
|
||||||
|
spilledMb: number;
|
||||||
|
}> = [];
|
||||||
|
for (const [baseId, c] of this.collections) {
|
||||||
|
out.push({
|
||||||
|
baseId,
|
||||||
|
rows: c.rowCount,
|
||||||
|
heapMb: +(c.heapBytes / (1024 * 1024)).toFixed(1),
|
||||||
|
spilledMb: +(c.spilledBytes / (1024 * 1024)).toFixed(1),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Apply a change envelope received from Redis pub/sub to the local
|
* Apply a change envelope received from Redis pub/sub to the local
|
||||||
* collection (if any). Rows that target bases not resident on this node
|
* collection (if any). Rows that target bases not resident on this node
|
||||||
@@ -507,6 +531,9 @@ export class BaseQueryCacheService
|
|||||||
baseId: baseId.slice(0, 8),
|
baseId: baseId.slice(0, 8),
|
||||||
findMs,
|
findMs,
|
||||||
loadMs,
|
loadMs,
|
||||||
|
rows: loaded.rowCount,
|
||||||
|
heapMb: +(loaded.heapBytes / (1024 * 1024)).toFixed(1),
|
||||||
|
spilledMb: +(loaded.spilledBytes / (1024 * 1024)).toFixed(1),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -113,8 +113,22 @@ export class CollectionLoader {
|
|||||||
(countResult.getRowObjects()[0] as { c: bigint | number }).c,
|
(countResult.getRowObjects()[0] as { c: bigint | number }).c,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const memoryResult = await connection.runAndReadAll(
|
||||||
|
`SELECT
|
||||||
|
COALESCE(sum(memory_usage_bytes), 0)::BIGINT AS used_bytes,
|
||||||
|
COALESCE(sum(temporary_storage_bytes), 0)::BIGINT AS spilled_bytes
|
||||||
|
FROM duckdb_memory()`,
|
||||||
|
);
|
||||||
|
const mem = memoryResult.getRowObjects()[0] as {
|
||||||
|
used_bytes: bigint | number;
|
||||||
|
spilled_bytes: bigint | number;
|
||||||
|
};
|
||||||
|
const heapBytes = Number(mem.used_bytes);
|
||||||
|
const spilledBytes = Number(mem.spilled_bytes);
|
||||||
|
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Loaded ${rowCount} rows for base ${baseId} (schemaVersion=${schemaVersion})`,
|
`Loaded ${rowCount} rows for base ${baseId} ` +
|
||||||
|
`(schemaVersion=${schemaVersion}, heap=${fmtMb(heapBytes)}MB, spilled=${fmtMb(spilledBytes)}MB)`,
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -125,6 +139,8 @@ export class CollectionLoader {
|
|||||||
connection,
|
connection,
|
||||||
lastAccessedAt: Date.now(),
|
lastAccessedAt: Date.now(),
|
||||||
rowCount,
|
rowCount,
|
||||||
|
heapBytes,
|
||||||
|
spilledBytes,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
try {
|
try {
|
||||||
@@ -150,3 +166,7 @@ export class CollectionLoader {
|
|||||||
function quoteIdent(name: string): string {
|
function quoteIdent(name: string): string {
|
||||||
return `"${name.replace(/"/g, '""')}"`;
|
return `"${name.replace(/"/g, '""')}"`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function fmtMb(bytes: number): string {
|
||||||
|
return (bytes / (1024 * 1024)).toFixed(1);
|
||||||
|
}
|
||||||
|
|||||||
@@ -29,6 +29,11 @@ export type LoadedCollection = {
|
|||||||
lastAccessedAt: number;
|
lastAccessedAt: number;
|
||||||
// cached; set by loader, maintained by applyChange
|
// cached; set by loader, maintained by applyChange
|
||||||
rowCount: number;
|
rowCount: number;
|
||||||
|
// Memory stats captured immediately after load. Static until next
|
||||||
|
// explicit refresh — see `BaseQueryCacheService.refreshMemoryStats` if you
|
||||||
|
// need up-to-date figures after many applyChange() mutations.
|
||||||
|
heapBytes: number;
|
||||||
|
spilledBytes: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type ChangeEnvelope =
|
export type ChangeEnvelope =
|
||||||
|
|||||||
Reference in New Issue
Block a user