From 5b96dfe6c9654b0bacda33a96cfd42cb2984cf56 Mon Sep 17 00:00:00 2001
From: Philipinho <16838612+Philipinho@users.noreply.github.com>
Date: Thu, 23 Apr 2026 14:07:36 +0100
Subject: [PATCH] feat(base): log duckdb heap + spill per base on cold load

---
 .../query-cache/base-query-cache.service.ts   | 27 +++++++++++++++++++
 .../base/query-cache/collection-loader.ts     | 22 ++++++++++++++-
 .../base/query-cache/query-cache.types.ts     |  5 ++++
 3 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/apps/server/src/core/base/query-cache/base-query-cache.service.ts b/apps/server/src/core/base/query-cache/base-query-cache.service.ts
index e01726dc..f2ea353d 100644
--- a/apps/server/src/core/base/query-cache/base-query-cache.service.ts
+++ b/apps/server/src/core/base/query-cache/base-query-cache.service.ts
@@ -298,6 +298,30 @@ export class BaseQueryCacheService
     return this.collections.get(baseId);
   }
 
+  // Returns the memory footprint of every currently resident collection.
+  residencySnapshot(): Array<{
+    baseId: string;
+    rows: number;
+    heapMb: number;
+    spilledMb: number;
+  }> {
+    const out: Array<{
+      baseId: string;
+      rows: number;
+      heapMb: number;
+      spilledMb: number;
+    }> = [];
+    for (const [baseId, c] of this.collections) {
+      out.push({
+        baseId,
+        rows: c.rowCount,
+        heapMb: +(c.heapBytes / (1024 * 1024)).toFixed(1),
+        spilledMb: +(c.spilledBytes / (1024 * 1024)).toFixed(1),
+      });
+    }
+    return out;
+  }
+
   /*
    * Apply a change envelope received from Redis pub/sub to the local
    * collection (if any). Rows that target bases not resident on this node
@@ -507,6 +531,9 @@ export class BaseQueryCacheService
           baseId: baseId.slice(0, 8),
           findMs,
           loadMs,
+          rows: loaded.rowCount,
+          heapMb: +(loaded.heapBytes / (1024 * 1024)).toFixed(1),
+          spilledMb: +(loaded.spilledBytes / (1024 * 1024)).toFixed(1),
         }),
       );
     }
diff --git a/apps/server/src/core/base/query-cache/collection-loader.ts b/apps/server/src/core/base/query-cache/collection-loader.ts
index 9be69867..395925c6 100644
--- a/apps/server/src/core/base/query-cache/collection-loader.ts
+++ b/apps/server/src/core/base/query-cache/collection-loader.ts
@@ -113,8 +113,22 @@ export class CollectionLoader {
         (countResult.getRowObjects()[0] as { c: bigint | number }).c,
       );
 
+      const memoryResult = await connection.runAndReadAll(
+        `SELECT
+           COALESCE(sum(memory_usage_bytes), 0)::BIGINT AS used_bytes,
+           COALESCE(sum(temporary_storage_bytes), 0)::BIGINT AS spilled_bytes
+         FROM duckdb_memory()`,
+      );
+      const mem = memoryResult.getRowObjects()[0] as {
+        used_bytes: bigint | number;
+        spilled_bytes: bigint | number;
+      };
+      const heapBytes = Number(mem.used_bytes);
+      const spilledBytes = Number(mem.spilled_bytes);
+
       this.logger.debug(
-        `Loaded ${rowCount} rows for base ${baseId} (schemaVersion=${schemaVersion})`,
+        `Loaded ${rowCount} rows for base ${baseId} ` +
+          `(schemaVersion=${schemaVersion}, heap=${fmtMb(heapBytes)}MB, spilled=${fmtMb(spilledBytes)}MB)`,
       );
 
       return {
@@ -125,6 +139,8 @@ export class CollectionLoader {
         connection,
         lastAccessedAt: Date.now(),
         rowCount,
+        heapBytes,
+        spilledBytes,
       };
     } catch (err) {
       try {
@@ -150,3 +166,7 @@ export class CollectionLoader {
 function quoteIdent(name: string): string {
   return `"${name.replace(/"/g, '""')}"`;
 }
+
+function fmtMb(bytes: number): string {
+  return (bytes / (1024 * 1024)).toFixed(1);
+}
diff --git a/apps/server/src/core/base/query-cache/query-cache.types.ts b/apps/server/src/core/base/query-cache/query-cache.types.ts
index 1f06c017..256dfd9d 100644
--- a/apps/server/src/core/base/query-cache/query-cache.types.ts
+++ b/apps/server/src/core/base/query-cache/query-cache.types.ts
@@ -29,6 +29,11 @@ export type LoadedCollection = {
   lastAccessedAt: number;
   // cached; set by loader, maintained by applyChange
   rowCount: number;
+  // Memory stats captured immediately after load. Static until next
+  // explicit refresh — see `BaseQueryCacheService.refreshMemoryStats` if you
+  // need up-to-date figures after many applyChange() mutations.
+  heapBytes: number;
+  spilledBytes: number;
 };
 
 export type ChangeEnvelope =