From fef77e5a21fb399b6eb858733df1804b4f5f93a4 Mon Sep 17 00:00:00 2001 From: atmn Date: Mon, 23 Feb 2026 15:00:18 +0000 Subject: [PATCH 1/6] feat: add periodic memory usage logging to server and workers Logs RSS, heap, external memory, and array buffers every 60s via Axiom logger with type: 'memory_log' for easy filtering. Includes delta tracking to spot growth trends. --- server/src/init.ts | 2 + server/src/utils/memoryMonitor.ts | 64 +++++++++++++++++++++++++++++++ server/src/workers.ts | 3 ++ 3 files changed, 69 insertions(+) create mode 100644 server/src/utils/memoryMonitor.ts diff --git a/server/src/init.ts b/server/src/init.ts index 10db2d8e2..41c0451eb 100644 --- a/server/src/init.ts +++ b/server/src/init.ts @@ -21,6 +21,7 @@ import { redirectToHono } from "./initHono.js"; import { auth } from "./utils/auth.js"; import { generateId } from "./utils/genUtils.js"; import { checkEnvVars } from "./utils/initUtils.js"; +import { startMemoryMonitor } from "./utils/memoryMonitor.js"; checkEnvVars(); // subscribeToOrgUpdates({ db }); @@ -140,6 +141,7 @@ const init = async () => { // Bind to 0.0.0.0 for AWS ECS/Docker containers server.listen(PORT, "0.0.0.0", () => { console.log(`Server running on port ${PORT}`); + startMemoryMonitor("server", 60_000); }); }; diff --git a/server/src/utils/memoryMonitor.ts b/server/src/utils/memoryMonitor.ts new file mode 100644 index 000000000..3dd5b2b6f --- /dev/null +++ b/server/src/utils/memoryMonitor.ts @@ -0,0 +1,64 @@ +/** + * Periodic memory usage logger for diagnosing memory leaks. + * + * Logs heap usage, RSS, external memory, and array buffers every interval. + * Ships to Axiom via the standard logger with type: "memory_log". + */ + +import { logger } from "@/external/logtail/logtailUtils.js"; + +const DEFAULT_INTERVAL_MS = 60_000; // 1 minute + +let previousRss = 0; +let intervalHandle: ReturnType | null = null; + +function toMB(bytes: number): number { + return Math.round((bytes / 1024 / 1024) * 10) / 10; +} + +function logMemoryUsage(label: string) { + const mem = process.memoryUsage(); + + logger.info("memory log", { + type: "memory_log", + label, + pid: process.pid, + rss_mb: toMB(mem.rss), + heap_used_mb: toMB(mem.heapUsed), + heap_total_mb: toMB(mem.heapTotal), + external_mb: toMB(mem.external), + array_buffers_mb: toMB(mem.arrayBuffers), + rss_delta_mb: previousRss ? toMB(mem.rss - previousRss) : 0, + }); + + previousRss = mem.rss; +} + +/** + * Start periodic memory logging. + * @param label - identifier for the process (e.g. "server", "worker") + * @param intervalMs - how often to log (default: 60s) + */ +export function startMemoryMonitor( + label: string, + intervalMs = DEFAULT_INTERVAL_MS, +) { + // Log immediately on start + logMemoryUsage(label); + + intervalHandle = setInterval(() => { + logMemoryUsage(label); + }, intervalMs); + + // Don't prevent process exit + if (intervalHandle.unref) { + intervalHandle.unref(); + } +} + +export function stopMemoryMonitor() { + if (intervalHandle) { + clearInterval(intervalHandle); + intervalHandle = null; + } +} diff --git a/server/src/workers.ts b/server/src/workers.ts index c1d56dc6d..a08d7e212 100644 --- a/server/src/workers.ts +++ b/server/src/workers.ts @@ -19,6 +19,8 @@ const NUM_PROCESSES = process.env.NODE_ENV === "development" ? 1 : 4; // Track if we're shutting down let isShuttingDown = false; +import { startMemoryMonitor } from "./utils/memoryMonitor.js"; + if (cluster.isPrimary) { await initInfisical(); @@ -94,6 +96,7 @@ if (cluster.isPrimary) { } else { // Worker process console.log(`[Worker ${process.pid}] Starting queue consumer...`); + startMemoryMonitor("worker", 60_000); // Auto-detect which queue implementation to use if (process.env.SQS_QUEUE_URL) { From 41b089a19af283783dc1841b01366b5a33c5978f Mon Sep 17 00:00:00 2001 From: atmn Date: Mon, 23 Feb 2026 15:40:30 +0000 Subject: [PATCH 2/6] fix: use Axiom logger for memory monitoring with type: memory_log --- server/src/utils/memoryMonitor.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/utils/memoryMonitor.ts b/server/src/utils/memoryMonitor.ts index 3dd5b2b6f..9c0978207 100644 --- a/server/src/utils/memoryMonitor.ts +++ b/server/src/utils/memoryMonitor.ts @@ -2,14 +2,13 @@ * Periodic memory usage logger for diagnosing memory leaks. * * Logs heap usage, RSS, external memory, and array buffers every interval. - * Ships to Axiom via the standard logger with type: "memory_log". + * Uses Axiom logger so metrics are queryable via type: "memory_log". */ -import { logger } from "@/external/logtail/logtailUtils.js"; +import { logger } from "../external/logtail/logtailUtils.js"; const DEFAULT_INTERVAL_MS = 60_000; // 1 minute -let previousRss = 0; let intervalHandle: ReturnType | null = null; function toMB(bytes: number): number { @@ -19,19 +18,16 @@ function toMB(bytes: number): number { function logMemoryUsage(label: string) { const mem = process.memoryUsage(); - logger.info("memory log", { + logger.info("memory_log", { type: "memory_log", label, pid: process.pid, - rss_mb: toMB(mem.rss), - heap_used_mb: toMB(mem.heapUsed), - heap_total_mb: toMB(mem.heapTotal), - external_mb: toMB(mem.external), - array_buffers_mb: toMB(mem.arrayBuffers), - rss_delta_mb: previousRss ? toMB(mem.rss - previousRss) : 0, + rssMB: toMB(mem.rss), + heapUsedMB: toMB(mem.heapUsed), + heapTotalMB: toMB(mem.heapTotal), + externalMB: toMB(mem.external), + arrayBuffersMB: toMB(mem.arrayBuffers), }); - - previousRss = mem.rss; } /** @@ -54,6 +50,10 @@ export function startMemoryMonitor( if (intervalHandle.unref) { intervalHandle.unref(); } + + console.log( + `[mem:${label}] Memory monitor started (every ${intervalMs / 1000}s)`, + ); } export function stopMemoryMonitor() { From 9d4c5b2ffd2ed8db80c1a73281ca6114e0908430 Mon Sep 17 00:00:00 2001 From: atmn Date: Mon, 23 Feb 2026 15:57:01 +0000 Subject: [PATCH 3/6] feat: add /v1/debug/heap-snapshot and /v1/debug/memory endpoints Locked to org_2rzkkRh7r5dBSaBC101QHG9KDgt and org_2vwdxwTdqxRrLEdUYddcynMv3n3. Uses Bun.generateHeapSnapshot() when available, falls back to v8. --- server/src/initHono.ts | 4 + .../src/internal/debug/heapSnapshotRoute.ts | 85 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 server/src/internal/debug/heapSnapshotRoute.ts diff --git a/server/src/initHono.ts b/server/src/initHono.ts index f68143ca9..e41d67339 100644 --- a/server/src/initHono.ts +++ b/server/src/initHono.ts @@ -18,6 +18,7 @@ import type { HonoEnv } from "./honoUtils/HonoEnv.js"; import { handleHealthCheck } from "./honoUtils/handleHealthCheck.js"; import { cliRouter } from "./internal/dev/cli/cliRouter.js"; import { handleOAuthCallback } from "./internal/orgs/handlers/stripeHandlers/handleOAuthCallback.js"; +import { heapSnapshotRouter } from "./internal/debug/heapSnapshotRoute.js"; import { apiRouter } from "./routers/apiRouter.js"; import { internalRouter } from "./routers/internalRouter.js"; import { publicRouter } from "./routers/publicRouter.js"; @@ -148,6 +149,9 @@ const createHonoApp = () => { // Public routes (no auth required) app.route("", publicRouter); + // Debug routes (auth handled internally) + app.route("/v1/debug", heapSnapshotRouter); + // API Middleware app.route("/v1", apiRouter); app.route("", internalRouter); diff --git a/server/src/internal/debug/heapSnapshotRoute.ts b/server/src/internal/debug/heapSnapshotRoute.ts new file mode 100644 index 000000000..a8115e25e --- /dev/null +++ b/server/src/internal/debug/heapSnapshotRoute.ts @@ -0,0 +1,85 @@ +import { writeFileSync, readFileSync, unlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Hono } from "hono"; +import { secretKeyMiddleware } from "@/honoMiddlewares/secretKeyMiddleware.js"; +import { orgConfigMiddleware } from "@/honoMiddlewares/orgConfigMiddleware.js"; +import type { HonoEnv } from "@/honoUtils/HonoEnv.js"; + +const ALLOWED_ORG_IDS = new Set([ + "org_2rzkkRh7r5dBSaBC101QHG9KDgt", + "org_2vwdxwTdqxRrLEdUYddcynMv3n3", +]); + +export const heapSnapshotRouter = new Hono(); + +heapSnapshotRouter.use("*", secretKeyMiddleware); +heapSnapshotRouter.use("*", orgConfigMiddleware); + +heapSnapshotRouter.get("/heap-snapshot", async (c) => { + const ctx = c.get("ctx"); + + if (!ALLOWED_ORG_IDS.has(ctx.org?.id)) { + return c.json({ error: "Forbidden" }, 403); + } + + // Bun-specific heap snapshot + if (typeof Bun !== "undefined" && typeof Bun.generateHeapSnapshot === "function") { + const snapshot = Bun.generateHeapSnapshot(); + return c.json({ + ok: true, + pid: process.pid, + timestamp: new Date().toISOString(), + snapshot, + }); + } + + // Node.js fallback using v8 + try { + const v8 = await import("node:v8"); + const snapshotPath = join(tmpdir(), `heap-${process.pid}-${Date.now()}.heapsnapshot`); + + v8.writeHeapSnapshot(snapshotPath); + + const data = readFileSync(snapshotPath); + unlinkSync(snapshotPath); + + return new Response(data, { + headers: { + "Content-Type": "application/json", + "Content-Disposition": `attachment; filename="heap-${process.pid}-${Date.now()}.heapsnapshot"`, + }, + }); + } catch (err) { + return c.json( + { + error: "Failed to generate heap snapshot", + detail: err instanceof Error ? err.message : String(err), + }, + 500, + ); + } +}); + +heapSnapshotRouter.get("/memory", async (c) => { + const ctx = c.get("ctx"); + + if (!ALLOWED_ORG_IDS.has(ctx.org?.id)) { + return c.json({ error: "Forbidden" }, 403); + } + + const mem = process.memoryUsage(); + + return c.json({ + ok: true, + pid: process.pid, + timestamp: new Date().toISOString(), + memory: { + rssMB: +(mem.rss / 1024 / 1024).toFixed(1), + heapUsedMB: +(mem.heapUsed / 1024 / 1024).toFixed(1), + heapTotalMB: +(mem.heapTotal / 1024 / 1024).toFixed(1), + externalMB: +(mem.external / 1024 / 1024).toFixed(1), + arrayBuffersMB: +(mem.arrayBuffers / 1024 / 1024).toFixed(1), + }, + }); +}); From 60fd48544730c98b133aade169658321d94ff105 Mon Sep 17 00:00:00 2001 From: atmn Date: Mon, 23 Feb 2026 15:58:32 +0000 Subject: [PATCH 4/6] fix: move debug routes to apiRouter for proper auth chain --- bun.lock | 2 +- server/src/initHono.ts | 4 ---- server/src/internal/debug/heapSnapshotRoute.ts | 17 +++++++++-------- server/src/routers/apiRouter.ts | 2 ++ 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/bun.lock b/bun.lock index e3c9c72c5..3b8799da6 100644 --- a/bun.lock +++ b/bun.lock @@ -101,7 +101,7 @@ }, "packages/autumn-js": { "name": "autumn-js", - "version": "1.0.0-beta.4", + "version": "1.0.0-beta.5", "dependencies": { "query-string": "^9.2.2", "rou3": "^0.6.1", diff --git a/server/src/initHono.ts b/server/src/initHono.ts index e41d67339..f68143ca9 100644 --- a/server/src/initHono.ts +++ b/server/src/initHono.ts @@ -18,7 +18,6 @@ import type { HonoEnv } from "./honoUtils/HonoEnv.js"; import { handleHealthCheck } from "./honoUtils/handleHealthCheck.js"; import { cliRouter } from "./internal/dev/cli/cliRouter.js"; import { handleOAuthCallback } from "./internal/orgs/handlers/stripeHandlers/handleOAuthCallback.js"; -import { heapSnapshotRouter } from "./internal/debug/heapSnapshotRoute.js"; import { apiRouter } from "./routers/apiRouter.js"; import { internalRouter } from "./routers/internalRouter.js"; import { publicRouter } from "./routers/publicRouter.js"; @@ -149,9 +148,6 @@ const createHonoApp = () => { // Public routes (no auth required) app.route("", publicRouter); - // Debug routes (auth handled internally) - app.route("/v1/debug", heapSnapshotRouter); - // API Middleware app.route("/v1", apiRouter); app.route("", internalRouter); diff --git a/server/src/internal/debug/heapSnapshotRoute.ts b/server/src/internal/debug/heapSnapshotRoute.ts index a8115e25e..52a7b6ed7 100644 --- a/server/src/internal/debug/heapSnapshotRoute.ts +++ b/server/src/internal/debug/heapSnapshotRoute.ts @@ -1,9 +1,7 @@ -import { writeFileSync, readFileSync, unlinkSync } from "node:fs"; +import { readFileSync, unlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Hono } from "hono"; -import { secretKeyMiddleware } from "@/honoMiddlewares/secretKeyMiddleware.js"; -import { orgConfigMiddleware } from "@/honoMiddlewares/orgConfigMiddleware.js"; import type { HonoEnv } from "@/honoUtils/HonoEnv.js"; const ALLOWED_ORG_IDS = new Set([ @@ -13,9 +11,6 @@ const ALLOWED_ORG_IDS = new Set([ export const heapSnapshotRouter = new Hono(); -heapSnapshotRouter.use("*", secretKeyMiddleware); -heapSnapshotRouter.use("*", orgConfigMiddleware); - heapSnapshotRouter.get("/heap-snapshot", async (c) => { const ctx = c.get("ctx"); @@ -24,7 +19,10 @@ heapSnapshotRouter.get("/heap-snapshot", async (c) => { } // Bun-specific heap snapshot - if (typeof Bun !== "undefined" && typeof Bun.generateHeapSnapshot === "function") { + if ( + typeof Bun !== "undefined" && + typeof Bun.generateHeapSnapshot === "function" + ) { const snapshot = Bun.generateHeapSnapshot(); return c.json({ ok: true, @@ -37,7 +35,10 @@ heapSnapshotRouter.get("/heap-snapshot", async (c) => { // Node.js fallback using v8 try { const v8 = await import("node:v8"); - const snapshotPath = join(tmpdir(), `heap-${process.pid}-${Date.now()}.heapsnapshot`); + const snapshotPath = join( + tmpdir(), + `heap-${process.pid}-${Date.now()}.heapsnapshot`, + ); v8.writeHeapSnapshot(snapshotPath); diff --git a/server/src/routers/apiRouter.ts b/server/src/routers/apiRouter.ts index 1b6596982..7e55baa38 100644 --- a/server/src/routers/apiRouter.ts +++ b/server/src/routers/apiRouter.ts @@ -34,6 +34,7 @@ import { honoProductRouter, migrationRouter, } from "../internal/products/productRouter.js"; +import { heapSnapshotRouter } from "../internal/debug/heapSnapshotRoute.js"; import { rpcRouter } from "./rpcRouter.js"; export const apiRouter = new Hono(); @@ -79,3 +80,4 @@ apiRouter.route("/events", eventsRouter); apiRouter.route("/configs", configsRouter); apiRouter.route("/components", componentsRouter); +apiRouter.route("/debug", heapSnapshotRouter); From eee4e3c5d6913b110065eee231a78069b0206e3f Mon Sep 17 00:00:00 2001 From: atmn Date: Mon, 23 Feb 2026 15:59:10 +0000 Subject: [PATCH 5/6] Revert "fix: move debug routes to apiRouter for proper auth chain" This reverts commit 60fd48544730c98b133aade169658321d94ff105. --- bun.lock | 2 +- server/src/initHono.ts | 4 ++++ server/src/internal/debug/heapSnapshotRoute.ts | 17 ++++++++--------- server/src/routers/apiRouter.ts | 2 -- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bun.lock b/bun.lock index 3b8799da6..e3c9c72c5 100644 --- a/bun.lock +++ b/bun.lock @@ -101,7 +101,7 @@ }, "packages/autumn-js": { "name": "autumn-js", - "version": "1.0.0-beta.5", + "version": "1.0.0-beta.4", "dependencies": { "query-string": "^9.2.2", "rou3": "^0.6.1", diff --git a/server/src/initHono.ts b/server/src/initHono.ts index f68143ca9..e41d67339 100644 --- a/server/src/initHono.ts +++ b/server/src/initHono.ts @@ -18,6 +18,7 @@ import type { HonoEnv } from "./honoUtils/HonoEnv.js"; import { handleHealthCheck } from "./honoUtils/handleHealthCheck.js"; import { cliRouter } from "./internal/dev/cli/cliRouter.js"; import { handleOAuthCallback } from "./internal/orgs/handlers/stripeHandlers/handleOAuthCallback.js"; +import { heapSnapshotRouter } from "./internal/debug/heapSnapshotRoute.js"; import { apiRouter } from "./routers/apiRouter.js"; import { internalRouter } from "./routers/internalRouter.js"; import { publicRouter } from "./routers/publicRouter.js"; @@ -148,6 +149,9 @@ const createHonoApp = () => { // Public routes (no auth required) app.route("", publicRouter); + // Debug routes (auth handled internally) + app.route("/v1/debug", heapSnapshotRouter); + // API Middleware app.route("/v1", apiRouter); app.route("", internalRouter); diff --git a/server/src/internal/debug/heapSnapshotRoute.ts b/server/src/internal/debug/heapSnapshotRoute.ts index 52a7b6ed7..a8115e25e 100644 --- a/server/src/internal/debug/heapSnapshotRoute.ts +++ b/server/src/internal/debug/heapSnapshotRoute.ts @@ -1,7 +1,9 @@ -import { readFileSync, unlinkSync } from "node:fs"; +import { writeFileSync, readFileSync, unlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Hono } from "hono"; +import { secretKeyMiddleware } from "@/honoMiddlewares/secretKeyMiddleware.js"; +import { orgConfigMiddleware } from "@/honoMiddlewares/orgConfigMiddleware.js"; import type { HonoEnv } from "@/honoUtils/HonoEnv.js"; const ALLOWED_ORG_IDS = new Set([ @@ -11,6 +13,9 @@ const ALLOWED_ORG_IDS = new Set([ export const heapSnapshotRouter = new Hono(); +heapSnapshotRouter.use("*", secretKeyMiddleware); +heapSnapshotRouter.use("*", orgConfigMiddleware); + heapSnapshotRouter.get("/heap-snapshot", async (c) => { const ctx = c.get("ctx"); @@ -19,10 +24,7 @@ heapSnapshotRouter.get("/heap-snapshot", async (c) => { } // Bun-specific heap snapshot - if ( - typeof Bun !== "undefined" && - typeof Bun.generateHeapSnapshot === "function" - ) { + if (typeof Bun !== "undefined" && typeof Bun.generateHeapSnapshot === "function") { const snapshot = Bun.generateHeapSnapshot(); return c.json({ ok: true, @@ -35,10 +37,7 @@ heapSnapshotRouter.get("/heap-snapshot", async (c) => { // Node.js fallback using v8 try { const v8 = await import("node:v8"); - const snapshotPath = join( - tmpdir(), - `heap-${process.pid}-${Date.now()}.heapsnapshot`, - ); + const snapshotPath = join(tmpdir(), `heap-${process.pid}-${Date.now()}.heapsnapshot`); v8.writeHeapSnapshot(snapshotPath); diff --git a/server/src/routers/apiRouter.ts b/server/src/routers/apiRouter.ts index 7e55baa38..1b6596982 100644 --- a/server/src/routers/apiRouter.ts +++ b/server/src/routers/apiRouter.ts @@ -34,7 +34,6 @@ import { honoProductRouter, migrationRouter, } from "../internal/products/productRouter.js"; -import { heapSnapshotRouter } from "../internal/debug/heapSnapshotRoute.js"; import { rpcRouter } from "./rpcRouter.js"; export const apiRouter = new Hono(); @@ -80,4 +79,3 @@ apiRouter.route("/events", eventsRouter); apiRouter.route("/configs", configsRouter); apiRouter.route("/components", componentsRouter); -apiRouter.route("/debug", heapSnapshotRouter); From 61c8b26fb9ea6b28b09345886670195e9e784d75 Mon Sep 17 00:00:00 2001 From: atmn Date: Mon, 23 Feb 2026 16:13:41 +0000 Subject: [PATCH 6/6] =?UTF-8?q?feat:=20add=20SIGUSR2=20heap=20snapshot=20h?= =?UTF-8?q?andler=20=E2=80=94=20sends=20snapshots=20to=20Discord?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a process receives SIGUSR2, it: 1. Takes a heap snapshot (Bun.generateHeapSnapshot or v8) 2. Sends it to Discord via webhook with memory stats 3. Cleans up the temp file Works for both server and worker processes. Usage: kill -USR2 --- server/src/init.ts | 2 + server/src/utils/heapSnapshotHandler.ts | 87 +++++++++++++++++++++++++ server/src/workers.ts | 2 + 3 files changed, 91 insertions(+) create mode 100644 server/src/utils/heapSnapshotHandler.ts diff --git a/server/src/init.ts b/server/src/init.ts index 41c0451eb..4862179af 100644 --- a/server/src/init.ts +++ b/server/src/init.ts @@ -21,6 +21,7 @@ import { redirectToHono } from "./initHono.js"; import { auth } from "./utils/auth.js"; import { generateId } from "./utils/genUtils.js"; import { checkEnvVars } from "./utils/initUtils.js"; +import { registerHeapSnapshotHandler } from "./utils/heapSnapshotHandler.js"; import { startMemoryMonitor } from "./utils/memoryMonitor.js"; checkEnvVars(); @@ -142,6 +143,7 @@ const init = async () => { server.listen(PORT, "0.0.0.0", () => { console.log(`Server running on port ${PORT}`); startMemoryMonitor("server", 60_000); + registerHeapSnapshotHandler("server"); }); }; diff --git a/server/src/utils/heapSnapshotHandler.ts b/server/src/utils/heapSnapshotHandler.ts new file mode 100644 index 000000000..5752c4bc9 --- /dev/null +++ b/server/src/utils/heapSnapshotHandler.ts @@ -0,0 +1,87 @@ +/** + * Registers a SIGUSR2 handler that takes a heap snapshot + * and sends it to Discord via webhook. + * + * Usage: kill -USR2 + * Or trigger from the cluster primary via IPC. + */ + +import { writeFileSync, readFileSync, unlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { logger } from "../external/logtail/logtailUtils.js"; + +const DISCORD_WEBHOOK_URL = process.env.DISCORD_FEEDBACK_WEBHOOK; + +async function sendToDiscord(filePath: string, label: string) { + if (!DISCORD_WEBHOOK_URL) { + logger.warn("DISCORD_FEEDBACK_WEBHOOK not configured, cannot send heap snapshot"); + return; + } + + const fileData = readFileSync(filePath); + const filename = `heap-${label}-pid${process.pid}-${Date.now()}.heapsnapshot`; + + const mem = process.memoryUsage(); + const toMB = (b: number) => (b / 1024 / 1024).toFixed(1); + + const summary = [ + `**Heap Snapshot — ${label} (pid ${process.pid})**`, + `RSS: ${toMB(mem.rss)}MB | Heap: ${toMB(mem.heapUsed)}/${toMB(mem.heapTotal)}MB`, + `External: ${toMB(mem.external)}MB | ArrayBuffers: ${toMB(mem.arrayBuffers)}MB`, + `File size: ${(fileData.length / 1024 / 1024).toFixed(1)}MB`, + "Open in Chrome DevTools → Memory → Load", + ].join("\n"); + + const formData = new FormData(); + formData.append("content", summary); + formData.append( + "files[0]", + new Blob([fileData], { type: "application/json" }), + filename, + ); + + try { + const res = await fetch(DISCORD_WEBHOOK_URL, { + method: "POST", + body: formData, + }); + + if (!res.ok) { + logger.error(`Failed to send heap snapshot to Discord: ${res.status} ${await res.text()}`); + } else { + logger.info(`Heap snapshot sent to Discord: ${filename}`); + } + } catch (err) { + logger.error(`Failed to send heap snapshot to Discord: ${err}`); + } +} + +export function registerHeapSnapshotHandler(label: string) { + process.on("SIGUSR2", async () => { + logger.info(`SIGUSR2 received — generating heap snapshot for ${label} (pid ${process.pid})`); + + const snapshotPath = join(tmpdir(), `heap-${label}-${process.pid}-${Date.now()}.heapsnapshot`); + + try { + if (typeof Bun !== "undefined" && typeof Bun.generateHeapSnapshot === "function") { + const snapshot = Bun.generateHeapSnapshot(); + writeFileSync(snapshotPath, JSON.stringify(snapshot)); + } else { + const v8 = await import("node:v8"); + v8.writeHeapSnapshot(snapshotPath); + } + + await sendToDiscord(snapshotPath, label); + + // Clean up + try { + unlinkSync(snapshotPath); + } catch {} + } catch (err) { + logger.error(`Failed to generate heap snapshot: ${err}`); + } + }); + + logger.info(`[${label}] Heap snapshot handler registered (send SIGUSR2 to pid ${process.pid})`); +} diff --git a/server/src/workers.ts b/server/src/workers.ts index a08d7e212..af0a2a091 100644 --- a/server/src/workers.ts +++ b/server/src/workers.ts @@ -19,6 +19,7 @@ const NUM_PROCESSES = process.env.NODE_ENV === "development" ? 1 : 4; // Track if we're shutting down let isShuttingDown = false; +import { registerHeapSnapshotHandler } from "./utils/heapSnapshotHandler.js"; import { startMemoryMonitor } from "./utils/memoryMonitor.js"; if (cluster.isPrimary) { @@ -97,6 +98,7 @@ if (cluster.isPrimary) { // Worker process console.log(`[Worker ${process.pid}] Starting queue consumer...`); startMemoryMonitor("worker", 60_000); + registerHeapSnapshotHandler("worker"); // Auto-detect which queue implementation to use if (process.env.SQS_QUEUE_URL) {