Merge pull request #798 from useautumn/jj/memory-diagnostics

fix: use Axiom logger for memory diagnostics

server/src/init.ts

@@ -21,6 +21,8 @@ import { redirectToHono } from "./initHono.js";
 import { auth } from "./utils/auth.js";
 import { generateId } from "./utils/genUtils.js";
 import { checkEnvVars } from "./utils/initUtils.js";
+import { registerHeapSnapshotHandler } from "./utils/heapSnapshotHandler.js";
+import { startMemoryMonitor } from "./utils/memoryMonitor.js";
 
 checkEnvVars();
 // subscribeToOrgUpdates({ db });
@@ -140,6 +142,8 @@ const init = async () => {
 	// Bind to 0.0.0.0 for AWS ECS/Docker containers
 	server.listen(PORT, "0.0.0.0", () => {
 		console.log(`Server running on port ${PORT}`);
+		startMemoryMonitor("server", 60_000);
+		registerHeapSnapshotHandler("server");
 	});
 };
 

server/src/initHono.ts

@@ -18,6 +18,7 @@ import type { HonoEnv } from "./honoUtils/HonoEnv.js";
 import { handleHealthCheck } from "./honoUtils/handleHealthCheck.js";
 import { cliRouter } from "./internal/dev/cli/cliRouter.js";
 import { handleOAuthCallback } from "./internal/orgs/handlers/stripeHandlers/handleOAuthCallback.js";
+import { heapSnapshotRouter } from "./internal/debug/heapSnapshotRoute.js";
 import { apiRouter } from "./routers/apiRouter.js";
 import { internalRouter } from "./routers/internalRouter.js";
 import { publicRouter } from "./routers/publicRouter.js";
@@ -148,6 +149,9 @@ const createHonoApp = () => {
 
 	// Public routes (no auth required)
 	app.route("", publicRouter);
+	// Debug routes (auth handled internally)
+	app.route("/v1/debug", heapSnapshotRouter);
+
 	// API Middleware
 	app.route("/v1", apiRouter);
 	app.route("", internalRouter);

server/src/internal/debug/heapSnapshotRoute.ts

@@ -0,0 +1,85 @@
+import { writeFileSync, readFileSync, unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Hono } from "hono";
+import { secretKeyMiddleware } from "@/honoMiddlewares/secretKeyMiddleware.js";
+import { orgConfigMiddleware } from "@/honoMiddlewares/orgConfigMiddleware.js";
+import type { HonoEnv } from "@/honoUtils/HonoEnv.js";
+
+const ALLOWED_ORG_IDS = new Set([
+	"org_2rzkkRh7r5dBSaBC101QHG9KDgt",
+	"org_2vwdxwTdqxRrLEdUYddcynMv3n3",
+]);
+
+export const heapSnapshotRouter = new Hono<HonoEnv>();
+
+heapSnapshotRouter.use("*", secretKeyMiddleware);
+heapSnapshotRouter.use("*", orgConfigMiddleware);
+
+heapSnapshotRouter.get("/heap-snapshot", async (c) => {
+	const ctx = c.get("ctx");
+
+	if (!ALLOWED_ORG_IDS.has(ctx.org?.id)) {
+		return c.json({ error: "Forbidden" }, 403);
+	}
+
+	// Bun-specific heap snapshot
+	if (typeof Bun !== "undefined" && typeof Bun.generateHeapSnapshot === "function") {
+		const snapshot = Bun.generateHeapSnapshot();
+		return c.json({
+			ok: true,
+			pid: process.pid,
+			timestamp: new Date().toISOString(),
+			snapshot,
+		});
+	}
+
+	// Node.js fallback using v8
+	try {
+		const v8 = await import("node:v8");
+		const snapshotPath = join(tmpdir(), `heap-${process.pid}-${Date.now()}.heapsnapshot`);
+
+		v8.writeHeapSnapshot(snapshotPath);
+
+		const data = readFileSync(snapshotPath);
+		unlinkSync(snapshotPath);
+
+		return new Response(data, {
+			headers: {
+				"Content-Type": "application/json",
+				"Content-Disposition": `attachment; filename="heap-${process.pid}-${Date.now()}.heapsnapshot"`,
+			},
+		});
+	} catch (err) {
+		return c.json(
+			{
+				error: "Failed to generate heap snapshot",
+				detail: err instanceof Error ? err.message : String(err),
+			},
+			500,
+		);
+	}
+});
+
+heapSnapshotRouter.get("/memory", async (c) => {
+	const ctx = c.get("ctx");
+
+	if (!ALLOWED_ORG_IDS.has(ctx.org?.id)) {
+		return c.json({ error: "Forbidden" }, 403);
+	}
+
+	const mem = process.memoryUsage();
+
+	return c.json({
+		ok: true,
+		pid: process.pid,
+		timestamp: new Date().toISOString(),
+		memory: {
+			rssMB: +(mem.rss / 1024 / 1024).toFixed(1),
+			heapUsedMB: +(mem.heapUsed / 1024 / 1024).toFixed(1),
+			heapTotalMB: +(mem.heapTotal / 1024 / 1024).toFixed(1),
+			externalMB: +(mem.external / 1024 / 1024).toFixed(1),
+			arrayBuffersMB: +(mem.arrayBuffers / 1024 / 1024).toFixed(1),
+		},
+	});
+});

server/src/utils/heapSnapshotHandler.ts

@@ -0,0 +1,87 @@
+/**
+ * Registers a SIGUSR2 handler that takes a heap snapshot
+ * and sends it to Discord via webhook.
+ *
+ * Usage: kill -USR2 <pid>
+ * Or trigger from the cluster primary via IPC.
+ */
+
+import { writeFileSync, readFileSync, unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { logger } from "../external/logtail/logtailUtils.js";
+
+const DISCORD_WEBHOOK_URL = process.env.DISCORD_FEEDBACK_WEBHOOK;
+
+async function sendToDiscord(filePath: string, label: string) {
+	if (!DISCORD_WEBHOOK_URL) {
+		logger.warn("DISCORD_FEEDBACK_WEBHOOK not configured, cannot send heap snapshot");
+		return;
+	}
+
+	const fileData = readFileSync(filePath);
+	const filename = `heap-${label}-pid${process.pid}-${Date.now()}.heapsnapshot`;
+
+	const mem = process.memoryUsage();
+	const toMB = (b: number) => (b / 1024 / 1024).toFixed(1);
+
+	const summary = [
+		`**Heap Snapshot — ${label} (pid ${process.pid})**`,
+		`RSS: ${toMB(mem.rss)}MB | Heap: ${toMB(mem.heapUsed)}/${toMB(mem.heapTotal)}MB`,
+		`External: ${toMB(mem.external)}MB | ArrayBuffers: ${toMB(mem.arrayBuffers)}MB`,
+		`File size: ${(fileData.length / 1024 / 1024).toFixed(1)}MB`,
+		"Open in Chrome DevTools → Memory → Load",
+	].join("\n");
+
+	const formData = new FormData();
+	formData.append("content", summary);
+	formData.append(
+		"files[0]",
+		new Blob([fileData], { type: "application/json" }),
+		filename,
+	);
+
+	try {
+		const res = await fetch(DISCORD_WEBHOOK_URL, {
+			method: "POST",
+			body: formData,
+		});
+
+		if (!res.ok) {
+			logger.error(`Failed to send heap snapshot to Discord: ${res.status} ${await res.text()}`);
+		} else {
+			logger.info(`Heap snapshot sent to Discord: ${filename}`);
+		}
+	} catch (err) {
+		logger.error(`Failed to send heap snapshot to Discord: ${err}`);
+	}
+}
+
+export function registerHeapSnapshotHandler(label: string) {
+	process.on("SIGUSR2", async () => {
+		logger.info(`SIGUSR2 received — generating heap snapshot for ${label} (pid ${process.pid})`);
+
+		const snapshotPath = join(tmpdir(), `heap-${label}-${process.pid}-${Date.now()}.heapsnapshot`);
+
+		try {
+			if (typeof Bun !== "undefined" && typeof Bun.generateHeapSnapshot === "function") {
+				const snapshot = Bun.generateHeapSnapshot();
+				writeFileSync(snapshotPath, JSON.stringify(snapshot));
+			} else {
+				const v8 = await import("node:v8");
+				v8.writeHeapSnapshot(snapshotPath);
+			}
+
+			await sendToDiscord(snapshotPath, label);
+
+			// Clean up
+			try {
+				unlinkSync(snapshotPath);
+			} catch {}
+		} catch (err) {
+			logger.error(`Failed to generate heap snapshot: ${err}`);
+		}
+	});
+
+	logger.info(`[${label}] Heap snapshot handler registered (send SIGUSR2 to pid ${process.pid})`);
+}

server/src/utils/memoryMonitor.ts

@@ -0,0 +1,64 @@
+/**
+ * Periodic memory usage logger for diagnosing memory leaks.
+ *
+ * Logs heap usage, RSS, external memory, and array buffers every interval.
+ * Uses Axiom logger so metrics are queryable via type: "memory_log".
+ */
+
+import { logger } from "../external/logtail/logtailUtils.js";
+
+const DEFAULT_INTERVAL_MS = 60_000; // 1 minute
+
+let intervalHandle: ReturnType<typeof setInterval> | null = null;
+
+function toMB(bytes: number): number {
+	return Math.round((bytes / 1024 / 1024) * 10) / 10;
+}
+
+function logMemoryUsage(label: string) {
+	const mem = process.memoryUsage();
+
+	logger.info("memory_log", {
+		type: "memory_log",
+		label,
+		pid: process.pid,
+		rssMB: toMB(mem.rss),
+		heapUsedMB: toMB(mem.heapUsed),
+		heapTotalMB: toMB(mem.heapTotal),
+		externalMB: toMB(mem.external),
+		arrayBuffersMB: toMB(mem.arrayBuffers),
+	});
+}
+
+/**
+ * Start periodic memory logging.
+ * @param label - identifier for the process (e.g. "server", "worker")
+ * @param intervalMs - how often to log (default: 60s)
+ */
+export function startMemoryMonitor(
+	label: string,
+	intervalMs = DEFAULT_INTERVAL_MS,
+) {
+	// Log immediately on start
+	logMemoryUsage(label);
+
+	intervalHandle = setInterval(() => {
+		logMemoryUsage(label);
+	}, intervalMs);
+
+	// Don't prevent process exit
+	if (intervalHandle.unref) {
+		intervalHandle.unref();
+	}
+
+	console.log(
+		`[mem:${label}] Memory monitor started (every ${intervalMs / 1000}s)`,
+	);
+}
+
+export function stopMemoryMonitor() {
+	if (intervalHandle) {
+		clearInterval(intervalHandle);
+		intervalHandle = null;
+	}
+}

server/src/workers.ts

@@ -19,6 +19,9 @@ const NUM_PROCESSES = process.env.NODE_ENV === "development" ? 1 : 4;
 // Track if we're shutting down
 let isShuttingDown = false;
 
+import { registerHeapSnapshotHandler } from "./utils/heapSnapshotHandler.js";
+import { startMemoryMonitor } from "./utils/memoryMonitor.js";
+
 if (cluster.isPrimary) {
 	await initInfisical();
 
@@ -94,6 +97,8 @@ if (cluster.isPrimary) {
 } else {
 	// Worker process
 	console.log(`[Worker ${process.pid}] Starting queue consumer...`);
+	startMemoryMonitor("worker", 60_000);
+	registerHeapSnapshotHandler("worker");
 
 	// Auto-detect which queue implementation to use
 	if (process.env.SQS_QUEUE_URL) {