added logs for partial cache missing

server/src/internal/billing/v2/actions/locks/checkoutSessionLock/checkCheckoutSessionLock.ts

@@ -1,5 +1,5 @@
 import type { BillingContext, BillingPlan } from "@autumn/shared";
-import { ErrCode, RecaseError } from "@autumn/shared";
+import { AppEnv, ErrCode, RecaseError } from "@autumn/shared";
 import type { AutumnContext } from "@/honoUtils/HonoEnv";
 import type { CreateAutumnCheckoutResult } from "@/internal/billing/v2/common/createAutumnCheckout";
 import { hashJson } from "@/utils/hash/hashJson";
@@ -61,6 +61,18 @@ export const checkCheckoutSessionLock = async <T extends BillingContext>({
 		return null;
 	}
 
+	if (ctx.env === AppEnv.Sandbox) {
+		ctx.logger.info(
+			`Sandbox: clearing checkout session lock for customer ${customerId} (session ${existingLock.checkoutSessionId}) to allow non-checkout billing action`,
+		);
+		await checkoutSessionLock.expireAndClear({
+			ctx,
+			customerId,
+			checkoutSessionId: existingLock.checkoutSessionId,
+		});
+		return null;
+	}
+
 	ctx.logger.info(
 		`Blocking non-checkout billing action for customer ${customerId} — checkout session ${existingLock.checkoutSessionId} still active`,
 	);

server/src/internal/customers/cache/fullSubject/actions/partial/getCachedPartialFullSubject.ts

@@ -10,6 +10,7 @@ import { applyLiveAggregatedBalances } from "../../balances/applyLiveAggregatedB
 import { getCachedFeatureBalancesBatch } from "../../balances/getCachedFeatureBalances.js";
 import { buildFullSubjectKey } from "../../builders/buildFullSubjectKey.js";
 import { buildFullSubjectViewEpochKey } from "../../builders/buildFullSubjectViewEpochKey.js";
+import { buildSharedFullSubjectBalanceKey } from "../../builders/buildSharedFullSubjectBalanceKey.js";
 import { filterNormalizedFullSubjectByFeatureIds } from "../../filterFullSubjectByFeatureIds.js";
 import {
 	type CachedFullSubject,
@@ -233,6 +234,28 @@ export const getCachedPartialFullSubject = async ({
 			source: "partial-incomplete",
 		});
 
+	if (featureBalancesOutcome.kind === "missing") {
+		const probeFeatureId =
+			meteredFeatureIdsToFetch[0] ?? cached.meteredFeatures[0];
+		if (probeFeatureId) {
+			const balanceKey = buildSharedFullSubjectBalanceKey({
+				orgId: org.id,
+				env,
+				customerId,
+				featureId: probeFeatureId,
+			});
+			const [hlenRaw, ttlRaw] = await Promise.all([
+				redisV2.hlen(balanceKey).catch(() => -99),
+				redisV2.ttl(balanceKey).catch(() => -99),
+			]);
+			const expectedIds =
+				cached.customerEntitlementIdsByFeatureId[probeFeatureId] ?? [];
+			ctx.logger.warn(
+				`[incomplete-debug] ${subjectLabel} feature=${probeFeatureId} reason=${featureBalancesOutcome.reason} hlen=${hlenRaw} ttl=${ttlRaw} expected=${expectedIds.join(",")}`,
+			);
+		}
+	}
+
 	const balancesPresent = await tryOrInvalidate({
 		ctx,
 		operation: () =>
@@ -240,7 +263,7 @@ export const getCachedPartialFullSubject = async ({
 				? undefined
 				: featureBalancesOutcome.value,
 		invalidate: invalidateIncomplete,
-		warnMessage: `[getCachedPartialFullSubject] Incomplete cache for ${subjectLabel}, source: ${source}`,
+		warnMessage: `[getCachedPartialFullSubject] Incomplete cache for ${subjectLabel}, source: ${source}, reason: ${featureBalancesOutcome.kind === "missing" ? featureBalancesOutcome.reason : "n/a"}`,
 	});
 	if (balancesPresent === undefined) {
 		return {

server/src/internal/customers/cache/fullSubject/balances/getCachedFeatureBalances.ts

@@ -17,11 +17,11 @@ export type FeatureBalanceResult = {
 
 export type FeatureBalanceOutcome =
 	| { kind: "ok"; value: FeatureBalanceResult }
-	| { kind: "missing" };
+	| { kind: "missing"; reason: string };
 
 export type FeatureBalancesBatchOutcome =
 	| { kind: "ok"; value: FeatureBalanceResult[] }
-	| { kind: "missing" };
+	| { kind: "missing"; reason: string };
 
 const readFeatureBalancesFromMaster = async ({
 	ctx,
@@ -82,12 +82,16 @@ export const getCachedFeatureBalance = async ({
 		redisInstance: redisV2,
 	});
 
-	if (!results) return { kind: "missing" };
+	if (!results) return { kind: "missing", reason: "single_pipeline_null" };
 
 	const balances: SubjectBalance[] = [];
 	for (let i = 0; i < customerEntitlementIds.length; i++) {
 		const entryJson = results[i];
-		if (!entryJson) return { kind: "missing" };
+		if (!entryJson)
+			return {
+				kind: "missing",
+				reason: `single_field_null:${featureId}:${customerEntitlementIds[i]}`,
+			};
 		try {
 			const parsedBalance = JSON.parse(entryJson) as SubjectBalance;
 			balances.push(
@@ -98,7 +102,10 @@ export const getCachedFeatureBalance = async ({
 				}),
 			);
 		} catch {
-			return { kind: "missing" };
+			return {
+				kind: "missing",
+				reason: `single_parse_failed:${featureId}:${customerEntitlementIds[i]}`,
+			};
 		}
 	}
 
@@ -145,7 +152,7 @@ export const getCachedFeatureBalancesBatch = async ({
 		redisInstance: redisV2,
 	});
 
-	if (!results) return { kind: "missing" };
+	if (!results) return { kind: "missing", reason: "batch_pipeline_null" };
 
 	const featureBalances: FeatureBalanceResult[] = [];
 
@@ -153,7 +160,11 @@ export const getCachedFeatureBalancesBatch = async ({
 		const customerEntitlementIds =
 			customerEntitlementIdsByFeatureId[featureIds[i]] ?? [];
 		const allValues = results[i]?.[1] as (string | null)[] | null;
-		if (!allValues) return { kind: "missing" };
+		if (!allValues)
+			return {
+				kind: "missing",
+				reason: `batch_hash_missing:${featureIds[i]}`,
+			};
 
 		let aggregated: AggregatedFeatureBalance | undefined;
 		let ceValues: (string | null)[];
@@ -176,11 +187,19 @@ export const getCachedFeatureBalancesBatch = async ({
 		}
 
 		if (ceValues.length !== customerEntitlementIds.length)
-			return { kind: "missing" };
+			return {
+				kind: "missing",
+				reason: `batch_length_mismatch:${featureIds[i]}:got=${ceValues.length}:expected=${customerEntitlementIds.length}`,
+			};
 
 		const balances: SubjectBalance[] = [];
-		for (const entryJson of ceValues) {
-			if (!entryJson) return { kind: "missing" };
+		for (let j = 0; j < ceValues.length; j++) {
+			const entryJson = ceValues[j];
+			if (!entryJson)
+				return {
+					kind: "missing",
+					reason: `batch_field_null:${featureIds[i]}:${customerEntitlementIds[j]}`,
+				};
 			try {
 				const parsedBalance = JSON.parse(entryJson) as SubjectBalance;
 				balances.push(
@@ -191,7 +210,10 @@ export const getCachedFeatureBalancesBatch = async ({
 					}),
 				);
 			} catch {
-				return { kind: "missing" };
+				return {
+					kind: "missing",
+					reason: `batch_parse_failed:${featureIds[i]}:${customerEntitlementIds[j]}`,
+				};
 			}
 		}
 

server/tests/_groups/temp.ts

@@ -2,14 +2,13 @@ import type { TestGroup } from "./types";
 
 export const temp: TestGroup = {
 	name: "temp",
-	description: "patch-style custom plan update coverage",
+	description: "failed retry tests",
 	tier: "domain",
 	paths: [
-		"integration/billing/update-subscription/custom-plan-patch/patch-update-items.test.ts",
-		"integration/billing/update-subscription/custom-plan-patch/patch-update-price.test.ts",
-		"integration/billing/update-subscription/custom-plan-patch/patch-update-paid-features.test.ts",
-		"integration/billing/update-subscription/custom-plan-patch/patch-update-items-carry-usage.test.ts",
-		"integration/billing/update-subscription/custom-plan-patch/patch-update-items-carry-rollover.test.ts",
-		"integration/billing/update-subscription/custom-plan-patch/patch-update-with-others.test.ts",
+		"integration/billing/setup-payment/setup-payment-with-plan.test.ts",
+		"integration/crud/customers/customer-processors.test.ts",
+		"integration/crud/customers/get-customer-aggregated-balances.test.ts",
+		"integration/billing/stripe-webhooks/subscription-updated/subscription-updated-past-due.test.ts",
+		"integration/billing/stripe-webhooks/invoice-created/invoice-created-entity-consumable.test.ts",
 	],
 };

server/tests/integration/crud/customers/get-customer.test.ts

@@ -1,17 +1,17 @@
 import { expect, test } from "bun:test";
 import {
-	type ApiCustomer,
-	ApiCustomerSchema,
-	type AttachParamsV1Input,
-	CustomerExpand,
+    type ApiCustomer,
+    ApiCustomerSchema,
+    type AttachParamsV1Input,
+    CustomerExpand,
 } from "@autumn/shared";
 import {
-	type ApiCustomerV5,
-	ApiCustomerV5Schema,
+    type ApiCustomerV5,
+    ApiCustomerV5Schema,
 } from "@shared/api/customers/apiCustomerV5";
 import {
-	type ApiCustomerV3,
-	ApiCustomerV3Schema,
+    type ApiCustomerV3,
+    ApiCustomerV3Schema,
 } from "@shared/api/customers/previousVersions/apiCustomerV3";
 import { expectBalanceCorrect } from "@tests/integration/utils/expectBalanceCorrect";
 import { expectFlagCorrect } from "@tests/integration/utils/expectFlagCorrect";
@@ -127,18 +127,18 @@ test.concurrent(`${chalk.yellowBright("get-customer: multi-entity customer retur
 	});
 
 	const refDir = `${import.meta.dir}/../../../references`;
-	await Bun.write(
-		`${refDir}/getCustomerV1Response.json`,
-		JSON.stringify(cusV1, null, 2),
-	);
-	await Bun.write(
-		`${refDir}/getCustomerV2_1Response.json`,
-		JSON.stringify(cusV2_1, null, 2),
-	);
-	await Bun.write(
-		`${refDir}/getCustomerV2_2Response.json`,
-		JSON.stringify(cusV2_2, null, 2),
-	);
+	// await Bun.write(
+	// 	`${refDir}/getCustomerV1Response.json`,
+	// 	JSON.stringify(cusV1, null, 2),
+	// );
+	// await Bun.write(
+	// 	`${refDir}/getCustomerV2_1Response.json`,
+	// 	JSON.stringify(cusV2_1, null, 2),
+	// );
+	// await Bun.write(
+	// 	`${refDir}/getCustomerV2_2Response.json`,
+	// 	JSON.stringify(cusV2_2, null, 2),
+	// );
 });
 
 test.concurrent(`${chalk.yellowBright("get-customer: expand empty array returns items")}`, async () => {