177
server/experiments/diffListEntitiesV2Responses.ts
Normal file
177
server/experiments/diffListEntitiesV2Responses.ts
Normal file
@@ -0,0 +1,177 @@
|
||||
// Run with `CHECK_ORG_ID=... CHECK_CUSTOMER_ID=... CHECK_LIMIT=200 bun run experiments/diffListEntitiesV2Responses.ts`
|
||||
import { AppEnv, type CusProductStatus, type SubjectQueryRow } from "@autumn/shared";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { initDrizzle } from "../src/db/initDrizzle.js";
|
||||
import { RELEVANT_STATUSES } from "../src/internal/customers/cusProducts/CusProductService.js";
|
||||
import { getFullSubjectRowsQuery } from "../src/internal/customers/repos/getFullSubject/getFullSubjectRowsQuery.js";
|
||||
import { mergeEntityAndCustomerSubjectRows } from "../src/internal/customers/repos/getFullSubject/mergeEntityAndCustomerSubjectRows.js";
|
||||
import { getCursorPaginatedEntitySubjectsQuery } from "../src/internal/entities/repos/cursorListEntitiesQuery.js";
|
||||
import { getCustomerLevelSubjectRowsQuery } from "../src/internal/entities/repos/customerLevelSubjectsQuery.js";
|
||||
|
||||
const ORG_ID = process.env.CHECK_ORG_ID as string;
|
||||
const CUSTOMER_ID = process.env.CHECK_CUSTOMER_ID as string;
|
||||
const LIMIT = Number(process.env.CHECK_LIMIT || 200);
|
||||
const ENV = AppEnv.Live;
|
||||
|
||||
const getCombinedQuery = ({ inStatuses }: { inStatuses: CusProductStatus[] }) => {
|
||||
const customerFilter = CUSTOMER_ID ? sql`AND c.id = ${CUSTOMER_ID}` : sql``;
|
||||
const leadingCtes = sql`
|
||||
WITH entity_records AS (
|
||||
SELECT e.*
|
||||
FROM entities e
|
||||
JOIN customers c ON c.internal_id = e.internal_customer_id
|
||||
WHERE e.org_id = ${ORG_ID} AND e.env = ${ENV}
|
||||
AND c.org_id = ${ORG_ID} AND c.env = ${ENV}
|
||||
${customerFilter}
|
||||
ORDER BY e.created_at DESC, e.id DESC
|
||||
LIMIT ${LIMIT + 1}
|
||||
),
|
||||
subject_records AS (
|
||||
SELECT er.internal_id AS subject_key, er.internal_customer_id, er.internal_id AS internal_entity_id,
|
||||
ROW_NUMBER() OVER (ORDER BY er.created_at DESC, er.id DESC) AS subject_order
|
||||
FROM entity_records er
|
||||
)
|
||||
`;
|
||||
return getFullSubjectRowsQuery({
|
||||
leadingCtes,
|
||||
inStatuses,
|
||||
includeInvoices: false,
|
||||
includeEntityAggregations: false,
|
||||
});
|
||||
};
|
||||
|
||||
const stableStringify = (value: unknown): string => {
|
||||
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
const entries = Object.entries(value as Record<string, unknown>)
|
||||
.filter(([, v]) => v !== undefined)
|
||||
.sort(([a], [b]) => (a < b ? -1 : 1))
|
||||
.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`);
|
||||
return `{${entries.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
|
||||
const sortByKey = (rows: Record<string, unknown>[], key: string) =>
|
||||
[...rows].sort((a, b) => (String(a[key]) < String(b[key]) ? -1 : 1));
|
||||
|
||||
// order-insensitive fields: combined query has no deterministic ORDER BY here
|
||||
const UNORDERED_FIELDS: Record<string, string> = {
|
||||
customer_entitlements: "id",
|
||||
customer_prices: "id",
|
||||
subscriptions: "stripe_id",
|
||||
entitlements: "id",
|
||||
rollovers: "id",
|
||||
replaceables: "id",
|
||||
};
|
||||
const ORDERED_FIELDS = [
|
||||
"customer",
|
||||
"entity",
|
||||
"customer_products",
|
||||
"extra_customer_entitlements",
|
||||
"products",
|
||||
"prices",
|
||||
"free_trials",
|
||||
];
|
||||
|
||||
const main = async () => {
|
||||
const { db } = initDrizzle();
|
||||
const inStatuses = RELEVANT_STATUSES;
|
||||
|
||||
const combinedRows = (await db.execute(
|
||||
getCombinedQuery({ inStatuses }),
|
||||
)) as unknown as SubjectQueryRow[];
|
||||
|
||||
const entityRows = (await db.execute(
|
||||
getCursorPaginatedEntitySubjectsQuery({
|
||||
orgId: ORG_ID,
|
||||
env: ENV,
|
||||
limit: LIMIT,
|
||||
cursor: null,
|
||||
inStatuses,
|
||||
customerId: CUSTOMER_ID || undefined,
|
||||
}),
|
||||
)) as unknown as SubjectQueryRow[];
|
||||
|
||||
const internalCustomerIds = [
|
||||
...new Set(entityRows.map((row) => row.customer.internal_id)),
|
||||
];
|
||||
const customerRows =
|
||||
internalCustomerIds.length > 0
|
||||
? ((await db.execute(
|
||||
getCustomerLevelSubjectRowsQuery({
|
||||
orgId: ORG_ID,
|
||||
env: ENV,
|
||||
internalCustomerIds,
|
||||
inStatuses,
|
||||
}),
|
||||
)) as unknown as SubjectQueryRow[])
|
||||
: [];
|
||||
const customerRowsByInternalId = new Map(
|
||||
customerRows.map((row) => [row.customer.internal_id, row]),
|
||||
);
|
||||
|
||||
const mergedRows = entityRows.map((entityRow) =>
|
||||
mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow: customerRowsByInternalId.get(entityRow.customer.internal_id),
|
||||
}),
|
||||
);
|
||||
|
||||
console.log(`combined: ${combinedRows.length} rows, merged: ${mergedRows.length} rows`);
|
||||
if (combinedRows.length !== mergedRows.length) throw new Error("row count mismatch");
|
||||
|
||||
const classifiedFields = new Set([
|
||||
...ORDERED_FIELDS,
|
||||
...Object.keys(UNORDERED_FIELDS),
|
||||
]);
|
||||
const unclassifiedFields = [
|
||||
...new Set(
|
||||
[...combinedRows, ...mergedRows].flatMap((row) =>
|
||||
Object.keys(row as Record<string, unknown>),
|
||||
),
|
||||
),
|
||||
].filter((field) => !classifiedFields.has(field));
|
||||
if (unclassifiedFields.length > 0) {
|
||||
console.log(
|
||||
`unclassified row fields (add to ORDERED_FIELDS or UNORDERED_FIELDS): ${unclassifiedFields.join(", ")}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let mismatches = 0;
|
||||
for (let i = 0; i < combinedRows.length; i++) {
|
||||
const combined = combinedRows[i] as unknown as Record<string, unknown>;
|
||||
const merged = mergedRows[i] as unknown as Record<string, unknown>;
|
||||
|
||||
for (const field of ORDERED_FIELDS) {
|
||||
const left = stableStringify(combined[field] ?? null);
|
||||
const right = stableStringify(merged[field] ?? null);
|
||||
if (left !== right) {
|
||||
mismatches++;
|
||||
console.log(`row ${i} entity=${(combined.entity as { id?: string })?.id} ORDERED field "${field}" differs`);
|
||||
if (mismatches <= 3) {
|
||||
console.log(` combined: ${left.slice(0, 500)}`);
|
||||
console.log(` merged: ${right.slice(0, 500)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [field, key] of Object.entries(UNORDERED_FIELDS)) {
|
||||
const left = stableStringify(sortByKey((combined[field] as Record<string, unknown>[]) ?? [], key));
|
||||
const right = stableStringify(sortByKey((merged[field] as Record<string, unknown>[]) ?? [], key));
|
||||
if (left !== right) {
|
||||
mismatches++;
|
||||
console.log(`row ${i} entity=${(combined.entity as { id?: string })?.id} UNORDERED field "${field}" differs`);
|
||||
if (mismatches <= 3) {
|
||||
console.log(` combined: ${left.slice(0, 500)}`);
|
||||
console.log(` merged: ${right.slice(0, 500)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(mismatches === 0 ? "ALL ROWS IDENTICAL" : `${mismatches} field mismatches`);
|
||||
process.exit(mismatches === 0 ? 0 : 1);
|
||||
};
|
||||
|
||||
await main();
|
||||
181
server/experiments/explainListEntitiesV2.ts
Normal file
181
server/experiments/explainListEntitiesV2.ts
Normal file
@@ -0,0 +1,181 @@
|
||||
import { AppEnv, type CusProductStatus } from "@autumn/shared";
|
||||
import { type SQL, sql } from "drizzle-orm";
|
||||
import { RELEVANT_STATUSES } from "../src/internal/customers/cusProducts/CusProductService.js";
|
||||
import { getFullSubjectRowsQuery } from "../src/internal/customers/repos/getFullSubject/getFullSubjectRowsQuery.js";
|
||||
import { getCursorPaginatedEntitySubjectsQuery } from "../src/internal/entities/repos/cursorListEntitiesQuery.js";
|
||||
import { getCustomerLevelSubjectRowsQuery } from "../src/internal/entities/repos/customerLevelSubjectsQuery.js";
|
||||
import {
|
||||
initDrizzle,
|
||||
prodTestCustomerId,
|
||||
prodTestOrgId,
|
||||
} from "./experimentEnv";
|
||||
|
||||
// Run with `bun run experiments/explainListEntitiesV2.ts` (flags: --explain, --skip-old)
|
||||
const ORG_ID = prodTestOrgId;
|
||||
const ENV = AppEnv.Live;
|
||||
const CUSTOMER_ID = prodTestCustomerId;
|
||||
const LIMIT = Number(process.env.LIMIT || 1000);
|
||||
|
||||
/** Pre-split combined hydration: same page CTE, hydrated without entityScopedOnly. */
|
||||
const getCombinedEntityPageQuery = ({
|
||||
orgId,
|
||||
env,
|
||||
customerId,
|
||||
limit,
|
||||
inStatuses,
|
||||
}: {
|
||||
orgId: string;
|
||||
env: AppEnv;
|
||||
customerId?: string;
|
||||
limit: number;
|
||||
inStatuses: CusProductStatus[];
|
||||
}) => {
|
||||
const customerFilter = customerId ? sql`AND c.id = ${customerId}` : sql``;
|
||||
|
||||
const leadingCtes = sql`
|
||||
WITH entity_records AS (
|
||||
SELECT e.*
|
||||
FROM entities e
|
||||
JOIN customers c
|
||||
ON c.internal_id = e.internal_customer_id
|
||||
WHERE e.org_id = ${orgId}
|
||||
AND e.env = ${env}
|
||||
AND c.org_id = ${orgId}
|
||||
AND c.env = ${env}
|
||||
${customerFilter}
|
||||
ORDER BY e.created_at DESC, e.id DESC
|
||||
LIMIT ${limit + 1}
|
||||
),
|
||||
|
||||
subject_records AS (
|
||||
SELECT
|
||||
er.internal_id AS subject_key,
|
||||
er.internal_customer_id,
|
||||
er.internal_id AS internal_entity_id,
|
||||
ROW_NUMBER() OVER (ORDER BY er.created_at DESC, er.id DESC) AS subject_order
|
||||
FROM entity_records er
|
||||
)
|
||||
`;
|
||||
|
||||
return getFullSubjectRowsQuery({
|
||||
leadingCtes,
|
||||
inStatuses,
|
||||
includeInvoices: false,
|
||||
includeEntityAggregations: false,
|
||||
});
|
||||
};
|
||||
|
||||
const printExplainPlan = async ({
|
||||
db,
|
||||
query,
|
||||
}: {
|
||||
db: ReturnType<typeof initDrizzle>["db"];
|
||||
query: SQL;
|
||||
}) => {
|
||||
const explainResult = await db.execute(
|
||||
sql`EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) ${query}`,
|
||||
);
|
||||
|
||||
for (const row of explainResult) {
|
||||
console.log((row as Record<string, unknown>)["QUERY PLAN"]);
|
||||
}
|
||||
};
|
||||
|
||||
const runMeasuredQuery = async ({
|
||||
db,
|
||||
label,
|
||||
query,
|
||||
withExplain,
|
||||
}: {
|
||||
db: ReturnType<typeof initDrizzle>["db"];
|
||||
label: string;
|
||||
query: SQL;
|
||||
withExplain: boolean;
|
||||
}) => {
|
||||
console.log(`\n=== ${label} ===\n`);
|
||||
|
||||
const startedAt = performance.now();
|
||||
const result = await db.execute(query);
|
||||
const elapsedMilliseconds = performance.now() - startedAt;
|
||||
|
||||
console.log(`Rows returned: ${result.length}`);
|
||||
console.log(`Wall-clock time: ${elapsedMilliseconds.toFixed(2)}ms\n`);
|
||||
|
||||
if (withExplain) {
|
||||
await printExplainPlan({
|
||||
db,
|
||||
query,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const { db } = initDrizzle();
|
||||
const inStatuses = RELEVANT_STATUSES;
|
||||
const withExplain = process.argv.includes("--explain");
|
||||
const skipOld = process.argv.includes("--skip-old");
|
||||
|
||||
console.log("=== LIST ENTITIES V2 SPLIT HYDRATION EXPERIMENT ===\n");
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{ orgId: ORG_ID, env: ENV, customerId: CUSTOMER_ID, limit: LIMIT },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const customerRows = await db.execute(
|
||||
sql`SELECT internal_id FROM customers
|
||||
WHERE org_id = ${ORG_ID} AND env = ${ENV} AND id = ${CUSTOMER_ID}`,
|
||||
);
|
||||
const internalCustomerId = (customerRows[0] as { internal_id?: string })
|
||||
?.internal_id;
|
||||
if (!internalCustomerId) {
|
||||
throw new Error(`Customer ${CUSTOMER_ID} not found in org ${ORG_ID}`);
|
||||
}
|
||||
|
||||
if (!skipOld) {
|
||||
await runMeasuredQuery({
|
||||
db,
|
||||
label: "OLD COMBINED QUERY (pre-split hydration)",
|
||||
query: getCombinedEntityPageQuery({
|
||||
orgId: ORG_ID,
|
||||
env: ENV,
|
||||
customerId: CUSTOMER_ID,
|
||||
limit: LIMIT,
|
||||
inStatuses,
|
||||
}),
|
||||
withExplain,
|
||||
});
|
||||
}
|
||||
|
||||
await runMeasuredQuery({
|
||||
db,
|
||||
label: "NEW QUERY A (entity-scoped page hydration)",
|
||||
query: getCursorPaginatedEntitySubjectsQuery({
|
||||
orgId: ORG_ID,
|
||||
env: ENV,
|
||||
limit: LIMIT,
|
||||
cursor: null,
|
||||
inStatuses,
|
||||
customerId: CUSTOMER_ID,
|
||||
}),
|
||||
withExplain,
|
||||
});
|
||||
|
||||
await runMeasuredQuery({
|
||||
db,
|
||||
label: "NEW QUERY B (customer-level hydration, once per customer)",
|
||||
query: getCustomerLevelSubjectRowsQuery({
|
||||
orgId: ORG_ID,
|
||||
env: ENV,
|
||||
internalCustomerIds: [internalCustomerId],
|
||||
inStatuses,
|
||||
}),
|
||||
withExplain,
|
||||
});
|
||||
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
await main();
|
||||
@@ -8,6 +8,7 @@ import { instrumentDrizzleClient } from "@kubiks/otel-drizzle";
|
||||
import type { SQLWrapper } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import pg, { type PoolConfig } from "pg";
|
||||
import { logger } from "../external/logtail/logtailUtils.js";
|
||||
import { otelConfig } from "../utils/otel/otelConfig.js";
|
||||
import { attachPoolErrorHandlers, registerPool } from "./pgPoolMonitor.js";
|
||||
|
||||
@@ -97,22 +98,72 @@ export const initDrizzle = ({
|
||||
// Strict latency limits in prod; relaxed locally so dev pool warm-up doesn't kill tests.
|
||||
const isProd = process.env.NODE_ENV === "production";
|
||||
|
||||
const poolMaxFromEnv = ({
|
||||
envVar,
|
||||
fallback,
|
||||
}: {
|
||||
envVar: string;
|
||||
fallback: number;
|
||||
}): number => {
|
||||
const parsed = Number(process.env[envVar]);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
};
|
||||
|
||||
const PGBOUNCER_MAX_CLIENT_CONN = 7_600;
|
||||
const BUDGETED_FLEET_PROCESSES = 150;
|
||||
const BUDGETED_NON_SERVER_CONNECTIONS = 80;
|
||||
const POOL_BUDGET_HEADROOM = 0.85;
|
||||
|
||||
const PROD_POOL_MAX = {
|
||||
critical: 22,
|
||||
general: 14,
|
||||
replica: 6,
|
||||
};
|
||||
|
||||
const criticalPoolMax = poolMaxFromEnv({
|
||||
envVar: "CRITICAL_DB_POOL_MAX",
|
||||
fallback: isProd ? PROD_POOL_MAX.critical : 10,
|
||||
});
|
||||
const generalPoolMax = poolMaxFromEnv({
|
||||
envVar: "GENERAL_DB_POOL_MAX",
|
||||
fallback: isProd ? PROD_POOL_MAX.general : 10,
|
||||
});
|
||||
const replicaPoolMax = poolMaxFromEnv({
|
||||
envVar: "REPLICA_DB_POOL_MAX",
|
||||
fallback: PROD_POOL_MAX.replica,
|
||||
});
|
||||
|
||||
const budgetedFleetConnections =
|
||||
BUDGETED_FLEET_PROCESSES *
|
||||
(criticalPoolMax + generalPoolMax + replicaPoolMax) +
|
||||
BUDGETED_NON_SERVER_CONNECTIONS;
|
||||
|
||||
if (
|
||||
budgetedFleetConnections >
|
||||
PGBOUNCER_MAX_CLIENT_CONN * POOL_BUDGET_HEADROOM
|
||||
) {
|
||||
logger.warn(
|
||||
`[initDrizzle] pool budget (${budgetedFleetConnections}) exceeds ${POOL_BUDGET_HEADROOM} of max_client_conn (${PGBOUNCER_MAX_CLIENT_CONN}) — lower the pool maxes or raise the ceiling`,
|
||||
);
|
||||
}
|
||||
|
||||
export const { db: dbCritical, client: clientCritical } = initDrizzle({
|
||||
name: "critical",
|
||||
maxConnections: isProd ? 100 : 10,
|
||||
maxConnections: criticalPoolMax,
|
||||
connectTimeout: isProd ? 2 : 30,
|
||||
databaseUrl: process.env.DATABASE_CRITICAL_URL,
|
||||
poolConfig: {
|
||||
application_name: "autumn-critical",
|
||||
query_timeout: isProd ? 2_000 : 30_000,
|
||||
// Keep 10 warm conns to avoid TLS-handshake stampedes on bursty traffic.
|
||||
min: 10,
|
||||
// Keep warm conns to avoid TLS-handshake stampedes on bursty traffic.
|
||||
min: Math.min(10, criticalPoolMax),
|
||||
},
|
||||
});
|
||||
|
||||
// -- General pool: used by all other endpoints --
|
||||
export const { db: dbGeneral, client: clientGeneral } = initDrizzle({
|
||||
name: "general",
|
||||
maxConnections: generalPoolMax,
|
||||
connectTimeout: isProd ? 5 : 30,
|
||||
});
|
||||
|
||||
@@ -122,7 +173,7 @@ const replicaResult = process.env.DATABASE_REPLICA_URL
|
||||
? initDrizzle({
|
||||
name: "replica",
|
||||
replica: true,
|
||||
maxConnections: 15,
|
||||
maxConnections: replicaPoolMax,
|
||||
connectTimeout: null,
|
||||
})
|
||||
: null;
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
setRateLimitKeyInContext,
|
||||
} from "@/internal/misc/rateLimiter/rateLimitFactory";
|
||||
import {
|
||||
getOrgAggregateType,
|
||||
getRateLimitType,
|
||||
RateLimitType,
|
||||
} from "../internal/misc/rateLimiter/rateLimitConfigs";
|
||||
@@ -39,8 +40,31 @@ export const rateLimitMiddleware = async (c: Context<HonoEnv>, next: Next) => {
|
||||
// 4. Get the appropriate limiter for this type
|
||||
const limiter = getLimiterForType(rateLimitType);
|
||||
|
||||
// 5. Apply rate limiting
|
||||
return await limiter(c as Context<Env>, next);
|
||||
const aggregateType = getOrgAggregateType(rateLimitType);
|
||||
if (!aggregateType) {
|
||||
// 5. Apply rate limiting
|
||||
return await limiter(c as Context<Env>, next);
|
||||
}
|
||||
|
||||
// 5. Org-aggregate limiter wraps the per-customer one; the key slot is
|
||||
// swapped between them since keyGenerator reads it at execution time.
|
||||
setRateLimitKeyInContext(
|
||||
c as Context,
|
||||
getRateLimitKey({ c, rateLimitType: aggregateType }),
|
||||
);
|
||||
const aggregateLimiter = getLimiterForType(aggregateType);
|
||||
|
||||
let innerResponse: Response | undefined;
|
||||
const aggregateResponse = await aggregateLimiter(
|
||||
c as Context<Env>,
|
||||
async () => {
|
||||
setRateLimitKeyInContext(c as Context, rateLimitKey);
|
||||
innerResponse = (await limiter(c as Context<Env>, next)) ?? undefined;
|
||||
},
|
||||
);
|
||||
|
||||
// hono-rate-limiter discards next()'s return, so re-surface an inner 429.
|
||||
return aggregateResponse ?? innerResponse;
|
||||
} catch (error) {
|
||||
ctx.logger.error(
|
||||
`Error checking rate limit, error: ${error}. Bypassing for now`,
|
||||
|
||||
@@ -74,6 +74,10 @@ export type RequestContext = {
|
||||
fullCustomer?: FullCustomer;
|
||||
rolloutSnapshot?: RolloutSnapshot;
|
||||
|
||||
/** Org is over its aggregate rate cap — check/track flows skip the DB and
|
||||
* serve their fail-open responses (allow / SQS queue) instead. */
|
||||
orgRateLimitDegraded?: boolean;
|
||||
|
||||
testOptions?: {
|
||||
skipCacheDeletion?: boolean;
|
||||
skipWebhooks?: boolean;
|
||||
|
||||
@@ -18,6 +18,18 @@ export const runCheckWithRollout = async ({
|
||||
body: ParsedCheckParams;
|
||||
requiredBalance: number;
|
||||
}): Promise<RunCheckResult<CheckData | CheckDataV2>> => {
|
||||
if (ctx.orgRateLimitDegraded) {
|
||||
return {
|
||||
checkData: null,
|
||||
response: getCheckFailOpenFallback({
|
||||
ctx,
|
||||
body,
|
||||
requiredBalance,
|
||||
error: new Error("org aggregate rate cap exceeded"),
|
||||
}) as Record<string, unknown>,
|
||||
};
|
||||
}
|
||||
|
||||
if (!isFullSubjectRolloutEnabled({ ctx })) {
|
||||
return runCheckLegacyFlow({ ctx, body, requiredBalance });
|
||||
}
|
||||
|
||||
@@ -24,6 +24,11 @@ export const runTrackWithRollout = async ({
|
||||
apiVersion?: ApiVersion;
|
||||
}): Promise<TrackResponseV3> => {
|
||||
if (shouldUseTrackV3({ ctx })) {
|
||||
if (ctx.orgRateLimitDegraded) {
|
||||
const queuedResponse = await queueTrack({ ctx, body });
|
||||
if (queuedResponse) return queuedResponse;
|
||||
}
|
||||
|
||||
return withRedisFailOpen<TrackResponseV3>({
|
||||
source: "runTrackWithRollout",
|
||||
run: () =>
|
||||
|
||||
@@ -2,8 +2,39 @@ import { type CusProductStatus, RELEVANT_STATUSES } from "@autumn/shared";
|
||||
import { type SQL, sql } from "drizzle-orm";
|
||||
import { getEntityAggregateFragments } from "./getEntityAggregateFragments.js";
|
||||
|
||||
const CUSTOMER_PRODUCT_LIMIT = 200;
|
||||
const EXTRA_CUSTOMER_ENTITLEMENT_LIMIT = 200;
|
||||
export const CUSTOMER_PRODUCT_LIMIT = 200;
|
||||
export const EXTRA_CUSTOMER_ENTITLEMENT_LIMIT = 200;
|
||||
|
||||
/** Aggregate CTE → SubjectQueryRow column. Each CTE must expose (subject_key, items). */
|
||||
const SUBJECT_AGGREGATES = [
|
||||
{ cte: "cus_products_agg", column: "customer_products" },
|
||||
{ cte: "cus_entitlements_agg", column: "customer_entitlements" },
|
||||
{ cte: "cus_prices_agg", column: "customer_prices" },
|
||||
{ cte: "extra_cus_entitlements_agg", column: "extra_customer_entitlements" },
|
||||
{ cte: "replaceables_agg", column: "replaceables" },
|
||||
{ cte: "rollovers_agg", column: "rollovers" },
|
||||
{ cte: "products_agg", column: "products" },
|
||||
{ cte: "entitlements_agg", column: "entitlements" },
|
||||
{ cte: "prices_agg", column: "prices" },
|
||||
{ cte: "free_trials_agg", column: "free_trials" },
|
||||
{ cte: "subscriptions_agg", column: "subscriptions" },
|
||||
] as const;
|
||||
|
||||
const aggregateSelects = sql.join(
|
||||
SUBJECT_AGGREGATES.map(({ cte, column }) =>
|
||||
sql.raw(`COALESCE(${cte}.items, '[]'::json) AS ${column}`),
|
||||
),
|
||||
sql`,
|
||||
`,
|
||||
);
|
||||
|
||||
const aggregateJoins = sql.join(
|
||||
SUBJECT_AGGREGATES.map(({ cte }) =>
|
||||
sql.raw(`LEFT JOIN ${cte} ON ${cte}.subject_key = sr.subject_key`),
|
||||
),
|
||||
sql`
|
||||
`,
|
||||
);
|
||||
|
||||
const emptyEntityFragments = {
|
||||
ctes: sql``,
|
||||
@@ -19,11 +50,14 @@ export const getFullSubjectRowsQuery = ({
|
||||
inStatuses,
|
||||
includeInvoices,
|
||||
includeEntityAggregations,
|
||||
entityScopedOnly = false,
|
||||
}: {
|
||||
leadingCtes: SQL;
|
||||
inStatuses: CusProductStatus[];
|
||||
includeInvoices: boolean;
|
||||
includeEntityAggregations: boolean;
|
||||
/** Only hydrate rows scoped to the subject's entity (requires non-null internal_entity_id on every subject). Customer-level rows must be merged back in separately. */
|
||||
entityScopedOnly?: boolean;
|
||||
}) => {
|
||||
const statusFilter =
|
||||
inStatuses.length > 0
|
||||
@@ -50,6 +84,29 @@ export const getFullSubjectRowsQuery = ({
|
||||
})
|
||||
: emptyEntityFragments;
|
||||
|
||||
const customerProductSubjectPredicate = entityScopedOnly
|
||||
? sql`cp.internal_entity_id = sr.internal_entity_id`
|
||||
: sql`cp.internal_customer_id = sr.internal_customer_id
|
||||
AND (
|
||||
(sr.internal_entity_id IS NULL AND cp.internal_entity_id IS NULL)
|
||||
OR
|
||||
(sr.internal_entity_id IS NOT NULL AND (
|
||||
cp.internal_entity_id IS NULL
|
||||
OR cp.internal_entity_id = sr.internal_entity_id
|
||||
))
|
||||
)`;
|
||||
|
||||
const customerEntitlementSubjectPredicate = entityScopedOnly
|
||||
? sql`AND ce.internal_entity_id = sr.internal_entity_id`
|
||||
: sql`AND (
|
||||
(sr.internal_entity_id IS NULL AND ce.internal_entity_id IS NULL)
|
||||
OR
|
||||
(sr.internal_entity_id IS NOT NULL AND (
|
||||
ce.internal_entity_id IS NULL
|
||||
OR ce.internal_entity_id = sr.internal_entity_id
|
||||
))
|
||||
)`;
|
||||
|
||||
const invoicesCte = includeInvoices
|
||||
? sql`,
|
||||
|
||||
@@ -83,7 +140,7 @@ export const getFullSubjectRowsQuery = ({
|
||||
${leadingCtes}
|
||||
,
|
||||
|
||||
subject_customer_records AS (
|
||||
subject_customer_records AS MATERIALIZED (
|
||||
SELECT DISTINCT c.*
|
||||
FROM customers c
|
||||
JOIN subject_records sr
|
||||
@@ -119,15 +176,7 @@ export const getFullSubjectRowsQuery = ({
|
||||
FROM customer_products cp
|
||||
JOIN products prod
|
||||
ON prod.internal_id = cp.internal_product_id
|
||||
WHERE cp.internal_customer_id = sr.internal_customer_id
|
||||
AND (
|
||||
(sr.internal_entity_id IS NULL AND cp.internal_entity_id IS NULL)
|
||||
OR
|
||||
(sr.internal_entity_id IS NOT NULL AND (
|
||||
cp.internal_entity_id IS NULL
|
||||
OR cp.internal_entity_id = sr.internal_entity_id
|
||||
))
|
||||
)
|
||||
WHERE ${customerProductSubjectPredicate}
|
||||
${statusFilter}
|
||||
) cp_candidates ON true
|
||||
),
|
||||
@@ -175,14 +224,7 @@ export const getFullSubjectRowsQuery = ({
|
||||
AND f.type = 'boolean'
|
||||
)
|
||||
)
|
||||
AND (
|
||||
(sr.internal_entity_id IS NULL AND ce.internal_entity_id IS NULL)
|
||||
OR
|
||||
(sr.internal_entity_id IS NOT NULL AND (
|
||||
ce.internal_entity_id IS NULL
|
||||
OR ce.internal_entity_id = sr.internal_entity_id
|
||||
))
|
||||
)
|
||||
${customerEntitlementSubjectPredicate}
|
||||
ORDER BY subject_entity_priority ASC, ce.id DESC
|
||||
LIMIT ${EXTRA_CUSTOMER_ENTITLEMENT_LIMIT}
|
||||
) ce_ordered ON true
|
||||
@@ -300,149 +342,147 @@ export const getFullSubjectRowsQuery = ({
|
||||
${entityFragments.freeTrialRefsUnion}
|
||||
) src ON ft.id = src.free_trial_id
|
||||
ORDER BY src.subject_key, ft.id
|
||||
),
|
||||
|
||||
cus_products_agg AS (
|
||||
SELECT
|
||||
cp.subject_key,
|
||||
json_agg(
|
||||
(
|
||||
row_to_json(cp)::jsonb
|
||||
- 'subject_key'
|
||||
- 'subject_entity_priority'
|
||||
- 'status_priority'
|
||||
- 'has_customer_prices'
|
||||
- 'product_is_add_on'
|
||||
- 'subject_rank'
|
||||
)::json
|
||||
ORDER BY
|
||||
cp.subject_entity_priority ASC,
|
||||
cp.status_priority ASC,
|
||||
cp.has_customer_prices DESC,
|
||||
cp.product_is_add_on ASC,
|
||||
cp.created_at DESC
|
||||
) AS items
|
||||
FROM cus_products cp
|
||||
GROUP BY cp.subject_key
|
||||
),
|
||||
|
||||
cus_entitlements_agg AS (
|
||||
SELECT
|
||||
ce.subject_key,
|
||||
json_agg((row_to_json(ce)::jsonb - 'subject_key')::json) AS items
|
||||
FROM cus_entitlements ce
|
||||
GROUP BY ce.subject_key
|
||||
),
|
||||
|
||||
cus_prices_agg AS (
|
||||
SELECT
|
||||
cpr.subject_key,
|
||||
json_agg((row_to_json(cpr)::jsonb - 'subject_key')::json) AS items
|
||||
FROM cus_prices cpr
|
||||
GROUP BY cpr.subject_key
|
||||
),
|
||||
|
||||
extra_cus_entitlements_agg AS (
|
||||
SELECT
|
||||
ece.subject_key,
|
||||
json_agg(
|
||||
(
|
||||
row_to_json(ece)::jsonb
|
||||
- 'subject_key'
|
||||
- 'subject_entity_priority'
|
||||
)::json
|
||||
ORDER BY ece.subject_entity_priority ASC, ece.id DESC
|
||||
) AS items
|
||||
FROM extra_cus_entitlements ece
|
||||
GROUP BY ece.subject_key
|
||||
),
|
||||
|
||||
replaceables_agg AS (
|
||||
SELECT
|
||||
ace.subject_key,
|
||||
json_agg(row_to_json(rep) ORDER BY rep.created_at ASC, rep.id ASC) AS items
|
||||
FROM cus_replaceables rep
|
||||
JOIN all_cus_ent_ids ace
|
||||
ON ace.id = rep.cus_ent_id
|
||||
GROUP BY ace.subject_key
|
||||
),
|
||||
|
||||
rollovers_agg AS (
|
||||
SELECT
|
||||
ace.subject_key,
|
||||
json_agg(
|
||||
row_to_json(ro)
|
||||
ORDER BY ro.expires_at ASC NULLS LAST, ro.id ASC
|
||||
) AS items
|
||||
FROM cus_rollovers ro
|
||||
JOIN all_cus_ent_ids ace
|
||||
ON ace.id = ro.cus_ent_id
|
||||
GROUP BY ace.subject_key
|
||||
),
|
||||
|
||||
products_agg AS (
|
||||
SELECT
|
||||
p.subject_key,
|
||||
json_agg(
|
||||
(row_to_json(p)::jsonb - 'internal_customer_id' - 'subject_key')::json
|
||||
ORDER BY p.internal_id
|
||||
) AS items
|
||||
FROM distinct_products p
|
||||
GROUP BY p.subject_key
|
||||
),
|
||||
|
||||
entitlements_agg AS (
|
||||
SELECT
|
||||
ent.subject_key,
|
||||
json_agg((row_to_json(ent)::jsonb - 'internal_customer_id' - 'subject_key')::json) AS items
|
||||
FROM distinct_entitlements ent
|
||||
GROUP BY ent.subject_key
|
||||
),
|
||||
|
||||
prices_agg AS (
|
||||
SELECT
|
||||
pr.subject_key,
|
||||
json_agg(
|
||||
(row_to_json(pr)::jsonb - 'internal_customer_id' - 'subject_key')::json
|
||||
ORDER BY pr.id
|
||||
) AS items
|
||||
FROM distinct_prices pr
|
||||
GROUP BY pr.subject_key
|
||||
),
|
||||
|
||||
free_trials_agg AS (
|
||||
SELECT
|
||||
ft.subject_key,
|
||||
json_agg(
|
||||
(row_to_json(ft)::jsonb - 'internal_customer_id' - 'subject_key')::json
|
||||
ORDER BY ft.id
|
||||
) AS items
|
||||
FROM distinct_free_trials ft
|
||||
GROUP BY ft.subject_key
|
||||
),
|
||||
|
||||
subscriptions_agg AS (
|
||||
SELECT
|
||||
cs.subject_key,
|
||||
json_agg(row_to_json(cs.subscription_row))
|
||||
FILTER (WHERE (cs.subscription_row).stripe_id IS NOT NULL) AS items
|
||||
FROM (
|
||||
SELECT DISTINCT
|
||||
cp.subject_key,
|
||||
s AS subscription_row
|
||||
FROM cus_products cp
|
||||
JOIN LATERAL unnest(cp.subscription_ids) AS cp_sub(stripe_id) ON true
|
||||
JOIN subscriptions s
|
||||
ON s.stripe_id = cp_sub.stripe_id
|
||||
) cs
|
||||
GROUP BY cs.subject_key
|
||||
)
|
||||
|
||||
SELECT
|
||||
row_to_json(scr) AS customer,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg(
|
||||
(
|
||||
row_to_json(cp)::jsonb
|
||||
- 'subject_key'
|
||||
- 'subject_entity_priority'
|
||||
- 'status_priority'
|
||||
- 'has_customer_prices'
|
||||
- 'product_is_add_on'
|
||||
- 'subject_rank'
|
||||
)::json
|
||||
ORDER BY
|
||||
cp.subject_entity_priority ASC,
|
||||
cp.status_priority ASC,
|
||||
cp.has_customer_prices DESC,
|
||||
cp.product_is_add_on ASC,
|
||||
cp.created_at DESC
|
||||
)
|
||||
FROM cus_products cp
|
||||
WHERE cp.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS customer_products,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg((row_to_json(ce)::jsonb - 'subject_key')::json)
|
||||
FROM cus_entitlements ce
|
||||
WHERE ce.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS customer_entitlements,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg((row_to_json(cpr)::jsonb - 'subject_key')::json)
|
||||
FROM cus_prices cpr
|
||||
WHERE cpr.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS customer_prices,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg(
|
||||
(
|
||||
row_to_json(ece)::jsonb
|
||||
- 'subject_key'
|
||||
- 'subject_entity_priority'
|
||||
)::json
|
||||
ORDER BY ece.subject_entity_priority ASC, ece.id DESC
|
||||
)
|
||||
FROM extra_cus_entitlements ece
|
||||
WHERE ece.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS extra_customer_entitlements,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg(row_to_json(rep) ORDER BY rep.created_at ASC, rep.id ASC)
|
||||
FROM cus_replaceables rep
|
||||
WHERE rep.cus_ent_id IN (
|
||||
SELECT ace.id
|
||||
FROM all_cus_ent_ids ace
|
||||
WHERE ace.subject_key = sr.subject_key
|
||||
)
|
||||
),
|
||||
'[]'::json
|
||||
) AS replaceables,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg(
|
||||
row_to_json(ro)
|
||||
ORDER BY ro.expires_at ASC NULLS LAST, ro.id ASC
|
||||
)
|
||||
FROM cus_rollovers ro
|
||||
WHERE ro.cus_ent_id IN (
|
||||
SELECT ace.id
|
||||
FROM all_cus_ent_ids ace
|
||||
WHERE ace.subject_key = sr.subject_key
|
||||
)
|
||||
),
|
||||
'[]'::json
|
||||
) AS rollovers,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg((row_to_json(p)::jsonb - 'internal_customer_id' - 'subject_key')::json)
|
||||
FROM distinct_products p
|
||||
WHERE p.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS products,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg((row_to_json(ent)::jsonb - 'internal_customer_id' - 'subject_key')::json)
|
||||
FROM distinct_entitlements ent
|
||||
WHERE ent.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS entitlements,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg((row_to_json(pr)::jsonb - 'internal_customer_id' - 'subject_key')::json)
|
||||
FROM distinct_prices pr
|
||||
WHERE pr.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS prices,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg((row_to_json(ft)::jsonb - 'internal_customer_id' - 'subject_key')::json)
|
||||
FROM distinct_free_trials ft
|
||||
WHERE ft.subject_key = sr.subject_key
|
||||
),
|
||||
'[]'::json
|
||||
) AS free_trials,
|
||||
|
||||
COALESCE(
|
||||
(
|
||||
SELECT json_agg(row_to_json(cs)) FILTER (WHERE cs.stripe_id IS NOT NULL)
|
||||
FROM (
|
||||
SELECT DISTINCT s.*
|
||||
FROM cus_products cp
|
||||
JOIN LATERAL unnest(cp.subscription_ids) AS cp_sub(stripe_id) ON true
|
||||
JOIN subscriptions s
|
||||
ON s.stripe_id = cp_sub.stripe_id
|
||||
WHERE cp.subject_key = sr.subject_key
|
||||
) cs
|
||||
),
|
||||
'[]'::json
|
||||
) AS subscriptions
|
||||
${aggregateSelects}
|
||||
|
||||
${invoicesSelect},
|
||||
|
||||
@@ -455,6 +495,7 @@ export const getFullSubjectRowsQuery = ({
|
||||
FROM subject_records sr
|
||||
JOIN subject_customer_records scr
|
||||
ON scr.internal_id = sr.internal_customer_id
|
||||
${aggregateJoins}
|
||||
LEFT JOIN entities er
|
||||
ON er.internal_id = sr.internal_entity_id
|
||||
ORDER BY sr.subject_order
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { SubjectQueryRow } from "@autumn/shared";
|
||||
import {
|
||||
CUSTOMER_PRODUCT_LIMIT,
|
||||
EXTRA_CUSTOMER_ENTITLEMENT_LIMIT,
|
||||
} from "./getFullSubjectRowsQuery.js";
|
||||
|
||||
const dedupeBy = <T>(rows: T[], getKey: (row: T) => string): T[] => {
|
||||
const seen = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (!seen.has(getKey(row))) seen.set(getKey(row), row);
|
||||
}
|
||||
return [...seen.values()];
|
||||
};
|
||||
|
||||
/** Dedupe, keep only referenced rows, and sort by key to mirror the SQL's DISTINCT ON ... ORDER BY output. */
|
||||
const mergeCatalog = <T>(
|
||||
rows: T[],
|
||||
getKey: (row: T) => string,
|
||||
keptKeys: Set<string | null>,
|
||||
): T[] =>
|
||||
dedupeBy(rows, getKey)
|
||||
.filter((row) => keptKeys.has(getKey(row)))
|
||||
.sort((left, right) => (getKey(left) < getKey(right) ? -1 : 1));
|
||||
|
||||
/**
|
||||
* Recombines an entityScopedOnly subject row with its customer's
|
||||
* customer-level row into the SubjectQueryRow the combined query would
|
||||
* produce. Entity rows concat before customer rows (subject_entity_priority
|
||||
* leads the SQL ranking), the caps apply to the combined arrays, and since
|
||||
* the SQL derives every other array AFTER the caps, rows referencing
|
||||
* capped-out customer products are dropped here too.
|
||||
*/
|
||||
export const mergeEntityAndCustomerSubjectRows = ({
|
||||
entityRow,
|
||||
customerRow,
|
||||
}: {
|
||||
entityRow: SubjectQueryRow;
|
||||
customerRow: SubjectQueryRow | undefined;
|
||||
}): SubjectQueryRow => {
|
||||
if (!customerRow) return entityRow;
|
||||
|
||||
// The entityScopedOnly query matches on internal_entity_id alone (adding the
|
||||
// customer predicate degrades its plan), so enforce the customer match here.
|
||||
// Dependent rows of any dropped product are filtered transitively below.
|
||||
const customerProducts = [
|
||||
...entityRow.customer_products.filter(
|
||||
(product) =>
|
||||
product.internal_customer_id === entityRow.customer.internal_id,
|
||||
),
|
||||
...customerRow.customer_products,
|
||||
].slice(0, CUSTOMER_PRODUCT_LIMIT);
|
||||
|
||||
const extraCustomerEntitlements = [
|
||||
...entityRow.extra_customer_entitlements,
|
||||
...customerRow.extra_customer_entitlements,
|
||||
].slice(0, EXTRA_CUSTOMER_ENTITLEMENT_LIMIT);
|
||||
|
||||
const keptProductIds = new Set<string | null>(
|
||||
customerProducts.map((product) => product.id),
|
||||
);
|
||||
|
||||
const customerEntitlements = [
|
||||
...entityRow.customer_entitlements,
|
||||
...customerRow.customer_entitlements,
|
||||
].filter((entitlement) => keptProductIds.has(entitlement.customer_product_id));
|
||||
|
||||
const customerPrices = [
|
||||
...entityRow.customer_prices,
|
||||
...customerRow.customer_prices,
|
||||
].filter((price) => keptProductIds.has(price.customer_product_id));
|
||||
|
||||
const keptCusEntIds = new Set(
|
||||
[...customerEntitlements, ...extraCustomerEntitlements].map((ce) => ce.id),
|
||||
);
|
||||
const keptRefs = {
|
||||
products: new Set<string | null>(
|
||||
customerProducts.map((p) => p.internal_product_id),
|
||||
),
|
||||
prices: new Set<string | null>(customerPrices.map((p) => p.price_id)),
|
||||
entitlements: new Set<string | null>(
|
||||
[...customerEntitlements, ...extraCustomerEntitlements].map(
|
||||
(ce) => ce.entitlement_id,
|
||||
),
|
||||
),
|
||||
freeTrials: new Set<string | null>(
|
||||
customerProducts.map((p) => p.free_trial_id),
|
||||
),
|
||||
subscriptionIds: new Set(
|
||||
customerProducts.flatMap((p) => p.subscription_ids ?? []),
|
||||
),
|
||||
};
|
||||
|
||||
// Explicit keys only (no spreads): a new required SubjectQueryRow field must
|
||||
// fail compilation here until this merge handles it.
|
||||
return {
|
||||
customer: entityRow.customer,
|
||||
entity: entityRow.entity,
|
||||
customer_products: customerProducts,
|
||||
customer_entitlements: customerEntitlements,
|
||||
customer_prices: customerPrices,
|
||||
extra_customer_entitlements: extraCustomerEntitlements,
|
||||
rollovers: [...entityRow.rollovers, ...customerRow.rollovers].filter(
|
||||
(rollover) => keptCusEntIds.has(rollover.cus_ent_id),
|
||||
),
|
||||
replaceables: [
|
||||
...entityRow.replaceables,
|
||||
...customerRow.replaceables,
|
||||
].filter((replaceable) => keptCusEntIds.has(replaceable.cus_ent_id)),
|
||||
products: mergeCatalog(
|
||||
[...entityRow.products, ...customerRow.products],
|
||||
(p) => p.internal_id,
|
||||
keptRefs.products,
|
||||
),
|
||||
entitlements: dedupeBy(
|
||||
[...entityRow.entitlements, ...customerRow.entitlements],
|
||||
(e) => e.id,
|
||||
).filter((e) => keptRefs.entitlements.has(e.id)),
|
||||
prices: mergeCatalog(
|
||||
[...entityRow.prices, ...customerRow.prices],
|
||||
(p) => p.id,
|
||||
keptRefs.prices,
|
||||
),
|
||||
free_trials: mergeCatalog(
|
||||
[...entityRow.free_trials, ...customerRow.free_trials],
|
||||
(ft) => ft.id,
|
||||
keptRefs.freeTrials,
|
||||
),
|
||||
subscriptions: dedupeBy(
|
||||
[...entityRow.subscriptions, ...customerRow.subscriptions],
|
||||
(s) => s.stripe_id ?? "",
|
||||
).filter(
|
||||
(s) =>
|
||||
s.stripe_id !== null && keptRefs.subscriptionIds.has(s.stripe_id),
|
||||
),
|
||||
};
|
||||
};
|
||||
@@ -30,6 +30,7 @@ import { resultToFullSubject } from "@/internal/customers/repos/getFullSubject/i
|
||||
import { getOrgPaginationMaxLimit } from "../../misc/edgeConfig/orgLimitsStore.js";
|
||||
import { getApiEntityBaseV2 } from "../entityUtils/getApiEntityV2/getApiEntityBaseV2.js";
|
||||
import { getCursorPaginatedEntitySubjectsQuery } from "../repos/cursorListEntitiesQuery.js";
|
||||
import { hydrateEntityRowsWithCustomerData } from "../repos/hydrateEntityRowsWithCustomerData.js";
|
||||
import {
|
||||
countEntitiesByOrgIdAndEnv,
|
||||
countFilteredEntitiesByOrgIdAndEnv,
|
||||
@@ -50,13 +51,21 @@ const getListEntitiesStatuses = ({
|
||||
const buildApiEntitiesFromRows = async ({
|
||||
ctx,
|
||||
rows,
|
||||
inStatuses,
|
||||
}: {
|
||||
ctx: RequestContext;
|
||||
rows: unknown[];
|
||||
inStatuses: CusProductStatus[];
|
||||
}) => {
|
||||
const fullSubjects = rows.map((row) =>
|
||||
const mergedRows = await hydrateEntityRowsWithCustomerData({
|
||||
ctx,
|
||||
entityRows: rows as unknown as SubjectQueryRow[],
|
||||
inStatuses,
|
||||
});
|
||||
|
||||
const fullSubjects = mergedRows.map((row) =>
|
||||
resultToFullSubject({
|
||||
row: row as unknown as SubjectQueryRow,
|
||||
row,
|
||||
entityIdRequested: true,
|
||||
}),
|
||||
);
|
||||
@@ -141,7 +150,11 @@ const runOffsetListEntities = async ({
|
||||
})
|
||||
: totalCount;
|
||||
|
||||
const entities = await buildApiEntitiesFromRows({ ctx, rows: subjectRows });
|
||||
const entities = await buildApiEntitiesFromRows({
|
||||
ctx,
|
||||
rows: subjectRows,
|
||||
inStatuses,
|
||||
});
|
||||
|
||||
const hasMore = body.offset + entities.length < totalFilteredCount;
|
||||
|
||||
@@ -207,7 +220,11 @@ export const handleListEntitiesV2 = createRoute({
|
||||
const hasMore = rows.length > body.limit;
|
||||
const pageRows = hasMore ? rows.slice(0, body.limit) : rows;
|
||||
|
||||
const entities = await buildApiEntitiesFromRows({ ctx, rows: pageRows });
|
||||
const entities = await buildApiEntitiesFromRows({
|
||||
ctx,
|
||||
rows: pageRows,
|
||||
inStatuses,
|
||||
});
|
||||
|
||||
const lastRow = pageRows[pageRows.length - 1] as
|
||||
| { entity?: { id?: string; created_at?: number | string } }
|
||||
|
||||
@@ -151,5 +151,6 @@ export const getCursorPaginatedEntitySubjectsQuery = ({
|
||||
inStatuses,
|
||||
includeInvoices: false,
|
||||
includeEntityAggregations: false,
|
||||
entityScopedOnly: true,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { AppEnv, CusProductStatus } from "@autumn/shared";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { getFullSubjectRowsQuery } from "@/internal/customers/repos/getFullSubject/getFullSubjectRowsQuery.js";
|
||||
|
||||
/** Hydrates customer-level subject rows once per customer, merged back into entity list rows by mergeEntityAndCustomerSubjectRows. */
|
||||
export const getCustomerLevelSubjectRowsQuery = ({
|
||||
orgId,
|
||||
env,
|
||||
internalCustomerIds,
|
||||
inStatuses,
|
||||
}: {
|
||||
orgId: string;
|
||||
env: AppEnv;
|
||||
internalCustomerIds: string[];
|
||||
inStatuses: CusProductStatus[];
|
||||
}) => {
|
||||
const idList = sql.join(
|
||||
internalCustomerIds.map((internalCustomerId) => sql`${internalCustomerId}`),
|
||||
sql`, `,
|
||||
);
|
||||
|
||||
const leadingCtes = sql`
|
||||
WITH subject_records AS (
|
||||
SELECT
|
||||
c.internal_id AS subject_key,
|
||||
c.internal_id AS internal_customer_id,
|
||||
NULL::text AS internal_entity_id,
|
||||
ROW_NUMBER() OVER (ORDER BY c.internal_id) AS subject_order
|
||||
FROM customers c
|
||||
WHERE c.internal_id IN (${idList})
|
||||
AND c.org_id = ${orgId}
|
||||
AND c.env = ${env}
|
||||
)
|
||||
`;
|
||||
|
||||
return getFullSubjectRowsQuery({
|
||||
leadingCtes,
|
||||
inStatuses,
|
||||
includeInvoices: false,
|
||||
includeEntityAggregations: false,
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { CusProductStatus, SubjectQueryRow } from "@autumn/shared";
|
||||
import type { RequestContext } from "@/honoUtils/HonoEnv.js";
|
||||
import { mergeEntityAndCustomerSubjectRows } from "@/internal/customers/repos/getFullSubject/mergeEntityAndCustomerSubjectRows.js";
|
||||
import { getCustomerLevelSubjectRowsQuery } from "./customerLevelSubjectsQuery.js";
|
||||
|
||||
/** Fetches customer-level data once per distinct customer on the page and merges it into each entityScopedOnly row. */
|
||||
export const hydrateEntityRowsWithCustomerData = async ({
|
||||
ctx,
|
||||
entityRows,
|
||||
inStatuses,
|
||||
}: {
|
||||
ctx: RequestContext;
|
||||
entityRows: SubjectQueryRow[];
|
||||
inStatuses: CusProductStatus[];
|
||||
}): Promise<SubjectQueryRow[]> => {
|
||||
if (entityRows.length === 0) return entityRows;
|
||||
|
||||
const internalCustomerIds = [
|
||||
...new Set(entityRows.map((row) => row.customer.internal_id)),
|
||||
];
|
||||
|
||||
const customerRows = (await ctx.db.execute(
|
||||
getCustomerLevelSubjectRowsQuery({
|
||||
orgId: ctx.org.id,
|
||||
env: ctx.env,
|
||||
internalCustomerIds,
|
||||
inStatuses,
|
||||
}),
|
||||
)) as unknown as SubjectQueryRow[];
|
||||
|
||||
const customerRowsByInternalId = new Map(
|
||||
customerRows.map((row) => [row.customer.internal_id, row]),
|
||||
);
|
||||
|
||||
return entityRows.map((entityRow) => {
|
||||
const customerRow = customerRowsByInternalId.get(
|
||||
entityRow.customer.internal_id,
|
||||
);
|
||||
if (!customerRow) {
|
||||
ctx.logger.warn(
|
||||
`[hydrateEntityRowsWithCustomerData] missing customer-level row for internal customer id ${entityRow.customer.internal_id}`,
|
||||
);
|
||||
}
|
||||
return mergeEntityAndCustomerSubjectRows({ entityRow, customerRow });
|
||||
});
|
||||
};
|
||||
@@ -174,6 +174,7 @@ export const getPaginatedEntitySubjectsQuery = ({
|
||||
inStatuses,
|
||||
includeInvoices: false,
|
||||
includeEntityAggregations: false,
|
||||
entityScopedOnly: true,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -13,8 +13,23 @@ export enum RateLimitType {
|
||||
ListCustomers = "list_customers",
|
||||
CustomerEntitiesGet = "customer_entities_get",
|
||||
Logs = "logs",
|
||||
TrackOrg = "track_org",
|
||||
CheckOrg = "check_org",
|
||||
EntitiesGetOrg = "entities_get_org",
|
||||
}
|
||||
|
||||
// Org-wide aggregate caps summed across all of an org's customers — the
|
||||
// per-customer limits never bind for many-customer storms (2026-06-08 incident).
|
||||
const ORG_AGGREGATE_TYPES: Partial<Record<RateLimitType, RateLimitType>> = {
|
||||
[RateLimitType.Track]: RateLimitType.TrackOrg,
|
||||
[RateLimitType.Check]: RateLimitType.CheckOrg,
|
||||
[RateLimitType.CustomerEntitiesGet]: RateLimitType.EntitiesGetOrg,
|
||||
};
|
||||
|
||||
export const getOrgAggregateType = (
|
||||
type: RateLimitType,
|
||||
): RateLimitType | undefined => ORG_AGGREGATE_TYPES[type];
|
||||
|
||||
type RoutePattern = {
|
||||
method: string;
|
||||
url: string;
|
||||
@@ -120,6 +135,22 @@ const RATE_LIMIT_ROUTE_GROUPS: RateLimitRouteGroup[] = [
|
||||
},
|
||||
];
|
||||
|
||||
// Check-group routes that can fail open (allowed: true) when an org is over
|
||||
// its aggregate cap; the establish routes in the group shed a 503 instead.
|
||||
const CHECK_FAIL_OPEN_PATTERNS: RoutePattern[] = [
|
||||
route({ method: "POST", url: "/v1/check" }),
|
||||
route({ method: "POST", url: "/v1/entitled" }),
|
||||
route({ method: "POST", url: "/v1/balances.check" }),
|
||||
];
|
||||
|
||||
export const isCheckFailOpenRoute = (c: Context<HonoEnv>): boolean => {
|
||||
const method = c.req.method;
|
||||
const path = c.req.path;
|
||||
return CHECK_FAIL_OPEN_PATTERNS.some((pattern) =>
|
||||
matchRoute({ url: path, method, pattern }),
|
||||
);
|
||||
};
|
||||
|
||||
export const getRateLimitType = (c: Context<HonoEnv>) => {
|
||||
const method = c.req.method;
|
||||
const path = c.req.path;
|
||||
@@ -154,6 +185,9 @@ export type RateLimitConfig = {
|
||||
windowMs: number;
|
||||
notInRedis: boolean;
|
||||
scope: RateLimitScope;
|
||||
// "degrade" = over-limit requests fail open (check -> allow, track -> SQS
|
||||
// queue) instead of 429, so the cap sheds DB load without losing events.
|
||||
overLimit?: "reject" | "degrade";
|
||||
};
|
||||
|
||||
export const resolveRateLimit = ({
|
||||
@@ -255,4 +289,29 @@ export const RATE_LIMIT_CONFIGS: Record<RateLimitType, RateLimitConfig> = {
|
||||
notInRedis: false,
|
||||
scope: RateLimitScope.Org,
|
||||
},
|
||||
// 60s windows sized ~1.5-2x the highest legit per-org peak observed over 7d
|
||||
// of prod traffic (check 157k/min, track 60k/min, entities.get 53k/min).
|
||||
[RateLimitType.TrackOrg]: {
|
||||
name: "track_org",
|
||||
limit: 120_000,
|
||||
windowMs: 60_000,
|
||||
notInRedis: false,
|
||||
scope: RateLimitScope.Org,
|
||||
overLimit: "degrade",
|
||||
},
|
||||
[RateLimitType.CheckOrg]: {
|
||||
name: "check_org",
|
||||
limit: 240_000,
|
||||
windowMs: 60_000,
|
||||
notInRedis: false,
|
||||
scope: RateLimitScope.Org,
|
||||
overLimit: "degrade",
|
||||
},
|
||||
[RateLimitType.EntitiesGetOrg]: {
|
||||
name: "entities_get_org",
|
||||
limit: 90_000,
|
||||
windowMs: 60_000,
|
||||
notInRedis: false,
|
||||
scope: RateLimitScope.Org,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
import type { ApiVersion } from "@autumn/shared";
|
||||
import type { Context } from "hono";
|
||||
import type { Context, Next } from "hono";
|
||||
import { rateLimiter } from "hono-rate-limiter";
|
||||
import { logger } from "@/external/logtail/logtailUtils.js";
|
||||
import { shouldUseRedis } from "@/external/redis/initRedis";
|
||||
import type { HonoEnv } from "@/honoUtils/HonoEnv";
|
||||
import {
|
||||
isCheckFailOpenRoute,
|
||||
RATE_LIMIT_CONFIGS,
|
||||
type RateLimitConfig,
|
||||
RateLimitScope,
|
||||
type RateLimitType,
|
||||
RateLimitType,
|
||||
resolveRateLimit,
|
||||
} from "./rateLimitConfigs";
|
||||
import { getOrgRateLimitOverride } from "./rateLimitOverridesStore";
|
||||
@@ -60,11 +61,38 @@ export const rateLimitFactory = ({
|
||||
return resolveRateLimit({ config, apiVersion }).limit;
|
||||
};
|
||||
|
||||
// Over-limit "degrade": fail open instead of 429 — check routes get the
|
||||
// allow-fallback via the ctx flag; establish routes shed a retryable 503.
|
||||
const degradeHandler = async (
|
||||
c: Context,
|
||||
next: Next,
|
||||
): Promise<Response | undefined> => {
|
||||
const honoContext = c as Context<HonoEnv>;
|
||||
const ctx = honoContext.get("ctx");
|
||||
|
||||
if (type === RateLimitType.CheckOrg && !isCheckFailOpenRoute(honoContext)) {
|
||||
return c.json(
|
||||
{
|
||||
message: "Service is temporarily unavailable, please retry shortly.",
|
||||
code: "service_unavailable",
|
||||
env: ctx?.env,
|
||||
},
|
||||
503,
|
||||
);
|
||||
}
|
||||
|
||||
if (ctx) ctx.orgRateLimitDegraded = true;
|
||||
c.header("Retry-After", undefined);
|
||||
await next();
|
||||
return;
|
||||
};
|
||||
|
||||
const options = {
|
||||
windowMs,
|
||||
limit: dynamicLimit,
|
||||
standardHeaders: "draft-6" as const,
|
||||
keyGenerator: getRateLimitKeyFromContext,
|
||||
...(config.overLimit === "degrade" && { handler: degradeHandler }),
|
||||
};
|
||||
|
||||
let inMemoryLimiter: ReturnType<typeof rateLimiter> | null = null;
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type {
|
||||
DbCustomerEntitlement,
|
||||
DbCustomerPrice,
|
||||
DbCustomerProduct,
|
||||
SubjectQueryRow,
|
||||
} from "@autumn/shared";
|
||||
import {
|
||||
CUSTOMER_PRODUCT_LIMIT,
|
||||
EXTRA_CUSTOMER_ENTITLEMENT_LIMIT,
|
||||
} from "@/internal/customers/repos/getFullSubject/getFullSubjectRowsQuery.js";
|
||||
import { mergeEntityAndCustomerSubjectRows } from "@/internal/customers/repos/getFullSubject/mergeEntityAndCustomerSubjectRows.js";
|
||||
|
||||
const createCustomerProduct = ({
|
||||
id,
|
||||
internalProductId = `prod_internal_${id}`,
|
||||
internalCustomerId = "cus_internal_1",
|
||||
freeTrialId = null,
|
||||
subscriptionIds = [],
|
||||
}: {
|
||||
id: string;
|
||||
internalProductId?: string;
|
||||
internalCustomerId?: string;
|
||||
freeTrialId?: string | null;
|
||||
subscriptionIds?: string[];
|
||||
}) =>
|
||||
({
|
||||
id,
|
||||
internal_product_id: internalProductId,
|
||||
internal_customer_id: internalCustomerId,
|
||||
free_trial_id: freeTrialId,
|
||||
subscription_ids: subscriptionIds,
|
||||
}) as DbCustomerProduct;
|
||||
|
||||
const createCustomerEntitlement = ({
|
||||
id,
|
||||
customerProductId,
|
||||
entitlementId = `ent_${id}`,
|
||||
}: {
|
||||
id: string;
|
||||
customerProductId: string | null;
|
||||
entitlementId?: string;
|
||||
}) =>
|
||||
({
|
||||
id,
|
||||
customer_product_id: customerProductId,
|
||||
entitlement_id: entitlementId,
|
||||
}) as DbCustomerEntitlement;
|
||||
|
||||
const createCustomerPrice = ({
|
||||
id,
|
||||
customerProductId,
|
||||
priceId = `price_${id}`,
|
||||
}: {
|
||||
id: string;
|
||||
customerProductId: string | null;
|
||||
priceId?: string;
|
||||
}) =>
|
||||
({
|
||||
id,
|
||||
customer_product_id: customerProductId,
|
||||
price_id: priceId,
|
||||
}) as DbCustomerPrice;
|
||||
|
||||
const createRow = (overrides: Partial<SubjectQueryRow> = {}): SubjectQueryRow =>
|
||||
({
|
||||
customer: { internal_id: "cus_internal_1" },
|
||||
customer_products: [],
|
||||
customer_entitlements: [],
|
||||
customer_prices: [],
|
||||
extra_customer_entitlements: [],
|
||||
replaceables: [],
|
||||
rollovers: [],
|
||||
products: [],
|
||||
entitlements: [],
|
||||
prices: [],
|
||||
free_trials: [],
|
||||
subscriptions: [],
|
||||
...overrides,
|
||||
}) as SubjectQueryRow;
|
||||
|
||||
describe("mergeEntityAndCustomerSubjectRows", () => {
|
||||
test("returns entity row unchanged when customer row is missing", () => {
|
||||
const entityRow = createRow({
|
||||
customer_products: [createCustomerProduct({ id: "cp_entity_1" })],
|
||||
});
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow: undefined,
|
||||
});
|
||||
|
||||
expect(merged).toBe(entityRow);
|
||||
});
|
||||
|
||||
test("drops entity-scoped products belonging to a different customer", () => {
|
||||
const entityRow = createRow({
|
||||
customer_products: [
|
||||
createCustomerProduct({ id: "cp_ours" }),
|
||||
createCustomerProduct({
|
||||
id: "cp_other_customer",
|
||||
internalCustomerId: "cus_internal_other",
|
||||
}),
|
||||
],
|
||||
});
|
||||
const customerRow = createRow();
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.customer_products.map((product) => product.id)).toEqual([
|
||||
"cp_ours",
|
||||
]);
|
||||
});
|
||||
|
||||
test("orders entity-scoped rows before customer-level rows", () => {
|
||||
const entityRow = createRow({
|
||||
customer_products: [
|
||||
createCustomerProduct({ id: "cp_entity_1" }),
|
||||
createCustomerProduct({ id: "cp_entity_2" }),
|
||||
],
|
||||
extra_customer_entitlements: [
|
||||
createCustomerEntitlement({
|
||||
id: "ce_extra_entity",
|
||||
customerProductId: null,
|
||||
}),
|
||||
],
|
||||
});
|
||||
const customerRow = createRow({
|
||||
customer_products: [createCustomerProduct({ id: "cp_customer_1" })],
|
||||
extra_customer_entitlements: [
|
||||
createCustomerEntitlement({
|
||||
id: "ce_extra_customer",
|
||||
customerProductId: null,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.customer_products.map((product) => product.id)).toEqual([
|
||||
"cp_entity_1",
|
||||
"cp_entity_2",
|
||||
"cp_customer_1",
|
||||
]);
|
||||
expect(
|
||||
merged.extra_customer_entitlements.map((entitlement) => entitlement.id),
|
||||
).toEqual(["ce_extra_entity", "ce_extra_customer"]);
|
||||
});
|
||||
|
||||
test("keeps customer fields from the entity row", () => {
|
||||
const entityRow = createRow({
|
||||
entity: { internal_id: "entity_internal_1" } as SubjectQueryRow["entity"],
|
||||
});
|
||||
const customerRow = createRow();
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.customer).toBe(entityRow.customer);
|
||||
expect(merged.entity).toBe(entityRow.entity);
|
||||
expect(merged.invoices).toBeUndefined();
|
||||
expect(merged.entity_aggregations).toBeUndefined();
|
||||
});
|
||||
|
||||
test("cap truncation drops customer-level products and all their dependent rows", () => {
|
||||
const entityProducts = Array.from(
|
||||
{ length: CUSTOMER_PRODUCT_LIMIT - 1 },
|
||||
(_, index) => createCustomerProduct({ id: `cp_entity_${index}` }),
|
||||
);
|
||||
const keptProduct = createCustomerProduct({
|
||||
id: "cp_customer_kept",
|
||||
freeTrialId: "ft_kept",
|
||||
subscriptionIds: ["sub_kept"],
|
||||
});
|
||||
const droppedProduct = createCustomerProduct({
|
||||
id: "cp_customer_dropped",
|
||||
freeTrialId: "ft_dropped",
|
||||
subscriptionIds: ["sub_dropped"],
|
||||
});
|
||||
|
||||
const entityRow = createRow({ customer_products: entityProducts });
|
||||
const customerRow = createRow({
|
||||
customer_products: [keptProduct, droppedProduct],
|
||||
customer_entitlements: [
|
||||
createCustomerEntitlement({
|
||||
id: "ce_kept",
|
||||
customerProductId: keptProduct.id,
|
||||
entitlementId: "ent_kept",
|
||||
}),
|
||||
createCustomerEntitlement({
|
||||
id: "ce_dropped",
|
||||
customerProductId: droppedProduct.id,
|
||||
entitlementId: "ent_dropped",
|
||||
}),
|
||||
],
|
||||
customer_prices: [
|
||||
createCustomerPrice({
|
||||
id: "cpr_kept",
|
||||
customerProductId: keptProduct.id,
|
||||
priceId: "price_kept",
|
||||
}),
|
||||
createCustomerPrice({
|
||||
id: "cpr_dropped",
|
||||
customerProductId: droppedProduct.id,
|
||||
priceId: "price_dropped",
|
||||
}),
|
||||
],
|
||||
rollovers: [
|
||||
{ id: "ro_kept", cus_ent_id: "ce_kept" },
|
||||
{ id: "ro_dropped", cus_ent_id: "ce_dropped" },
|
||||
] as SubjectQueryRow["rollovers"],
|
||||
replaceables: [
|
||||
{ id: "rep_kept", cus_ent_id: "ce_kept" },
|
||||
{ id: "rep_dropped", cus_ent_id: "ce_dropped" },
|
||||
] as SubjectQueryRow["replaceables"],
|
||||
products: [
|
||||
{ internal_id: keptProduct.internal_product_id },
|
||||
{ internal_id: droppedProduct.internal_product_id },
|
||||
] as SubjectQueryRow["products"],
|
||||
entitlements: [
|
||||
{ id: "ent_kept" },
|
||||
{ id: "ent_dropped" },
|
||||
] as SubjectQueryRow["entitlements"],
|
||||
prices: [
|
||||
{ id: "price_kept" },
|
||||
{ id: "price_dropped" },
|
||||
] as SubjectQueryRow["prices"],
|
||||
free_trials: [
|
||||
{ id: "ft_kept" },
|
||||
{ id: "ft_dropped" },
|
||||
] as SubjectQueryRow["free_trials"],
|
||||
subscriptions: [
|
||||
{ stripe_id: "sub_kept" },
|
||||
{ stripe_id: "sub_dropped" },
|
||||
] as SubjectQueryRow["subscriptions"],
|
||||
});
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.customer_products).toHaveLength(CUSTOMER_PRODUCT_LIMIT);
|
||||
expect(
|
||||
merged.customer_products[merged.customer_products.length - 1]?.id,
|
||||
).toBe(keptProduct.id);
|
||||
expect(merged.customer_entitlements.map((row) => row.id)).toEqual([
|
||||
"ce_kept",
|
||||
]);
|
||||
expect(merged.customer_prices.map((row) => row.id)).toEqual(["cpr_kept"]);
|
||||
expect(merged.rollovers.map((row) => row.id)).toEqual(["ro_kept"]);
|
||||
expect(merged.replaceables.map((row) => row.id)).toEqual(["rep_kept"]);
|
||||
expect(merged.products.map((row) => row.internal_id)).toContain(
|
||||
keptProduct.internal_product_id,
|
||||
);
|
||||
expect(merged.products.map((row) => row.internal_id)).not.toContain(
|
||||
droppedProduct.internal_product_id,
|
||||
);
|
||||
expect(merged.entitlements.map((row) => row.id)).toEqual(["ent_kept"]);
|
||||
expect(merged.prices.map((row) => row.id)).toEqual(["price_kept"]);
|
||||
expect(merged.free_trials.map((row) => row.id)).toEqual(["ft_kept"]);
|
||||
expect(merged.subscriptions.map((row) => row.stripe_id)).toEqual([
|
||||
"sub_kept",
|
||||
]);
|
||||
});
|
||||
|
||||
test("extras cap truncation drops the dropped extras' rollovers and entitlement refs", () => {
|
||||
const entityExtras = Array.from(
|
||||
{ length: EXTRA_CUSTOMER_ENTITLEMENT_LIMIT },
|
||||
(_, index) =>
|
||||
createCustomerEntitlement({
|
||||
id: `ce_extra_entity_${index}`,
|
||||
customerProductId: null,
|
||||
entitlementId: `ent_extra_entity_${index}`,
|
||||
}),
|
||||
);
|
||||
const droppedExtra = createCustomerEntitlement({
|
||||
id: "ce_extra_customer_dropped",
|
||||
customerProductId: null,
|
||||
entitlementId: "ent_extra_dropped",
|
||||
});
|
||||
|
||||
const entityRow = createRow({
|
||||
extra_customer_entitlements: entityExtras,
|
||||
entitlements: entityExtras.map(
|
||||
(extra) =>
|
||||
({
|
||||
id: extra.entitlement_id,
|
||||
}) as SubjectQueryRow["entitlements"][number],
|
||||
),
|
||||
});
|
||||
const customerRow = createRow({
|
||||
extra_customer_entitlements: [droppedExtra],
|
||||
rollovers: [
|
||||
{ id: "ro_dropped", cus_ent_id: droppedExtra.id },
|
||||
] as SubjectQueryRow["rollovers"],
|
||||
entitlements: [
|
||||
{ id: droppedExtra.entitlement_id },
|
||||
] as SubjectQueryRow["entitlements"],
|
||||
});
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.extra_customer_entitlements).toHaveLength(
|
||||
EXTRA_CUSTOMER_ENTITLEMENT_LIMIT,
|
||||
);
|
||||
expect(
|
||||
merged.extra_customer_entitlements.map((row) => row.id),
|
||||
).not.toContain(droppedExtra.id);
|
||||
expect(merged.rollovers).toHaveLength(0);
|
||||
expect(merged.entitlements.map((row) => row.id)).not.toContain(
|
||||
droppedExtra.entitlement_id,
|
||||
);
|
||||
});
|
||||
|
||||
test("dedupes shared catalog rows and subscriptions across both rows", () => {
|
||||
const entityProduct = createCustomerProduct({
|
||||
id: "cp_entity_1",
|
||||
internalProductId: "prod_shared",
|
||||
freeTrialId: "ft_shared",
|
||||
subscriptionIds: ["sub_shared"],
|
||||
});
|
||||
const customerProduct = createCustomerProduct({
|
||||
id: "cp_customer_1",
|
||||
internalProductId: "prod_shared",
|
||||
freeTrialId: "ft_shared",
|
||||
subscriptionIds: ["sub_shared"],
|
||||
});
|
||||
const sharedCatalog = {
|
||||
products: [{ internal_id: "prod_shared" }] as SubjectQueryRow["products"],
|
||||
prices: [{ id: "price_shared" }] as SubjectQueryRow["prices"],
|
||||
entitlements: [{ id: "ent_shared" }] as SubjectQueryRow["entitlements"],
|
||||
free_trials: [{ id: "ft_shared" }] as SubjectQueryRow["free_trials"],
|
||||
subscriptions: [
|
||||
{ stripe_id: "sub_shared" },
|
||||
] as SubjectQueryRow["subscriptions"],
|
||||
};
|
||||
|
||||
const entityRow = createRow({
|
||||
customer_products: [entityProduct],
|
||||
customer_entitlements: [
|
||||
createCustomerEntitlement({
|
||||
id: "ce_entity",
|
||||
customerProductId: entityProduct.id,
|
||||
entitlementId: "ent_shared",
|
||||
}),
|
||||
],
|
||||
customer_prices: [
|
||||
createCustomerPrice({
|
||||
id: "cpr_entity",
|
||||
customerProductId: entityProduct.id,
|
||||
priceId: "price_shared",
|
||||
}),
|
||||
],
|
||||
...sharedCatalog,
|
||||
});
|
||||
const customerRow = createRow({
|
||||
customer_products: [customerProduct],
|
||||
customer_entitlements: [
|
||||
createCustomerEntitlement({
|
||||
id: "ce_customer",
|
||||
customerProductId: customerProduct.id,
|
||||
entitlementId: "ent_shared",
|
||||
}),
|
||||
],
|
||||
customer_prices: [
|
||||
createCustomerPrice({
|
||||
id: "cpr_customer",
|
||||
customerProductId: customerProduct.id,
|
||||
priceId: "price_shared",
|
||||
}),
|
||||
],
|
||||
...sharedCatalog,
|
||||
});
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.products).toHaveLength(1);
|
||||
expect(merged.prices).toHaveLength(1);
|
||||
expect(merged.entitlements).toHaveLength(1);
|
||||
expect(merged.free_trials).toHaveLength(1);
|
||||
expect(merged.subscriptions).toHaveLength(1);
|
||||
expect(merged.customer_entitlements.map((row) => row.id)).toEqual([
|
||||
"ce_entity",
|
||||
"ce_customer",
|
||||
]);
|
||||
});
|
||||
|
||||
test("sorts merged catalog rows by id to mirror DISTINCT ON ordering", () => {
|
||||
const entityProduct = createCustomerProduct({
|
||||
id: "cp_entity_1",
|
||||
internalProductId: "prod_b",
|
||||
});
|
||||
const customerProduct = createCustomerProduct({
|
||||
id: "cp_customer_1",
|
||||
internalProductId: "prod_a",
|
||||
});
|
||||
|
||||
const entityRow = createRow({
|
||||
customer_products: [entityProduct],
|
||||
products: [{ internal_id: "prod_b" }] as SubjectQueryRow["products"],
|
||||
});
|
||||
const customerRow = createRow({
|
||||
customer_products: [customerProduct],
|
||||
products: [{ internal_id: "prod_a" }] as SubjectQueryRow["products"],
|
||||
});
|
||||
|
||||
const merged = mergeEntityAndCustomerSubjectRows({
|
||||
entityRow,
|
||||
customerRow,
|
||||
});
|
||||
|
||||
expect(merged.products.map((row) => row.internal_id)).toEqual([
|
||||
"prod_a",
|
||||
"prod_b",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,17 +1,23 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { describe, expect, mock, test } from "bun:test";
|
||||
import {
|
||||
ErrCode,
|
||||
type Feature,
|
||||
FeatureType,
|
||||
FeatureUsageType,
|
||||
} from "@autumn/shared";
|
||||
import {
|
||||
getModelCreditCost,
|
||||
getModelCreditCostBreakdown,
|
||||
} from "@/internal/features/aiCreditSystemUtils.js";
|
||||
import { getCreditCost } from "@/internal/features/creditSystemUtils.js";
|
||||
|
||||
// Uses custom/* models so pricing resolves offline (no models.dev fetch).
|
||||
mock.module("@/internal/features/utils/getModelPricing.js", () => ({
|
||||
getModelsDevPricing: async () => ({}),
|
||||
}));
|
||||
|
||||
const { getModelCreditCost, getModelCreditCostBreakdown } = await import(
|
||||
"@/internal/features/aiCreditSystemUtils.js"
|
||||
);
|
||||
const { getCreditCost } = await import(
|
||||
"@/internal/features/creditSystemUtils.js"
|
||||
);
|
||||
|
||||
// custom/* models price from model_markups; pricing data is mocked empty.
|
||||
const CUSTOM_MODEL = "custom/foo";
|
||||
|
||||
const aiCreditFeature: Feature = {
|
||||
|
||||
54
server/tests/unit/rate-limits/org-aggregate-config.test.ts
Normal file
54
server/tests/unit/rate-limits/org-aggregate-config.test.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
getOrgAggregateType,
|
||||
RATE_LIMIT_CONFIGS,
|
||||
RateLimitScope,
|
||||
RateLimitType,
|
||||
} from "@/internal/misc/rateLimiter/rateLimitConfigs.js";
|
||||
|
||||
describe("org aggregate rate limits", () => {
|
||||
test("high-volume per-customer types map to an org aggregate", () => {
|
||||
expect(getOrgAggregateType(RateLimitType.Track)).toBe(
|
||||
RateLimitType.TrackOrg,
|
||||
);
|
||||
expect(getOrgAggregateType(RateLimitType.Check)).toBe(
|
||||
RateLimitType.CheckOrg,
|
||||
);
|
||||
expect(getOrgAggregateType(RateLimitType.CustomerEntitiesGet)).toBe(
|
||||
RateLimitType.EntitiesGetOrg,
|
||||
);
|
||||
});
|
||||
|
||||
test("types without an aggregate return undefined", () => {
|
||||
expect(getOrgAggregateType(RateLimitType.General)).toBeUndefined();
|
||||
expect(getOrgAggregateType(RateLimitType.Attach)).toBeUndefined();
|
||||
expect(getOrgAggregateType(RateLimitType.TrackOrg)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("aggregate configs are org-scoped, redis-backed, 60s windows", () => {
|
||||
const aggregates = [
|
||||
RateLimitType.TrackOrg,
|
||||
RateLimitType.CheckOrg,
|
||||
RateLimitType.EntitiesGetOrg,
|
||||
];
|
||||
for (const type of aggregates) {
|
||||
const config = RATE_LIMIT_CONFIGS[type];
|
||||
expect(config.scope).toBe(RateLimitScope.Org);
|
||||
expect(config.notInRedis).toBe(false);
|
||||
expect(config.windowMs).toBe(60_000);
|
||||
expect(config.limit).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test("check/track aggregates degrade (fail open) instead of rejecting", () => {
|
||||
expect(RATE_LIMIT_CONFIGS[RateLimitType.CheckOrg].overLimit).toBe(
|
||||
"degrade",
|
||||
);
|
||||
expect(RATE_LIMIT_CONFIGS[RateLimitType.TrackOrg].overLimit).toBe(
|
||||
"degrade",
|
||||
);
|
||||
expect(
|
||||
RATE_LIMIT_CONFIGS[RateLimitType.EntitiesGetOrg].overLimit,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -19,7 +19,7 @@ export const PAGINATION_CONFIGS: Record<PaginationType, PaginationConfig> = {
|
||||
},
|
||||
[PaginationType.ListEntities]: {
|
||||
defaultLimit: PaginationDefaults.DefaultLimit,
|
||||
maxLimit: PaginationDefaults.MaxLimit,
|
||||
maxLimit: PaginationDefaults.SchemaHardCeiling,
|
||||
},
|
||||
[PaginationType.SearchCustomers]: {
|
||||
defaultLimit: PaginationDefaults.DefaultLimit,
|
||||
@@ -30,5 +30,3 @@ export const PAGINATION_CONFIGS: Record<PaginationType, PaginationConfig> = {
|
||||
maxLimit: PaginationDefaults.MaxLimit,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
|
||||
1
shared/drizzle/0011_easy_spot.sql
Normal file
1
shared/drizzle/0011_easy_spot.sql
Normal file
@@ -0,0 +1 @@
|
||||
CREATE INDEX CONCURRENTLY "idx_entities_customer_created_at" ON "entities" USING btree ("internal_customer_id","created_at" DESC,"id" DESC);
|
||||
7629
shared/drizzle/meta/0011_snapshot.json
Normal file
7629
shared/drizzle/meta/0011_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -78,6 +78,13 @@
|
||||
"when": 1780990532501,
|
||||
"tag": "0010_magenta_misty_knight",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 11,
|
||||
"version": "7",
|
||||
"when": 1781085888296,
|
||||
"tag": "0011_easy_spot",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -76,5 +76,10 @@ export const entities = pgTable(
|
||||
sql`${table.created_at} DESC`,
|
||||
sql`${table.id} DESC`,
|
||||
),
|
||||
index("idx_entities_customer_created_at").on(
|
||||
table.internal_customer_id,
|
||||
sql`${table.created_at} DESC`,
|
||||
sql`${table.id} DESC`,
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user