fix: v1.2 cus feature schema for python

This commit is contained in:
John Yeo
2025-11-20 16:58:07 +00:00
parent ec9c473c33
commit 631b12d5f9
14 changed files with 314 additions and 61 deletions

View File

@@ -0,0 +1,17 @@
import { CusEntService } from "../src/internal/customers/cusProducts/cusEnts/CusEntitlementService";
import { initDrizzle } from "../src/db/initDrizzle";
import { clearCusEntsFromCache } from "../src/cron/cronUtils";
const main = async () => {
const { db } = initDrizzle();
const cusEnts = await CusEntService.getActiveResetPassed({
db,
batchSize: 500,
});
await clearCusEntsFromCache({ cusEnts });
};
await main();
process.exit(0);

View File

@@ -0,0 +1,103 @@
-- batchDeleteCustomers.lua
-- Atomically deletes multiple customers and all their associated entity caches
-- ARGV[1]: JSON array of {orgId, env, customerId} objects
-- Returns: number of keys deleted
local customersJson = ARGV[1]
local customers = cjson.decode(customersJson)
local allKeysToDelete = {}
-- Helper function to add balance-related keys for a cache key
local function addBalanceKeys(keysToDelete, cacheKey, featureIds)
if not featureIds or #featureIds == 0 then
return
end
for _, featureId in ipairs(featureIds) do
local balanceKey = buildBalanceCacheKey(cacheKey, featureId)
table.insert(keysToDelete, balanceKey)
-- Get the balance HSET to find breakdown/rollover counts
local balanceData = redis.call("HGETALL", balanceKey)
if balanceData and #balanceData > 0 then
-- Convert array to hash table
local balanceHash = {}
for i = 1, #balanceData, 2 do
balanceHash[balanceData[i]] = balanceData[i + 1]
end
-- Delete rollover keys
local rolloverCount = tonumber(balanceHash["_rollover_count"]) or 0
for i = 0, rolloverCount - 1 do
table.insert(keysToDelete, buildRolloverCacheKey(cacheKey, featureId, i))
end
-- Delete breakdown keys
local breakdownCount = tonumber(balanceHash["_breakdown_count"]) or 0
for i = 0, breakdownCount - 1 do
table.insert(keysToDelete, buildBreakdownCacheKey(cacheKey, featureId, i))
end
end
end
end
-- Process each customer
for _, customerInfo in ipairs(customers) do
local orgId = customerInfo.orgId
local env = customerInfo.env
local customerId = customerInfo.customerId
-- Build versioned cache key using shared utility
local cacheKey = buildCustomerCacheKey(orgId, env, customerId)
-- Get the customer base JSON to find entity and feature IDs
local baseJson = redis.call("GET", cacheKey)
-- Skip if customer not in cache
if baseJson then
table.insert(allKeysToDelete, cacheKey)
local success, customer = pcall(cjson.decode, baseJson)
if success and customer then
local entityIds = customer._entityIds or {}
local balanceFeatureIds = customer._balanceFeatureIds or {}
-- Add customer balance keys (with rollover/breakdown)
addBalanceKeys(allKeysToDelete, cacheKey, balanceFeatureIds)
-- Process each entity
for _, entityId in ipairs(entityIds) do
local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId)
table.insert(allKeysToDelete, entityCacheKey)
-- Get entity to find its feature IDs
local entityJson = redis.call("GET", entityCacheKey)
if entityJson then
local entitySuccess, entity = pcall(cjson.decode, entityJson)
if entitySuccess and entity then
local entityFeatureIds = entity._balanceFeatureIds or {}
-- Add entity balance keys (with rollover/breakdown)
addBalanceKeys(allKeysToDelete, entityCacheKey, entityFeatureIds)
end
end
end
end
end
end
-- Use UNLINK instead of DEL for async deletion (non-blocking)
local deletedCount = 0
if #allKeysToDelete > 0 then
-- UNLINK has a limit, so batch in chunks of 1000 keys
local chunkSize = 1000
for i = 1, #allKeysToDelete, chunkSize do
local chunk = {}
for j = i, math.min(i + chunkSize - 1, #allKeysToDelete) do
table.insert(chunk, allKeysToDelete[j])
end
deletedCount = deletedCount + redis.call("UNLINK", unpack(chunk))
end
end
return deletedCount

View File

@@ -101,6 +101,13 @@ const deleteCustomerScript = readFileSync(
);
export const DELETE_CUSTOMER_SCRIPT = `${CACHE_KEY_UTILS}\n${deleteCustomerScript}`;
// Prepend cache key utils to BATCH_DELETE_CUSTOMERS_SCRIPT
const batchDeleteCustomersScript = readFileSync(
join(__dirname, "cusLuaScripts/batchDeleteCustomers.lua"),
"utf-8",
);
export const BATCH_DELETE_CUSTOMERS_SCRIPT = `${CACHE_KEY_UTILS}\n${batchDeleteCustomersScript}`;
// ============================================================================
// ENTITY SCRIPTS
// ============================================================================

View File

@@ -282,11 +282,13 @@ local function mergeFeatureBalances(targetBalance, sourceBalance)
-- Merge rollover balances
if sourceBalance.rollovers and #sourceBalance.rollovers > 0 then
-- Both have rollovers, merge them
for i, targetRollover in ipairs(targetBalance.rollovers) do
local sourceRollover = sourceBalance.rollovers[i]
if sourceRollover then
targetRollover.balance = toNum(targetRollover.balance) + toNum(sourceRollover.balance)
-- Both have rollovers, merge them
if targetBalance.rollovers and #targetBalance.rollovers > 0 then
for i, targetRollover in ipairs(targetBalance.rollovers) do
local sourceRollover = sourceBalance.rollovers[i]
if sourceRollover then
targetRollover.balance = toNum(targetRollover.balance) + toNum(sourceRollover.balance)
end
end
end
end

View File

@@ -5,17 +5,15 @@ import { format } from "date-fns";
import { initDrizzle } from "../db/initDrizzle.js";
import { CusEntService } from "../internal/customers/cusProducts/cusEnts/CusEntitlementService.js";
import { notNullish } from "../utils/genUtils.js";
import { resetCustomerEntitlement } from "./cronUtils.js";
import {
clearCusEntsFromCache,
resetCustomerEntitlement,
} from "./cronUtils.js";
import { runProductCron } from "./productCron/runProductCron.js";
const { db, client } = initDrizzle();
export const cronTask = async () => {
console.log(
"\n----------------------------------\nRUNNING RESET CRON:",
format(new UTCDate(), "yyyy-MM-dd HH:mm:ss"),
);
try {
const cusEnts: ResetCusEnt[] = await CusEntService.getActiveResetPassed({
db,
@@ -43,6 +41,8 @@ export const cronTask = async () => {
data: toUpsert as CustomerEntitlement[],
});
console.log(`Upserted ${toUpsert.length} short entitlements`);
await clearCusEntsFromCache({ cusEnts: batch });
}
console.log(

View File

@@ -5,6 +5,7 @@ import {
type FullCusEntWithProduct,
type FullEntitlement,
getStartingBalance,
notNullish,
type Organization,
type ResetCusEnt,
} from "@autumn/shared";
@@ -19,12 +20,11 @@ import { getRelatedCusPrice } from "@/internal/customers/cusProducts/cusEnts/cus
import { getRolloverUpdates } from "@/internal/customers/cusProducts/cusEnts/cusRollovers/rolloverUtils.js";
import { getResetBalancesUpdate } from "@/internal/customers/cusProducts/cusEnts/groupByUtils.js";
import { CusPriceService } from "@/internal/customers/cusProducts/cusPrices/CusPriceService.js";
import { OrgService } from "@/internal/orgs/OrgService.js";
import { getEntOptions } from "@/internal/products/prices/priceUtils.js";
import { getNextResetAt } from "@/utils/timeUtils.js";
import type { DrizzleCli } from "../db/initDrizzle.js";
import { RolloverService } from "../internal/customers/cusProducts/cusEnts/cusRollovers/RolloverService.js";
import { deleteCachedApiCustomer } from "../internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.js";
import { batchDeleteCachedCustomers } from "../internal/customers/cusUtils/apiCusCacheUtils/batchDeleteCachedCustomers.js";
const checkSubAnchor = async ({
db,
@@ -134,16 +134,16 @@ const handleShortDurationCusEnt = async ({
`Reseting short cus ent (${cusEnt.feature_id}) [${ent.interval}], customer: ${cusEnt.customer_id}, org: ${cusEnt.customer.org_id}`,
);
const org = await OrgService.get({
db,
orgId: cusEnt.customer.org_id,
});
// const org = await OrgService.get({
// db,
// orgId: cusEnt.customer.org_id,
// });
await deleteCachedApiCustomer({
customerId: cusEnt.customer.id!,
orgId: org.id,
env: cusEnt.customer.env,
});
// await deleteCachedApiCustomer({
// customerId: cusEnt.customer.id!,
// orgId: org.id,
// env: cusEnt.customer.env,
// });
return newCusEnt;
};
@@ -294,19 +294,37 @@ export const resetCustomerEntitlement = async ({
)}`,
);
const org = await OrgService.get({
db,
orgId: cusEnt.customer.org_id,
});
// const org = await OrgService.get({
// db,
// orgId: cusEnt.customer.org_id,
// });
await deleteCachedApiCustomer({
customerId: cusEnt.customer.id!,
orgId: org.id,
env: cusEnt.customer.env,
});
// await deleteCachedApiCustomer({
// customerId: cusEnt.customer.id!,
// orgId: org.id,
// env: cusEnt.customer.env,
// });
} catch (error: any) {
console.log(
`Failed to reset ${cusEnt.id} | ${cusEnt.customer_id} | ${cusEnt.feature_id}, error: ${error}`,
);
}
};
export const clearCusEntsFromCache = async ({
cusEnts,
}: {
cusEnts: ResetCusEnt[];
}) => {
const customersToDelete = cusEnts
.filter((ce) => notNullish(ce.customer.id))
.map((cusEnt) => ({
orgId: cusEnt.customer.org_id,
env: cusEnt.customer.env,
customerId: cusEnt.customer.id!,
}));
if (customersToDelete.length === 0) return;
await batchDeleteCachedCustomers({ customers: customersToDelete });
};

View File

@@ -4,10 +4,11 @@ import {
customerPrices,
customerProducts,
customers,
notNullish,
} from "@autumn/shared";
import { and, eq, inArray, isNotNull, lt, notExists, sql } from "drizzle-orm";
import { db } from "@/db/initDrizzle.js";
import { deleteCachedApiCustomer } from "../../internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer";
import { batchDeleteCachedCustomers } from "../../internal/customers/cusUtils/apiCusCacheUtils/batchDeleteCachedCustomers";
export const runProductCron = async () => {
console.log("Running product cron");
@@ -64,17 +65,26 @@ export const runProductCron = async () => {
`Expired batch of ${i + batch.length}/${results.length} customer products`,
);
const clearCachePromises = [];
for (const result of batch) {
clearCachePromises.push(
deleteCachedApiCustomer({
customerId: result.customers.id ?? "",
orgId: result.customers.org_id,
env: result.customers.env,
}),
);
}
await Promise.all(clearCachePromises);
await batchDeleteCachedCustomers({
customers: batch
.filter((r) => notNullish(r.customers.id))
.map((r) => ({
orgId: r.customers.org_id,
env: r.customers.env,
customerId: r.customers.id!,
})),
});
// const clearCachePromises = [];
// for (const result of batch) {
// clearCachePromises.push(
// deleteCachedApiCustomer({
// customerId: result.customers.id ?? "",
// orgId: result.customers.org_id,
// env: result.customers.env,
// }),
// );
// }
// await Promise.all(clearCachePromises);
}
return results;

View File

@@ -1,5 +1,6 @@
import { Redis } from "ioredis";
import {
BATCH_DELETE_CUSTOMERS_SCRIPT,
DELETE_CUSTOMER_SCRIPT,
GET_CUSTOMER_SCRIPT,
GET_ENTITY_SCRIPT,
@@ -93,6 +94,11 @@ redis.defineCommand("deleteCustomer", {
lua: DELETE_CUSTOMER_SCRIPT,
});
redis.defineCommand("batchDeleteCustomers", {
numberOfKeys: 0,
lua: BATCH_DELETE_CUSTOMERS_SCRIPT,
});
// Add type definitions
declare module "ioredis" {
interface RedisCommander {
@@ -157,6 +163,7 @@ declare module "ioredis" {
env: string,
customerId: string,
): Promise<number>;
batchDeleteCustomers(customersJson: string): Promise<number>;
}
}

View File

@@ -0,0 +1,75 @@
import * as Sentry from "@sentry/bun";
import { redis } from "@/external/redis/initRedis.js";
/**
* Batch delete multiple customer caches in one Redis operation
* Much more efficient than calling deleteCachedApiCustomer multiple times
* @param customers Array of {orgId, env, customerId} to delete
* @returns Number of keys deleted
*/
export const batchDeleteCachedCustomers = async ({
customers,
}: {
customers: Array<{
orgId: string;
env: string;
customerId: string;
}>;
}): Promise<number> => {
if (redis.status !== "ready") {
console.warn("❗️ Redis not ready, skipping batch cache deletion", {
status: redis.status,
count: customers.length,
});
return 0;
}
if (customers.length === 0) {
return 0;
}
try {
// Group customers by orgId to avoid Redis Cluster hash slot errors
// All keys in a Lua script must be in the same hash slot (same {orgId})
const customersByOrg = new Map<string, typeof customers>();
for (const customer of customers) {
const key = customer.orgId;
if (!customersByOrg.has(key)) {
customersByOrg.set(key, []);
}
customersByOrg.get(key)?.push(customer);
}
// Use pipeline to batch all org deletions into one network round trip
const pipeline = redis.pipeline();
for (const orgCustomers of customersByOrg.values()) {
pipeline.batchDeleteCustomers(JSON.stringify(orgCustomers));
}
const results = await pipeline.exec();
// Sum up all deleted counts
let totalDeleted = 0;
if (results) {
for (const [error, result] of results) {
if (error) {
console.error("Error in pipeline batch delete:", error);
throw error;
}
totalDeleted += result as number;
}
}
console.log(
`Batch deleted ${totalDeleted} cache keys for ${customers.length} customers across ${customersByOrg.size} orgs`,
);
return totalDeleted;
} catch (error) {
console.error("Error batch deleting customers:", error);
Sentry.captureException(error);
throw error;
}
};

View File

@@ -152,7 +152,6 @@ describe(`${chalk.yellowBright(`${testCase}: Testing rollovers for feature item
expect(nonCachedMsgesFeature.rollovers[0].balance).toBe(expectedRollover);
}
});
return;
test("should reset again and have correct rollovers", async () => {
await resetAndGetCusEnt({

View File

@@ -1,5 +1,8 @@
import type { Customer } from "@autumn/shared";
import { resetCustomerEntitlement } from "@/cron/cronUtils.js";
import {
clearCusEntsFromCache,
resetCustomerEntitlement,
} from "@/cron/cronUtils.js";
import type { DrizzleCli } from "@/db/initDrizzle.js";
import { CusEntService } from "@/internal/customers/cusProducts/cusEnts/CusEntitlementService.js";
import { cusProductToCusEnt } from "@/internal/customers/cusProducts/cusProductUtils/convertCusProduct.js";
@@ -28,14 +31,18 @@ export const resetAndGetCusEnt = async ({
featureId,
});
const resetCusEnt = {
...cusEnt!,
customer,
};
const updatedCusEnt = await resetCustomerEntitlement({
db,
cusEnt: {
...cusEnt!,
customer,
},
cusEnt: resetCusEnt,
});
await clearCusEntsFromCache({ cusEnts: [resetCusEnt] });
if (updatedCusEnt) {
await CusEntService.upsert({
db,

View File

@@ -84,7 +84,7 @@ const toV3Type = ({ feature }: { feature?: ApiFeatureV1 }) => {
return ApiFeatureType.ContinuousUse;
}
} else if (feature?.type === FeatureType.CreditSystem) {
return ApiFeatureType.CreditSystem;
return ApiFeatureType.SingleUsage;
} else {
return ApiFeatureType.Static;
}

View File

@@ -68,6 +68,12 @@ export function UsageReset({ showBillingLabel = false }: UsageResetProps) {
<SelectContent>
{Object.values(isFeaturePrice ? BillingInterval : EntInterval)
.filter((i) => {
if (
i === EntInterval.Minute &&
itemToEntInterval({ item }) !== EntInterval.Minute
) {
return false;
}
if (isFeaturePrice && item.usage_model === UsageModel.PayPerUse) {
return i !== BillingInterval.OneOff;
}

View File

@@ -95,22 +95,24 @@ export const SelectResetCycle = () => {
<span className="block truncate overflow-hidden text-ellipsis max-w-full">
{getIntervalText({
interval,
intervalCount: item!.interval_count,
intervalCount: item?.interval_count ?? 1,
})}
</span>
</SelectValue>
</SelectTrigger>
<SelectContent className="w-48">
{Object.values(EntInterval).map((intervalOption) => {
const isSelected = intervalOption === interval;
return (
<SelectIntervalItem
key={intervalOption}
interval={intervalOption}
isSelected={isSelected}
/>
);
})}
{Object.values(EntInterval)
.filter((i) => i !== EntInterval.Minute)
.map((intervalOption) => {
const isSelected = intervalOption === interval;
return (
<SelectIntervalItem
key={intervalOption}
interval={intervalOption}
isSelected={isSelected}
/>
);
})}
<CustomiseIntervalPopover />
</SelectContent>
</Select>