fix: v1.2 cus feature schema for python
This commit is contained in:
17
server/experiments/batchDelete.ts
Normal file
17
server/experiments/batchDelete.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { CusEntService } from "../src/internal/customers/cusProducts/cusEnts/CusEntitlementService";
|
||||
import { initDrizzle } from "../src/db/initDrizzle";
|
||||
import { clearCusEntsFromCache } from "../src/cron/cronUtils";
|
||||
|
||||
const main = async () => {
|
||||
const { db } = initDrizzle();
|
||||
const cusEnts = await CusEntService.getActiveResetPassed({
|
||||
db,
|
||||
batchSize: 500,
|
||||
});
|
||||
|
||||
|
||||
await clearCusEntsFromCache({ cusEnts });
|
||||
};
|
||||
|
||||
await main();
|
||||
process.exit(0);
|
||||
103
server/src/_luaScripts/cusLuaScripts/batchDeleteCustomers.lua
Normal file
103
server/src/_luaScripts/cusLuaScripts/batchDeleteCustomers.lua
Normal file
@@ -0,0 +1,103 @@
|
||||
-- batchDeleteCustomers.lua
|
||||
-- Atomically deletes multiple customers and all their associated entity caches
|
||||
-- ARGV[1]: JSON array of {orgId, env, customerId} objects
|
||||
-- Returns: number of keys deleted
|
||||
|
||||
local customersJson = ARGV[1]
|
||||
local customers = cjson.decode(customersJson)
|
||||
local allKeysToDelete = {}
|
||||
|
||||
-- Helper function to add balance-related keys for a cache key
|
||||
local function addBalanceKeys(keysToDelete, cacheKey, featureIds)
|
||||
if not featureIds or #featureIds == 0 then
|
||||
return
|
||||
end
|
||||
|
||||
for _, featureId in ipairs(featureIds) do
|
||||
local balanceKey = buildBalanceCacheKey(cacheKey, featureId)
|
||||
table.insert(keysToDelete, balanceKey)
|
||||
|
||||
-- Get the balance HSET to find breakdown/rollover counts
|
||||
local balanceData = redis.call("HGETALL", balanceKey)
|
||||
if balanceData and #balanceData > 0 then
|
||||
-- Convert array to hash table
|
||||
local balanceHash = {}
|
||||
for i = 1, #balanceData, 2 do
|
||||
balanceHash[balanceData[i]] = balanceData[i + 1]
|
||||
end
|
||||
|
||||
-- Delete rollover keys
|
||||
local rolloverCount = tonumber(balanceHash["_rollover_count"]) or 0
|
||||
for i = 0, rolloverCount - 1 do
|
||||
table.insert(keysToDelete, buildRolloverCacheKey(cacheKey, featureId, i))
|
||||
end
|
||||
|
||||
-- Delete breakdown keys
|
||||
local breakdownCount = tonumber(balanceHash["_breakdown_count"]) or 0
|
||||
for i = 0, breakdownCount - 1 do
|
||||
table.insert(keysToDelete, buildBreakdownCacheKey(cacheKey, featureId, i))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- Process each customer
|
||||
for _, customerInfo in ipairs(customers) do
|
||||
local orgId = customerInfo.orgId
|
||||
local env = customerInfo.env
|
||||
local customerId = customerInfo.customerId
|
||||
|
||||
-- Build versioned cache key using shared utility
|
||||
local cacheKey = buildCustomerCacheKey(orgId, env, customerId)
|
||||
|
||||
-- Get the customer base JSON to find entity and feature IDs
|
||||
local baseJson = redis.call("GET", cacheKey)
|
||||
|
||||
-- Skip if customer not in cache
|
||||
if baseJson then
|
||||
table.insert(allKeysToDelete, cacheKey)
|
||||
local success, customer = pcall(cjson.decode, baseJson)
|
||||
if success and customer then
|
||||
local entityIds = customer._entityIds or {}
|
||||
local balanceFeatureIds = customer._balanceFeatureIds or {}
|
||||
|
||||
-- Add customer balance keys (with rollover/breakdown)
|
||||
addBalanceKeys(allKeysToDelete, cacheKey, balanceFeatureIds)
|
||||
|
||||
-- Process each entity
|
||||
for _, entityId in ipairs(entityIds) do
|
||||
local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId)
|
||||
table.insert(allKeysToDelete, entityCacheKey)
|
||||
|
||||
-- Get entity to find its feature IDs
|
||||
local entityJson = redis.call("GET", entityCacheKey)
|
||||
if entityJson then
|
||||
local entitySuccess, entity = pcall(cjson.decode, entityJson)
|
||||
if entitySuccess and entity then
|
||||
local entityFeatureIds = entity._balanceFeatureIds or {}
|
||||
|
||||
-- Add entity balance keys (with rollover/breakdown)
|
||||
addBalanceKeys(allKeysToDelete, entityCacheKey, entityFeatureIds)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- Use UNLINK instead of DEL for async deletion (non-blocking)
|
||||
local deletedCount = 0
|
||||
if #allKeysToDelete > 0 then
|
||||
-- UNLINK has a limit, so batch in chunks of 1000 keys
|
||||
local chunkSize = 1000
|
||||
for i = 1, #allKeysToDelete, chunkSize do
|
||||
local chunk = {}
|
||||
for j = i, math.min(i + chunkSize - 1, #allKeysToDelete) do
|
||||
table.insert(chunk, allKeysToDelete[j])
|
||||
end
|
||||
deletedCount = deletedCount + redis.call("UNLINK", unpack(chunk))
|
||||
end
|
||||
end
|
||||
|
||||
return deletedCount
|
||||
|
||||
@@ -101,6 +101,13 @@ const deleteCustomerScript = readFileSync(
|
||||
);
|
||||
export const DELETE_CUSTOMER_SCRIPT = `${CACHE_KEY_UTILS}\n${deleteCustomerScript}`;
|
||||
|
||||
// Prepend cache key utils to BATCH_DELETE_CUSTOMERS_SCRIPT
|
||||
const batchDeleteCustomersScript = readFileSync(
|
||||
join(__dirname, "cusLuaScripts/batchDeleteCustomers.lua"),
|
||||
"utf-8",
|
||||
);
|
||||
export const BATCH_DELETE_CUSTOMERS_SCRIPT = `${CACHE_KEY_UTILS}\n${batchDeleteCustomersScript}`;
|
||||
|
||||
// ============================================================================
|
||||
// ENTITY SCRIPTS
|
||||
// ============================================================================
|
||||
|
||||
@@ -282,11 +282,13 @@ local function mergeFeatureBalances(targetBalance, sourceBalance)
|
||||
|
||||
-- Merge rollover balances
|
||||
if sourceBalance.rollovers and #sourceBalance.rollovers > 0 then
|
||||
-- Both have rollovers, merge them
|
||||
for i, targetRollover in ipairs(targetBalance.rollovers) do
|
||||
local sourceRollover = sourceBalance.rollovers[i]
|
||||
if sourceRollover then
|
||||
targetRollover.balance = toNum(targetRollover.balance) + toNum(sourceRollover.balance)
|
||||
-- Both have rollovers, merge them
|
||||
if targetBalance.rollovers and #targetBalance.rollovers > 0 then
|
||||
for i, targetRollover in ipairs(targetBalance.rollovers) do
|
||||
local sourceRollover = sourceBalance.rollovers[i]
|
||||
if sourceRollover then
|
||||
targetRollover.balance = toNum(targetRollover.balance) + toNum(sourceRollover.balance)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -5,17 +5,15 @@ import { format } from "date-fns";
|
||||
import { initDrizzle } from "../db/initDrizzle.js";
|
||||
import { CusEntService } from "../internal/customers/cusProducts/cusEnts/CusEntitlementService.js";
|
||||
import { notNullish } from "../utils/genUtils.js";
|
||||
import { resetCustomerEntitlement } from "./cronUtils.js";
|
||||
import {
|
||||
clearCusEntsFromCache,
|
||||
resetCustomerEntitlement,
|
||||
} from "./cronUtils.js";
|
||||
import { runProductCron } from "./productCron/runProductCron.js";
|
||||
|
||||
const { db, client } = initDrizzle();
|
||||
|
||||
export const cronTask = async () => {
|
||||
console.log(
|
||||
"\n----------------------------------\nRUNNING RESET CRON:",
|
||||
format(new UTCDate(), "yyyy-MM-dd HH:mm:ss"),
|
||||
);
|
||||
|
||||
try {
|
||||
const cusEnts: ResetCusEnt[] = await CusEntService.getActiveResetPassed({
|
||||
db,
|
||||
@@ -43,6 +41,8 @@ export const cronTask = async () => {
|
||||
data: toUpsert as CustomerEntitlement[],
|
||||
});
|
||||
console.log(`Upserted ${toUpsert.length} short entitlements`);
|
||||
|
||||
await clearCusEntsFromCache({ cusEnts: batch });
|
||||
}
|
||||
|
||||
console.log(
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
type FullCusEntWithProduct,
|
||||
type FullEntitlement,
|
||||
getStartingBalance,
|
||||
notNullish,
|
||||
type Organization,
|
||||
type ResetCusEnt,
|
||||
} from "@autumn/shared";
|
||||
@@ -19,12 +20,11 @@ import { getRelatedCusPrice } from "@/internal/customers/cusProducts/cusEnts/cus
|
||||
import { getRolloverUpdates } from "@/internal/customers/cusProducts/cusEnts/cusRollovers/rolloverUtils.js";
|
||||
import { getResetBalancesUpdate } from "@/internal/customers/cusProducts/cusEnts/groupByUtils.js";
|
||||
import { CusPriceService } from "@/internal/customers/cusProducts/cusPrices/CusPriceService.js";
|
||||
import { OrgService } from "@/internal/orgs/OrgService.js";
|
||||
import { getEntOptions } from "@/internal/products/prices/priceUtils.js";
|
||||
import { getNextResetAt } from "@/utils/timeUtils.js";
|
||||
import type { DrizzleCli } from "../db/initDrizzle.js";
|
||||
import { RolloverService } from "../internal/customers/cusProducts/cusEnts/cusRollovers/RolloverService.js";
|
||||
import { deleteCachedApiCustomer } from "../internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.js";
|
||||
import { batchDeleteCachedCustomers } from "../internal/customers/cusUtils/apiCusCacheUtils/batchDeleteCachedCustomers.js";
|
||||
|
||||
const checkSubAnchor = async ({
|
||||
db,
|
||||
@@ -134,16 +134,16 @@ const handleShortDurationCusEnt = async ({
|
||||
`Reseting short cus ent (${cusEnt.feature_id}) [${ent.interval}], customer: ${cusEnt.customer_id}, org: ${cusEnt.customer.org_id}`,
|
||||
);
|
||||
|
||||
const org = await OrgService.get({
|
||||
db,
|
||||
orgId: cusEnt.customer.org_id,
|
||||
});
|
||||
// const org = await OrgService.get({
|
||||
// db,
|
||||
// orgId: cusEnt.customer.org_id,
|
||||
// });
|
||||
|
||||
await deleteCachedApiCustomer({
|
||||
customerId: cusEnt.customer.id!,
|
||||
orgId: org.id,
|
||||
env: cusEnt.customer.env,
|
||||
});
|
||||
// await deleteCachedApiCustomer({
|
||||
// customerId: cusEnt.customer.id!,
|
||||
// orgId: org.id,
|
||||
// env: cusEnt.customer.env,
|
||||
// });
|
||||
|
||||
return newCusEnt;
|
||||
};
|
||||
@@ -294,19 +294,37 @@ export const resetCustomerEntitlement = async ({
|
||||
)}`,
|
||||
);
|
||||
|
||||
const org = await OrgService.get({
|
||||
db,
|
||||
orgId: cusEnt.customer.org_id,
|
||||
});
|
||||
// const org = await OrgService.get({
|
||||
// db,
|
||||
// orgId: cusEnt.customer.org_id,
|
||||
// });
|
||||
|
||||
await deleteCachedApiCustomer({
|
||||
customerId: cusEnt.customer.id!,
|
||||
orgId: org.id,
|
||||
env: cusEnt.customer.env,
|
||||
});
|
||||
// await deleteCachedApiCustomer({
|
||||
// customerId: cusEnt.customer.id!,
|
||||
// orgId: org.id,
|
||||
// env: cusEnt.customer.env,
|
||||
// });
|
||||
} catch (error: any) {
|
||||
console.log(
|
||||
`Failed to reset ${cusEnt.id} | ${cusEnt.customer_id} | ${cusEnt.feature_id}, error: ${error}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const clearCusEntsFromCache = async ({
|
||||
cusEnts,
|
||||
}: {
|
||||
cusEnts: ResetCusEnt[];
|
||||
}) => {
|
||||
const customersToDelete = cusEnts
|
||||
.filter((ce) => notNullish(ce.customer.id))
|
||||
.map((cusEnt) => ({
|
||||
orgId: cusEnt.customer.org_id,
|
||||
env: cusEnt.customer.env,
|
||||
customerId: cusEnt.customer.id!,
|
||||
}));
|
||||
|
||||
if (customersToDelete.length === 0) return;
|
||||
|
||||
await batchDeleteCachedCustomers({ customers: customersToDelete });
|
||||
};
|
||||
|
||||
@@ -4,10 +4,11 @@ import {
|
||||
customerPrices,
|
||||
customerProducts,
|
||||
customers,
|
||||
notNullish,
|
||||
} from "@autumn/shared";
|
||||
import { and, eq, inArray, isNotNull, lt, notExists, sql } from "drizzle-orm";
|
||||
import { db } from "@/db/initDrizzle.js";
|
||||
import { deleteCachedApiCustomer } from "../../internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer";
|
||||
import { batchDeleteCachedCustomers } from "../../internal/customers/cusUtils/apiCusCacheUtils/batchDeleteCachedCustomers";
|
||||
|
||||
export const runProductCron = async () => {
|
||||
console.log("Running product cron");
|
||||
@@ -64,17 +65,26 @@ export const runProductCron = async () => {
|
||||
`Expired batch of ${i + batch.length}/${results.length} customer products`,
|
||||
);
|
||||
|
||||
const clearCachePromises = [];
|
||||
for (const result of batch) {
|
||||
clearCachePromises.push(
|
||||
deleteCachedApiCustomer({
|
||||
customerId: result.customers.id ?? "",
|
||||
orgId: result.customers.org_id,
|
||||
env: result.customers.env,
|
||||
}),
|
||||
);
|
||||
}
|
||||
await Promise.all(clearCachePromises);
|
||||
await batchDeleteCachedCustomers({
|
||||
customers: batch
|
||||
.filter((r) => notNullish(r.customers.id))
|
||||
.map((r) => ({
|
||||
orgId: r.customers.org_id,
|
||||
env: r.customers.env,
|
||||
customerId: r.customers.id!,
|
||||
})),
|
||||
});
|
||||
// const clearCachePromises = [];
|
||||
// for (const result of batch) {
|
||||
// clearCachePromises.push(
|
||||
// deleteCachedApiCustomer({
|
||||
// customerId: result.customers.id ?? "",
|
||||
// orgId: result.customers.org_id,
|
||||
// env: result.customers.env,
|
||||
// }),
|
||||
// );
|
||||
// }
|
||||
// await Promise.all(clearCachePromises);
|
||||
}
|
||||
|
||||
return results;
|
||||
|
||||
7
server/src/external/redis/initRedis.ts
vendored
7
server/src/external/redis/initRedis.ts
vendored
@@ -1,5 +1,6 @@
|
||||
import { Redis } from "ioredis";
|
||||
import {
|
||||
BATCH_DELETE_CUSTOMERS_SCRIPT,
|
||||
DELETE_CUSTOMER_SCRIPT,
|
||||
GET_CUSTOMER_SCRIPT,
|
||||
GET_ENTITY_SCRIPT,
|
||||
@@ -93,6 +94,11 @@ redis.defineCommand("deleteCustomer", {
|
||||
lua: DELETE_CUSTOMER_SCRIPT,
|
||||
});
|
||||
|
||||
redis.defineCommand("batchDeleteCustomers", {
|
||||
numberOfKeys: 0,
|
||||
lua: BATCH_DELETE_CUSTOMERS_SCRIPT,
|
||||
});
|
||||
|
||||
// Add type definitions
|
||||
declare module "ioredis" {
|
||||
interface RedisCommander {
|
||||
@@ -157,6 +163,7 @@ declare module "ioredis" {
|
||||
env: string,
|
||||
customerId: string,
|
||||
): Promise<number>;
|
||||
batchDeleteCustomers(customersJson: string): Promise<number>;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import * as Sentry from "@sentry/bun";
|
||||
import { redis } from "@/external/redis/initRedis.js";
|
||||
|
||||
/**
|
||||
* Batch delete multiple customer caches in one Redis operation
|
||||
* Much more efficient than calling deleteCachedApiCustomer multiple times
|
||||
* @param customers Array of {orgId, env, customerId} to delete
|
||||
* @returns Number of keys deleted
|
||||
*/
|
||||
export const batchDeleteCachedCustomers = async ({
|
||||
customers,
|
||||
}: {
|
||||
customers: Array<{
|
||||
orgId: string;
|
||||
env: string;
|
||||
customerId: string;
|
||||
}>;
|
||||
}): Promise<number> => {
|
||||
if (redis.status !== "ready") {
|
||||
console.warn("❗️ Redis not ready, skipping batch cache deletion", {
|
||||
status: redis.status,
|
||||
count: customers.length,
|
||||
});
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (customers.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
// Group customers by orgId to avoid Redis Cluster hash slot errors
|
||||
// All keys in a Lua script must be in the same hash slot (same {orgId})
|
||||
const customersByOrg = new Map<string, typeof customers>();
|
||||
|
||||
for (const customer of customers) {
|
||||
const key = customer.orgId;
|
||||
if (!customersByOrg.has(key)) {
|
||||
customersByOrg.set(key, []);
|
||||
}
|
||||
customersByOrg.get(key)?.push(customer);
|
||||
}
|
||||
|
||||
// Use pipeline to batch all org deletions into one network round trip
|
||||
const pipeline = redis.pipeline();
|
||||
|
||||
for (const orgCustomers of customersByOrg.values()) {
|
||||
pipeline.batchDeleteCustomers(JSON.stringify(orgCustomers));
|
||||
}
|
||||
|
||||
const results = await pipeline.exec();
|
||||
|
||||
// Sum up all deleted counts
|
||||
let totalDeleted = 0;
|
||||
if (results) {
|
||||
for (const [error, result] of results) {
|
||||
if (error) {
|
||||
console.error("Error in pipeline batch delete:", error);
|
||||
throw error;
|
||||
}
|
||||
totalDeleted += result as number;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Batch deleted ${totalDeleted} cache keys for ${customers.length} customers across ${customersByOrg.size} orgs`,
|
||||
);
|
||||
|
||||
return totalDeleted;
|
||||
} catch (error) {
|
||||
console.error("Error batch deleting customers:", error);
|
||||
Sentry.captureException(error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -152,7 +152,6 @@ describe(`${chalk.yellowBright(`${testCase}: Testing rollovers for feature item
|
||||
expect(nonCachedMsgesFeature.rollovers[0].balance).toBe(expectedRollover);
|
||||
}
|
||||
});
|
||||
return;
|
||||
|
||||
test("should reset again and have correct rollovers", async () => {
|
||||
await resetAndGetCusEnt({
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import type { Customer } from "@autumn/shared";
|
||||
import { resetCustomerEntitlement } from "@/cron/cronUtils.js";
|
||||
import {
|
||||
clearCusEntsFromCache,
|
||||
resetCustomerEntitlement,
|
||||
} from "@/cron/cronUtils.js";
|
||||
import type { DrizzleCli } from "@/db/initDrizzle.js";
|
||||
import { CusEntService } from "@/internal/customers/cusProducts/cusEnts/CusEntitlementService.js";
|
||||
import { cusProductToCusEnt } from "@/internal/customers/cusProducts/cusProductUtils/convertCusProduct.js";
|
||||
@@ -28,14 +31,18 @@ export const resetAndGetCusEnt = async ({
|
||||
featureId,
|
||||
});
|
||||
|
||||
const resetCusEnt = {
|
||||
...cusEnt!,
|
||||
customer,
|
||||
};
|
||||
|
||||
const updatedCusEnt = await resetCustomerEntitlement({
|
||||
db,
|
||||
cusEnt: {
|
||||
...cusEnt!,
|
||||
customer,
|
||||
},
|
||||
cusEnt: resetCusEnt,
|
||||
});
|
||||
|
||||
await clearCusEntsFromCache({ cusEnts: [resetCusEnt] });
|
||||
|
||||
if (updatedCusEnt) {
|
||||
await CusEntService.upsert({
|
||||
db,
|
||||
|
||||
@@ -84,7 +84,7 @@ const toV3Type = ({ feature }: { feature?: ApiFeatureV1 }) => {
|
||||
return ApiFeatureType.ContinuousUse;
|
||||
}
|
||||
} else if (feature?.type === FeatureType.CreditSystem) {
|
||||
return ApiFeatureType.CreditSystem;
|
||||
return ApiFeatureType.SingleUsage;
|
||||
} else {
|
||||
return ApiFeatureType.Static;
|
||||
}
|
||||
|
||||
@@ -68,6 +68,12 @@ export function UsageReset({ showBillingLabel = false }: UsageResetProps) {
|
||||
<SelectContent>
|
||||
{Object.values(isFeaturePrice ? BillingInterval : EntInterval)
|
||||
.filter((i) => {
|
||||
if (
|
||||
i === EntInterval.Minute &&
|
||||
itemToEntInterval({ item }) !== EntInterval.Minute
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (isFeaturePrice && item.usage_model === UsageModel.PayPerUse) {
|
||||
return i !== BillingInterval.OneOff;
|
||||
}
|
||||
|
||||
@@ -95,22 +95,24 @@ export const SelectResetCycle = () => {
|
||||
<span className="block truncate overflow-hidden text-ellipsis max-w-full">
|
||||
{getIntervalText({
|
||||
interval,
|
||||
intervalCount: item!.interval_count,
|
||||
intervalCount: item?.interval_count ?? 1,
|
||||
})}
|
||||
</span>
|
||||
</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent className="w-48">
|
||||
{Object.values(EntInterval).map((intervalOption) => {
|
||||
const isSelected = intervalOption === interval;
|
||||
return (
|
||||
<SelectIntervalItem
|
||||
key={intervalOption}
|
||||
interval={intervalOption}
|
||||
isSelected={isSelected}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
{Object.values(EntInterval)
|
||||
.filter((i) => i !== EntInterval.Minute)
|
||||
.map((intervalOption) => {
|
||||
const isSelected = intervalOption === interval;
|
||||
return (
|
||||
<SelectIntervalItem
|
||||
key={intervalOption}
|
||||
interval={intervalOption}
|
||||
isSelected={isSelected}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
<CustomiseIntervalPopover />
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
Reference in New Issue
Block a user