From 48b33abc2f99fd9ac598b735d6a4d64a5e88b4c9 Mon Sep 17 00:00:00 2001 From: John Yeo Date: Mon, 10 Nov 2025 09:17:18 +0000 Subject: [PATCH] fix: versioning for cached customer --- scripts/testGroups/g1.sh | 44 +-- server/src/_luaScripts/cacheConfig.ts | 21 ++ server/src/_luaScripts/cacheKeyUtils.lua | 19 + .../cusLuaScripts/deleteCustomer.lua | 12 +- .../_luaScripts/cusLuaScripts/getCustomer.lua | 13 +- .../cusLuaScripts/loadCusFeatures.lua | 6 +- .../_luaScripts/cusLuaScripts/setCustomer.lua | 23 +- .../cusLuaScripts/setCustomerDetails.lua | 14 +- .../cusLuaScripts/setCustomerProducts.lua | 14 +- .../deductionLuaScripts/batchDeduction.lua | 7 +- .../entityLuaScripts/getEntity.lua | 326 ++---------------- .../entityLuaScripts/setEntitiesBatch.lua | 17 +- .../entityLuaScripts/setEntity.lua | 25 +- .../entityLuaScripts/setEntityProducts.lua | 16 +- server/src/_luaScripts/luaScripts.ts | 51 ++- server/src/external/autumn/autumnCli.ts | 14 +- .../stripe/handleStripeWebhookEvent.ts | 55 +-- server/src/init.ts | 2 +- .../api/check/checkUtils/getCheckData.ts | 41 --- .../setUsage/getSetUsageDeductions.ts | 3 + .../internal/balances/track/handleTrack.ts | 2 - .../track/redisTrackUtils/BatchingManager.ts | 10 +- .../track/redisTrackUtils/deductFromCache.ts | 8 - .../redisTrackUtils/executeBatchDeduction.ts | 5 +- .../deductRpc/performDeductionV2.sql | 16 +- .../track/trackUtils/runDeductionTx.ts | 15 +- .../attach/attachUtils/handleAttachErrors.ts | 78 +---- .../deleteCachedApiCustomer.ts | 13 +- .../apiCusCacheUtils/getCachedApiCustomer.ts | 14 +- .../setCachedApiCusDetails.ts | 13 +- .../setCachedApiCusProducts.ts | 28 +- .../apiCusCacheUtils/setCachedApiCustomer.ts | 11 +- .../getApiCusFeature/getApiCusFeatures.ts | 6 +- .../getCusFeaturesResponse.ts | 6 +- .../cusUtils/getOrCreateApiCustomer.ts | 5 + .../handlers/handlePostCustomerV2.ts | 2 - .../apiEntityCacheUtils/getCachedApiEntity.ts | 13 +- server/src/utils/importUtils/updateUsages.ts | 13 +- .../track/basic/track-basic10.test.ts | 13 +- .../track/legacy/track-legacy3.test.ts | 2 +- .../balances/track/misc/track-misc3.test.ts | 63 ++++ server/tsconfig.build.json | 3 +- server/tsconfig.json | 2 +- shared/utils/cusEntUtils/getRolloverFields.ts | 6 +- .../cusProductUtils/filterCusProductUtils.ts | 2 +- .../featureUtils/apiFeatureToDbFeature.ts | 2 +- shared/utils/index.ts | 3 + shared/utils/orgUtils/convertOrgUtils.ts | 9 + shared/utils/productV3Utils/mapToProductV3.ts | 2 - .../productItemUtils/productV3ItemUtils.ts | 2 +- 50 files changed, 426 insertions(+), 664 deletions(-) create mode 100644 server/src/_luaScripts/cacheConfig.ts create mode 100644 server/src/_luaScripts/cacheKeyUtils.lua create mode 100644 server/tests/balances/track/misc/track-misc3.test.ts create mode 100644 shared/utils/orgUtils/convertOrgUtils.ts diff --git a/scripts/testGroups/g1.sh b/scripts/testGroups/g1.sh index d09a71e23..5f949f608 100755 --- a/scripts/testGroups/g1.sh +++ b/scripts/testGroups/g1.sh @@ -14,26 +14,26 @@ fi # Run tests using TypeScript runner with compact mode # Adjust --max to control concurrency (default: 6) -BUN_PARALLEL_COMPACT \ - 'server/tests/balances/track/basic' \ - 'server/tests/balances/track/concurrency' \ - 'server/tests/balances/track/allocated' \ - 'server/tests/balances/track/credit-systems' \ - 'server/tests/balances/track/entity-balances' \ - 'server/tests/balances/track/entity-products' \ - 'server/tests/balances/track/legacy' \ - 'server/tests/balances/check/basic' \ - 'server/tests/balances/check/credit-systems' \ - 'server/tests/balances/check/misc' \ - # BUN_PARALLEL_COMPACT \ -# 'server/tests/attach/basic' \ -# 'server/tests/attach/entities' \ -# 'server/tests/attach/upgrade' \ -# 'server/tests/attach/downgrade' \ -# 'server/tests/attach/free' \ -# 'server/tests/attach/addOn' \ -# 'server/tests/attach/entities' \ -# 'server/tests/attach/checkout' \ -# 'server/tests/attach/misc' \ -# --max=6 \ \ No newline at end of file +# 'server/tests/balances/track/basic' \ +# 'server/tests/balances/track/concurrency' \ +# 'server/tests/balances/track/allocated' \ +# 'server/tests/balances/track/credit-systems' \ +# 'server/tests/balances/track/entity-balances' \ +# 'server/tests/balances/track/entity-products' \ +# 'server/tests/balances/track/legacy' \ +# 'server/tests/balances/check/basic' \ +# 'server/tests/balances/check/credit-systems' \ +# 'server/tests/balances/check/misc' \ + +BUN_PARALLEL_COMPACT \ + 'server/tests/attach/basic' \ + 'server/tests/attach/entities' \ + 'server/tests/attach/upgrade' \ + 'server/tests/attach/downgrade' \ + 'server/tests/attach/free' \ + 'server/tests/attach/addOn' \ + 'server/tests/attach/entities' \ + 'server/tests/attach/checkout' \ + 'server/tests/attach/misc' \ + --max=6 \ \ No newline at end of file diff --git a/server/src/_luaScripts/cacheConfig.ts b/server/src/_luaScripts/cacheConfig.ts new file mode 100644 index 000000000..9487c8ecc --- /dev/null +++ b/server/src/_luaScripts/cacheConfig.ts @@ -0,0 +1,21 @@ +import { ApiVersion } from "@autumn/shared"; + +/** + * Cache configuration constants + * These values are injected into Lua scripts at load time for optimal performance + */ + +/** + * Customer cache version (applies to both customer and entity caches) + * Increment this (change to a newer ApiVersion) when customer/entity cache structure changes + * Old caches will be orphaned and expire after CACHE_TTL_SECONDS + * + * Format: customer:{version}:{customerId} or customer:{version}:{customerId}:entity:{entityId} + */ +export const CACHE_CUSTOMER_VERSION = ApiVersion.V1_2; + +/** + * Cache time-to-live in seconds (7 days) + * All customer and entity caches will expire after this duration + */ +export const CACHE_TTL_SECONDS = 7 * 24 * 60 * 60; // 7 days = 604800 seconds diff --git a/server/src/_luaScripts/cacheKeyUtils.lua b/server/src/_luaScripts/cacheKeyUtils.lua new file mode 100644 index 000000000..c8058a8de --- /dev/null +++ b/server/src/_luaScripts/cacheKeyUtils.lua @@ -0,0 +1,19 @@ +-- cacheKeyUtils.lua +-- Shared cache key builders for customer and entity caches +-- Version placeholder {CUSTOMER_VERSION} is replaced at load time + +-- Cache TTL constant (replaced at load time) +local CACHE_TTL_SECONDS = {TTL_SECONDS} + +-- Build customer cache key with version +-- Returns: {orgId}:env:customer:{version}:customerId +local function buildCustomerCacheKey(orgId, env, customerId) + return "{" .. orgId .. "}:" .. env .. ":customer:{CUSTOMER_VERSION}:" .. customerId +end + +-- Build entity cache key with version +-- Returns: {orgId}:env:customer:{version}:customerId:entity:entityId +local function buildEntityCacheKey(orgId, env, customerId, entityId) + return "{" .. orgId .. "}:" .. env .. ":customer:{CUSTOMER_VERSION}:" .. customerId .. ":entity:" .. entityId +end + diff --git a/server/src/_luaScripts/cusLuaScripts/deleteCustomer.lua b/server/src/_luaScripts/cusLuaScripts/deleteCustomer.lua index 4a15f1b16..fdb89e621 100644 --- a/server/src/_luaScripts/cusLuaScripts/deleteCustomer.lua +++ b/server/src/_luaScripts/cusLuaScripts/deleteCustomer.lua @@ -1,9 +1,17 @@ -- deleteCustomer.lua -- Atomically deletes a customer and all its associated entity caches --- KEYS[1]: customer cache key pattern (e.g., "{org_id}:env:customer:customer_id") +-- ARGV[1]: org_id +-- ARGV[2]: env +-- ARGV[3]: customer_id -- Returns: number of keys deleted -local basePattern = KEYS[1] .. "*" +local orgId = ARGV[1] +local env = ARGV[2] +local customerId = ARGV[3] + +-- Build versioned cache key using shared utility +local cacheKey = buildCustomerCacheKey(orgId, env, customerId) +local basePattern = cacheKey .. "*" local keysToDelete = {} -- Scan for all keys matching the pattern diff --git a/server/src/_luaScripts/cusLuaScripts/getCustomer.lua b/server/src/_luaScripts/cusLuaScripts/getCustomer.lua index 4c0599f6f..e84acef32 100644 --- a/server/src/_luaScripts/cusLuaScripts/getCustomer.lua +++ b/server/src/_luaScripts/cusLuaScripts/getCustomer.lua @@ -1,10 +1,9 @@ -- getCustomer.lua -- Atomically retrieves a customer object from Redis, reconstructing from base JSON and feature HSETs -- Merges master customer features with entity features (unless skipEntityMerge is true) --- KEYS[1]: cache key (e.g., "org_id:env:customer:customer_id") --- ARGV[1]: org_id (for building entity cache keys) --- ARGV[2]: env (for building entity cache keys) --- ARGV[3]: customer_id (for building entity cache keys) +-- ARGV[1]: org_id +-- ARGV[2]: env +-- ARGV[3]: customer_id -- ARGV[4]: skipEntityMerge (optional, "true" to skip merging with entities) -- Helper function to merge products array by product ID and normalized status @@ -93,12 +92,14 @@ local function mergeProducts(productsArray) return mergedProducts end -local cacheKey = KEYS[1] local orgId = ARGV[1] local env = ARGV[2] local customerId = ARGV[3] local skipEntityMerge = ARGV[4] == "true" +-- Build versioned cache key using shared utility +local cacheKey = buildCustomerCacheKey(orgId, env, customerId) + -- Load features based on merge mode -- If skipEntityMerge is true, only load customer's own features (no entity merging) -- If skipEntityMerge is false, load merged features (customer + entities) @@ -131,7 +132,7 @@ local entityIds = baseCustomer._entityIds or {} -- Build entity base data map for product access local entityBaseData = {} for _, entityId in ipairs(entityIds) do - local entityCacheKey = "{" .. orgId .. "}:" .. env .. ":customer:" .. customerId .. ":entity:" .. entityId + local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) local entityBaseJson = redis.call("GET", entityCacheKey) if entityBaseJson then diff --git a/server/src/_luaScripts/cusLuaScripts/loadCusFeatures.lua b/server/src/_luaScripts/cusLuaScripts/loadCusFeatures.lua index cad0f4d0f..bb30830d6 100644 --- a/server/src/_luaScripts/cusLuaScripts/loadCusFeatures.lua +++ b/server/src/_luaScripts/cusLuaScripts/loadCusFeatures.lua @@ -163,8 +163,8 @@ end -- Parameters: cacheKey (customer cache key), orgId, env, customerId, entityId -- Returns: merged features table (entity + customer) or nil local function loadEntityLevelFeatures(cacheKey, orgId, env, customerId, entityId) - -- Build entity cache key - local entityCacheKey = "{" .. orgId .. "}:" .. env .. ":customer:" .. customerId .. ":entity:" .. entityId + -- Build versioned entity cache key using shared utility + local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) -- Get entity base JSON local entityBaseJson = redis.call("GET", entityCacheKey) @@ -411,7 +411,7 @@ local function loadCusFeatures(cacheKey, orgId, env, customerId, entityId) local entityBaseData = {} -- {[entityId] = entityBase} - Store entity base for product access for _, entityId in ipairs(entityIds) do - local entityCacheKey = "{" .. orgId .. "}:" .. env .. ":customer:" .. customerId .. ":entity:" .. entityId + local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) local entityBaseJson = redis.call("GET", entityCacheKey) if entityBaseJson then diff --git a/server/src/_luaScripts/cusLuaScripts/setCustomer.lua b/server/src/_luaScripts/cusLuaScripts/setCustomer.lua index ee165cccc..7dec873af 100644 --- a/server/src/_luaScripts/cusLuaScripts/setCustomer.lua +++ b/server/src/_luaScripts/cusLuaScripts/setCustomer.lua @@ -1,15 +1,18 @@ -- setCustomer.lua -- Atomically stores a customer object with base data as JSON and features/breakdowns as HSETs -- Separates master customer features from entity features --- KEYS[1]: cache key (e.g., "org_id:env:customer:customer_id") -- ARGV[1]: serialized customer data JSON string --- ARGV[2]: org_id (for building entity cache keys) --- ARGV[3]: env (for building entity cache keys) +-- ARGV[2]: org_id +-- ARGV[3]: env +-- ARGV[4]: customer_id -local cacheKey = KEYS[1] local customerDataJson = ARGV[1] local orgId = ARGV[2] local env = ARGV[3] +local customerId = ARGV[4] + +-- Build versioned cache key using shared utility +local cacheKey = buildCustomerCacheKey(orgId, env, customerId) -- Check if complete cache already exists if checkCacheExists(cacheKey) then @@ -60,9 +63,10 @@ local baseCustomer = { _entityIds = entityIds } --- Store base customer as JSON +-- Store base customer as JSON with TTL local baseKey = cacheKey redis.call("SET", baseKey, cjson.encode(baseCustomer)) +redis.call("EXPIRE", baseKey, CACHE_TTL_SECONDS) -- Helper function to convert values to strings, handling cjson.null local function toString(value) @@ -95,7 +99,7 @@ if customerData.features then creditSchemaJson = cjson.encode(featureData.credit_schema) end - -- Store all top-level feature fields in a single HSET call + -- Store all top-level feature fields in a single HSET call with TTL redis.call("HSET", featureKey, "id", toString(featureData.id), "type", toString(featureData.type), @@ -113,8 +117,9 @@ if customerData.features then "_breakdown_count", toString(breakdownCount), "_rollover_count", toString(rolloverCount) ) + redis.call("EXPIRE", featureKey, CACHE_TTL_SECONDS) - -- Store each rollover item as separate HSET (single call per rollover) + -- Store each rollover item as separate HSET with TTL (single call per rollover) if featureData.rollovers then for index, rolloverItem in ipairs(featureData.rollovers) do local rolloverKey = cacheKey .. ":features:" .. featureId .. ":rollover:" .. (index - 1) @@ -123,10 +128,11 @@ if customerData.features then "balance", toString(rolloverItem.balance), "expires_at", toString(rolloverItem.expires_at) ) + redis.call("EXPIRE", rolloverKey, CACHE_TTL_SECONDS) end end - -- Store each breakdown item as separate HSET (single call per breakdown) + -- Store each breakdown item as separate HSET with TTL (single call per breakdown) if featureData.breakdown then for index, breakdownItem in ipairs(featureData.breakdown) do local breakdownKey = cacheKey .. ":features:" .. featureId .. ":breakdown:" .. (index - 1) @@ -141,6 +147,7 @@ if customerData.features then "usage_limit", toString(breakdownItem.usage_limit), "overage_allowed", toString(breakdownItem.overage_allowed) ) + redis.call("EXPIRE", breakdownKey, CACHE_TTL_SECONDS) end end end diff --git a/server/src/_luaScripts/cusLuaScripts/setCustomerDetails.lua b/server/src/_luaScripts/cusLuaScripts/setCustomerDetails.lua index a477a41ee..5055e6c5e 100644 --- a/server/src/_luaScripts/cusLuaScripts/setCustomerDetails.lua +++ b/server/src/_luaScripts/cusLuaScripts/setCustomerDetails.lua @@ -1,10 +1,17 @@ -- setCustomerDetails.lua -- Updates only the customer detail fields (name, email, etc.) in the customer cache --- KEYS[1]: cache key (e.g., "org_id:env:customer:customer_id") -- ARGV[1]: serialized customer details JSON string (object with name, email, etc.) +-- ARGV[2]: org_id +-- ARGV[3]: env +-- ARGV[4]: customer_id -local cacheKey = KEYS[1] local detailsJson = ARGV[1] +local orgId = ARGV[2] +local env = ARGV[3] +local customerId = ARGV[4] + +-- Build versioned cache key using shared utility +local cacheKey = buildCustomerCacheKey(orgId, env, customerId) local baseKey = cacheKey -- Get base customer JSON @@ -31,8 +38,9 @@ if details.metadata ~= nil then baseCustomer.metadata = details.metadata end --- Store updated base customer as JSON +-- Store updated base customer as JSON and extend TTL redis.call("SET", baseKey, cjson.encode(baseCustomer)) +redis.call("EXPIRE", baseKey, CACHE_TTL_SECONDS) return "OK" diff --git a/server/src/_luaScripts/cusLuaScripts/setCustomerProducts.lua b/server/src/_luaScripts/cusLuaScripts/setCustomerProducts.lua index 2f91810c5..fd49c136f 100644 --- a/server/src/_luaScripts/cusLuaScripts/setCustomerProducts.lua +++ b/server/src/_luaScripts/cusLuaScripts/setCustomerProducts.lua @@ -1,10 +1,17 @@ -- setCustomerProducts.lua -- Updates only the products array in the customer cache --- KEYS[1]: cache key (e.g., "org_id:env:customer:customer_id") -- ARGV[1]: serialized products array JSON string +-- ARGV[2]: org_id +-- ARGV[3]: env +-- ARGV[4]: customer_id -local cacheKey = KEYS[1] local productsJson = ARGV[1] +local orgId = ARGV[2] +local env = ARGV[3] +local customerId = ARGV[4] + +-- Build versioned cache key using shared utility +local cacheKey = buildCustomerCacheKey(orgId, env, customerId) local baseKey = cacheKey -- Get base customer JSON @@ -20,8 +27,9 @@ local products = cjson.decode(productsJson) -- Update only the products array baseCustomer.products = products --- Store updated base customer as JSON +-- Store updated base customer as JSON and extend TTL redis.call("SET", baseKey, cjson.encode(baseCustomer)) +redis.call("EXPIRE", baseKey, CACHE_TTL_SECONDS) return "OK" diff --git a/server/src/_luaScripts/deductionLuaScripts/batchDeduction.lua b/server/src/_luaScripts/deductionLuaScripts/batchDeduction.lua index 7d0d95adb..31fa2e6f0 100644 --- a/server/src/_luaScripts/deductionLuaScripts/batchDeduction.lua +++ b/server/src/_luaScripts/deductionLuaScripts/batchDeduction.lua @@ -2,7 +2,6 @@ -- Atomically processes a batch of track requests for a customer -- Each request can deduct from multiple features -- --- KEYS[1]: cache key (e.g., "org_id:env:customer:customer_id") -- ARGV[1]: JSON array of requests: -- [ -- { @@ -18,12 +17,14 @@ -- ARGV[3]: env -- ARGV[4]: customer_id -local cacheKey = KEYS[1] local requestsJson = ARGV[1] local orgId = ARGV[2] local env = ARGV[3] local customerId = ARGV[4] +-- Build versioned customer cache key using shared utility +local cacheKey = buildCustomerCacheKey(orgId, env, customerId) + -- Parse requests local requests = cjson.decode(requestsJson) @@ -972,7 +973,7 @@ local entityIds = baseCustomer._entityIds or {} -- Load all entity features: { [entityId] = { [featureId] = entityFeature } } local entityFeatureStates = {} for _, entityId in ipairs(entityIds) do - local entityCacheKey = "{" .. orgId .. "}:" .. env .. ":customer:" .. customerId .. ":entity:" .. entityId + local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) local entityBaseJson = redis.call("GET", entityCacheKey) if entityBaseJson then diff --git a/server/src/_luaScripts/entityLuaScripts/getEntity.lua b/server/src/_luaScripts/entityLuaScripts/getEntity.lua index 99c4d1d85..31e557533 100644 --- a/server/src/_luaScripts/entityLuaScripts/getEntity.lua +++ b/server/src/_luaScripts/entityLuaScripts/getEntity.lua @@ -1,9 +1,8 @@ -- getEntity.lua -- Atomically retrieves an entity object from Redis, reconstructing from base JSON and feature HSETs -- Merges entity features with customer features (unless skipCustomerMerge is true) --- KEYS[1]: cache key (e.g., "{org_id}:env:customer:customer_id:entity:entity_id") --- ARGV[1]: org_id (for building customer cache keys) --- ARGV[2]: env (for building customer cache keys) +-- ARGV[1]: org_id +-- ARGV[2]: env -- ARGV[3]: customerId -- ARGV[4]: entityId -- ARGV[5]: skipCustomerMerge (optional, "true" to skip merging with customer) @@ -61,318 +60,45 @@ local function mergeCustomerProductsIntoEntity(entityProducts, customerProducts) return mergedProducts end -local cacheKey = KEYS[1] -local baseKey = cacheKey local orgId = ARGV[1] local env = ARGV[2] local customerId = ARGV[3] local entityId = ARGV[4] local skipCustomerMerge = ARGV[5] == "true" +-- Build versioned entity cache key using shared utility +local entityCacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) + -- Get base entity JSON -local baseJson = redis.call("GET", baseKey) +local baseJson = redis.call("GET", entityCacheKey) if not baseJson then return nil end local baseEntity = cjson.decode(baseJson) -local entityFeatureIds = baseEntity._featureIds or {} + +-- Build customer cache key for feature loading +local customerCacheKey = buildCustomerCacheKey(orgId, env, customerId) -- ============================================================================ --- FETCH ENTITY FEATURES +-- LOAD FEATURES USING loadCusFeatures -- ============================================================================ -local entityFeatures = {} +local mergedFeatures -for _, featureId in ipairs(entityFeatureIds) do - local featureKey = cacheKey .. ":features:" .. featureId - local featureHash = redis.call("HGETALL", featureKey) - - -- If feature key is missing, return nil (partial eviction detected) - if #featureHash == 0 then - return nil - end - - -- Convert HGETALL result (flat array) to table - local featureData = {} - for i = 1, #featureHash, 2 do - local key = featureHash[i] - local value = featureHash[i + 1] - - -- Check for null first before parsing - if value == "null" then - featureData[key] = cjson.null - elseif key == "balance" or key == "usage" or key == "included_usage" or key == "usage_limit" or key == "interval_count" or key == "next_reset_at" or key == "_breakdown_count" or key == "_rollover_count" then - featureData[key] = tonumber(value) - elseif key == "unlimited" or key == "overage_allowed" then - featureData[key] = (value == "true") - elseif key == "credit_schema" then - -- Parse credit_schema JSON array - if value ~= "" then - featureData[key] = cjson.decode(value) - else - featureData[key] = cjson.null - end - else - featureData[key] = value - end - end - - -- Get rollover count - local rolloverCount = featureData._rollover_count or 0 - featureData._rollover_count = nil -- Remove from final output - - -- Fetch rollover items - local rollovers = {} - for i = 0, rolloverCount - 1 do - local rolloverKey = cacheKey .. ":features:" .. featureId .. ":rollover:" .. i - local rolloverHash = redis.call("HGETALL", rolloverKey) - - -- If rollover key is missing, return nil (partial eviction detected) - if #rolloverHash == 0 then - return nil - end - - local rolloverData = {} - for j = 1, #rolloverHash, 2 do - local key = rolloverHash[j] - local value = rolloverHash[j + 1] - - if value == "null" then - rolloverData[key] = cjson.null - elseif key == "balance" or key == "expires_at" then - rolloverData[key] = tonumber(value) - else - rolloverData[key] = value - end - end - table.insert(rollovers, rolloverData) - end - - if #rollovers > 0 then - featureData.rollovers = rollovers - end - - -- Get breakdown count - local breakdownCount = featureData._breakdown_count or 0 - featureData._breakdown_count = nil -- Remove from final output - - -- Fetch breakdown items - local breakdown = {} - for i = 0, breakdownCount - 1 do - local breakdownKey = cacheKey .. ":features:" .. featureId .. ":breakdown:" .. i - local breakdownHash = redis.call("HGETALL", breakdownKey) - - -- If breakdown key is missing, return nil (partial eviction detected) - if #breakdownHash == 0 then - return nil - end - - local breakdownData = {} - for j = 1, #breakdownHash, 2 do - local key = breakdownHash[j] - local value = breakdownHash[j + 1] - - if value == "null" then - breakdownData[key] = cjson.null - elseif key == "balance" or key == "usage" or key == "included_usage" or key == "usage_limit" or key == "interval_count" or key == "next_reset_at" then - breakdownData[key] = tonumber(value) - elseif key == "overage_allowed" then - breakdownData[key] = (value == "true") - else - breakdownData[key] = value - end - end - table.insert(breakdown, breakdownData) - end - - if #breakdown > 0 then - featureData.breakdown = breakdown - end - - entityFeatures[featureId] = featureData +if skipCustomerMerge then + -- Load only entity's own features (no customer merging) + -- We'll use loadCusFeatures with "__CUSTOMER_ONLY__" mode on the entity cache key + -- This is a bit of a hack but works with the current structure + mergedFeatures = loadCusFeatures(entityCacheKey, orgId, env, customerId, "__CUSTOMER_ONLY__") +else + -- Load entity-level merged features (entity + customer) + -- loadCusFeatures handles this when entityId is provided + mergedFeatures = loadCusFeatures(customerCacheKey, orgId, env, customerId, entityId) end --- ============================================================================ --- FETCH CUSTOMER MASTER FEATURES (no entity aggregation) --- Skip if skipCustomerMerge is true --- ============================================================================ -local customerFeatures = {} -local customerBase = nil -- Store customer base for product access - -if not skipCustomerMerge and customerId then - local customerCacheKey = "{" .. orgId .. "}:" .. env .. ":customer:" .. customerId - local customerBaseJson = redis.call("GET", customerCacheKey) - - if customerBaseJson then - customerBase = cjson.decode(customerBaseJson) - local customerFeatureIds = customerBase._featureIds or {} - - for _, featureId in ipairs(customerFeatureIds) do - local customerFeatureKey = customerCacheKey .. ":features:" .. featureId - local customerFeatureHash = redis.call("HGETALL", customerFeatureKey) - - if #customerFeatureHash > 0 then - -- Parse customer feature - local customerFeature = {} - for i = 1, #customerFeatureHash, 2 do - local key = customerFeatureHash[i] - local value = customerFeatureHash[i + 1] - - if value == "null" then - customerFeature[key] = cjson.null - elseif key == "balance" or key == "usage" or key == "included_usage" or key == "usage_limit" or key == "interval_count" or key == "next_reset_at" or key == "_breakdown_count" or key == "_rollover_count" then - customerFeature[key] = tonumber(value) - elseif key == "unlimited" or key == "overage_allowed" then - customerFeature[key] = (value == "true") - elseif key == "credit_schema" then - if value ~= "" then - customerFeature[key] = cjson.decode(value) - else - customerFeature[key] = cjson.null - end - else - customerFeature[key] = value - end - end - - -- Fetch rollover items - local rolloverCount = customerFeature._rollover_count or 0 - customerFeature._rollover_count = nil - local rollovers = {} - - for i = 0, rolloverCount - 1 do - local rolloverKey = customerFeatureKey .. ":rollover:" .. i - local rolloverHash = redis.call("HGETALL", rolloverKey) - - if #rolloverHash > 0 then - local rolloverData = {} - for j = 1, #rolloverHash, 2 do - local key = rolloverHash[j] - local value = rolloverHash[j + 1] - - if value == "null" then - rolloverData[key] = cjson.null - elseif key == "balance" or key == "expires_at" then - rolloverData[key] = tonumber(value) - else - rolloverData[key] = value - end - end - table.insert(rollovers, rolloverData) - end - end - - if #rollovers > 0 then - customerFeature.rollovers = rollovers - end - - -- Fetch breakdown items - local breakdownCount = customerFeature._breakdown_count or 0 - customerFeature._breakdown_count = nil - local breakdown = {} - - for i = 0, breakdownCount - 1 do - local breakdownKey = customerFeatureKey .. ":breakdown:" .. i - local breakdownHash = redis.call("HGETALL", breakdownKey) - - if #breakdownHash > 0 then - local breakdownData = {} - for j = 1, #breakdownHash, 2 do - local key = breakdownHash[j] - local value = breakdownHash[j + 1] - - if value == "null" then - breakdownData[key] = cjson.null - elseif key == "balance" or key == "usage" or key == "included_usage" or key == "usage_limit" or key == "interval_count" or key == "next_reset_at" then - breakdownData[key] = tonumber(value) - elseif key == "overage_allowed" then - breakdownData[key] = (value == "true") - else - breakdownData[key] = value - end - end - table.insert(breakdown, breakdownData) - end - end - - if #breakdown > 0 then - customerFeature.breakdown = breakdown - end - - customerFeatures[featureId] = customerFeature - end - end - end -end - --- ============================================================================ --- MERGE CUSTOMER AND ENTITY FEATURES --- ============================================================================ -local mergedFeatures = {} - --- First, add all customer features (inherited) -for featureId, customerFeature in pairs(customerFeatures) do - mergedFeatures[featureId] = customerFeature -end - --- Then, merge or add entity features -for featureId, entityFeature in pairs(entityFeatures) do - local customerFeature = customerFeatures[featureId] - - if customerFeature then - -- Both customer and entity have this feature - merge balances - if not entityFeature.unlimited and not customerFeature.unlimited then - entityFeature.balance = toNum(entityFeature.balance) + toNum(customerFeature.balance) - entityFeature.usage = toNum(entityFeature.usage) + toNum(customerFeature.usage) - entityFeature.included_usage = toNum(entityFeature.included_usage) + toNum(customerFeature.included_usage) - entityFeature.usage_limit = toNum(entityFeature.usage_limit) + toNum(customerFeature.usage_limit) - - -- Use minimum next_reset_at (earliest reset time) - if type(entityFeature.next_reset_at) == "number" and type(customerFeature.next_reset_at) == "number" then - if customerFeature.next_reset_at < entityFeature.next_reset_at then - entityFeature.next_reset_at = customerFeature.next_reset_at - end - elseif type(customerFeature.next_reset_at) == "number" then - entityFeature.next_reset_at = customerFeature.next_reset_at - end - - -- Merge breakdown balances - if entityFeature.breakdown and customerFeature.breakdown then - for i, entityBreakdown in ipairs(entityFeature.breakdown) do - local customerBreakdown = customerFeature.breakdown[i] - if customerBreakdown then - entityBreakdown.balance = toNum(entityBreakdown.balance) + toNum(customerBreakdown.balance) - entityBreakdown.usage = toNum(entityBreakdown.usage) + toNum(customerBreakdown.usage) - entityBreakdown.included_usage = toNum(entityBreakdown.included_usage) + toNum(customerBreakdown.included_usage) - entityBreakdown.usage_limit = toNum(entityBreakdown.usage_limit) + toNum(customerBreakdown.usage_limit) - - -- Use minimum next_reset_at for breakdown - if type(entityBreakdown.next_reset_at) == "number" and type(customerBreakdown.next_reset_at) == "number" then - if customerBreakdown.next_reset_at < entityBreakdown.next_reset_at then - entityBreakdown.next_reset_at = customerBreakdown.next_reset_at - end - elseif type(customerBreakdown.next_reset_at) == "number" then - entityBreakdown.next_reset_at = customerBreakdown.next_reset_at - end - end - end - end - - -- Merge rollover balances - if entityFeature.rollovers and customerFeature.rollovers then - for i, entityRollover in ipairs(entityFeature.rollovers) do - local customerRollover = customerFeature.rollovers[i] - if customerRollover then - entityRollover.balance = toNum(entityRollover.balance) + toNum(customerRollover.balance) - end - end - end - end - mergedFeatures[featureId] = entityFeature - else - -- Only entity has this feature - use entity's feature - mergedFeatures[featureId] = entityFeature - end +-- If features loading failed (partial eviction), return nil +if not mergedFeatures then + return nil end -- ============================================================================ @@ -384,9 +110,11 @@ end local entityProducts = baseEntity.products or {} if not skipCustomerMerge then - -- Get customer products if customer base exists + -- Get customer products local customerProducts = nil - if customerBase and customerBase.products then + local customerBaseJson = redis.call("GET", customerCacheKey) + if customerBaseJson then + local customerBase = cjson.decode(customerBaseJson) customerProducts = customerBase.products end diff --git a/server/src/_luaScripts/entityLuaScripts/setEntitiesBatch.lua b/server/src/_luaScripts/entityLuaScripts/setEntitiesBatch.lua index d0906445e..bf632eabf 100644 --- a/server/src/_luaScripts/entityLuaScripts/setEntitiesBatch.lua +++ b/server/src/_luaScripts/entityLuaScripts/setEntitiesBatch.lua @@ -1,6 +1,5 @@ -- setEntitiesBatch.lua -- Atomically stores multiple entity objects in a single call --- KEYS: none (we'll build keys dynamically) -- ARGV[1]: JSON array of entity data objects: [{entityId: "...", entityData: {...}}, ...] -- ARGV[2]: org_id -- ARGV[3]: env @@ -25,9 +24,9 @@ for _, entityWrapper in ipairs(entities) do local entityId = entityWrapper.entityId local entityData = entityWrapper.entityData - -- Build cache key for this entity (includes customer_id for hierarchy) + -- Build versioned cache key for this entity using shared utility local customerId = entityData.customer_id - local cacheKey = "{" .. orgId .. "}:" .. env .. ":customer:" .. customerId .. ":entity:" .. entityId + local cacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) -- Extract feature IDs for tracking local featureIds = {} @@ -49,8 +48,9 @@ for _, entityWrapper in ipairs(entities) do _featureIds = featureIds } - -- Store base entity as JSON + -- Store base entity as JSON with TTL redis.call("SET", cacheKey, cjson.encode(baseEntity)) + redis.call("EXPIRE", cacheKey, CACHE_TTL_SECONDS) -- Store each feature as HSET if entityData.features then @@ -75,7 +75,7 @@ for _, entityWrapper in ipairs(entities) do creditSchemaJson = cjson.encode(featureData.credit_schema) end - -- Store all top-level feature fields in a single HSET call + -- Store all top-level feature fields in a single HSET call with TTL redis.call("HSET", featureKey, "id", toString(featureData.id), "type", toString(featureData.type), @@ -93,8 +93,9 @@ for _, entityWrapper in ipairs(entities) do "_breakdown_count", toString(breakdownCount), "_rollover_count", toString(rolloverCount) ) + redis.call("EXPIRE", featureKey, CACHE_TTL_SECONDS) - -- Store each rollover item as separate HSET (single call per rollover) + -- Store each rollover item as separate HSET with TTL (single call per rollover) if featureData.rollovers then for index, rolloverItem in ipairs(featureData.rollovers) do local rolloverKey = cacheKey .. ":features:" .. featureId .. ":rollover:" .. (index - 1) @@ -103,10 +104,11 @@ for _, entityWrapper in ipairs(entities) do "balance", toString(rolloverItem.balance), "expires_at", toString(rolloverItem.expires_at) ) + redis.call("EXPIRE", rolloverKey, CACHE_TTL_SECONDS) end end - -- Store each breakdown item as separate HSET (single call per breakdown) + -- Store each breakdown item as separate HSET with TTL (single call per breakdown) if featureData.breakdown then for index, breakdownItem in ipairs(featureData.breakdown) do local breakdownKey = cacheKey .. ":features:" .. featureId .. ":breakdown:" .. (index - 1) @@ -121,6 +123,7 @@ for _, entityWrapper in ipairs(entities) do "usage_limit", toString(breakdownItem.usage_limit), "overage_allowed", toString(breakdownItem.overage_allowed) ) + redis.call("EXPIRE", breakdownKey, CACHE_TTL_SECONDS) end end end diff --git a/server/src/_luaScripts/entityLuaScripts/setEntity.lua b/server/src/_luaScripts/entityLuaScripts/setEntity.lua index c6f70d422..995bc9620 100644 --- a/server/src/_luaScripts/entityLuaScripts/setEntity.lua +++ b/server/src/_luaScripts/entityLuaScripts/setEntity.lua @@ -1,10 +1,19 @@ -- setEntity.lua -- Atomically stores an entity object with base data as JSON and features/breakdowns as HSETs --- KEYS[1]: cache key (e.g., "{org_id}:env:customer:customer_id:entity:entity_id") -- ARGV[1]: serialized entity data JSON string +-- ARGV[2]: org_id +-- ARGV[3]: env +-- ARGV[4]: customer_id +-- ARGV[5]: entity_id -local cacheKey = KEYS[1] local entityDataJson = ARGV[1] +local orgId = ARGV[2] +local env = ARGV[3] +local customerId = ARGV[4] +local entityId = ARGV[5] + +-- Build versioned cache key using shared utility +local cacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) -- Check if complete cache already exists if checkCacheExists(cacheKey) then @@ -37,9 +46,10 @@ local baseEntity = { _featureIds = featureIds } --- Store base entity as JSON +-- Store base entity as JSON with TTL local baseKey = cacheKey redis.call("SET", baseKey, cjson.encode(baseEntity)) +redis.call("EXPIRE", baseKey, CACHE_TTL_SECONDS) -- Helper function to convert values to strings, handling cjson.null local function toString(value) @@ -72,7 +82,7 @@ if entityData.features then creditSchemaJson = cjson.encode(featureData.credit_schema) end - -- Store all top-level feature fields in a single HSET call + -- Store all top-level feature fields in a single HSET call with TTL redis.call("HSET", featureKey, "id", toString(featureData.id), "type", toString(featureData.type), @@ -90,8 +100,9 @@ if entityData.features then "_breakdown_count", toString(breakdownCount), "_rollover_count", toString(rolloverCount) ) + redis.call("EXPIRE", featureKey, CACHE_TTL_SECONDS) - -- Store each rollover item as separate HSET (single call per rollover) + -- Store each rollover item as separate HSET with TTL (single call per rollover) if featureData.rollovers then for index, rolloverItem in ipairs(featureData.rollovers) do local rolloverKey = cacheKey .. ":features:" .. featureId .. ":rollover:" .. (index - 1) @@ -100,10 +111,11 @@ if entityData.features then "balance", toString(rolloverItem.balance), "expires_at", toString(rolloverItem.expires_at) ) + redis.call("EXPIRE", rolloverKey, CACHE_TTL_SECONDS) end end - -- Store each breakdown item as separate HSET (single call per breakdown) + -- Store each breakdown item as separate HSET with TTL (single call per breakdown) if featureData.breakdown then for index, breakdownItem in ipairs(featureData.breakdown) do local breakdownKey = cacheKey .. ":features:" .. featureId .. ":breakdown:" .. (index - 1) @@ -118,6 +130,7 @@ if entityData.features then "usage_limit", toString(breakdownItem.usage_limit), "overage_allowed", toString(breakdownItem.overage_allowed) ) + redis.call("EXPIRE", breakdownKey, CACHE_TTL_SECONDS) end end end diff --git a/server/src/_luaScripts/entityLuaScripts/setEntityProducts.lua b/server/src/_luaScripts/entityLuaScripts/setEntityProducts.lua index c47f20685..343f0af2a 100644 --- a/server/src/_luaScripts/entityLuaScripts/setEntityProducts.lua +++ b/server/src/_luaScripts/entityLuaScripts/setEntityProducts.lua @@ -1,10 +1,19 @@ -- setEntityProducts.lua -- Updates only the products array in the entity cache --- KEYS[1]: cache key (e.g., "{org_id}:env:customer:customer_id:entity:entity_id") -- ARGV[1]: serialized products array JSON string +-- ARGV[2]: org_id +-- ARGV[3]: env +-- ARGV[4]: customer_id +-- ARGV[5]: entity_id -local cacheKey = KEYS[1] local productsJson = ARGV[1] +local orgId = ARGV[2] +local env = ARGV[3] +local customerId = ARGV[4] +local entityId = ARGV[5] + +-- Build versioned cache key using shared utility +local cacheKey = buildEntityCacheKey(orgId, env, customerId, entityId) local baseKey = cacheKey -- Get base entity JSON @@ -20,8 +29,9 @@ local products = cjson.decode(productsJson) -- Update only the products array baseEntity.products = products --- Store updated base entity as JSON +-- Store updated base entity as JSON and extend TTL redis.call("SET", baseKey, cjson.encode(baseEntity)) +redis.call("EXPIRE", baseKey, CACHE_TTL_SECONDS) return "OK" diff --git a/server/src/_luaScripts/luaScripts.ts b/server/src/_luaScripts/luaScripts.ts index 915c85030..e3becf169 100644 --- a/server/src/_luaScripts/luaScripts.ts +++ b/server/src/_luaScripts/luaScripts.ts @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { CACHE_CUSTOMER_VERSION, CACHE_TTL_SECONDS } from "./cacheConfig.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -9,6 +10,18 @@ const __dirname = dirname(__filename); // SHARED LUA FUNCTIONS // ============================================================================ +// Load cache key utilities and inject version constants +const CACHE_KEY_UTILS_RAW = readFileSync( + join(__dirname, "cacheKeyUtils.lua"), + "utf-8", +); + +// Inject cache version and TTL constants into cache key utils +const CACHE_KEY_UTILS = CACHE_KEY_UTILS_RAW.replace( + /{CUSTOMER_VERSION}/g, + CACHE_CUSTOMER_VERSION, +).replace("{TTL_SECONDS}", CACHE_TTL_SECONDS.toString()); + // Load shared feature loading function (used by customer, entity, and deduction scripts) const LOAD_CUS_FEATURES = readFileSync( join(__dirname, "cusLuaScripts/loadCusFeatures.lua"), @@ -25,34 +38,40 @@ const CHECK_CACHE_EXISTS = readFileSync( "utf-8", ); -// Prepend loadCusFeatures to GET_CUSTOMER_SCRIPT so it can use the function +// Prepend cache key utils and loadCusFeatures to GET_CUSTOMER_SCRIPT const getCustomerScript = readFileSync( join(__dirname, "cusLuaScripts/getCustomer.lua"), "utf-8", ); -export const GET_CUSTOMER_SCRIPT = `${LOAD_CUS_FEATURES}\n${getCustomerScript}`; +export const GET_CUSTOMER_SCRIPT = `${CACHE_KEY_UTILS}\n${LOAD_CUS_FEATURES}\n${getCustomerScript}`; -// Prepend validation function to SET_CUSTOMER_SCRIPT +// Prepend cache key utils and validation function to SET_CUSTOMER_SCRIPT const setCustomerScript = readFileSync( join(__dirname, "cusLuaScripts/setCustomer.lua"), "utf-8", ); -export const SET_CUSTOMER_SCRIPT = `${CHECK_CACHE_EXISTS}\n${setCustomerScript}`; +export const SET_CUSTOMER_SCRIPT = `${CACHE_KEY_UTILS}\n${CHECK_CACHE_EXISTS}\n${setCustomerScript}`; -export const SET_CUSTOMER_PRODUCTS_SCRIPT = readFileSync( +// Prepend cache key utils to SET_CUSTOMER_PRODUCTS_SCRIPT +const setCustomerProductsScript = readFileSync( join(__dirname, "cusLuaScripts/setCustomerProducts.lua"), "utf-8", ); +export const SET_CUSTOMER_PRODUCTS_SCRIPT = `${CACHE_KEY_UTILS}\n${setCustomerProductsScript}`; -export const SET_CUSTOMER_DETAILS_SCRIPT = readFileSync( +// Prepend cache key utils to SET_CUSTOMER_DETAILS_SCRIPT +const setCustomerDetailsScript = readFileSync( join(__dirname, "cusLuaScripts/setCustomerDetails.lua"), "utf-8", ); +export const SET_CUSTOMER_DETAILS_SCRIPT = `${CACHE_KEY_UTILS}\n${setCustomerDetailsScript}`; -export const DELETE_CUSTOMER_SCRIPT = readFileSync( +// Prepend cache key utils to DELETE_CUSTOMER_SCRIPT +const deleteCustomerScript = readFileSync( join(__dirname, "cusLuaScripts/deleteCustomer.lua"), "utf-8", ); +export const DELETE_CUSTOMER_SCRIPT = `${CACHE_KEY_UTILS}\n${deleteCustomerScript}`; // ============================================================================ // ENTITY SCRIPTS @@ -64,29 +83,33 @@ const CHECK_ENTITY_CACHE_EXISTS = readFileSync( "utf-8", ); -// Prepend loadCusFeatures to GET_ENTITY_SCRIPT so it can use the function +// Prepend cache key utils and loadCusFeatures to GET_ENTITY_SCRIPT const getEntityScript = readFileSync( join(__dirname, "entityLuaScripts/getEntity.lua"), "utf-8", ); -export const GET_ENTITY_SCRIPT = `${LOAD_CUS_FEATURES}\n${getEntityScript}`; +export const GET_ENTITY_SCRIPT = `${CACHE_KEY_UTILS}\n${LOAD_CUS_FEATURES}\n${getEntityScript}`; -// Prepend validation function to SET_ENTITY_SCRIPT +// Prepend cache key utils and validation function to SET_ENTITY_SCRIPT const setEntityScript = readFileSync( join(__dirname, "entityLuaScripts/setEntity.lua"), "utf-8", ); -export const SET_ENTITY_SCRIPT = `${CHECK_ENTITY_CACHE_EXISTS}\n${setEntityScript}`; +export const SET_ENTITY_SCRIPT = `${CACHE_KEY_UTILS}\n${CHECK_ENTITY_CACHE_EXISTS}\n${setEntityScript}`; -export const SET_ENTITIES_BATCH_SCRIPT = readFileSync( +// Prepend cache key utils to SET_ENTITIES_BATCH_SCRIPT +const setEntitiesBatchScript = readFileSync( join(__dirname, "entityLuaScripts/setEntitiesBatch.lua"), "utf-8", ); +export const SET_ENTITIES_BATCH_SCRIPT = `${CACHE_KEY_UTILS}\n${setEntitiesBatchScript}`; -export const SET_ENTITY_PRODUCTS_SCRIPT = readFileSync( +// Prepend cache key utils to SET_ENTITY_PRODUCTS_SCRIPT +const setEntityProductsScript = readFileSync( join(__dirname, "entityLuaScripts/setEntityProducts.lua"), "utf-8", ); +export const SET_ENTITY_PRODUCTS_SCRIPT = `${CACHE_KEY_UTILS}\n${setEntityProductsScript}`; // ============================================================================ // DEDUCTION SCRIPTS @@ -99,7 +122,7 @@ const batchDeduction = readFileSync( ); export function getBatchDeductionScript(): string { - return `${LOAD_CUS_FEATURES}\n${batchDeduction}`; + return `${CACHE_KEY_UTILS}\n${LOAD_CUS_FEATURES}\n${batchDeduction}`; } export const BATCH_DEDUCTION_SCRIPT = getBatchDeductionScript(); diff --git a/server/src/external/autumn/autumnCli.ts b/server/src/external/autumn/autumnCli.ts index f86f50e72..f72af7129 100644 --- a/server/src/external/autumn/autumnCli.ts +++ b/server/src/external/autumn/autumnCli.ts @@ -6,6 +6,7 @@ dotenv.config(); import { type ApiEntity, type AttachBody, + type CreateCustomerParams, type CreateEntityParams, type CreateRewardProgram, CusExpand, @@ -305,24 +306,17 @@ export class AutumnInt { }, create: async ({ - id, - email, - name, withAutumnId = true, expand = [], + ...customerData }: { - id: string; - email?: string; - name?: string; withAutumnId?: boolean; expand?: CusExpand[]; - }) => { + } & CreateCustomerParams) => { const data = await this.post( `/customers?with_autumn_id=${withAutumnId ? "true" : "false"}${expand && expand.length > 0 ? `&expand=${expand.join(",")}` : ""}`, { - id, - email, - name, + ...customerData, }, ); return data; diff --git a/server/src/external/stripe/handleStripeWebhookEvent.ts b/server/src/external/stripe/handleStripeWebhookEvent.ts index cb3bc87f3..2b4b6821c 100644 --- a/server/src/external/stripe/handleStripeWebhookEvent.ts +++ b/server/src/external/stripe/handleStripeWebhookEvent.ts @@ -7,7 +7,6 @@ import { unsetOrgStripeKeys } from "@/internal/orgs/orgUtils.js"; import type { ExtendedRequest } from "@/utils/models/Request.js"; import type { AutumnContext } from "../../honoUtils/HonoEnv.js"; import { deleteCachedApiCustomer } from "../../internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.js"; -import { setCachedApiCusProducts } from "../../internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusProducts.js"; import type { Logger } from "../logtail/logtailUtils.js"; import { handleCheckoutSessionCompleted } from "./webhookHandlers/handleCheckoutCompleted.js"; import { handleCusDiscountDeleted } from "./webhookHandlers/handleCusDiscountDeleted.js"; @@ -82,30 +81,38 @@ const handleStripeWebhookRefresh = async ({ return; } - if (updateProductEvents.includes(eventType)) { - const fullCus = await CusService.getFull({ - db, - idOrInternalId: cus.id!, - orgId: org.id, - env, - withEntities: true, - withSubs: true, - }); + logger.info(`Attempting delete cached api customer! ${eventType}`); + await deleteCachedApiCustomer({ + customerId: cus.id!, + orgId: org.id, + env, + source: `handleStripeWebhookRefresh: ${eventType}`, + }); - await setCachedApiCusProducts({ - ctx, - fullCus, - customerId: cus.id!, - }); - } else { - logger.info(`Attempting delete cached api customer! ${eventType}`); - await deleteCachedApiCustomer({ - customerId: cus.id!, - orgId: org.id, - env, - source: `handleStripeWebhookRefresh: ${eventType}`, - }); - } + // if (updateProductEvents.includes(eventType)) { + // const fullCus = await CusService.getFull({ + // db, + // idOrInternalId: cus.id!, + // orgId: org.id, + // env, + // withEntities: true, + // withSubs: true, + // }); + + // await setCachedApiCusProducts({ + // ctx, + // fullCus, + // customerId: cus.id!, + // }); + // } else { + // logger.info(`Attempting delete cached api customer! ${eventType}`); + // await deleteCachedApiCustomer({ + // customerId: cus.id!, + // orgId: org.id, + // env, + // source: `handleStripeWebhookRefresh: ${eventType}`, + // }); + // } } }; diff --git a/server/src/init.ts b/server/src/init.ts index 27e017958..67c7dcd24 100644 --- a/server/src/init.ts +++ b/server/src/init.ts @@ -154,7 +154,7 @@ const init = async () => { await Promise.all([ClickHouseManager.getInstance()]); // Initialize database functions - // await initializeDatabaseFunctions(); + await initializeDatabaseFunctions(); app.use(async (req: any, res: any, next: any) => { // Add Render region identifier headers for load balancer verification diff --git a/server/src/internal/api/check/checkUtils/getCheckData.ts b/server/src/internal/api/check/checkUtils/getCheckData.ts index df1291b35..08d500cb0 100644 --- a/server/src/internal/api/check/checkUtils/getCheckData.ts +++ b/server/src/internal/api/check/checkUtils/getCheckData.ts @@ -156,52 +156,11 @@ export const getCheckData = async ({ apiEntity, }); - // const filteredCusEnts = cusEnts.filter((cusEnt) => - // cusEntMatchesFeature({ cusEnt, feature: featureToUse }), - // ); - return { customerId: customer_id, entityId: entity_id, cusFeature: apiEntity.features?.[featureToUse.id], - // cusEnts: filteredCusEnts, originalFeature: feature, featureToUse, - // cusProducts, - // entity: customer.entity, - // allFeatures, - // entity: customer.entity, }; }; - -// if (entity_id) { -// const cusFeature = apiCustomer.features[feature.id]; -// } - -// const cusProducts = customer.customer_products; - -// let cusEnts = cusProductsToCusEnts({ cusProducts }); - -// if (customer.entity) { -// cusEnts = cusEnts.filter((cusEnt) => -// cusEntMatchesEntity({ -// cusEnt, -// entity: customer.entity!, -// features: allFeatures, -// }), -// ); -// } - -// const inStatuses = org.config.include_past_due -// ? [CusProductStatus.Active, CusProductStatus.PastDue] -// : [CusProductStatus.Active]; - -// const customer = await getOrCreateCustomer({ -// req: ctx as ExtendedRequest, -// customerId: customer_id, -// customerData: customer_data, -// inStatuses, -// entityId: entity_id, -// entityData: body.entity_data, -// withCache: true, -// }); diff --git a/server/src/internal/balances/setUsage/getSetUsageDeductions.ts b/server/src/internal/balances/setUsage/getSetUsageDeductions.ts index 1ad9488e7..ff6187055 100644 --- a/server/src/internal/balances/setUsage/getSetUsageDeductions.ts +++ b/server/src/internal/balances/setUsage/getSetUsageDeductions.ts @@ -7,6 +7,7 @@ import { FeatureNotFoundError, FeatureType, type FullCustomerEntitlement, + orgToInStatuses, RecaseError, type SetUsageParams, sumValues, @@ -72,6 +73,7 @@ export const getSetUsageDeductions = async ({ cusProducts: fullCus.customer_products, reverseOrder: org.config?.reverse_deduction_order, featureId: feature.id, + inStatuses: orgToInStatuses({ org }), }); // ========================================== @@ -143,6 +145,7 @@ export const getSetUsageDeductions = async ({ cusProducts: fullCus.customer_products, reverseOrder: org.config?.reverse_deduction_order, featureId: deductionFeature.id, + inStatuses: orgToInStatuses({ org }), }); const { unlimited } = getUnlimitedAndUsageAllowed({ diff --git a/server/src/internal/balances/track/handleTrack.ts b/server/src/internal/balances/track/handleTrack.ts index 1d207b14d..ced325fba 100644 --- a/server/src/internal/balances/track/handleTrack.ts +++ b/server/src/internal/balances/track/handleTrack.ts @@ -1,6 +1,5 @@ import { ApiVersion, - CusProductStatus, ErrCode, InsufficientBalanceError, isContUseFeature, @@ -50,7 +49,6 @@ const executePostgresTracking = async ({ customerData: body.customer_data, entityId: body.entity_id, entityData: body.entity_data, - inStatuses: [CusProductStatus.Active, CusProductStatus.PastDue], withEntities: true, }); diff --git a/server/src/internal/balances/track/redisTrackUtils/BatchingManager.ts b/server/src/internal/balances/track/redisTrackUtils/BatchingManager.ts index 6f4de7d74..acfd7b8ef 100644 --- a/server/src/internal/balances/track/redisTrackUtils/BatchingManager.ts +++ b/server/src/internal/balances/track/redisTrackUtils/BatchingManager.ts @@ -142,13 +142,6 @@ export class BatchingManager { const requests = batch.requests; const batchSize = requests.length; - // Build cache key from batch context (always customer cache key for the Lua script) - const cacheKey = buildCachedApiCustomerKey({ - customerId: batch.customerId, - orgId: batch.orgId, - env: batch.env, - }); - const batchType = batch.entityId ? `entity ${batch.entityId}` : "customer-level"; @@ -157,11 +150,10 @@ export class BatchingManager { ); try { - // Execute batch Lua script + // Execute batch Lua script (Lua builds cache key internally) // All requests in this batch have the same entityId (batch-level) const result = await executeBatchDeduction({ redis, - cacheKey, requests: requests.map((r) => ({ featureDeductions: r.featureDeductions, overageBehavior: r.overageBehavior, diff --git a/server/src/internal/balances/track/redisTrackUtils/deductFromCache.ts b/server/src/internal/balances/track/redisTrackUtils/deductFromCache.ts index 71e3013f3..d0242dc2c 100644 --- a/server/src/internal/balances/track/redisTrackUtils/deductFromCache.ts +++ b/server/src/internal/balances/track/redisTrackUtils/deductFromCache.ts @@ -1,7 +1,6 @@ import { redis } from "../../../../external/redis/initRedis.js"; import type { AutumnContext } from "../../../../honoUtils/HonoEnv.js"; import { tryRedisWrite } from "../../../../utils/cacheUtils/cacheUtils.js"; -import { buildCachedApiCustomerKey } from "../../../customers/cusUtils/apiCusCacheUtils/getCachedApiCustomer.js"; import { executeBatchDeduction } from "./executeBatchDeduction.js"; /** @@ -28,17 +27,10 @@ export const deductFromCache = async ({ }): Promise => { const { org, env } = ctx; - const cacheKey = buildCachedApiCustomerKey({ - customerId, - orgId: org.id, - env, - }); - // Execute Redis deduction directly (no batching to avoid race conditions) await tryRedisWrite(async () => { const result = await executeBatchDeduction({ redis, - cacheKey, requests: [ { featureDeductions: [ diff --git a/server/src/internal/balances/track/redisTrackUtils/executeBatchDeduction.ts b/server/src/internal/balances/track/redisTrackUtils/executeBatchDeduction.ts index b3e749588..35e8a212d 100644 --- a/server/src/internal/balances/track/redisTrackUtils/executeBatchDeduction.ts +++ b/server/src/internal/balances/track/redisTrackUtils/executeBatchDeduction.ts @@ -35,14 +35,12 @@ interface BatchDeductionResult { */ export const executeBatchDeduction = async ({ redis, - cacheKey, requests, orgId, env, customerId, }: { redis: Redis; - cacheKey: string; requests: BatchRequest[]; orgId: string; env: string; @@ -52,8 +50,7 @@ export const executeBatchDeduction = async ({ // Execute Lua script (hot reload in dev) const result = await redis.eval( getBatchDeductionScript(), - 1, // number of keys - cacheKey, // KEYS[1] + 0, // No KEYS, all params in ARGV JSON.stringify(requests), // ARGV[1] orgId, // ARGV[2] env, // ARGV[3] diff --git a/server/src/internal/balances/track/trackUtils/deductRpc/performDeductionV2.sql b/server/src/internal/balances/track/trackUtils/deductRpc/performDeductionV2.sql index 37d83beb7..400d9fbd9 100644 --- a/server/src/internal/balances/track/trackUtils/deductRpc/performDeductionV2.sql +++ b/server/src/internal/balances/track/trackUtils/deductRpc/performDeductionV2.sql @@ -211,6 +211,8 @@ BEGIN END IF; -- Track in updates_json + -- Convert deducted (credit amount) back to original feature amount + -- If credit_cost is NULL or 1, no conversion needed (not a credit system) updates_json := jsonb_set( updates_json, ARRAY[ent_id], @@ -218,11 +220,12 @@ BEGIN 'balance', new_balance, 'entities', new_entities, 'adjustment', new_adjustment, - 'deducted', deducted + 'deducted', CASE WHEN credit_cost IS NULL OR credit_cost = 1 THEN deducted ELSE deducted / credit_cost END ) ); - remaining_amount := remaining_amount - (deducted / credit_cost); + -- Subtract from remaining_amount (convert credit amount back to feature amount) + remaining_amount := remaining_amount - CASE WHEN credit_cost IS NULL OR credit_cost = 1 THEN deducted ELSE deducted / credit_cost END; END IF; END LOOP; @@ -287,6 +290,8 @@ BEGIN END IF; -- Update or create entry in updates_json + -- Convert deducted (credit amount) back to original feature amount + -- If credit_cost is NULL or 1, no conversion needed (not a credit system) IF updates_json ? ent_id THEN -- Update existing entry (entitlement was updated in both passes) updates_json := jsonb_set( @@ -296,7 +301,7 @@ BEGIN 'balance', new_balance, 'entities', new_entities, 'adjustment', new_adjustment, - 'deducted', (updates_json->ent_id->>'deducted')::numeric + deducted + 'deducted', (updates_json->ent_id->>'deducted')::numeric + CASE WHEN credit_cost IS NULL OR credit_cost = 1 THEN deducted ELSE deducted / credit_cost END ) ); ELSE @@ -308,12 +313,13 @@ BEGIN 'balance', new_balance, 'entities', new_entities, 'adjustment', new_adjustment, - 'deducted', deducted + 'deducted', CASE WHEN credit_cost IS NULL OR credit_cost = 1 THEN deducted ELSE deducted / credit_cost END ) ); END IF; - remaining_amount := remaining_amount - (deducted / credit_cost); + -- Subtract from remaining_amount (convert credit amount back to feature amount) + remaining_amount := remaining_amount - CASE WHEN credit_cost IS NULL OR credit_cost = 1 THEN deducted ELSE deducted / credit_cost END; END IF; END LOOP; END IF; diff --git a/server/src/internal/balances/track/trackUtils/runDeductionTx.ts b/server/src/internal/balances/track/trackUtils/runDeductionTx.ts index ebd1de8ce..db1260a97 100644 --- a/server/src/internal/balances/track/trackUtils/runDeductionTx.ts +++ b/server/src/internal/balances/track/trackUtils/runDeductionTx.ts @@ -12,9 +12,9 @@ import { InternalError, notNullish, nullish, + orgToInStatuses, updateCusEntInFullCus, } from "@autumn/shared"; -import chalk from "chalk"; import { sql } from "drizzle-orm"; import type { DrizzleCli } from "../../../../db/initDrizzle.js"; import type { AutumnContext } from "../../../../honoUtils/HonoEnv.js"; @@ -118,6 +118,7 @@ export const deductFromCusEnts = async ({ featureIds: relevantFeatures.map((f) => f.id), reverseOrder: org.config?.reverse_deduction_order, entity: fullCus.entity, + inStatuses: orgToInStatuses({ org }), }); if (printLogs) { @@ -385,7 +386,7 @@ export const runDeductionTx = async ( "../redisTrackUtils/deductFromCache.js" ); - const printLogs = true; + const printLogs = false; for (const [featureId, deductedAmount] of Object.entries( actualDeductions, @@ -401,12 +402,10 @@ export const runDeductionTx = async ( }); if (printLogs) { - logger.info( - `[REDIS] Deduced users from cache: ${chalk.yellow(actualDeductions.users)}`, - ); - // logger.info( - // `[REDIS] balance after deduction for ${featureId}: ${chalk.yellow(balance)}`, - // ); + console.log("Deducted from Redis cache", { + featureId, + deductedAmount, + }); } } } diff --git a/server/src/internal/customers/attach/attachUtils/handleAttachErrors.ts b/server/src/internal/customers/attach/attachUtils/handleAttachErrors.ts index 3a7f741b0..013d078c7 100644 --- a/server/src/internal/customers/attach/attachUtils/handleAttachErrors.ts +++ b/server/src/internal/customers/attach/attachUtils/handleAttachErrors.ts @@ -2,17 +2,11 @@ import { type AttachBody, AttachBranch, type AttachConfig, - AttachErrCode, BillingType, - cusProductsToCusEnts, - cusProductToPrices, ErrCode, - type FullCusProduct, - getStartingBalance, - type UsagePriceConfig + ErrCode, + type UsagePriceConfig, } from "@autumn/shared"; -import { Decimal } from "decimal.js"; import { StatusCodes } from "http-status-codes"; -import { findPriceForFeature } from "@/internal/products/prices/priceUtils/findPriceUtils.js"; import { getBillingType, getEntOptions, @@ -23,7 +17,6 @@ import RecaseError from "@/utils/errorUtils.js"; import { notNullish, nullOrUndefined } from "@/utils/genUtils.js"; import type { AttachParams } from "../../cusProducts/AttachParams.js"; import type { AttachFlags } from "../models/AttachFlags.js"; -import { attachParamToCusProducts } from "./convertAttachParams.js"; import { handleMultiAttachErrors } from "./handleAttachErrors/handleMultiAttachErrors.js"; const handleNonCheckoutErrors = ({ @@ -144,66 +137,6 @@ const handlePrepaidErrors = async ({ } }; -const handleUpdateQuantityErrors = async ({ - attachParams, -}: { - attachParams: AttachParams; -}) => { - const { curMainProduct, curSameProduct } = attachParamToCusProducts({ - attachParams, - }); - - if (!curSameProduct && !curMainProduct) { - return; - } - - const cusProduct = (curSameProduct || curMainProduct) as FullCusProduct; - const cusEnts = cusProductsToCusEnts({ cusProducts: [cusProduct] }); - const prices = cusProductToPrices({ cusProduct }); - - for (const option of attachParams.optionsList) { - const price = findPriceForFeature({ - prices, - internalFeatureId: option.internal_feature_id!, - }); - - if (!price) continue; - - const totalQuantity = - option.quantity! * (price?.config as UsagePriceConfig).billing_units!; - - const totalUsage = cusEnts - .reduce((acc, curr) => { - if ( - curr.entitlement.internal_feature_id === option.internal_feature_id - ) { - const allowance = getStartingBalance({ - entitlement: curr.entitlement, - options: cusProduct.options.find( - (o) => o.internal_feature_id === option.internal_feature_id, - ), - relatedPrice: price, - }); - - const usage = new Decimal(allowance!).minus(curr.balance!); - - return acc.plus(usage); - } - - return acc; - }, new Decimal(0)) - .toNumber(); - - if (totalUsage > totalQuantity) { - throw new RecaseError({ - message: `Current usage for ${option.feature_id} is ${totalUsage}, can't update to ${totalQuantity}`, - code: AttachErrCode.InvalidOptions, - statusCode: StatusCodes.BAD_REQUEST, - }); - } - } -}; - export const handleAttachErrors = async ({ attachParams, attachBody, @@ -237,12 +170,7 @@ export const handleAttachErrors = async ({ AttachBranch.Downgrade, ]; - - - - if ( - upgradeDowngradeFlows.includes(branch) - ) { + if (upgradeDowngradeFlows.includes(branch)) { handleNonCheckoutErrors({ flags, config, diff --git a/server/src/internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.ts b/server/src/internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.ts index d1a22c259..34aba22c4 100644 --- a/server/src/internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.ts +++ b/server/src/internal/customers/cusUtils/apiCusCacheUtils/deleteCachedApiCustomer.ts @@ -1,7 +1,6 @@ import { DELETE_CUSTOMER_SCRIPT } from "@lua/luaScripts.js"; import { redis } from "@/external/redis/initRedis.js"; import { logger } from "../../../../external/logtail/logtailUtils.js"; -import { buildCachedApiCustomerKey } from "./getCachedApiCustomer.js"; /** * Delete all cached ApiCustomer data from Redis @@ -29,17 +28,13 @@ export const deleteCachedApiCustomer = async ({ if (!customerId) return; - const cacheKey = buildCachedApiCustomerKey({ - customerId, - orgId, - env, - }); - try { const deletedCount = await redis.eval( DELETE_CUSTOMER_SCRIPT, - 1, - cacheKey, // The base pattern: {orgId}:env:customer:customerId + 0, // No KEYS, all params in ARGV + orgId, + env, + customerId, ); logger.info( diff --git a/server/src/internal/customers/cusUtils/apiCusCacheUtils/getCachedApiCustomer.ts b/server/src/internal/customers/cusUtils/apiCusCacheUtils/getCachedApiCustomer.ts index f9f850e35..84585473d 100644 --- a/server/src/internal/customers/cusUtils/apiCusCacheUtils/getCachedApiCustomer.ts +++ b/server/src/internal/customers/cusUtils/apiCusCacheUtils/getCachedApiCustomer.ts @@ -5,6 +5,7 @@ import { type CustomerLegacyData, filterOutEntitiesFromCusProducts, } from "@autumn/shared"; +import { CACHE_CUSTOMER_VERSION } from "@lua/cacheConfig.js"; import { GET_CUSTOMER_SCRIPT } from "@lua/luaScripts.js"; import { redis } from "../../../../external/redis/initRedis.js"; import type { AutumnContext } from "../../../../honoUtils/HonoEnv.js"; @@ -13,7 +14,6 @@ import { tryRedisRead, } from "../../../../utils/cacheUtils/cacheUtils.js"; import { CusService } from "../../CusService.js"; -import { RELEVANT_STATUSES } from "../../cusProducts/CusProductService.js"; import { getApiCustomerBase } from "../apiCusUtils/getApiCustomerBase.js"; import { setCachedApiCustomer } from "./setCachedApiCustomer.js"; @@ -26,7 +26,7 @@ export const buildCachedApiCustomerKey = ({ orgId: string; env: string; }) => { - return `{${orgId}}:${env}:customer:${customerId}`; + return `{${orgId}}:${env}:customer:${CACHE_CUSTOMER_VERSION}:${customerId}`; }; /** @@ -49,19 +49,12 @@ export const getCachedApiCustomer = async ({ }): Promise<{ apiCustomer: ApiCustomer; legacyData: CustomerLegacyData }> => { const { org, env, db } = ctx; - const cacheKey = buildCachedApiCustomerKey({ - customerId, - orgId: org.id, - env, - }); - // Try to get from cache using Lua script (unless skipCache is true) if (!skipCache) { const cachedResult = await tryRedisRead(() => redis.eval( GET_CUSTOMER_SCRIPT, - 1, - cacheKey, + 0, // No KEYS, all params in ARGV org.id, env, customerId, @@ -93,7 +86,6 @@ export const getCachedApiCustomer = async ({ idOrInternalId: customerId, orgId: org.id, env: env as AppEnv, - inStatuses: RELEVANT_STATUSES, withEntities: true, withSubs: true, }); diff --git a/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusDetails.ts b/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusDetails.ts index 96ffba8b5..70f596a0f 100644 --- a/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusDetails.ts +++ b/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusDetails.ts @@ -3,7 +3,6 @@ import { SET_CUSTOMER_DETAILS_SCRIPT } from "@lua/luaScripts.js"; import { redis } from "../../../../external/redis/initRedis.js"; import type { AutumnContext } from "../../../../honoUtils/HonoEnv.js"; import { tryRedisWrite } from "../../../../utils/cacheUtils/cacheUtils.js"; -import { buildCachedApiCustomerKey } from "./getCachedApiCustomer.js"; /** * Update customer detail fields in Redis cache if key exists @@ -25,13 +24,7 @@ export const setCachedApiCusDetails = async ({ }): Promise => { const { org, env, logger } = ctx; - // Build the cache key const customerId = customer.id || (customer as FullCustomer).internal_id; - const cacheKey = buildCachedApiCustomerKey({ - customerId, - orgId: org.id, - env, - }); let wasUpdated = false; @@ -39,9 +32,11 @@ export const setCachedApiCusDetails = async ({ await tryRedisWrite(async () => { const result = await redis.eval( SET_CUSTOMER_DETAILS_SCRIPT, - 1, - cacheKey, + 0, // No KEYS, all params in ARGV JSON.stringify(updates), + org.id, + env, + customerId, ); if (result === "OK") { diff --git a/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusProducts.ts b/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusProducts.ts index 971ce34c5..673406a46 100644 --- a/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusProducts.ts +++ b/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCusProducts.ts @@ -10,9 +10,7 @@ import { import { redis } from "../../../../external/redis/initRedis.js"; import type { AutumnContext } from "../../../../honoUtils/HonoEnv.js"; import { tryRedisWrite } from "../../../../utils/cacheUtils/cacheUtils.js"; -import { buildCachedApiEntityKey } from "../../../entities/entityUtils/apiEntityCacheUtils/getCachedApiEntity.js"; import { getApiCusProducts } from "../apiCusUtils/getApiCusProduct/getApiCusProducts.js"; -import { buildCachedApiCustomerKey } from "./getCachedApiCustomer.js"; /** * Set customer products cache in Redis with all entities @@ -30,12 +28,6 @@ export const setCachedApiCusProducts = async ({ }) => { const { org, env, logger } = ctx; - const cacheKey = buildCachedApiCustomerKey({ - customerId, - orgId: org.id, - env, - }); - // Build master api customer products (customer-level products only) const { apiCusProducts: masterApiCusProducts } = await getApiCusProducts({ ctx, @@ -52,9 +44,11 @@ export const setCachedApiCusProducts = async ({ // Update customer products await redis.eval( SET_CUSTOMER_PRODUCTS_SCRIPT, - 1, - cacheKey, + 0, // No KEYS, all params in ARGV JSON.stringify(masterApiCusProducts), + org.id, + env, + customerId, ); logger.info( `Updated customer products cache for customer ${customerId} (${masterApiCusProducts.length} products)`, @@ -78,18 +72,14 @@ export const setCachedApiCusProducts = async ({ }, }); - const entityCacheKey = buildCachedApiEntityKey({ - entityId: entity.id, - customerId, - orgId: org.id, - env, - }); - await redis.eval( SET_ENTITY_PRODUCTS_SCRIPT, - 1, - entityCacheKey, + 0, // No KEYS, all params in ARGV JSON.stringify(entityProducts), + org.id, + env, + customerId, + entity.id, ); logger.info( `Updated entity products cache for entity ${entity.id} (${entityProducts.length} products)`, diff --git a/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCustomer.ts b/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCustomer.ts index e7b21e400..406a43e47 100644 --- a/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCustomer.ts +++ b/server/src/internal/customers/cusUtils/apiCusCacheUtils/setCachedApiCustomer.ts @@ -13,7 +13,6 @@ import type { AutumnContext } from "../../../../honoUtils/HonoEnv.js"; import { tryRedisWrite } from "../../../../utils/cacheUtils/cacheUtils.js"; import { getApiEntityBase } from "../../../entities/entityUtils/apiEntityUtils/getApiEntityBase.js"; import { getApiCustomerBase } from "../apiCusUtils/getApiCustomerBase.js"; -import { buildCachedApiCustomerKey } from "./getCachedApiCustomer.js"; /** * Set customer cache in Redis with all entities @@ -33,12 +32,6 @@ export const setCachedApiCustomer = async ({ }) => { const { org, env, logger } = ctx; - const cacheKey = buildCachedApiCustomerKey({ - customerId, - orgId: org.id, - env, - }); - // Build master api customer (customer-level features only) const { apiCustomer: masterApiCustomer, legacyData } = await getApiCustomerBase({ @@ -82,8 +75,7 @@ export const setCachedApiCustomer = async ({ await tryRedisWrite(async () => { await redis.eval( SET_CUSTOMER_SCRIPT, - 1, - cacheKey, + 0, // No KEYS, all params in ARGV JSON.stringify({ ...masterApiCustomer, entities: fullCus.entities, @@ -91,6 +83,7 @@ export const setCachedApiCustomer = async ({ }), org.id, env, + customerId, ); if (entityBatch.length > 0) { diff --git a/server/src/internal/customers/cusUtils/apiCusUtils/getApiCusFeature/getApiCusFeatures.ts b/server/src/internal/customers/cusUtils/apiCusUtils/getApiCusFeature/getApiCusFeatures.ts index 76a5d3542..c32e20900 100644 --- a/server/src/internal/customers/cusUtils/apiCusUtils/getApiCusFeature/getApiCusFeatures.ts +++ b/server/src/internal/customers/cusUtils/apiCusUtils/getApiCusFeature/getApiCusFeatures.ts @@ -1,9 +1,9 @@ import { type ApiCusFeature, - CusProductStatus, cusProductsToCusEnts, type FullCusEntWithFullCusProduct, type FullCustomer, + orgToInStatuses, } from "@autumn/shared"; import type { RequestContext } from "@/honoUtils/HonoEnv.js"; import { getApiCusFeature } from "./getApiCusFeature.js"; @@ -19,9 +19,7 @@ export const getApiCusFeatures = async ({ const cusEntsWithCusProduct = cusProductsToCusEnts({ cusProducts: fullCus.customer_products, - inStatuses: org.config.include_past_due - ? [CusProductStatus.Active, CusProductStatus.PastDue] - : [CusProductStatus.Active], + inStatuses: orgToInStatuses({ org }), entity: fullCus.entity, }); diff --git a/server/src/internal/customers/cusUtils/cusFeatureResponseUtils/getCusFeaturesResponse.ts b/server/src/internal/customers/cusUtils/cusFeatureResponseUtils/getCusFeaturesResponse.ts index eafc747e5..5b434ceb3 100644 --- a/server/src/internal/customers/cusUtils/cusFeatureResponseUtils/getCusFeaturesResponse.ts +++ b/server/src/internal/customers/cusUtils/cusFeatureResponseUtils/getCusFeaturesResponse.ts @@ -5,6 +5,7 @@ import { type Entity, type FullCusProduct, type Organization, + orgToInStatuses, } from "@autumn/shared"; import { balancesToFeatureResponse } from "./balancesToFeatureResponse.js"; import { getCusBalances } from "./getCusBalances.js"; @@ -20,7 +21,10 @@ export const getCusFeaturesResponse = async ({ entity?: Entity; apiVersion: ApiVersionClass; }) => { - const cusEnts = cusProductsToCusEnts({ cusProducts }) as any; + const cusEnts = cusProductsToCusEnts({ + cusProducts, + inStatuses: orgToInStatuses({ org }), + }) as any; const balances = await getCusBalances({ cusEntsWithCusProduct: cusEnts, diff --git a/server/src/internal/customers/cusUtils/getOrCreateApiCustomer.ts b/server/src/internal/customers/cusUtils/getOrCreateApiCustomer.ts index c3786c14d..2c6405d32 100644 --- a/server/src/internal/customers/cusUtils/getOrCreateApiCustomer.ts +++ b/server/src/internal/customers/cusUtils/getOrCreateApiCustomer.ts @@ -161,6 +161,11 @@ export const getOrCreateApiCustomer = async ({ env: ctx.env, }); + await getCachedApiCustomer({ + ctx, + customerId, + }); + const apiEntity = ApiEntitySchema.parse(newEntity); apiCustomer.entities = [...(apiCustomer.entities || []), apiEntity]; } diff --git a/server/src/internal/customers/handlers/handlePostCustomerV2.ts b/server/src/internal/customers/handlers/handlePostCustomerV2.ts index 2fc595f3c..ded1fdc42 100644 --- a/server/src/internal/customers/handlers/handlePostCustomerV2.ts +++ b/server/src/internal/customers/handlers/handlePostCustomerV2.ts @@ -39,8 +39,6 @@ export const handlePostCustomer = createRoute({ customerData: createCusParams, }); - console.log("Expand:", expand); - const apiCustomer = await getApiCustomer({ ctx, customerId: createCusParams.id || "", diff --git a/server/src/internal/entities/entityUtils/apiEntityCacheUtils/getCachedApiEntity.ts b/server/src/internal/entities/entityUtils/apiEntityCacheUtils/getCachedApiEntity.ts index 0fd96e888..5266eabad 100644 --- a/server/src/internal/entities/entityUtils/apiEntityCacheUtils/getCachedApiEntity.ts +++ b/server/src/internal/entities/entityUtils/apiEntityCacheUtils/getCachedApiEntity.ts @@ -5,6 +5,7 @@ import { type FullCustomer, filterEntityLevelCusProducts, } from "@autumn/shared"; +import { CACHE_CUSTOMER_VERSION } from "@lua/cacheConfig.js"; import { GET_ENTITY_SCRIPT } from "@lua/luaScripts.js"; import { redis } from "@/external/redis/initRedis.js"; import type { AutumnContext } from "@/honoUtils/HonoEnv.js"; @@ -28,7 +29,7 @@ export const buildCachedApiEntityKey = ({ orgId: string; env: string; }) => { - return `{${orgId}}:${env}:customer:${customerId}:entity:${entityId}`; + return `{${orgId}}:${env}:customer:${CACHE_CUSTOMER_VERSION}:${customerId}:entity:${entityId}`; }; /** @@ -53,20 +54,12 @@ export const getCachedApiEntity = async ({ }): Promise<{ apiEntity: ApiEntity }> => { const { org, env, db } = ctx; - const cacheKey = buildCachedApiEntityKey({ - entityId, - customerId, - orgId: org.id, - env, - }); - // Try to get from cache using Lua script (unless skipCache is true) if (!skipCache) { const cachedResult = await tryRedisRead(() => redis.eval( GET_ENTITY_SCRIPT, - 1, // number of keys - cacheKey, // KEYS[1] + 0, // No KEYS, all params in ARGV org.id, // ARGV[1] env, // ARGV[2] customerId, // ARGV[3] diff --git a/server/src/utils/importUtils/updateUsages.ts b/server/src/utils/importUtils/updateUsages.ts index 6d90d7455..626294cbd 100644 --- a/server/src/utils/importUtils/updateUsages.ts +++ b/server/src/utils/importUtils/updateUsages.ts @@ -1,8 +1,7 @@ -import { DrizzleCli } from "@/db/initDrizzle.js"; -import { CusEntService } from "@/internal/customers/cusProducts/cusEnts/CusEntitlementService.js"; +import { cusProductsToCusEnts, type FullCustomer } from "@autumn/shared"; +import type { DrizzleCli } from "@/db/initDrizzle.js"; import { RELEVANT_STATUSES } from "@/internal/customers/cusProducts/CusProductService.js"; -import { cusProductsToCusEnts } from "@autumn/shared"; -import { FullCustomer } from "@autumn/shared"; +import { CusEntService } from "@/internal/customers/cusProducts/cusEnts/CusEntitlementService.js"; export const updateUsages = async ({ featureId, @@ -15,7 +14,7 @@ export const updateUsages = async ({ fullCus: FullCustomer; db: DrizzleCli; }) => { - let cusEnts = cusProductsToCusEnts({ + const cusEnts = cusProductsToCusEnts({ cusProducts: fullCus.customer_products, inStatuses: RELEVANT_STATUSES, featureId, @@ -24,8 +23,8 @@ export const updateUsages = async ({ throw new Error(`No cus ent for ${featureId}`); } - let cusEnt = cusEnts[0]; - let newBalance = cusEnt.balance! - usage; + const cusEnt = cusEnts[0]; + const newBalance = cusEnt.balance! - usage; await CusEntService.update({ db, diff --git a/server/tests/balances/track/basic/track-basic10.test.ts b/server/tests/balances/track/basic/track-basic10.test.ts index 54c227bcf..49e85358e 100644 --- a/server/tests/balances/track/basic/track-basic10.test.ts +++ b/server/tests/balances/track/basic/track-basic10.test.ts @@ -42,6 +42,13 @@ describe(`${chalk.yellowBright(`${testCase}: Testing deduction order with monthl const autumnV1: AutumnInt = new AutumnInt({ version: ApiVersion.V1_2 }); beforeAll(async () => { + await initProductsV0({ + ctx, + products: [monthlyProduct], + prefix: testCase, + customerId, + }); + await initCustomerV3({ ctx, customerId, @@ -49,12 +56,6 @@ describe(`${chalk.yellowBright(`${testCase}: Testing deduction order with monthl attachPm: "success", }); - await initProductsV0({ - ctx, - products: [monthlyProduct], - prefix: testCase, - }); - // Attach monthly product first await autumnV1.attach({ customer_id: customerId, diff --git a/server/tests/balances/track/legacy/track-legacy3.test.ts b/server/tests/balances/track/legacy/track-legacy3.test.ts index c31f6424c..1d6e0aa95 100644 --- a/server/tests/balances/track/legacy/track-legacy3.test.ts +++ b/server/tests/balances/track/legacy/track-legacy3.test.ts @@ -1,4 +1,4 @@ -import { beforeAll } from "bun:test"; +import { beforeAll, describe, test } from "bun:test"; import { ProductItemInterval } from "@autumn/shared"; import chalk from "chalk"; import ctx from "tests/utils/testInitUtils/createTestContext.js"; diff --git a/server/tests/balances/track/misc/track-misc3.test.ts b/server/tests/balances/track/misc/track-misc3.test.ts new file mode 100644 index 000000000..9cd9e0ce3 --- /dev/null +++ b/server/tests/balances/track/misc/track-misc3.test.ts @@ -0,0 +1,63 @@ +// import { beforeAll, describe, expect, test } from "bun:test"; +// import { ApiVersion } from "@autumn/shared"; +// import chalk from "chalk"; +// import { TestFeature } from "tests/setup/v2Features.js"; +// import ctx from "tests/utils/testInitUtils/createTestContext.js"; +// import { AutumnInt } from "@/external/autumn/autumnCli.js"; +// import { EventService } from "../../../../src/internal/api/events/EventService.js"; +// import { constructFeatureItem } from "../../../../src/utils/scriptUtils/constructItem.js"; +// import { constructProduct } from "../../../../src/utils/scriptUtils/createTestProducts.js"; +// import { timeout } from "../../../utils/genUtils.js"; + +// const proProduct = constructProduct({ +// type: "pro", +// items: [constructFeatureItem({ featureId: TestFeature.Messages })], +// }); + +// describe(`${chalk.yellowBright("track-misc3: testing track auto creates customer and entity")}`, () => { +// const customerId = "track-misc3"; +// const autumnV1: AutumnInt = new AutumnInt({ version: ApiVersion.V1_2 }); + +// beforeAll(async () => { +// try { +// await autumnV1.customers.delete(customerId); +// } catch { +// // Ignore if customer doesn't exist +// } +// }); + +// test("should track event for customer / entity and have properties set", async () => { +// await autumnV1.track({ +// customer_id: customerId, +// customer_data: { +// name: "track-misc2", +// email: "track-misc2@test.com", +// }, +// feature_id: TestFeature.Messages, +// value: 5, +// properties: { +// hello: "world", +// foo: "bar", +// }, +// }); + +// const customer = await autumnV1.customers.get(customerId, { +// with_autumn_id: true, +// }); + +// await timeout(2000); + +// const events = await EventService.getByCustomerId({ +// db: ctx.db, +// orgId: ctx.org.id, +// internalCustomerId: customer.autumn_id!, +// env: ctx.env, +// }); + +// expect(events).toHaveLength(1); +// expect(events?.[0].properties).toMatchObject({ +// hello: "world", +// foo: "bar", +// }); +// }); +// }); diff --git a/server/tsconfig.build.json b/server/tsconfig.build.json index 2f05895e3..c84cd78c3 100644 --- a/server/tsconfig.build.json +++ b/server/tsconfig.build.json @@ -21,7 +21,8 @@ "noEmitOnError": false, "paths": { "@/*": ["src/*"], - "@emails/*": ["emails/*"] + "@emails/*": ["emails/*"], + "@lua/*": ["src/_luaScripts/*"], } }, "include": ["src", "emails", "src/external/clickhouse/queries"], diff --git a/server/tsconfig.json b/server/tsconfig.json index 68cc1b6d6..746db577e 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -21,8 +21,8 @@ "paths": { "@/*": ["src/*"], "@shared/*": ["../shared/*"], - "@scripts/*": ["scripts/*"], "@emails/*": ["emails/*"], + "@scripts/*": ["scripts/*"], "@lua/*": ["src/_luaScripts/*"], } }, diff --git a/shared/utils/cusEntUtils/getRolloverFields.ts b/shared/utils/cusEntUtils/getRolloverFields.ts index 22eda94b0..0347fc9da 100644 --- a/shared/utils/cusEntUtils/getRolloverFields.ts +++ b/shared/utils/cusEntUtils/getRolloverFields.ts @@ -26,7 +26,7 @@ export const getRolloverFields = ({ if (cusEnt.entitlement.entity_feature_id) { if (entityId) { return rollovers.reduce( - (acc, rollover) => { + (acc: RolloverFields, rollover) => { if (rollover.entities[entityId]) { return { balance: acc.balance + rollover.entities[entityId].balance, @@ -51,7 +51,7 @@ export const getRolloverFields = ({ ); } else { return rollovers.reduce( - (acc, rollover) => { + (acc: RolloverFields, rollover) => { let newBalance = 0; let newUsage = 0; @@ -82,7 +82,7 @@ export const getRolloverFields = ({ } } else { return rollovers.reduce( - (acc, rollover) => { + (acc: RolloverFields, rollover) => { return { balance: acc.balance + rollover.balance, usage: acc.usage + rollover.usage, diff --git a/shared/utils/cusProductUtils/filterCusProductUtils.ts b/shared/utils/cusProductUtils/filterCusProductUtils.ts index f35a22474..c3bb5dd1a 100644 --- a/shared/utils/cusProductUtils/filterCusProductUtils.ts +++ b/shared/utils/cusProductUtils/filterCusProductUtils.ts @@ -1,8 +1,8 @@ -import { notNullish, nullish } from "@utils/utils.js"; import type { Entity } from "../../models/cusModels/entityModels/entityModels.js"; import type { FullCustomerEntitlement } from "../../models/cusProductModels/cusEntModels/cusEntModels.js"; import type { FullCusProduct } from "../../models/cusProductModels/cusProductModels.js"; import type { Organization } from "../../models/orgModels/orgTable.js"; +import { notNullish } from "../utils.js"; /** * Filter customer products by entity diff --git a/shared/utils/featureUtils/apiFeatureToDbFeature.ts b/shared/utils/featureUtils/apiFeatureToDbFeature.ts index ce22864f9..1aa7b3cd6 100644 --- a/shared/utils/featureUtils/apiFeatureToDbFeature.ts +++ b/shared/utils/featureUtils/apiFeatureToDbFeature.ts @@ -35,7 +35,7 @@ export const apiFeatureToDbFeature = ({ } if (apiFeature.credit_schema) { - newConfig.schema = apiFeature.credit_schema.map((credit) => ({ + newConfig.schema = apiFeature.credit_schema.map((credit: { metered_feature_id: string; credit_cost: number }) => ({ metered_feature_id: credit.metered_feature_id, credit_amount: credit.credit_cost, })); diff --git a/shared/utils/index.ts b/shared/utils/index.ts index 39c70a351..726a57905 100644 --- a/shared/utils/index.ts +++ b/shared/utils/index.ts @@ -21,6 +21,8 @@ export * from "./featureUtils/apiFeatureToDbFeature.js"; export * from "./featureUtils/convertFeatureUtils.js"; // Feature utils export * from "./featureUtils.js"; +// Org utils +export * from "./orgUtils/convertOrgUtils.js"; // Product utils export * from "./productUtils/convertUtils.js"; export * from "./productUtils/priceUtils/convertAmountUtils.js"; @@ -32,4 +34,5 @@ export * from "./productV2Utils/productItemUtils/getItemType.js"; export * from "./productV2Utils/productItemUtils/mapToItem.js"; export * from "./productV2Utils/productItemUtils/productItemUtils.js"; export * from "./productV2Utils/productV2ToV1.js"; +export * from "./productV3Utils/productItemUtils/productV3ItemUtils.js"; export * from "./utils.js"; diff --git a/shared/utils/orgUtils/convertOrgUtils.ts b/shared/utils/orgUtils/convertOrgUtils.ts new file mode 100644 index 000000000..b68d39e9e --- /dev/null +++ b/shared/utils/orgUtils/convertOrgUtils.ts @@ -0,0 +1,9 @@ +import { CusProductStatus } from "../../models/cusProductModels/cusProductEnums.js"; +import type { Organization } from "../../models/orgModels/orgTable.js"; + +export const orgToInStatuses = ({ org }: { org: Organization }) => { + if (org.config.include_past_due) { + return [CusProductStatus.Active, CusProductStatus.PastDue]; + } + return [CusProductStatus.Active]; +}; diff --git a/shared/utils/productV3Utils/mapToProductV3.ts b/shared/utils/productV3Utils/mapToProductV3.ts index f372ba88d..f44162f40 100644 --- a/shared/utils/productV3Utils/mapToProductV3.ts +++ b/shared/utils/productV3Utils/mapToProductV3.ts @@ -16,5 +16,3 @@ export function mapToProductV3({ product }: { product: ProductV2 }) { }; return productV3; } - -export * from "./productItemUtils/productV3ItemUtils.js"; diff --git a/shared/utils/productV3Utils/productItemUtils/productV3ItemUtils.ts b/shared/utils/productV3Utils/productItemUtils/productV3ItemUtils.ts index d3b8fae6f..a57abac80 100644 --- a/shared/utils/productV3Utils/productItemUtils/productV3ItemUtils.ts +++ b/shared/utils/productV3Utils/productItemUtils/productV3ItemUtils.ts @@ -1,4 +1,4 @@ -import type { FixedPriceConfig } from "@models/productModels/priceModels/priceConfig/fixedPriceConfig.js"; +import type { FixedPriceConfig } from "../../../models/productModels/priceModels/priceConfig/fixedPriceConfig.js"; import type { ProductItem, ProductItemInterval,