chore: init fail open track
This commit is contained in:
6
server/src/external/aws/s3/adminS3Config.ts
vendored
6
server/src/external/aws/s3/adminS3Config.ts
vendored
@@ -6,6 +6,7 @@ export const ADMIN_CUSTOMER_BLOCK_CONFIG_KEY =
|
|||||||
export const ADMIN_ORG_LIMITS_CONFIG_KEY = "admin/org-limits-config.json";
|
export const ADMIN_ORG_LIMITS_CONFIG_KEY = "admin/org-limits-config.json";
|
||||||
export const ADMIN_REDIS_V2_CACHE_CONFIG_KEY =
|
export const ADMIN_REDIS_V2_CACHE_CONFIG_KEY =
|
||||||
"admin/redis-v2-cache-config.json";
|
"admin/redis-v2-cache-config.json";
|
||||||
|
export const ADMIN_JOB_QUEUE_CONFIG_KEY = "admin/job-queue-config.json";
|
||||||
|
|
||||||
const bucket = process.env.S3_BUCKET || "autumn-prod-server";
|
const bucket = process.env.S3_BUCKET || "autumn-prod-server";
|
||||||
const region = process.env.S3_REGION || "us-east-2";
|
const region = process.env.S3_REGION || "us-east-2";
|
||||||
@@ -42,6 +43,11 @@ export const getAdminEdgeConfigSources = () => ({
|
|||||||
label: "V2 Redis Instance",
|
label: "V2 Redis Instance",
|
||||||
key: ADMIN_REDIS_V2_CACHE_CONFIG_KEY,
|
key: ADMIN_REDIS_V2_CACHE_CONFIG_KEY,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: "job-queues",
|
||||||
|
label: "Job Queues",
|
||||||
|
key: ADMIN_JOB_QUEUE_CONFIG_KEY,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: "stripe-sync",
|
id: "stripe-sync",
|
||||||
label: "Stripe Sync",
|
label: "Stripe Sync",
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import "./internal/misc/customerBlocks/customerBlockStore.js";
|
|||||||
import "./internal/misc/edgeConfig/orgLimitsStore.js";
|
import "./internal/misc/edgeConfig/orgLimitsStore.js";
|
||||||
import "./internal/misc/stripeSync/stripeSyncStore.js";
|
import "./internal/misc/stripeSync/stripeSyncStore.js";
|
||||||
import "./internal/misc/redisV2Cache/redisV2CacheStore.js";
|
import "./internal/misc/redisV2Cache/redisV2CacheStore.js";
|
||||||
|
import "./internal/misc/jobQueues/jobQueueStore.js";
|
||||||
import { closeStripeSyncEngine } from "@autumn/stripe-sync";
|
import { closeStripeSyncEngine } from "@autumn/stripe-sync";
|
||||||
import {
|
import {
|
||||||
startRedisMonitor,
|
startRedisMonitor,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type { HonoEnv } from "../../honoUtils/HonoEnv";
|
|||||||
import { handleGetAdminCustomerBlockConfig } from "./handleGetAdminCustomerBlockConfig";
|
import { handleGetAdminCustomerBlockConfig } from "./handleGetAdminCustomerBlockConfig";
|
||||||
import { handleGetAdminEdgeConfigSources } from "./handleGetAdminEdgeConfigSources";
|
import { handleGetAdminEdgeConfigSources } from "./handleGetAdminEdgeConfigSources";
|
||||||
import { handleGetAdminFeatureFlagsConfig } from "./handleGetAdminFeatureFlagsConfig";
|
import { handleGetAdminFeatureFlagsConfig } from "./handleGetAdminFeatureFlagsConfig";
|
||||||
|
import { handleGetAdminJobQueueConfig } from "./handleGetAdminJobQueueConfig";
|
||||||
import { handleGetAdminOrgLimitsConfig } from "./handleGetAdminOrgLimitsConfig";
|
import { handleGetAdminOrgLimitsConfig } from "./handleGetAdminOrgLimitsConfig";
|
||||||
import { handleGetAdminOrgRequestBlock } from "./handleGetAdminOrgRequestBlock";
|
import { handleGetAdminOrgRequestBlock } from "./handleGetAdminOrgRequestBlock";
|
||||||
import { handleGetAdminRequestBlockConfig } from "./handleGetAdminRequestBlockConfig";
|
import { handleGetAdminRequestBlockConfig } from "./handleGetAdminRequestBlockConfig";
|
||||||
@@ -17,6 +18,7 @@ import { handleListAdminUsers } from "./handleListAdminUsers";
|
|||||||
import { handleListOAuthClients } from "./handleListOAuthClients";
|
import { handleListOAuthClients } from "./handleListOAuthClients";
|
||||||
import { handleUpsertAdminCustomerBlockConfig } from "./handleUpsertAdminCustomerBlockConfig";
|
import { handleUpsertAdminCustomerBlockConfig } from "./handleUpsertAdminCustomerBlockConfig";
|
||||||
import { handleUpsertAdminFeatureFlagsConfig } from "./handleUpsertAdminFeatureFlagsConfig";
|
import { handleUpsertAdminFeatureFlagsConfig } from "./handleUpsertAdminFeatureFlagsConfig";
|
||||||
|
import { handleUpsertAdminJobQueueConfig } from "./handleUpsertAdminJobQueueConfig";
|
||||||
import { handleUpsertAdminOrgLimitsConfig } from "./handleUpsertAdminOrgLimitsConfig";
|
import { handleUpsertAdminOrgLimitsConfig } from "./handleUpsertAdminOrgLimitsConfig";
|
||||||
import { handleUpsertAdminOrgRequestBlock } from "./handleUpsertAdminOrgRequestBlock";
|
import { handleUpsertAdminOrgRequestBlock } from "./handleUpsertAdminOrgRequestBlock";
|
||||||
import { handleUpsertAdminRequestBlockConfig } from "./handleUpsertAdminRequestBlockConfig";
|
import { handleUpsertAdminRequestBlockConfig } from "./handleUpsertAdminRequestBlockConfig";
|
||||||
@@ -67,6 +69,8 @@ honoAdminRouter.put(
|
|||||||
);
|
);
|
||||||
honoAdminRouter.get("/org-limits-config", ...handleGetAdminOrgLimitsConfig);
|
honoAdminRouter.get("/org-limits-config", ...handleGetAdminOrgLimitsConfig);
|
||||||
honoAdminRouter.put("/org-limits-config", ...handleUpsertAdminOrgLimitsConfig);
|
honoAdminRouter.put("/org-limits-config", ...handleUpsertAdminOrgLimitsConfig);
|
||||||
|
honoAdminRouter.get("/job-queue-config", ...handleGetAdminJobQueueConfig);
|
||||||
|
honoAdminRouter.put("/job-queue-config", ...handleUpsertAdminJobQueueConfig);
|
||||||
honoAdminRouter.get("/stripe-sync-config", ...handleGetAdminStripeSyncConfig);
|
honoAdminRouter.get("/stripe-sync-config", ...handleGetAdminStripeSyncConfig);
|
||||||
honoAdminRouter.put(
|
honoAdminRouter.put(
|
||||||
"/stripe-sync-config",
|
"/stripe-sync-config",
|
||||||
|
|||||||
22
server/src/internal/admin/handleGetAdminJobQueueConfig.ts
Normal file
22
server/src/internal/admin/handleGetAdminJobQueueConfig.ts
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
import { createRoute } from "@/honoMiddlewares/routeHandler.js";
|
||||||
|
import {
|
||||||
|
getJobQueueConfigFromSource,
|
||||||
|
getJobQueueConfigStatus,
|
||||||
|
KNOWN_JOB_QUEUES,
|
||||||
|
} from "@/internal/misc/jobQueues/jobQueueStore.js";
|
||||||
|
|
||||||
|
export const handleGetAdminJobQueueConfig = createRoute({
|
||||||
|
handler: async (c) => {
|
||||||
|
const status = getJobQueueConfigStatus();
|
||||||
|
const config = await getJobQueueConfigFromSource();
|
||||||
|
|
||||||
|
return c.json({
|
||||||
|
...config,
|
||||||
|
knownQueues: KNOWN_JOB_QUEUES,
|
||||||
|
configHealthy: status.healthy,
|
||||||
|
configConfigured: status.configured,
|
||||||
|
lastSuccessAt: status.lastSuccessAt ?? null,
|
||||||
|
error: status.error ?? null,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
14
server/src/internal/admin/handleUpsertAdminJobQueueConfig.ts
Normal file
14
server/src/internal/admin/handleUpsertAdminJobQueueConfig.ts
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
import { createRoute } from "@/honoMiddlewares/routeHandler.js";
|
||||||
|
import { JobQueueConfigSchema } from "@/internal/misc/jobQueues/jobQueueSchemas.js";
|
||||||
|
import { updateFullJobQueueConfig } from "@/internal/misc/jobQueues/jobQueueStore.js";
|
||||||
|
|
||||||
|
export const handleUpsertAdminJobQueueConfig = createRoute({
|
||||||
|
body: JobQueueConfigSchema,
|
||||||
|
handler: async (c) => {
|
||||||
|
const body = c.req.valid("json");
|
||||||
|
|
||||||
|
await updateFullJobQueueConfig({ config: body });
|
||||||
|
|
||||||
|
return c.json({ success: true });
|
||||||
|
},
|
||||||
|
});
|
||||||
40
server/src/internal/balances/track/runQueuedTrack.ts
Normal file
40
server/src/internal/balances/track/runQueuedTrack.ts
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
import { ErrCode, RecaseError, type ApiVersion, type TrackParams } from "@autumn/shared";
|
||||||
|
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||||
|
import { getTrackFeatureDeductionsForBody } from "./utils/getFeatureDeductions.js";
|
||||||
|
import { runTrackV3 } from "./v3/runTrackV3.js";
|
||||||
|
|
||||||
|
export const runQueuedTrack = async ({
|
||||||
|
ctx,
|
||||||
|
body,
|
||||||
|
apiVersion,
|
||||||
|
}: {
|
||||||
|
ctx: AutumnContext;
|
||||||
|
body: TrackParams;
|
||||||
|
apiVersion?: ApiVersion;
|
||||||
|
}) => {
|
||||||
|
const featureDeductions = getTrackFeatureDeductionsForBody({ ctx, body });
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runTrackV3({
|
||||||
|
ctx,
|
||||||
|
body,
|
||||||
|
featureDeductions,
|
||||||
|
apiVersion,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (
|
||||||
|
!(error instanceof RecaseError) ||
|
||||||
|
error.code !== ErrCode.DuplicateIdempotencyKey
|
||||||
|
) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.logger.info("[track] queued replay already applied", {
|
||||||
|
type: "track_queue_replay_duplicate",
|
||||||
|
customer_id: body.customer_id,
|
||||||
|
entity_id: body.entity_id,
|
||||||
|
feature_id: body.feature_id,
|
||||||
|
event_name: body.event_name,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -1,10 +1,7 @@
|
|||||||
import type { ApiVersion, TrackParams, TrackResponseV3 } from "@autumn/shared";
|
import type { ApiVersion, TrackParams, TrackResponseV3 } from "@autumn/shared";
|
||||||
import { Result } from "better-result";
|
import { withRedisFailOpen } from "@/external/redis/utils/withRedisFailOpen.js";
|
||||||
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||||
import {
|
import { isFullSubjectRolloutEnabled } from "@/internal/misc/rollouts/fullSubjectRolloutUtils.js";
|
||||||
isFullSubjectRolloutEnabled,
|
|
||||||
isRetryableFullSubjectRolloutError,
|
|
||||||
} from "@/internal/misc/rollouts/fullSubjectRolloutUtils.js";
|
|
||||||
import type { FeatureDeduction } from "../utils/types/featureDeduction.js";
|
import type { FeatureDeduction } from "../utils/types/featureDeduction.js";
|
||||||
import { runTrackV2 } from "./runTrackV2.js";
|
import { runTrackV2 } from "./runTrackV2.js";
|
||||||
import { queueTrack } from "./utils/queueTrack.js";
|
import { queueTrack } from "./utils/queueTrack.js";
|
||||||
@@ -27,29 +24,21 @@ export const runTrackWithRollout = async ({
|
|||||||
apiVersion?: ApiVersion;
|
apiVersion?: ApiVersion;
|
||||||
}): Promise<TrackResponseV3> => {
|
}): Promise<TrackResponseV3> => {
|
||||||
if (shouldUseTrackV3({ ctx })) {
|
if (shouldUseTrackV3({ ctx })) {
|
||||||
const result = await Result.tryPromise({
|
return withRedisFailOpen<TrackResponseV3>({
|
||||||
try: () =>
|
source: "runTrackWithRollout",
|
||||||
|
run: () =>
|
||||||
runTrackV3({
|
runTrackV3({
|
||||||
ctx,
|
ctx,
|
||||||
body,
|
body,
|
||||||
featureDeductions,
|
featureDeductions,
|
||||||
apiVersion,
|
apiVersion,
|
||||||
}),
|
}),
|
||||||
catch: (error) => error,
|
fallback: async (error) => {
|
||||||
|
const queuedResponse = await queueTrack({ ctx, body });
|
||||||
|
if (queuedResponse) return queuedResponse;
|
||||||
|
throw error;
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (Result.isOk(result)) return result.value;
|
|
||||||
|
|
||||||
const error = result.error;
|
|
||||||
if (!isRetryableFullSubjectRolloutError({ error })) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
const queuedResponse = await queueTrack({ ctx, body });
|
|
||||||
|
|
||||||
if (queuedResponse) return queuedResponse;
|
|
||||||
|
|
||||||
throw error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return runTrackV2({
|
return runTrackV2({
|
||||||
|
|||||||
@@ -23,11 +23,14 @@ export const queueTrack = async ({
|
|||||||
await addTaskToQueue({
|
await addTaskToQueue({
|
||||||
jobName: JobName.Track,
|
jobName: JobName.Track,
|
||||||
queueUrl,
|
queueUrl,
|
||||||
messageGroupId: `${ctx.org.id}:${ctx.env}:${body.customer_id}`,
|
messageGroupId: `${ctx.org.id}:${ctx.env}:${body.customer_id}:${body.entity_id ?? "none"}`,
|
||||||
messageDeduplicationId: body.idempotency_key,
|
messageDeduplicationId: body.idempotency_key || ctx.id,
|
||||||
payload: {
|
payload: {
|
||||||
orgId: ctx.org.id,
|
orgId: ctx.org.id,
|
||||||
env: ctx.env,
|
env: ctx.env,
|
||||||
|
customerId: body.customer_id,
|
||||||
|
entityId: body.entity_id,
|
||||||
|
requestId: ctx.id,
|
||||||
apiVersion: ctx.apiVersion.value,
|
apiVersion: ctx.apiVersion.value,
|
||||||
body,
|
body,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import {
|
|||||||
RedisDeductionError,
|
RedisDeductionError,
|
||||||
RedisDeductionErrorCode,
|
RedisDeductionErrorCode,
|
||||||
} from "../../utils/types/redisDeductionError.js";
|
} from "../../utils/types/redisDeductionError.js";
|
||||||
import { queueTrack } from "../utils/queueTrack.js";
|
|
||||||
import { runPostgresTrackV3 } from "./runPostgresTrackV3.js";
|
import { runPostgresTrackV3 } from "./runPostgresTrackV3.js";
|
||||||
|
|
||||||
/** Handles errors from V2 Redis deduction. Falls back to Postgres V3 path. */
|
/** Handles errors from V2 Redis deduction. Falls back to Postgres V3 path. */
|
||||||
@@ -58,8 +57,6 @@ export const handleRedisTrackErrorV3 = async ({
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (error.isRedisUnavailable()) {
|
if (error.isRedisUnavailable()) {
|
||||||
const queuedResponse = await queueTrack({ ctx, body });
|
|
||||||
if (queuedResponse) return queuedResponse;
|
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
14
server/src/internal/misc/jobQueues/jobQueueSchemas.ts
Normal file
14
server/src/internal/misc/jobQueues/jobQueueSchemas.ts
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
import { z } from "zod/v4";
|
||||||
|
|
||||||
|
export const JobQueueConfigSchema = z.object({
|
||||||
|
queues: z
|
||||||
|
.record(
|
||||||
|
z.string(),
|
||||||
|
z.object({
|
||||||
|
enabled: z.boolean().default(false),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.default({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type JobQueueConfig = z.infer<typeof JobQueueConfigSchema>;
|
||||||
55
server/src/internal/misc/jobQueues/jobQueueStore.ts
Normal file
55
server/src/internal/misc/jobQueues/jobQueueStore.ts
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
import { ADMIN_JOB_QUEUE_CONFIG_KEY } from "@/external/aws/s3/adminS3Config.js";
|
||||||
|
import { registerEdgeConfig } from "@/internal/misc/edgeConfig/edgeConfigRegistry.js";
|
||||||
|
import { createEdgeConfigStore } from "@/internal/misc/edgeConfig/edgeConfigStore.js";
|
||||||
|
import {
|
||||||
|
type JobQueueConfig,
|
||||||
|
JobQueueConfigSchema,
|
||||||
|
} from "./jobQueueSchemas.js";
|
||||||
|
|
||||||
|
export const JOB_QUEUE_IDS = {
|
||||||
|
primary: "primary",
|
||||||
|
track: "track",
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export const KNOWN_JOB_QUEUES = [
|
||||||
|
{
|
||||||
|
id: JOB_QUEUE_IDS.primary,
|
||||||
|
label: "Primary Queue",
|
||||||
|
description: "Shared SQS queue for standard background jobs.",
|
||||||
|
defaultEnabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: JOB_QUEUE_IDS.track,
|
||||||
|
label: "Track Replay Queue",
|
||||||
|
description: "Dedicated async track replay queue used during fail-open recovery.",
|
||||||
|
defaultEnabled: true,
|
||||||
|
},
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const store = createEdgeConfigStore<JobQueueConfig>({
|
||||||
|
s3Key: ADMIN_JOB_QUEUE_CONFIG_KEY,
|
||||||
|
schema: JobQueueConfigSchema,
|
||||||
|
defaultValue: () => ({ queues: {} }),
|
||||||
|
});
|
||||||
|
|
||||||
|
registerEdgeConfig({ store });
|
||||||
|
|
||||||
|
export const isJobQueueEnabled = ({
|
||||||
|
queue,
|
||||||
|
defaultEnabled = true,
|
||||||
|
}: {
|
||||||
|
queue: string;
|
||||||
|
defaultEnabled?: boolean;
|
||||||
|
}) => store.get().queues[queue]?.enabled ?? defaultEnabled;
|
||||||
|
|
||||||
|
export const getJobQueueConfigStatus = () => store.getStatus();
|
||||||
|
|
||||||
|
export const getJobQueueConfigFromSource = async () => store.readFromSource();
|
||||||
|
|
||||||
|
export const updateFullJobQueueConfig = async ({
|
||||||
|
config,
|
||||||
|
}: {
|
||||||
|
config: JobQueueConfig;
|
||||||
|
}) => {
|
||||||
|
await store.writeToSource({ config });
|
||||||
|
};
|
||||||
@@ -18,10 +18,11 @@ export const createWorkerContext = async ({
|
|||||||
orgId?: string;
|
orgId?: string;
|
||||||
env?: AppEnv;
|
env?: AppEnv;
|
||||||
customerId?: string;
|
customerId?: string;
|
||||||
|
requestId?: string;
|
||||||
};
|
};
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
}) => {
|
}) => {
|
||||||
const { orgId, env, customerId } = payload;
|
const { orgId, env, customerId, requestId } = payload;
|
||||||
if (!orgId || !env) return;
|
if (!orgId || !env) return;
|
||||||
|
|
||||||
// Fetch org with features once for all items
|
// Fetch org with features once for all items
|
||||||
@@ -74,7 +75,7 @@ export const createWorkerContext = async ({
|
|||||||
logger: workerLogger,
|
logger: workerLogger,
|
||||||
redisV2: resolveRedisV2(),
|
redisV2: resolveRedisV2(),
|
||||||
|
|
||||||
id: generateId("job"),
|
id: requestId || generateId("job"),
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
isPublic: false,
|
isPublic: false,
|
||||||
authType: AuthType.Worker,
|
authType: AuthType.Worker,
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ import * as Sentry from "@sentry/bun";
|
|||||||
import { type DrizzleCli, initDrizzle } from "@/db/initDrizzle.js";
|
import { type DrizzleCli, initDrizzle } from "@/db/initDrizzle.js";
|
||||||
import { logger } from "@/external/logtail/logtailUtils.js";
|
import { logger } from "@/external/logtail/logtailUtils.js";
|
||||||
import { verifyCacheConsistency } from "@/internal/billing/v2/workflows/verifyCacheConsistency/verifyCacheConsistency.js";
|
import { verifyCacheConsistency } from "@/internal/billing/v2/workflows/verifyCacheConsistency/verifyCacheConsistency.js";
|
||||||
|
import {
|
||||||
|
isJobQueueEnabled,
|
||||||
|
JOB_QUEUE_IDS,
|
||||||
|
} from "@/internal/misc/jobQueues/jobQueueStore.js";
|
||||||
import { generateId } from "@/utils/genUtils.js";
|
import { generateId } from "@/utils/genUtils.js";
|
||||||
import { withTimeout } from "@/utils/withTimeout.js";
|
import { withTimeout } from "@/utils/withTimeout.js";
|
||||||
import { hatchet } from "../external/hatchet/initHatchet.js";
|
import { hatchet } from "../external/hatchet/initHatchet.js";
|
||||||
@@ -21,7 +25,7 @@ import { processMessage, type SqsJob } from "./processMessage.js";
|
|||||||
|
|
||||||
// ============ Shared State ============
|
// ============ Shared State ============
|
||||||
let isRunning = true;
|
let isRunning = true;
|
||||||
let abortController: AbortController;
|
const abortControllers = new Set<AbortController>();
|
||||||
|
|
||||||
// Process recycling — exit after processing this many messages to prevent memory leaks
|
// Process recycling — exit after processing this many messages to prevent memory leaks
|
||||||
const MAX_MESSAGES_BEFORE_RECYCLE = 50_000;
|
const MAX_MESSAGES_BEFORE_RECYCLE = 50_000;
|
||||||
@@ -47,18 +51,20 @@ const logPrefix = ({ queueUrl }: { queueUrl: string }) =>
|
|||||||
|
|
||||||
// ============ Polling Loop (per-queue, per-loop state) ============
|
// ============ Polling Loop (per-queue, per-loop state) ============
|
||||||
|
|
||||||
const startPollingLoop = async ({
|
export const startPollingLoop = async ({
|
||||||
db,
|
db,
|
||||||
queueUrl,
|
queueUrl,
|
||||||
isFifo,
|
isFifo,
|
||||||
getSqsClientFn,
|
getSqsClientFn,
|
||||||
recreateSqsClientFn,
|
recreateSqsClientFn,
|
||||||
|
shouldPoll = () => true,
|
||||||
}: {
|
}: {
|
||||||
db: DrizzleCli;
|
db: DrizzleCli;
|
||||||
queueUrl: string;
|
queueUrl: string;
|
||||||
isFifo: boolean;
|
isFifo: boolean;
|
||||||
getSqsClientFn: () => SQSClient;
|
getSqsClientFn: () => SQSClient;
|
||||||
recreateSqsClientFn: () => SQSClient;
|
recreateSqsClientFn: () => SQSClient;
|
||||||
|
shouldPoll?: () => boolean;
|
||||||
}) => {
|
}) => {
|
||||||
// Per-loop state
|
// Per-loop state
|
||||||
let messagesProcessed = 0;
|
let messagesProcessed = 0;
|
||||||
@@ -70,6 +76,8 @@ const startPollingLoop = async ({
|
|||||||
let consecutiveZeroMessageIntervals = 0;
|
let consecutiveZeroMessageIntervals = 0;
|
||||||
|
|
||||||
const prefix = logPrefix({ queueUrl });
|
const prefix = logPrefix({ queueUrl });
|
||||||
|
let abortController = new AbortController();
|
||||||
|
abortControllers.add(abortController);
|
||||||
|
|
||||||
const alertZeroMessages = () => {
|
const alertZeroMessages = () => {
|
||||||
const minutes = consecutiveZeroMessageIntervals;
|
const minutes = consecutiveZeroMessageIntervals;
|
||||||
@@ -105,6 +113,13 @@ const startPollingLoop = async ({
|
|||||||
};
|
};
|
||||||
|
|
||||||
const logStatsAndCheckZeroMessages = () => {
|
const logStatsAndCheckZeroMessages = () => {
|
||||||
|
if (!shouldPoll()) {
|
||||||
|
consecutiveZeroMessageIntervals = 0;
|
||||||
|
messagesProcessed = 0;
|
||||||
|
lastStatsTime = Date.now();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const elapsedSeconds = ((Date.now() - lastStatsTime) / 1000).toFixed(0);
|
const elapsedSeconds = ((Date.now() - lastStatsTime) / 1000).toFixed(0);
|
||||||
const mem = process.memoryUsage();
|
const mem = process.memoryUsage();
|
||||||
console.log(
|
console.log(
|
||||||
@@ -245,6 +260,7 @@ const startPollingLoop = async ({
|
|||||||
);
|
);
|
||||||
consecutiveEmptyPolls = 0;
|
consecutiveEmptyPolls = 0;
|
||||||
abortController = new AbortController();
|
abortController = new AbortController();
|
||||||
|
abortControllers.add(abortController);
|
||||||
return recreateSqsClientFn();
|
return recreateSqsClientFn();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,6 +286,7 @@ const startPollingLoop = async ({
|
|||||||
console.warn(`${prefix} Repeated errors - recreating SQS client`);
|
console.warn(`${prefix} Repeated errors - recreating SQS client`);
|
||||||
consecutiveEmptyPolls = 0;
|
consecutiveEmptyPolls = 0;
|
||||||
abortController = new AbortController();
|
abortController = new AbortController();
|
||||||
|
abortControllers.add(abortController);
|
||||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||||
return recreateSqsClientFn();
|
return recreateSqsClientFn();
|
||||||
}
|
}
|
||||||
@@ -284,6 +301,12 @@ const startPollingLoop = async ({
|
|||||||
|
|
||||||
while (isRunning) {
|
while (isRunning) {
|
||||||
try {
|
try {
|
||||||
|
if (!shouldPoll()) {
|
||||||
|
consecutiveEmptyPolls = 0;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const response = await sqs.send(createReceiveCommand(), {
|
const response = await sqs.send(createReceiveCommand(), {
|
||||||
abortSignal: abortController.signal,
|
abortSignal: abortController.signal,
|
||||||
});
|
});
|
||||||
@@ -348,6 +371,7 @@ const startPollingLoop = async ({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
abortControllers.delete(abortController);
|
||||||
clearInterval(statsInterval);
|
clearInterval(statsInterval);
|
||||||
console.log(`${prefix} Stopped`);
|
console.log(`${prefix} Stopped`);
|
||||||
};
|
};
|
||||||
@@ -370,7 +394,9 @@ export const initWorkers = async ({
|
|||||||
const shutdown = async () => {
|
const shutdown = async () => {
|
||||||
console.log(`[SQS Worker ${process.pid}] Shutting down...`);
|
console.log(`[SQS Worker ${process.pid}] Shutting down...`);
|
||||||
isRunning = false;
|
isRunning = false;
|
||||||
if (abortController) abortController.abort();
|
for (const controller of abortControllers) {
|
||||||
|
controller.abort();
|
||||||
|
}
|
||||||
|
|
||||||
const isProd = process.env.NODE_ENV === "production";
|
const isProd = process.env.NODE_ENV === "production";
|
||||||
if (isProd) {
|
if (isProd) {
|
||||||
@@ -386,20 +412,36 @@ export const initWorkers = async ({
|
|||||||
process.on("SIGTERM", shutdown);
|
process.on("SIGTERM", shutdown);
|
||||||
process.on("SIGINT", shutdown);
|
process.on("SIGINT", shutdown);
|
||||||
|
|
||||||
abortController = new AbortController();
|
|
||||||
|
|
||||||
const startupDurationMs = Date.now() - startupStartedAt;
|
const startupDurationMs = Date.now() - startupStartedAt;
|
||||||
console.log(
|
console.log(
|
||||||
`[Worker ${process.pid}] ${queueImplementation} worker ready in ${startupDurationMs}ms`,
|
`[Worker ${process.pid}] ${queueImplementation} worker ready in ${startupDurationMs}ms`,
|
||||||
);
|
);
|
||||||
|
const pollingLoops = [
|
||||||
|
startPollingLoop({
|
||||||
|
db,
|
||||||
|
queueUrl: QUEUE_URL,
|
||||||
|
isFifo: QUEUE_URL.endsWith(".fifo"),
|
||||||
|
getSqsClientFn: getSqsClient,
|
||||||
|
recreateSqsClientFn: recreateSqsClient,
|
||||||
|
shouldPoll: () => isJobQueueEnabled({ queue: JOB_QUEUE_IDS.primary }),
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
await startPollingLoop({
|
const trackQueueUrl = process.env.TRACK_SQS_QUEUE_URL;
|
||||||
db,
|
if (trackQueueUrl) {
|
||||||
queueUrl: QUEUE_URL,
|
pollingLoops.push(
|
||||||
isFifo: QUEUE_URL.endsWith(".fifo"),
|
startPollingLoop({
|
||||||
getSqsClientFn: getSqsClient,
|
db,
|
||||||
recreateSqsClientFn: recreateSqsClient,
|
queueUrl: trackQueueUrl,
|
||||||
});
|
isFifo: trackQueueUrl.endsWith(".fifo"),
|
||||||
|
getSqsClientFn: getSqsClient,
|
||||||
|
recreateSqsClientFn: recreateSqsClient,
|
||||||
|
shouldPoll: () => isJobQueueEnabled({ queue: JOB_QUEUE_IDS.track }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(pollingLoops);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const initHatchetWorker = async () => {
|
export const initHatchetWorker = async () => {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { runActionHandlerTask } from "@/internal/analytics/runActionHandlerTask.
|
|||||||
import { autoTopup } from "@/internal/balances/autoTopUp/autoTopup.js";
|
import { autoTopup } from "@/internal/balances/autoTopUp/autoTopup.js";
|
||||||
import { runInsertEventBatch } from "@/internal/balances/events/runInsertEventBatch.js";
|
import { runInsertEventBatch } from "@/internal/balances/events/runInsertEventBatch.js";
|
||||||
import { expireLock } from "@/internal/balances/finalizeLock/expireLock.js";
|
import { expireLock } from "@/internal/balances/finalizeLock/expireLock.js";
|
||||||
|
import { runQueuedTrack } from "@/internal/balances/track/runQueuedTrack.js";
|
||||||
import { refreshEntityAggregateCache } from "@/internal/balances/utils/refreshEntityAggregate/index.js";
|
import { refreshEntityAggregateCache } from "@/internal/balances/utils/refreshEntityAggregate/index.js";
|
||||||
import { syncItemV3 } from "@/internal/balances/utils/sync/syncItemV3.js";
|
import { syncItemV3 } from "@/internal/balances/utils/sync/syncItemV3.js";
|
||||||
import { syncItemV4 } from "@/internal/balances/utils/sync/syncItemV4.js";
|
import { syncItemV4 } from "@/internal/balances/utils/sync/syncItemV4.js";
|
||||||
@@ -175,6 +176,20 @@ export const processMessage = async ({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (job.name === JobName.Track) {
|
||||||
|
if (!ctx) {
|
||||||
|
workerLogger.error("No context found for track job");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await runQueuedTrack({
|
||||||
|
ctx,
|
||||||
|
body: job.data.body,
|
||||||
|
apiVersion: job.data.apiVersion,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (job.name === JobName.RefreshEntityAggregate) {
|
if (job.name === JobName.RefreshEntityAggregate) {
|
||||||
if (!ctx) {
|
if (!ctx) {
|
||||||
workerLogger.error("No context found for refresh entity aggregate job");
|
workerLogger.error("No context found for refresh entity aggregate job");
|
||||||
|
|||||||
@@ -69,6 +69,9 @@ export interface Payloads {
|
|||||||
[JobName.Track]: {
|
[JobName.Track]: {
|
||||||
orgId: string;
|
orgId: string;
|
||||||
env: AppEnv;
|
env: AppEnv;
|
||||||
|
customerId: string;
|
||||||
|
entityId?: string;
|
||||||
|
requestId: string;
|
||||||
apiVersion: ApiVersion;
|
apiVersion: ApiVersion;
|
||||||
body: TrackParams;
|
body: TrackParams;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
import "./internal/misc/requestBlocks/requestBlockStore.js";
|
import "./internal/misc/requestBlocks/requestBlockStore.js";
|
||||||
import "./internal/misc/rollouts/rolloutConfigStore.js";
|
import "./internal/misc/rollouts/rolloutConfigStore.js";
|
||||||
import "./internal/misc/redisV2Cache/redisV2CacheStore.js";
|
import "./internal/misc/redisV2Cache/redisV2CacheStore.js";
|
||||||
|
import "./internal/misc/jobQueues/jobQueueStore.js";
|
||||||
|
|
||||||
// Number of worker processes (defaults to CPU cores)
|
// Number of worker processes (defaults to CPU cores)
|
||||||
const NUM_PROCESSES = process.env.NODE_ENV === "development" ? 3 : 4;
|
const NUM_PROCESSES = process.env.NODE_ENV === "development" ? 3 : 4;
|
||||||
|
|||||||
246
server/tests/integration/admin/job-queue-config.test.ts
Normal file
246
server/tests/integration/admin/job-queue-config.test.ts
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import { AppEnv, ErrCode } from "@autumn/shared";
|
||||||
|
import { Hono } from "hono";
|
||||||
|
import { errorMiddleware } from "@/honoMiddlewares/errorMiddleware.js";
|
||||||
|
import type { HonoEnv } from "@/honoUtils/HonoEnv.js";
|
||||||
|
|
||||||
|
type MockJobQueueConfig = {
|
||||||
|
queues: Record<string, { enabled: boolean }>;
|
||||||
|
};
|
||||||
|
|
||||||
|
type MockStatus = {
|
||||||
|
healthy: boolean;
|
||||||
|
configured: boolean;
|
||||||
|
lastSuccessAt: string | null;
|
||||||
|
error: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const mockState = {
|
||||||
|
config: {
|
||||||
|
queues: {
|
||||||
|
primary: { enabled: true },
|
||||||
|
track: { enabled: false },
|
||||||
|
},
|
||||||
|
} as MockJobQueueConfig,
|
||||||
|
status: {
|
||||||
|
healthy: true,
|
||||||
|
configured: true,
|
||||||
|
lastSuccessAt: "2026-04-24T10:00:00.000Z",
|
||||||
|
error: null,
|
||||||
|
} as MockStatus,
|
||||||
|
updateCalls: [] as unknown[],
|
||||||
|
};
|
||||||
|
|
||||||
|
mock.module("@/internal/misc/jobQueues/jobQueueStore.js", () => ({
|
||||||
|
KNOWN_JOB_QUEUES: [
|
||||||
|
{
|
||||||
|
id: "primary",
|
||||||
|
label: "Primary Queue",
|
||||||
|
description: "Shared SQS queue for standard background jobs.",
|
||||||
|
defaultEnabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "track",
|
||||||
|
label: "Track Replay Queue",
|
||||||
|
description:
|
||||||
|
"Dedicated async track replay queue used during fail-open recovery.",
|
||||||
|
defaultEnabled: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
getJobQueueConfigFromSource: async () => mockState.config,
|
||||||
|
getJobQueueConfigStatus: () => mockState.status,
|
||||||
|
updateFullJobQueueConfig: async ({ config }: { config: unknown }) => {
|
||||||
|
mockState.updateCalls.push(config);
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { handleGetAdminJobQueueConfig } from "@/internal/admin/handleGetAdminJobQueueConfig.js";
|
||||||
|
import { handleUpsertAdminJobQueueConfig } from "@/internal/admin/handleUpsertAdminJobQueueConfig.js";
|
||||||
|
|
||||||
|
const buildApp = () => {
|
||||||
|
const app = new Hono<HonoEnv>();
|
||||||
|
|
||||||
|
app.use("*", async (c, next) => {
|
||||||
|
c.set("ctx", {
|
||||||
|
env: AppEnv.Sandbox,
|
||||||
|
org: { slug: "tests-org" },
|
||||||
|
logger: {
|
||||||
|
warn: () => undefined,
|
||||||
|
error: () => undefined,
|
||||||
|
},
|
||||||
|
} as any);
|
||||||
|
await next();
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get("/admin/job-queue-config", ...handleGetAdminJobQueueConfig);
|
||||||
|
app.put("/admin/job-queue-config", ...handleUpsertAdminJobQueueConfig);
|
||||||
|
app.onError(errorMiddleware);
|
||||||
|
|
||||||
|
return app;
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("admin job queue config", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
mockState.config = {
|
||||||
|
queues: {
|
||||||
|
primary: { enabled: true },
|
||||||
|
track: { enabled: false },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
mockState.status = {
|
||||||
|
healthy: true,
|
||||||
|
configured: true,
|
||||||
|
lastSuccessAt: "2026-04-24T10:00:00.000Z",
|
||||||
|
error: null,
|
||||||
|
};
|
||||||
|
mockState.updateCalls = [];
|
||||||
|
});
|
||||||
|
|
||||||
|
test("GET returns the stored config, status, and known queues", async () => {
|
||||||
|
const app = buildApp();
|
||||||
|
|
||||||
|
const response = await app.request("http://localhost/admin/job-queue-config");
|
||||||
|
const body = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(body).toMatchObject({
|
||||||
|
queues: {
|
||||||
|
primary: { enabled: true },
|
||||||
|
track: { enabled: false },
|
||||||
|
},
|
||||||
|
configHealthy: true,
|
||||||
|
configConfigured: true,
|
||||||
|
lastSuccessAt: "2026-04-24T10:00:00.000Z",
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
expect(body.knownQueues).toEqual([
|
||||||
|
{
|
||||||
|
id: "primary",
|
||||||
|
label: "Primary Queue",
|
||||||
|
description: "Shared SQS queue for standard background jobs.",
|
||||||
|
defaultEnabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "track",
|
||||||
|
label: "Track Replay Queue",
|
||||||
|
description:
|
||||||
|
"Dedicated async track replay queue used during fail-open recovery.",
|
||||||
|
defaultEnabled: true,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("GET handles an unconfigured empty config", async () => {
|
||||||
|
mockState.config = { queues: {} };
|
||||||
|
mockState.status = {
|
||||||
|
healthy: false,
|
||||||
|
configured: false,
|
||||||
|
lastSuccessAt: null,
|
||||||
|
error: "missing s3 object",
|
||||||
|
};
|
||||||
|
|
||||||
|
const app = buildApp();
|
||||||
|
const response = await app.request("http://localhost/admin/job-queue-config");
|
||||||
|
const body = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(body).toMatchObject({
|
||||||
|
queues: {},
|
||||||
|
configHealthy: false,
|
||||||
|
configConfigured: false,
|
||||||
|
lastSuccessAt: null,
|
||||||
|
error: "missing s3 object",
|
||||||
|
});
|
||||||
|
expect(body.knownQueues).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("PUT saves a validated config payload", async () => {
|
||||||
|
const app = buildApp();
|
||||||
|
|
||||||
|
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
queues: {
|
||||||
|
primary: { enabled: false },
|
||||||
|
track: { enabled: true },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(await response.json()).toEqual({ success: true });
|
||||||
|
expect(mockState.updateCalls).toEqual([
|
||||||
|
{
|
||||||
|
queues: {
|
||||||
|
primary: { enabled: false },
|
||||||
|
track: { enabled: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("PUT preserves unknown queues for future config expansion", async () => {
|
||||||
|
const app = buildApp();
|
||||||
|
|
||||||
|
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
queues: {
|
||||||
|
reports: { enabled: false },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(mockState.updateCalls).toEqual([
|
||||||
|
{
|
||||||
|
queues: {
|
||||||
|
reports: { enabled: false },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("PUT accepts an empty payload and writes the schema default", async () => {
|
||||||
|
const app = buildApp();
|
||||||
|
|
||||||
|
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(mockState.updateCalls).toEqual([{ queues: {} }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("PUT rejects invalid queue payloads", async () => {
|
||||||
|
const app = buildApp();
|
||||||
|
|
||||||
|
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
queues: {
|
||||||
|
track: { enabled: "yes" },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const body = await response.json();
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
expect(body.code).toBe(ErrCode.InvalidInputs);
|
||||||
|
expect(mockState.updateCalls).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
35
server/tests/integration/queue/fixtures/livePollingWorker.ts
Normal file
35
server/tests/integration/queue/fixtures/livePollingWorker.ts
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
import { SQSClient } from "@aws-sdk/client-sqs";
|
||||||
|
import { startPollingLoop } from "@/queue/initWorkers.js";
|
||||||
|
|
||||||
|
const queueUrl = process.env.TEST_QUEUE_URL;
|
||||||
|
const endpoint = process.env.TEST_SQS_ENDPOINT;
|
||||||
|
const shouldPoll = process.env.TEST_SHOULD_POLL === "true";
|
||||||
|
|
||||||
|
if (!queueUrl || !endpoint) {
|
||||||
|
throw new Error("TEST_QUEUE_URL and TEST_SQS_ENDPOINT are required");
|
||||||
|
}
|
||||||
|
|
||||||
|
const createClient = () =>
|
||||||
|
new SQSClient({
|
||||||
|
region: "us-east-1",
|
||||||
|
endpoint,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: "x",
|
||||||
|
secretAccessKey: "x",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
let client = createClient();
|
||||||
|
|
||||||
|
await startPollingLoop({
|
||||||
|
db: {} as never,
|
||||||
|
queueUrl,
|
||||||
|
isFifo: queueUrl.endsWith(".fifo"),
|
||||||
|
getSqsClientFn: () => client,
|
||||||
|
recreateSqsClientFn: () => {
|
||||||
|
client.destroy();
|
||||||
|
client = createClient();
|
||||||
|
return client;
|
||||||
|
},
|
||||||
|
shouldPoll: () => shouldPoll,
|
||||||
|
});
|
||||||
271
server/tests/integration/queue/live-worker-process.test.ts
Normal file
271
server/tests/integration/queue/live-worker-process.test.ts
Normal file
@@ -0,0 +1,271 @@
|
|||||||
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { spawn, type ChildProcess } from "node:child_process";
|
||||||
|
import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
|
||||||
|
import {
|
||||||
|
CreateQueueCommand,
|
||||||
|
DeleteQueueCommand,
|
||||||
|
GetQueueAttributesCommand,
|
||||||
|
SendMessageCommand,
|
||||||
|
SQSClient,
|
||||||
|
} from "@aws-sdk/client-sqs";
|
||||||
|
|
||||||
|
const ELASTICMQ_JAR = `${process.env.HOME}/.autumn-agent/elasticmq/elasticmq.jar`;
|
||||||
|
const WORKER_FIXTURE_PATH = new URL(
|
||||||
|
"./fixtures/livePollingWorker.ts",
|
||||||
|
import.meta.url,
|
||||||
|
).pathname;
|
||||||
|
|
||||||
|
const children: ChildProcess[] = [];
|
||||||
|
const queueUrls: string[] = [];
|
||||||
|
let elasticmq: ChildProcess | null = null;
|
||||||
|
let tempDir: string | null = null;
|
||||||
|
let sqsEndpoint = "";
|
||||||
|
let sqs: SQSClient;
|
||||||
|
|
||||||
|
const waitForExit = async ({
|
||||||
|
child,
|
||||||
|
forceKillAfterMs = 1_000,
|
||||||
|
}: {
|
||||||
|
child: ChildProcess;
|
||||||
|
forceKillAfterMs?: number;
|
||||||
|
}) => {
|
||||||
|
if (child.exitCode !== null || child.signalCode !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
child.kill("SIGKILL");
|
||||||
|
}, forceKillAfterMs);
|
||||||
|
|
||||||
|
child.once("exit", () => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const waitFor = async ({
|
||||||
|
check,
|
||||||
|
timeoutMs = 8_000,
|
||||||
|
intervalMs = 200,
|
||||||
|
}: {
|
||||||
|
check: () => Promise<boolean>;
|
||||||
|
timeoutMs?: number;
|
||||||
|
intervalMs?: number;
|
||||||
|
}) => {
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
if (await check()) return;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Condition not met within ${timeoutMs}ms`);
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const port = 19_000 + Math.floor(Math.random() * 1_000);
|
||||||
|
const statsPort = port + 1;
|
||||||
|
tempDir = await mkdtemp(join(tmpdir(), "elasticmq-"));
|
||||||
|
const configPath = join(tempDir, "elasticmq.conf");
|
||||||
|
sqsEndpoint = `http://127.0.0.1:${port}`;
|
||||||
|
|
||||||
|
await writeFile(
|
||||||
|
configPath,
|
||||||
|
`include classpath("application.conf")
|
||||||
|
node-address {
|
||||||
|
protocol = http
|
||||||
|
host = "127.0.0.1"
|
||||||
|
port = ${port}
|
||||||
|
context-path = ""
|
||||||
|
}
|
||||||
|
rest-sqs {
|
||||||
|
enabled = true
|
||||||
|
bind-port = ${port}
|
||||||
|
bind-hostname = "127.0.0.1"
|
||||||
|
sqs-limits = strict
|
||||||
|
}
|
||||||
|
generate-node-address = false
|
||||||
|
rest-stats {
|
||||||
|
enabled = true
|
||||||
|
bind-port = ${statsPort}
|
||||||
|
bind-hostname = "127.0.0.1"
|
||||||
|
}
|
||||||
|
queues {}
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
|
||||||
|
elasticmq = spawn(
|
||||||
|
"java",
|
||||||
|
[`-Dconfig.file=${configPath}`, "-jar", ELASTICMQ_JAR],
|
||||||
|
{
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
sqs = new SQSClient({
|
||||||
|
region: "us-east-1",
|
||||||
|
endpoint: sqsEndpoint,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: "x",
|
||||||
|
secretAccessKey: "x",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitFor({
|
||||||
|
timeoutMs: 15_000,
|
||||||
|
check: async () => {
|
||||||
|
try {
|
||||||
|
const response = await fetch(
|
||||||
|
`${sqsEndpoint}/?Action=ListQueues&Version=2012-11-05`,
|
||||||
|
);
|
||||||
|
return response.ok;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
if (elasticmq) {
|
||||||
|
elasticmq.kill("SIGTERM");
|
||||||
|
await waitForExit({ child: elasticmq });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tempDir) {
|
||||||
|
await rm(tempDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const createTestQueue = async () => {
|
||||||
|
const name = `autumn-live-${randomUUID()}.fifo`;
|
||||||
|
const response = await sqs.send(
|
||||||
|
new CreateQueueCommand({
|
||||||
|
QueueName: name,
|
||||||
|
Attributes: {
|
||||||
|
FifoQueue: "true",
|
||||||
|
ContentBasedDeduplication: "true",
|
||||||
|
VisibilityTimeout: "5",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!response.QueueUrl) {
|
||||||
|
throw new Error("Failed to create test queue");
|
||||||
|
}
|
||||||
|
|
||||||
|
queueUrls.push(response.QueueUrl);
|
||||||
|
return response.QueueUrl;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getQueueCounts = async ({ queueUrl }: { queueUrl: string }) => {
|
||||||
|
const response = await sqs.send(
|
||||||
|
new GetQueueAttributesCommand({
|
||||||
|
QueueUrl: queueUrl,
|
||||||
|
AttributeNames: [
|
||||||
|
"ApproximateNumberOfMessages",
|
||||||
|
"ApproximateNumberOfMessagesNotVisible",
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
visible: Number.parseInt(
|
||||||
|
response.Attributes?.ApproximateNumberOfMessages ?? "0",
|
||||||
|
10,
|
||||||
|
),
|
||||||
|
notVisible: Number.parseInt(
|
||||||
|
response.Attributes?.ApproximateNumberOfMessagesNotVisible ?? "0",
|
||||||
|
10,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const startWorker = ({
|
||||||
|
queueUrl,
|
||||||
|
shouldPoll,
|
||||||
|
}: {
|
||||||
|
queueUrl: string;
|
||||||
|
shouldPoll: boolean;
|
||||||
|
}) => {
|
||||||
|
const child = spawn(
|
||||||
|
"bun",
|
||||||
|
[WORKER_FIXTURE_PATH],
|
||||||
|
{
|
||||||
|
cwd: process.cwd(),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
TEST_QUEUE_URL: queueUrl,
|
||||||
|
TEST_SQS_ENDPOINT: sqsEndpoint,
|
||||||
|
TEST_SHOULD_POLL: shouldPoll ? "true" : "false",
|
||||||
|
},
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
children.push(child);
|
||||||
|
return child;
|
||||||
|
};
|
||||||
|
|
||||||
|
const sendTestMessage = async ({ queueUrl }: { queueUrl: string }) => {
|
||||||
|
await sqs.send(
|
||||||
|
new SendMessageCommand({
|
||||||
|
QueueUrl: queueUrl,
|
||||||
|
MessageBody: JSON.stringify({
|
||||||
|
name: "integration-test-job",
|
||||||
|
data: {},
|
||||||
|
}),
|
||||||
|
MessageGroupId: "test",
|
||||||
|
MessageDeduplicationId: randomUUID(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const child of children.splice(0)) {
|
||||||
|
child.kill("SIGTERM");
|
||||||
|
await waitForExit({ child });
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const queueUrl of queueUrls.splice(0)) {
|
||||||
|
await sqs.send(
|
||||||
|
new DeleteQueueCommand({
|
||||||
|
QueueUrl: queueUrl,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("live worker process queue polling", () => {
|
||||||
|
test("enabled worker consumes a live SQS message", async () => {
|
||||||
|
const queueUrl = await createTestQueue();
|
||||||
|
|
||||||
|
startWorker({ queueUrl, shouldPoll: true });
|
||||||
|
await sendTestMessage({ queueUrl });
|
||||||
|
|
||||||
|
await waitFor({
|
||||||
|
check: async () => {
|
||||||
|
const counts = await getQueueCounts({ queueUrl });
|
||||||
|
return counts.visible === 0 && counts.notVisible === 0;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}, 15_000);
|
||||||
|
|
||||||
|
test("disabled worker leaves the live SQS message untouched", async () => {
|
||||||
|
const queueUrl = await createTestQueue();
|
||||||
|
|
||||||
|
startWorker({ queueUrl, shouldPoll: false });
|
||||||
|
await sendTestMessage({ queueUrl });
|
||||||
|
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||||
|
|
||||||
|
const counts = await getQueueCounts({ queueUrl });
|
||||||
|
expect(counts.visible).toBe(1);
|
||||||
|
expect(counts.notVisible).toBe(0);
|
||||||
|
}, 15_000);
|
||||||
|
});
|
||||||
89
server/tests/integration/queue/start-polling-loop.test.ts
Normal file
89
server/tests/integration/queue/start-polling-loop.test.ts
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||||
|
import { startPollingLoop } from "@/queue/initWorkers.js";
|
||||||
|
|
||||||
|
const originalSetTimeout = globalThis.setTimeout;
|
||||||
|
|
||||||
|
const makeAbortError = () => {
|
||||||
|
const error = new Error("aborted") as Error & { name: string };
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("startPollingLoop", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
globalThis.setTimeout = (((callback: TimerHandler) => {
|
||||||
|
if (typeof callback === "function") {
|
||||||
|
callback();
|
||||||
|
}
|
||||||
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||||
|
}) as unknown) as typeof setTimeout;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.setTimeout = originalSetTimeout;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not poll while the queue is disabled", async () => {
|
||||||
|
let shouldPollCalls = 0;
|
||||||
|
let sendCalls = 0;
|
||||||
|
|
||||||
|
await startPollingLoop({
|
||||||
|
db: {} as never,
|
||||||
|
queueUrl: "https://sqs.eu-west-1.amazonaws.com/123/track.fifo",
|
||||||
|
isFifo: true,
|
||||||
|
getSqsClientFn: () =>
|
||||||
|
({
|
||||||
|
send: async () => {
|
||||||
|
sendCalls++;
|
||||||
|
throw makeAbortError();
|
||||||
|
},
|
||||||
|
}) as never,
|
||||||
|
recreateSqsClientFn: () =>
|
||||||
|
({
|
||||||
|
send: async () => {
|
||||||
|
sendCalls++;
|
||||||
|
throw makeAbortError();
|
||||||
|
},
|
||||||
|
}) as never,
|
||||||
|
shouldPoll: () => {
|
||||||
|
shouldPollCalls++;
|
||||||
|
return shouldPollCalls > 1;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(shouldPollCalls).toBeGreaterThan(1);
|
||||||
|
expect(sendCalls).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("polls immediately when the queue is enabled", async () => {
|
||||||
|
let shouldPollCalls = 0;
|
||||||
|
let sendCalls = 0;
|
||||||
|
|
||||||
|
await startPollingLoop({
|
||||||
|
db: {} as never,
|
||||||
|
queueUrl: "https://sqs.eu-west-1.amazonaws.com/123/primary.fifo",
|
||||||
|
isFifo: true,
|
||||||
|
getSqsClientFn: () =>
|
||||||
|
({
|
||||||
|
send: async () => {
|
||||||
|
sendCalls++;
|
||||||
|
throw makeAbortError();
|
||||||
|
},
|
||||||
|
}) as never,
|
||||||
|
recreateSqsClientFn: () =>
|
||||||
|
({
|
||||||
|
send: async () => {
|
||||||
|
sendCalls++;
|
||||||
|
throw makeAbortError();
|
||||||
|
},
|
||||||
|
}) as never,
|
||||||
|
shouldPoll: () => {
|
||||||
|
shouldPollCalls++;
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(shouldPollCalls).toBe(1);
|
||||||
|
expect(sendCalls).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -7,21 +7,9 @@ import {
|
|||||||
} from "@/internal/balances/utils/types/redisDeductionError.js";
|
} from "@/internal/balances/utils/types/redisDeductionError.js";
|
||||||
|
|
||||||
const mockState = {
|
const mockState = {
|
||||||
queueCalls: [] as Record<string, unknown>[],
|
|
||||||
postgresCalls: [] as Record<string, unknown>[],
|
postgresCalls: [] as Record<string, unknown>[],
|
||||||
};
|
};
|
||||||
|
|
||||||
mock.module("@/internal/balances/track/utils/queueTrack.js", () => ({
|
|
||||||
queueTrack: async (args: Record<string, unknown>) => {
|
|
||||||
mockState.queueCalls.push(args);
|
|
||||||
return {
|
|
||||||
customer_id: "cus_123",
|
|
||||||
feature_id: "messages",
|
|
||||||
balance: null,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
mock.module(
|
mock.module(
|
||||||
"@/internal/balances/track/v3/runPostgresTrackV3.js",
|
"@/internal/balances/track/v3/runPostgresTrackV3.js",
|
||||||
() => ({
|
() => ({
|
||||||
@@ -45,31 +33,29 @@ const ctx = {
|
|||||||
|
|
||||||
describe("handleRedisTrackErrorV3", () => {
|
describe("handleRedisTrackErrorV3", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mockState.queueCalls = [];
|
|
||||||
mockState.postgresCalls = [];
|
mockState.postgresCalls = [];
|
||||||
});
|
});
|
||||||
|
|
||||||
test("queues track instead of falling back to Postgres when Redis is unavailable", async () => {
|
test("rethrows when Redis is unavailable", async () => {
|
||||||
const response = await handleRedisTrackErrorV3({
|
const error = new RedisDeductionError({
|
||||||
ctx,
|
message: "Redis not ready for deduction",
|
||||||
error: new RedisDeductionError({
|
code: RedisDeductionErrorCode.RedisUnavailable,
|
||||||
message: "Redis not ready for deduction",
|
|
||||||
code: RedisDeductionErrorCode.RedisUnavailable,
|
|
||||||
}),
|
|
||||||
body: {
|
|
||||||
customer_id: "cus_123",
|
|
||||||
feature_id: "messages",
|
|
||||||
value: 1,
|
|
||||||
},
|
|
||||||
fullSubject: {} as never,
|
|
||||||
featureDeductions: [],
|
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(mockState.queueCalls).toHaveLength(1);
|
await expect(
|
||||||
|
handleRedisTrackErrorV3({
|
||||||
|
ctx,
|
||||||
|
error,
|
||||||
|
body: {
|
||||||
|
customer_id: "cus_123",
|
||||||
|
feature_id: "messages",
|
||||||
|
value: 1,
|
||||||
|
},
|
||||||
|
fullSubject: {} as never,
|
||||||
|
featureDeductions: [],
|
||||||
|
}),
|
||||||
|
).rejects.toBe(error);
|
||||||
|
|
||||||
expect(mockState.postgresCalls).toHaveLength(0);
|
expect(mockState.postgresCalls).toHaveLength(0);
|
||||||
expect(response).toMatchObject({
|
|
||||||
customer_id: "cus_123",
|
|
||||||
balance: null,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -42,6 +42,10 @@ mock.module("@/internal/balances/track/utils/queueTrack.js", () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
mock.module("@/external/redis/initUtils/redisV2Availability.js", () => ({
|
||||||
|
shouldUseRedisV2: () => true,
|
||||||
|
}));
|
||||||
|
|
||||||
import { runTrackWithRollout } from "@/internal/balances/track/runTrackWithRollout.js";
|
import { runTrackWithRollout } from "@/internal/balances/track/runTrackWithRollout.js";
|
||||||
|
|
||||||
const ctx = {
|
const ctx = {
|
||||||
|
|||||||
75
server/tests/unit/balances/track/queueTrack.test.ts
Normal file
75
server/tests/unit/balances/track/queueTrack.test.ts
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import { ApiVersion, ApiVersionClass, AppEnv } from "@autumn/shared";
|
||||||
|
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||||
|
|
||||||
|
const mockState = {
|
||||||
|
queueCalls: [] as Record<string, unknown>[],
|
||||||
|
};
|
||||||
|
|
||||||
|
mock.module("@/queue/queueUtils.js", () => ({
|
||||||
|
addTaskToQueue: async (args: Record<string, unknown>) => {
|
||||||
|
mockState.queueCalls.push(args);
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
mock.module("@/internal/balances/track/utils/getQueuedTrackResponse.js", () => ({
|
||||||
|
getQueuedTrackResponse: () => ({
|
||||||
|
customer_id: "cus_123",
|
||||||
|
value: 2,
|
||||||
|
balance: null,
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { queueTrack } from "@/internal/balances/track/utils/queueTrack.js";
|
||||||
|
|
||||||
|
describe("queueTrack", () => {
|
||||||
|
const originalTrackQueueUrl = process.env.TRACK_SQS_QUEUE_URL;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
mockState.queueCalls = [];
|
||||||
|
process.env.TRACK_SQS_QUEUE_URL =
|
||||||
|
"https://sqs.eu-west-1.amazonaws.com/123456789012/track-dev.fifo";
|
||||||
|
});
|
||||||
|
|
||||||
|
test("queues track with request identity and entity-scoped grouping", async () => {
|
||||||
|
const ctx = {
|
||||||
|
id: "req_123",
|
||||||
|
org: { id: "org_123" },
|
||||||
|
env: AppEnv.Sandbox,
|
||||||
|
apiVersion: new ApiVersionClass(ApiVersion.V2_1),
|
||||||
|
logger: {
|
||||||
|
warn: mock(() => {}),
|
||||||
|
},
|
||||||
|
} as unknown as AutumnContext;
|
||||||
|
|
||||||
|
await queueTrack({
|
||||||
|
ctx,
|
||||||
|
body: {
|
||||||
|
customer_id: "cus_123",
|
||||||
|
entity_id: "ent_123",
|
||||||
|
feature_id: "messages",
|
||||||
|
value: 2,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockState.queueCalls).toHaveLength(1);
|
||||||
|
expect(mockState.queueCalls[0]).toMatchObject({
|
||||||
|
queueUrl:
|
||||||
|
"https://sqs.eu-west-1.amazonaws.com/123456789012/track-dev.fifo",
|
||||||
|
messageGroupId: "org_123:sandbox:cus_123:ent_123",
|
||||||
|
messageDeduplicationId: "req_123",
|
||||||
|
payload: {
|
||||||
|
orgId: "org_123",
|
||||||
|
env: AppEnv.Sandbox,
|
||||||
|
customerId: "cus_123",
|
||||||
|
entityId: "ent_123",
|
||||||
|
requestId: "req_123",
|
||||||
|
apiVersion: ApiVersion.V2_1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.env.TRACK_SQS_QUEUE_URL = originalTrackQueueUrl;
|
||||||
|
});
|
||||||
|
});
|
||||||
88
server/tests/unit/balances/track/runQueuedTrack.test.ts
Normal file
88
server/tests/unit/balances/track/runQueuedTrack.test.ts
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import { ApiVersion, ApiVersionClass, AppEnv, ErrCode, RecaseError } from "@autumn/shared";
|
||||||
|
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||||
|
|
||||||
|
const mockState = {
|
||||||
|
runTrackV3Calls: [] as Record<string, unknown>[],
|
||||||
|
getFeatureDeductionCalls: [] as Record<string, unknown>[],
|
||||||
|
runTrackV3Error: null as unknown,
|
||||||
|
};
|
||||||
|
|
||||||
|
mock.module("@/internal/balances/track/utils/getFeatureDeductions.js", () => ({
|
||||||
|
getTrackFeatureDeductionsForBody: (args: Record<string, unknown>) => {
|
||||||
|
mockState.getFeatureDeductionCalls.push(args);
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
mock.module("@/internal/balances/track/v3/runTrackV3.js", () => ({
|
||||||
|
runTrackV3: async (args: Record<string, unknown>) => {
|
||||||
|
mockState.runTrackV3Calls.push(args);
|
||||||
|
if (mockState.runTrackV3Error) throw mockState.runTrackV3Error;
|
||||||
|
return { customer_id: "cus_123", balance: null };
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { runQueuedTrack } from "@/internal/balances/track/runQueuedTrack.js";
|
||||||
|
|
||||||
|
const ctx = {
|
||||||
|
id: "req_123",
|
||||||
|
env: AppEnv.Sandbox,
|
||||||
|
org: { id: "org_123" },
|
||||||
|
apiVersion: new ApiVersionClass(ApiVersion.V2_1),
|
||||||
|
logger: {
|
||||||
|
info: mock(() => {}),
|
||||||
|
},
|
||||||
|
} as unknown as AutumnContext;
|
||||||
|
|
||||||
|
describe("runQueuedTrack", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
mockState.runTrackV3Calls = [];
|
||||||
|
mockState.getFeatureDeductionCalls = [];
|
||||||
|
mockState.runTrackV3Error = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("replays queued track through runTrackV3", async () => {
|
||||||
|
await runQueuedTrack({
|
||||||
|
ctx,
|
||||||
|
body: {
|
||||||
|
customer_id: "cus_123",
|
||||||
|
feature_id: "messages",
|
||||||
|
value: 1,
|
||||||
|
},
|
||||||
|
apiVersion: ApiVersion.V2_1,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockState.getFeatureDeductionCalls).toHaveLength(1);
|
||||||
|
expect(mockState.runTrackV3Calls).toHaveLength(1);
|
||||||
|
expect(mockState.runTrackV3Calls[0]).toMatchObject({
|
||||||
|
ctx,
|
||||||
|
body: {
|
||||||
|
customer_id: "cus_123",
|
||||||
|
feature_id: "messages",
|
||||||
|
},
|
||||||
|
featureDeductions: [],
|
||||||
|
apiVersion: ApiVersion.V2_1,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("treats duplicate idempotency as already applied", async () => {
|
||||||
|
mockState.runTrackV3Error = new RecaseError({
|
||||||
|
message: "duplicate",
|
||||||
|
code: ErrCode.DuplicateIdempotencyKey,
|
||||||
|
statusCode: 409,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runQueuedTrack({
|
||||||
|
ctx,
|
||||||
|
body: {
|
||||||
|
customer_id: "cus_123",
|
||||||
|
feature_id: "messages",
|
||||||
|
value: 1,
|
||||||
|
},
|
||||||
|
apiVersion: ApiVersion.V2_1,
|
||||||
|
}),
|
||||||
|
).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,20 +1,14 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||||
import { ApiVersion, AppEnv } from "@autumn/shared";
|
import { ApiVersion, AppEnv } from "@autumn/shared";
|
||||||
|
import type { SQSClient } from "@aws-sdk/client-sqs";
|
||||||
|
|
||||||
const mockState = {
|
const mockState = {
|
||||||
commands: [] as Record<string, unknown>[],
|
commands: [] as Record<string, unknown>[],
|
||||||
|
originalSend: null as null | SQSClient["send"],
|
||||||
};
|
};
|
||||||
|
|
||||||
mock.module("@/queue/initSqs.js", () => ({
|
|
||||||
QUEUE_URL: "https://sqs.eu-west-1.amazonaws.com/123456789012/primary.fifo",
|
|
||||||
getSqsClient: () => ({
|
|
||||||
send: async (command: { input: Record<string, unknown> }) => {
|
|
||||||
mockState.commands.push(command.input);
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
}));
|
|
||||||
|
|
||||||
import { JobName } from "@/queue/JobName.js";
|
import { JobName } from "@/queue/JobName.js";
|
||||||
|
import { getSqsClient } from "@/queue/initSqs.js";
|
||||||
import { addTaskToQueue } from "@/queue/queueUtils.js";
|
import { addTaskToQueue } from "@/queue/queueUtils.js";
|
||||||
|
|
||||||
describe("addTaskToQueue queue override", () => {
|
describe("addTaskToQueue queue override", () => {
|
||||||
@@ -23,11 +17,21 @@ describe("addTaskToQueue queue override", () => {
|
|||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mockState.commands = [];
|
mockState.commands = [];
|
||||||
|
const sqsClient = getSqsClient();
|
||||||
|
mockState.originalSend = sqsClient.send.bind(sqsClient);
|
||||||
|
sqsClient.send = (async (command: { input: Record<string, unknown> }) => {
|
||||||
|
mockState.commands.push(command.input);
|
||||||
|
return {};
|
||||||
|
}) as typeof sqsClient.send;
|
||||||
delete process.env.SQS_QUEUE_URL;
|
delete process.env.SQS_QUEUE_URL;
|
||||||
delete process.env.QUEUE_URL;
|
delete process.env.QUEUE_URL;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
if (mockState.originalSend) {
|
||||||
|
const sqsClient = getSqsClient();
|
||||||
|
sqsClient.send = mockState.originalSend as typeof sqsClient.send;
|
||||||
|
}
|
||||||
process.env.SQS_QUEUE_URL = originalSqsQueueUrl;
|
process.env.SQS_QUEUE_URL = originalSqsQueueUrl;
|
||||||
process.env.QUEUE_URL = originalQueueUrl;
|
process.env.QUEUE_URL = originalQueueUrl;
|
||||||
});
|
});
|
||||||
@@ -42,6 +46,8 @@ describe("addTaskToQueue queue override", () => {
|
|||||||
payload: {
|
payload: {
|
||||||
orgId: "org_123",
|
orgId: "org_123",
|
||||||
env: AppEnv.Sandbox,
|
env: AppEnv.Sandbox,
|
||||||
|
customerId: "cus_123",
|
||||||
|
requestId: "req_123",
|
||||||
apiVersion: ApiVersion.V2_1,
|
apiVersion: ApiVersion.V2_1,
|
||||||
body: {
|
body: {
|
||||||
customer_id: "cus_123",
|
customer_id: "cus_123",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { useAxiosInstance } from "@/services/useAxiosInstance";
|
|||||||
import { CustomerBlockDialog } from "./CustomerBlockDialog";
|
import { CustomerBlockDialog } from "./CustomerBlockDialog";
|
||||||
import { EdgeConfigDialog } from "./EdgeConfigDialog";
|
import { EdgeConfigDialog } from "./EdgeConfigDialog";
|
||||||
import { FeatureFlagsDialog } from "./FeatureFlagsDialog";
|
import { FeatureFlagsDialog } from "./FeatureFlagsDialog";
|
||||||
|
import { JobQueuesDialog } from "./JobQueuesDialog";
|
||||||
import { OrgLimitsDialog } from "./OrgLimitsDialog";
|
import { OrgLimitsDialog } from "./OrgLimitsDialog";
|
||||||
import { RawEdgeConfigDialog } from "./RawEdgeConfigDialog";
|
import { RawEdgeConfigDialog } from "./RawEdgeConfigDialog";
|
||||||
import { RedisV2CacheDialog } from "./RedisV2CacheDialog";
|
import { RedisV2CacheDialog } from "./RedisV2CacheDialog";
|
||||||
@@ -27,6 +28,7 @@ export function EdgeConfigTab() {
|
|||||||
const [featureFlagsOpen, setFeatureFlagsOpen] = useState(false);
|
const [featureFlagsOpen, setFeatureFlagsOpen] = useState(false);
|
||||||
const [customerBlockOpen, setCustomerBlockOpen] = useState(false);
|
const [customerBlockOpen, setCustomerBlockOpen] = useState(false);
|
||||||
const [orgLimitsOpen, setOrgLimitsOpen] = useState(false);
|
const [orgLimitsOpen, setOrgLimitsOpen] = useState(false);
|
||||||
|
const [jobQueuesOpen, setJobQueuesOpen] = useState(false);
|
||||||
const [stripeSyncOpen, setStripeSyncOpen] = useState(false);
|
const [stripeSyncOpen, setStripeSyncOpen] = useState(false);
|
||||||
const [redisV2CacheOpen, setRedisV2CacheOpen] = useState(false);
|
const [redisV2CacheOpen, setRedisV2CacheOpen] = useState(false);
|
||||||
|
|
||||||
@@ -156,6 +158,23 @@ export function EdgeConfigTab() {
|
|||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center justify-between border-t border-border p-4 last:border-b-0">
|
||||||
|
<div className="flex flex-col gap-0.5">
|
||||||
|
<div className="text-sm font-medium text-t1">Job Queues</div>
|
||||||
|
<div className="text-xs text-t3">
|
||||||
|
Pause or resume worker consumption for shared and dedicated SQS
|
||||||
|
queues.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
size="sm"
|
||||||
|
onClick={() => setJobQueuesOpen(true)}
|
||||||
|
>
|
||||||
|
Edit
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="flex items-center justify-between border-t border-border p-4 last:border-b-0">
|
<div className="flex items-center justify-between border-t border-border p-4 last:border-b-0">
|
||||||
<div className="flex flex-col gap-0.5">
|
<div className="flex flex-col gap-0.5">
|
||||||
<div className="text-sm font-medium text-t1">Stripe Sync</div>
|
<div className="text-sm font-medium text-t1">Stripe Sync</div>
|
||||||
@@ -215,6 +234,11 @@ export function EdgeConfigTab() {
|
|||||||
|
|
||||||
<OrgLimitsDialog open={orgLimitsOpen} onOpenChange={setOrgLimitsOpen} />
|
<OrgLimitsDialog open={orgLimitsOpen} onOpenChange={setOrgLimitsOpen} />
|
||||||
|
|
||||||
|
<JobQueuesDialog
|
||||||
|
open={jobQueuesOpen}
|
||||||
|
onOpenChange={setJobQueuesOpen}
|
||||||
|
/>
|
||||||
|
|
||||||
<StripeSyncDialog
|
<StripeSyncDialog
|
||||||
open={stripeSyncOpen}
|
open={stripeSyncOpen}
|
||||||
onOpenChange={setStripeSyncOpen}
|
onOpenChange={setStripeSyncOpen}
|
||||||
|
|||||||
235
vite/src/views/admin/components/JobQueuesDialog.tsx
Normal file
235
vite/src/views/admin/components/JobQueuesDialog.tsx
Normal file
@@ -0,0 +1,235 @@
|
|||||||
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import { Badge } from "@/components/v2/badges/Badge";
|
||||||
|
import { Button } from "@/components/v2/buttons/Button";
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogFooter,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from "@/components/v2/dialogs/Dialog";
|
||||||
|
import { Switch } from "@/components/ui/switch";
|
||||||
|
import { useAxiosInstance } from "@/services/useAxiosInstance";
|
||||||
|
import { getBackendErr } from "@/utils/genUtils";
|
||||||
|
|
||||||
|
type QueueEntry = {
|
||||||
|
enabled: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type KnownQueue = {
|
||||||
|
id: string;
|
||||||
|
label: string;
|
||||||
|
description: string;
|
||||||
|
defaultEnabled: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type JobQueueConfig = {
|
||||||
|
queues: Record<string, QueueEntry>;
|
||||||
|
knownQueues: KnownQueue[];
|
||||||
|
configHealthy: boolean;
|
||||||
|
configConfigured: boolean;
|
||||||
|
lastSuccessAt: string | null;
|
||||||
|
error: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_CONFIG: JobQueueConfig = {
|
||||||
|
queues: {},
|
||||||
|
knownQueues: [],
|
||||||
|
configHealthy: false,
|
||||||
|
configConfigured: false,
|
||||||
|
lastSuccessAt: null,
|
||||||
|
error: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
export function JobQueuesDialog({
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
}: {
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
}) {
|
||||||
|
const axiosInstance = useAxiosInstance();
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [config, setConfig] = useState<JobQueueConfig>(DEFAULT_CONFIG);
|
||||||
|
const [initialEnabledByQueue, setInitialEnabledByQueue] = useState<
|
||||||
|
Record<string, boolean>
|
||||||
|
>({});
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open) return;
|
||||||
|
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
|
||||||
|
void axiosInstance
|
||||||
|
.get<JobQueueConfig>("/admin/job-queue-config")
|
||||||
|
.then(({ data }) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
const nextConfig = { ...DEFAULT_CONFIG, ...data };
|
||||||
|
setConfig(nextConfig);
|
||||||
|
setInitialEnabledByQueue(
|
||||||
|
Object.fromEntries(
|
||||||
|
nextConfig.knownQueues.map((queue) => [
|
||||||
|
queue.id,
|
||||||
|
nextConfig.queues[queue.id]?.enabled ?? queue.defaultEnabled,
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
if (!cancelled) {
|
||||||
|
toast.error(getBackendErr(error, "Failed to load job queue config"));
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!cancelled) setLoading(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [axiosInstance, open]);
|
||||||
|
|
||||||
|
const effectiveQueues = useMemo(
|
||||||
|
() =>
|
||||||
|
config.knownQueues.map((queue) => ({
|
||||||
|
...queue,
|
||||||
|
enabled:
|
||||||
|
config.queues[queue.id]?.enabled ?? queue.defaultEnabled,
|
||||||
|
})),
|
||||||
|
[config],
|
||||||
|
);
|
||||||
|
|
||||||
|
const toggleQueue = ({
|
||||||
|
queueId,
|
||||||
|
enabled,
|
||||||
|
}: {
|
||||||
|
queueId: string;
|
||||||
|
enabled: boolean;
|
||||||
|
}) => {
|
||||||
|
setConfig((current) => ({
|
||||||
|
...current,
|
||||||
|
queues: {
|
||||||
|
...current.queues,
|
||||||
|
[queueId]: { enabled },
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
|
||||||
|
const dirty = effectiveQueues.some(
|
||||||
|
(queue) => queue.enabled !== initialEnabledByQueue[queue.id],
|
||||||
|
);
|
||||||
|
|
||||||
|
const handleSave = async () => {
|
||||||
|
setSaving(true);
|
||||||
|
try {
|
||||||
|
await axiosInstance.put("/admin/job-queue-config", {
|
||||||
|
queues: Object.fromEntries(
|
||||||
|
effectiveQueues.map((queue) => [
|
||||||
|
queue.id,
|
||||||
|
{ enabled: queue.enabled },
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
toast.success("Job queue config saved");
|
||||||
|
onOpenChange(false);
|
||||||
|
} catch (error) {
|
||||||
|
toast.error(getBackendErr(error, "Failed to save job queue config"));
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||||
|
<DialogContent className="max-w-2xl bg-card">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>Job Queues</DialogTitle>
|
||||||
|
<DialogDescription>
|
||||||
|
Control which SQS queues workers actively poll. Disabling a queue
|
||||||
|
pauses consumption without changing producers.
|
||||||
|
</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
|
||||||
|
{loading ? (
|
||||||
|
<div className="py-8 text-center text-sm text-t3">Loading...</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex flex-col gap-4">
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
{effectiveQueues.map((queue) => (
|
||||||
|
<div
|
||||||
|
key={queue.id}
|
||||||
|
className="flex items-center justify-between rounded-lg border border-border p-3"
|
||||||
|
>
|
||||||
|
<div className="flex flex-col gap-0.5 pr-4">
|
||||||
|
<div className="text-sm font-medium text-t1">
|
||||||
|
{queue.label}
|
||||||
|
</div>
|
||||||
|
<div className="text-xs text-t3">
|
||||||
|
{queue.description}
|
||||||
|
</div>
|
||||||
|
<div className="text-[11px] text-t3">
|
||||||
|
Default: {queue.defaultEnabled ? "enabled" : "disabled"}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<Switch
|
||||||
|
checked={queue.enabled}
|
||||||
|
onCheckedChange={(enabled) =>
|
||||||
|
toggleQueue({ queueId: queue.id, enabled })
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="rounded-lg border border-border p-3 text-xs text-t3">
|
||||||
|
<div className="mb-2 flex items-center gap-2">
|
||||||
|
<Badge
|
||||||
|
variant="muted"
|
||||||
|
className={
|
||||||
|
config.configHealthy
|
||||||
|
? "border-emerald-200 bg-emerald-50 text-emerald-700"
|
||||||
|
: "border-amber-200 bg-amber-50 text-amber-700"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{config.configHealthy
|
||||||
|
? "Config healthy"
|
||||||
|
: "Config unavailable"}
|
||||||
|
</Badge>
|
||||||
|
{config.lastSuccessAt && (
|
||||||
|
<span>
|
||||||
|
Last refresh:{" "}
|
||||||
|
{new Date(config.lastSuccessAt).toLocaleString()}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
{config.configConfigured === false
|
||||||
|
? "S3 job queue config is not configured."
|
||||||
|
: config.error ||
|
||||||
|
"Queue polling changes propagate to workers within ~60 seconds."}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<DialogFooter>
|
||||||
|
<Button variant="secondary" onClick={() => onOpenChange(false)}>
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={handleSave}
|
||||||
|
isLoading={saving}
|
||||||
|
disabled={loading || !dirty}
|
||||||
|
>
|
||||||
|
Save
|
||||||
|
</Button>
|
||||||
|
</DialogFooter>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user