chore: init fail open track
This commit is contained in:
6
server/src/external/aws/s3/adminS3Config.ts
vendored
6
server/src/external/aws/s3/adminS3Config.ts
vendored
@@ -6,6 +6,7 @@ export const ADMIN_CUSTOMER_BLOCK_CONFIG_KEY =
|
||||
export const ADMIN_ORG_LIMITS_CONFIG_KEY = "admin/org-limits-config.json";
|
||||
export const ADMIN_REDIS_V2_CACHE_CONFIG_KEY =
|
||||
"admin/redis-v2-cache-config.json";
|
||||
export const ADMIN_JOB_QUEUE_CONFIG_KEY = "admin/job-queue-config.json";
|
||||
|
||||
const bucket = process.env.S3_BUCKET || "autumn-prod-server";
|
||||
const region = process.env.S3_REGION || "us-east-2";
|
||||
@@ -42,6 +43,11 @@ export const getAdminEdgeConfigSources = () => ({
|
||||
label: "V2 Redis Instance",
|
||||
key: ADMIN_REDIS_V2_CACHE_CONFIG_KEY,
|
||||
},
|
||||
{
|
||||
id: "job-queues",
|
||||
label: "Job Queues",
|
||||
key: ADMIN_JOB_QUEUE_CONFIG_KEY,
|
||||
},
|
||||
{
|
||||
id: "stripe-sync",
|
||||
label: "Stripe Sync",
|
||||
|
||||
@@ -25,6 +25,7 @@ import "./internal/misc/customerBlocks/customerBlockStore.js";
|
||||
import "./internal/misc/edgeConfig/orgLimitsStore.js";
|
||||
import "./internal/misc/stripeSync/stripeSyncStore.js";
|
||||
import "./internal/misc/redisV2Cache/redisV2CacheStore.js";
|
||||
import "./internal/misc/jobQueues/jobQueueStore.js";
|
||||
import { closeStripeSyncEngine } from "@autumn/stripe-sync";
|
||||
import {
|
||||
startRedisMonitor,
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { HonoEnv } from "../../honoUtils/HonoEnv";
|
||||
import { handleGetAdminCustomerBlockConfig } from "./handleGetAdminCustomerBlockConfig";
|
||||
import { handleGetAdminEdgeConfigSources } from "./handleGetAdminEdgeConfigSources";
|
||||
import { handleGetAdminFeatureFlagsConfig } from "./handleGetAdminFeatureFlagsConfig";
|
||||
import { handleGetAdminJobQueueConfig } from "./handleGetAdminJobQueueConfig";
|
||||
import { handleGetAdminOrgLimitsConfig } from "./handleGetAdminOrgLimitsConfig";
|
||||
import { handleGetAdminOrgRequestBlock } from "./handleGetAdminOrgRequestBlock";
|
||||
import { handleGetAdminRequestBlockConfig } from "./handleGetAdminRequestBlockConfig";
|
||||
@@ -17,6 +18,7 @@ import { handleListAdminUsers } from "./handleListAdminUsers";
|
||||
import { handleListOAuthClients } from "./handleListOAuthClients";
|
||||
import { handleUpsertAdminCustomerBlockConfig } from "./handleUpsertAdminCustomerBlockConfig";
|
||||
import { handleUpsertAdminFeatureFlagsConfig } from "./handleUpsertAdminFeatureFlagsConfig";
|
||||
import { handleUpsertAdminJobQueueConfig } from "./handleUpsertAdminJobQueueConfig";
|
||||
import { handleUpsertAdminOrgLimitsConfig } from "./handleUpsertAdminOrgLimitsConfig";
|
||||
import { handleUpsertAdminOrgRequestBlock } from "./handleUpsertAdminOrgRequestBlock";
|
||||
import { handleUpsertAdminRequestBlockConfig } from "./handleUpsertAdminRequestBlockConfig";
|
||||
@@ -67,6 +69,8 @@ honoAdminRouter.put(
|
||||
);
|
||||
honoAdminRouter.get("/org-limits-config", ...handleGetAdminOrgLimitsConfig);
|
||||
honoAdminRouter.put("/org-limits-config", ...handleUpsertAdminOrgLimitsConfig);
|
||||
honoAdminRouter.get("/job-queue-config", ...handleGetAdminJobQueueConfig);
|
||||
honoAdminRouter.put("/job-queue-config", ...handleUpsertAdminJobQueueConfig);
|
||||
honoAdminRouter.get("/stripe-sync-config", ...handleGetAdminStripeSyncConfig);
|
||||
honoAdminRouter.put(
|
||||
"/stripe-sync-config",
|
||||
|
||||
22
server/src/internal/admin/handleGetAdminJobQueueConfig.ts
Normal file
22
server/src/internal/admin/handleGetAdminJobQueueConfig.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { createRoute } from "@/honoMiddlewares/routeHandler.js";
|
||||
import {
|
||||
getJobQueueConfigFromSource,
|
||||
getJobQueueConfigStatus,
|
||||
KNOWN_JOB_QUEUES,
|
||||
} from "@/internal/misc/jobQueues/jobQueueStore.js";
|
||||
|
||||
export const handleGetAdminJobQueueConfig = createRoute({
|
||||
handler: async (c) => {
|
||||
const status = getJobQueueConfigStatus();
|
||||
const config = await getJobQueueConfigFromSource();
|
||||
|
||||
return c.json({
|
||||
...config,
|
||||
knownQueues: KNOWN_JOB_QUEUES,
|
||||
configHealthy: status.healthy,
|
||||
configConfigured: status.configured,
|
||||
lastSuccessAt: status.lastSuccessAt ?? null,
|
||||
error: status.error ?? null,
|
||||
});
|
||||
},
|
||||
});
|
||||
14
server/src/internal/admin/handleUpsertAdminJobQueueConfig.ts
Normal file
14
server/src/internal/admin/handleUpsertAdminJobQueueConfig.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { createRoute } from "@/honoMiddlewares/routeHandler.js";
|
||||
import { JobQueueConfigSchema } from "@/internal/misc/jobQueues/jobQueueSchemas.js";
|
||||
import { updateFullJobQueueConfig } from "@/internal/misc/jobQueues/jobQueueStore.js";
|
||||
|
||||
export const handleUpsertAdminJobQueueConfig = createRoute({
|
||||
body: JobQueueConfigSchema,
|
||||
handler: async (c) => {
|
||||
const body = c.req.valid("json");
|
||||
|
||||
await updateFullJobQueueConfig({ config: body });
|
||||
|
||||
return c.json({ success: true });
|
||||
},
|
||||
});
|
||||
40
server/src/internal/balances/track/runQueuedTrack.ts
Normal file
40
server/src/internal/balances/track/runQueuedTrack.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { ErrCode, RecaseError, type ApiVersion, type TrackParams } from "@autumn/shared";
|
||||
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||
import { getTrackFeatureDeductionsForBody } from "./utils/getFeatureDeductions.js";
|
||||
import { runTrackV3 } from "./v3/runTrackV3.js";
|
||||
|
||||
export const runQueuedTrack = async ({
|
||||
ctx,
|
||||
body,
|
||||
apiVersion,
|
||||
}: {
|
||||
ctx: AutumnContext;
|
||||
body: TrackParams;
|
||||
apiVersion?: ApiVersion;
|
||||
}) => {
|
||||
const featureDeductions = getTrackFeatureDeductionsForBody({ ctx, body });
|
||||
|
||||
try {
|
||||
await runTrackV3({
|
||||
ctx,
|
||||
body,
|
||||
featureDeductions,
|
||||
apiVersion,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof RecaseError) ||
|
||||
error.code !== ErrCode.DuplicateIdempotencyKey
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
ctx.logger.info("[track] queued replay already applied", {
|
||||
type: "track_queue_replay_duplicate",
|
||||
customer_id: body.customer_id,
|
||||
entity_id: body.entity_id,
|
||||
feature_id: body.feature_id,
|
||||
event_name: body.event_name,
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -1,10 +1,7 @@
|
||||
import type { ApiVersion, TrackParams, TrackResponseV3 } from "@autumn/shared";
|
||||
import { Result } from "better-result";
|
||||
import { withRedisFailOpen } from "@/external/redis/utils/withRedisFailOpen.js";
|
||||
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||
import {
|
||||
isFullSubjectRolloutEnabled,
|
||||
isRetryableFullSubjectRolloutError,
|
||||
} from "@/internal/misc/rollouts/fullSubjectRolloutUtils.js";
|
||||
import { isFullSubjectRolloutEnabled } from "@/internal/misc/rollouts/fullSubjectRolloutUtils.js";
|
||||
import type { FeatureDeduction } from "../utils/types/featureDeduction.js";
|
||||
import { runTrackV2 } from "./runTrackV2.js";
|
||||
import { queueTrack } from "./utils/queueTrack.js";
|
||||
@@ -27,29 +24,21 @@ export const runTrackWithRollout = async ({
|
||||
apiVersion?: ApiVersion;
|
||||
}): Promise<TrackResponseV3> => {
|
||||
if (shouldUseTrackV3({ ctx })) {
|
||||
const result = await Result.tryPromise({
|
||||
try: () =>
|
||||
return withRedisFailOpen<TrackResponseV3>({
|
||||
source: "runTrackWithRollout",
|
||||
run: () =>
|
||||
runTrackV3({
|
||||
ctx,
|
||||
body,
|
||||
featureDeductions,
|
||||
apiVersion,
|
||||
}),
|
||||
catch: (error) => error,
|
||||
fallback: async (error) => {
|
||||
const queuedResponse = await queueTrack({ ctx, body });
|
||||
if (queuedResponse) return queuedResponse;
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
|
||||
if (Result.isOk(result)) return result.value;
|
||||
|
||||
const error = result.error;
|
||||
if (!isRetryableFullSubjectRolloutError({ error })) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const queuedResponse = await queueTrack({ ctx, body });
|
||||
|
||||
if (queuedResponse) return queuedResponse;
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
return runTrackV2({
|
||||
|
||||
@@ -23,11 +23,14 @@ export const queueTrack = async ({
|
||||
await addTaskToQueue({
|
||||
jobName: JobName.Track,
|
||||
queueUrl,
|
||||
messageGroupId: `${ctx.org.id}:${ctx.env}:${body.customer_id}`,
|
||||
messageDeduplicationId: body.idempotency_key,
|
||||
messageGroupId: `${ctx.org.id}:${ctx.env}:${body.customer_id}:${body.entity_id ?? "none"}`,
|
||||
messageDeduplicationId: body.idempotency_key || ctx.id,
|
||||
payload: {
|
||||
orgId: ctx.org.id,
|
||||
env: ctx.env,
|
||||
customerId: body.customer_id,
|
||||
entityId: body.entity_id,
|
||||
requestId: ctx.id,
|
||||
apiVersion: ctx.apiVersion.value,
|
||||
body,
|
||||
},
|
||||
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
RedisDeductionError,
|
||||
RedisDeductionErrorCode,
|
||||
} from "../../utils/types/redisDeductionError.js";
|
||||
import { queueTrack } from "../utils/queueTrack.js";
|
||||
import { runPostgresTrackV3 } from "./runPostgresTrackV3.js";
|
||||
|
||||
/** Handles errors from V2 Redis deduction. Falls back to Postgres V3 path. */
|
||||
@@ -58,8 +57,6 @@ export const handleRedisTrackErrorV3 = async ({
|
||||
}
|
||||
|
||||
if (error.isRedisUnavailable()) {
|
||||
const queuedResponse = await queueTrack({ ctx, body });
|
||||
if (queuedResponse) return queuedResponse;
|
||||
throw error;
|
||||
}
|
||||
|
||||
|
||||
14
server/src/internal/misc/jobQueues/jobQueueSchemas.ts
Normal file
14
server/src/internal/misc/jobQueues/jobQueueSchemas.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { z } from "zod/v4";
|
||||
|
||||
export const JobQueueConfigSchema = z.object({
|
||||
queues: z
|
||||
.record(
|
||||
z.string(),
|
||||
z.object({
|
||||
enabled: z.boolean().default(false),
|
||||
}),
|
||||
)
|
||||
.default({}),
|
||||
});
|
||||
|
||||
export type JobQueueConfig = z.infer<typeof JobQueueConfigSchema>;
|
||||
55
server/src/internal/misc/jobQueues/jobQueueStore.ts
Normal file
55
server/src/internal/misc/jobQueues/jobQueueStore.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
import { ADMIN_JOB_QUEUE_CONFIG_KEY } from "@/external/aws/s3/adminS3Config.js";
|
||||
import { registerEdgeConfig } from "@/internal/misc/edgeConfig/edgeConfigRegistry.js";
|
||||
import { createEdgeConfigStore } from "@/internal/misc/edgeConfig/edgeConfigStore.js";
|
||||
import {
|
||||
type JobQueueConfig,
|
||||
JobQueueConfigSchema,
|
||||
} from "./jobQueueSchemas.js";
|
||||
|
||||
export const JOB_QUEUE_IDS = {
|
||||
primary: "primary",
|
||||
track: "track",
|
||||
} as const;
|
||||
|
||||
export const KNOWN_JOB_QUEUES = [
|
||||
{
|
||||
id: JOB_QUEUE_IDS.primary,
|
||||
label: "Primary Queue",
|
||||
description: "Shared SQS queue for standard background jobs.",
|
||||
defaultEnabled: true,
|
||||
},
|
||||
{
|
||||
id: JOB_QUEUE_IDS.track,
|
||||
label: "Track Replay Queue",
|
||||
description: "Dedicated async track replay queue used during fail-open recovery.",
|
||||
defaultEnabled: true,
|
||||
},
|
||||
] as const;
|
||||
|
||||
const store = createEdgeConfigStore<JobQueueConfig>({
|
||||
s3Key: ADMIN_JOB_QUEUE_CONFIG_KEY,
|
||||
schema: JobQueueConfigSchema,
|
||||
defaultValue: () => ({ queues: {} }),
|
||||
});
|
||||
|
||||
registerEdgeConfig({ store });
|
||||
|
||||
export const isJobQueueEnabled = ({
|
||||
queue,
|
||||
defaultEnabled = true,
|
||||
}: {
|
||||
queue: string;
|
||||
defaultEnabled?: boolean;
|
||||
}) => store.get().queues[queue]?.enabled ?? defaultEnabled;
|
||||
|
||||
export const getJobQueueConfigStatus = () => store.getStatus();
|
||||
|
||||
export const getJobQueueConfigFromSource = async () => store.readFromSource();
|
||||
|
||||
export const updateFullJobQueueConfig = async ({
|
||||
config,
|
||||
}: {
|
||||
config: JobQueueConfig;
|
||||
}) => {
|
||||
await store.writeToSource({ config });
|
||||
};
|
||||
@@ -18,10 +18,11 @@ export const createWorkerContext = async ({
|
||||
orgId?: string;
|
||||
env?: AppEnv;
|
||||
customerId?: string;
|
||||
requestId?: string;
|
||||
};
|
||||
logger: Logger;
|
||||
}) => {
|
||||
const { orgId, env, customerId } = payload;
|
||||
const { orgId, env, customerId, requestId } = payload;
|
||||
if (!orgId || !env) return;
|
||||
|
||||
// Fetch org with features once for all items
|
||||
@@ -74,7 +75,7 @@ export const createWorkerContext = async ({
|
||||
logger: workerLogger,
|
||||
redisV2: resolveRedisV2(),
|
||||
|
||||
id: generateId("job"),
|
||||
id: requestId || generateId("job"),
|
||||
timestamp: Date.now(),
|
||||
isPublic: false,
|
||||
authType: AuthType.Worker,
|
||||
|
||||
@@ -12,6 +12,10 @@ import * as Sentry from "@sentry/bun";
|
||||
import { type DrizzleCli, initDrizzle } from "@/db/initDrizzle.js";
|
||||
import { logger } from "@/external/logtail/logtailUtils.js";
|
||||
import { verifyCacheConsistency } from "@/internal/billing/v2/workflows/verifyCacheConsistency/verifyCacheConsistency.js";
|
||||
import {
|
||||
isJobQueueEnabled,
|
||||
JOB_QUEUE_IDS,
|
||||
} from "@/internal/misc/jobQueues/jobQueueStore.js";
|
||||
import { generateId } from "@/utils/genUtils.js";
|
||||
import { withTimeout } from "@/utils/withTimeout.js";
|
||||
import { hatchet } from "../external/hatchet/initHatchet.js";
|
||||
@@ -21,7 +25,7 @@ import { processMessage, type SqsJob } from "./processMessage.js";
|
||||
|
||||
// ============ Shared State ============
|
||||
let isRunning = true;
|
||||
let abortController: AbortController;
|
||||
const abortControllers = new Set<AbortController>();
|
||||
|
||||
// Process recycling — exit after processing this many messages to prevent memory leaks
|
||||
const MAX_MESSAGES_BEFORE_RECYCLE = 50_000;
|
||||
@@ -47,18 +51,20 @@ const logPrefix = ({ queueUrl }: { queueUrl: string }) =>
|
||||
|
||||
// ============ Polling Loop (per-queue, per-loop state) ============
|
||||
|
||||
const startPollingLoop = async ({
|
||||
export const startPollingLoop = async ({
|
||||
db,
|
||||
queueUrl,
|
||||
isFifo,
|
||||
getSqsClientFn,
|
||||
recreateSqsClientFn,
|
||||
shouldPoll = () => true,
|
||||
}: {
|
||||
db: DrizzleCli;
|
||||
queueUrl: string;
|
||||
isFifo: boolean;
|
||||
getSqsClientFn: () => SQSClient;
|
||||
recreateSqsClientFn: () => SQSClient;
|
||||
shouldPoll?: () => boolean;
|
||||
}) => {
|
||||
// Per-loop state
|
||||
let messagesProcessed = 0;
|
||||
@@ -70,6 +76,8 @@ const startPollingLoop = async ({
|
||||
let consecutiveZeroMessageIntervals = 0;
|
||||
|
||||
const prefix = logPrefix({ queueUrl });
|
||||
let abortController = new AbortController();
|
||||
abortControllers.add(abortController);
|
||||
|
||||
const alertZeroMessages = () => {
|
||||
const minutes = consecutiveZeroMessageIntervals;
|
||||
@@ -105,6 +113,13 @@ const startPollingLoop = async ({
|
||||
};
|
||||
|
||||
const logStatsAndCheckZeroMessages = () => {
|
||||
if (!shouldPoll()) {
|
||||
consecutiveZeroMessageIntervals = 0;
|
||||
messagesProcessed = 0;
|
||||
lastStatsTime = Date.now();
|
||||
return;
|
||||
}
|
||||
|
||||
const elapsedSeconds = ((Date.now() - lastStatsTime) / 1000).toFixed(0);
|
||||
const mem = process.memoryUsage();
|
||||
console.log(
|
||||
@@ -245,6 +260,7 @@ const startPollingLoop = async ({
|
||||
);
|
||||
consecutiveEmptyPolls = 0;
|
||||
abortController = new AbortController();
|
||||
abortControllers.add(abortController);
|
||||
return recreateSqsClientFn();
|
||||
}
|
||||
|
||||
@@ -270,6 +286,7 @@ const startPollingLoop = async ({
|
||||
console.warn(`${prefix} Repeated errors - recreating SQS client`);
|
||||
consecutiveEmptyPolls = 0;
|
||||
abortController = new AbortController();
|
||||
abortControllers.add(abortController);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
return recreateSqsClientFn();
|
||||
}
|
||||
@@ -284,6 +301,12 @@ const startPollingLoop = async ({
|
||||
|
||||
while (isRunning) {
|
||||
try {
|
||||
if (!shouldPoll()) {
|
||||
consecutiveEmptyPolls = 0;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
continue;
|
||||
}
|
||||
|
||||
const response = await sqs.send(createReceiveCommand(), {
|
||||
abortSignal: abortController.signal,
|
||||
});
|
||||
@@ -348,6 +371,7 @@ const startPollingLoop = async ({
|
||||
}
|
||||
}
|
||||
|
||||
abortControllers.delete(abortController);
|
||||
clearInterval(statsInterval);
|
||||
console.log(`${prefix} Stopped`);
|
||||
};
|
||||
@@ -370,7 +394,9 @@ export const initWorkers = async ({
|
||||
const shutdown = async () => {
|
||||
console.log(`[SQS Worker ${process.pid}] Shutting down...`);
|
||||
isRunning = false;
|
||||
if (abortController) abortController.abort();
|
||||
for (const controller of abortControllers) {
|
||||
controller.abort();
|
||||
}
|
||||
|
||||
const isProd = process.env.NODE_ENV === "production";
|
||||
if (isProd) {
|
||||
@@ -386,20 +412,36 @@ export const initWorkers = async ({
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
abortController = new AbortController();
|
||||
|
||||
const startupDurationMs = Date.now() - startupStartedAt;
|
||||
console.log(
|
||||
`[Worker ${process.pid}] ${queueImplementation} worker ready in ${startupDurationMs}ms`,
|
||||
);
|
||||
const pollingLoops = [
|
||||
startPollingLoop({
|
||||
db,
|
||||
queueUrl: QUEUE_URL,
|
||||
isFifo: QUEUE_URL.endsWith(".fifo"),
|
||||
getSqsClientFn: getSqsClient,
|
||||
recreateSqsClientFn: recreateSqsClient,
|
||||
shouldPoll: () => isJobQueueEnabled({ queue: JOB_QUEUE_IDS.primary }),
|
||||
}),
|
||||
];
|
||||
|
||||
await startPollingLoop({
|
||||
db,
|
||||
queueUrl: QUEUE_URL,
|
||||
isFifo: QUEUE_URL.endsWith(".fifo"),
|
||||
getSqsClientFn: getSqsClient,
|
||||
recreateSqsClientFn: recreateSqsClient,
|
||||
});
|
||||
const trackQueueUrl = process.env.TRACK_SQS_QUEUE_URL;
|
||||
if (trackQueueUrl) {
|
||||
pollingLoops.push(
|
||||
startPollingLoop({
|
||||
db,
|
||||
queueUrl: trackQueueUrl,
|
||||
isFifo: trackQueueUrl.endsWith(".fifo"),
|
||||
getSqsClientFn: getSqsClient,
|
||||
recreateSqsClientFn: recreateSqsClient,
|
||||
shouldPoll: () => isJobQueueEnabled({ queue: JOB_QUEUE_IDS.track }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await Promise.all(pollingLoops);
|
||||
};
|
||||
|
||||
export const initHatchetWorker = async () => {
|
||||
|
||||
@@ -10,6 +10,7 @@ import { runActionHandlerTask } from "@/internal/analytics/runActionHandlerTask.
|
||||
import { autoTopup } from "@/internal/balances/autoTopUp/autoTopup.js";
|
||||
import { runInsertEventBatch } from "@/internal/balances/events/runInsertEventBatch.js";
|
||||
import { expireLock } from "@/internal/balances/finalizeLock/expireLock.js";
|
||||
import { runQueuedTrack } from "@/internal/balances/track/runQueuedTrack.js";
|
||||
import { refreshEntityAggregateCache } from "@/internal/balances/utils/refreshEntityAggregate/index.js";
|
||||
import { syncItemV3 } from "@/internal/balances/utils/sync/syncItemV3.js";
|
||||
import { syncItemV4 } from "@/internal/balances/utils/sync/syncItemV4.js";
|
||||
@@ -175,6 +176,20 @@ export const processMessage = async ({
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.name === JobName.Track) {
|
||||
if (!ctx) {
|
||||
workerLogger.error("No context found for track job");
|
||||
return;
|
||||
}
|
||||
|
||||
await runQueuedTrack({
|
||||
ctx,
|
||||
body: job.data.body,
|
||||
apiVersion: job.data.apiVersion,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.name === JobName.RefreshEntityAggregate) {
|
||||
if (!ctx) {
|
||||
workerLogger.error("No context found for refresh entity aggregate job");
|
||||
|
||||
@@ -69,6 +69,9 @@ export interface Payloads {
|
||||
[JobName.Track]: {
|
||||
orgId: string;
|
||||
env: AppEnv;
|
||||
customerId: string;
|
||||
entityId?: string;
|
||||
requestId: string;
|
||||
apiVersion: ApiVersion;
|
||||
body: TrackParams;
|
||||
};
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
import "./internal/misc/requestBlocks/requestBlockStore.js";
|
||||
import "./internal/misc/rollouts/rolloutConfigStore.js";
|
||||
import "./internal/misc/redisV2Cache/redisV2CacheStore.js";
|
||||
import "./internal/misc/jobQueues/jobQueueStore.js";
|
||||
|
||||
// Number of worker processes (defaults to CPU cores)
|
||||
const NUM_PROCESSES = process.env.NODE_ENV === "development" ? 3 : 4;
|
||||
|
||||
246
server/tests/integration/admin/job-queue-config.test.ts
Normal file
246
server/tests/integration/admin/job-queue-config.test.ts
Normal file
@@ -0,0 +1,246 @@
|
||||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import { AppEnv, ErrCode } from "@autumn/shared";
|
||||
import { Hono } from "hono";
|
||||
import { errorMiddleware } from "@/honoMiddlewares/errorMiddleware.js";
|
||||
import type { HonoEnv } from "@/honoUtils/HonoEnv.js";
|
||||
|
||||
type MockJobQueueConfig = {
|
||||
queues: Record<string, { enabled: boolean }>;
|
||||
};
|
||||
|
||||
type MockStatus = {
|
||||
healthy: boolean;
|
||||
configured: boolean;
|
||||
lastSuccessAt: string | null;
|
||||
error: string | null;
|
||||
};
|
||||
|
||||
const mockState = {
|
||||
config: {
|
||||
queues: {
|
||||
primary: { enabled: true },
|
||||
track: { enabled: false },
|
||||
},
|
||||
} as MockJobQueueConfig,
|
||||
status: {
|
||||
healthy: true,
|
||||
configured: true,
|
||||
lastSuccessAt: "2026-04-24T10:00:00.000Z",
|
||||
error: null,
|
||||
} as MockStatus,
|
||||
updateCalls: [] as unknown[],
|
||||
};
|
||||
|
||||
mock.module("@/internal/misc/jobQueues/jobQueueStore.js", () => ({
|
||||
KNOWN_JOB_QUEUES: [
|
||||
{
|
||||
id: "primary",
|
||||
label: "Primary Queue",
|
||||
description: "Shared SQS queue for standard background jobs.",
|
||||
defaultEnabled: true,
|
||||
},
|
||||
{
|
||||
id: "track",
|
||||
label: "Track Replay Queue",
|
||||
description:
|
||||
"Dedicated async track replay queue used during fail-open recovery.",
|
||||
defaultEnabled: true,
|
||||
},
|
||||
],
|
||||
getJobQueueConfigFromSource: async () => mockState.config,
|
||||
getJobQueueConfigStatus: () => mockState.status,
|
||||
updateFullJobQueueConfig: async ({ config }: { config: unknown }) => {
|
||||
mockState.updateCalls.push(config);
|
||||
},
|
||||
}));
|
||||
|
||||
import { handleGetAdminJobQueueConfig } from "@/internal/admin/handleGetAdminJobQueueConfig.js";
|
||||
import { handleUpsertAdminJobQueueConfig } from "@/internal/admin/handleUpsertAdminJobQueueConfig.js";
|
||||
|
||||
const buildApp = () => {
|
||||
const app = new Hono<HonoEnv>();
|
||||
|
||||
app.use("*", async (c, next) => {
|
||||
c.set("ctx", {
|
||||
env: AppEnv.Sandbox,
|
||||
org: { slug: "tests-org" },
|
||||
logger: {
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
},
|
||||
} as any);
|
||||
await next();
|
||||
});
|
||||
|
||||
app.get("/admin/job-queue-config", ...handleGetAdminJobQueueConfig);
|
||||
app.put("/admin/job-queue-config", ...handleUpsertAdminJobQueueConfig);
|
||||
app.onError(errorMiddleware);
|
||||
|
||||
return app;
|
||||
};
|
||||
|
||||
describe("admin job queue config", () => {
|
||||
beforeEach(() => {
|
||||
mockState.config = {
|
||||
queues: {
|
||||
primary: { enabled: true },
|
||||
track: { enabled: false },
|
||||
},
|
||||
};
|
||||
mockState.status = {
|
||||
healthy: true,
|
||||
configured: true,
|
||||
lastSuccessAt: "2026-04-24T10:00:00.000Z",
|
||||
error: null,
|
||||
};
|
||||
mockState.updateCalls = [];
|
||||
});
|
||||
|
||||
test("GET returns the stored config, status, and known queues", async () => {
|
||||
const app = buildApp();
|
||||
|
||||
const response = await app.request("http://localhost/admin/job-queue-config");
|
||||
const body = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body).toMatchObject({
|
||||
queues: {
|
||||
primary: { enabled: true },
|
||||
track: { enabled: false },
|
||||
},
|
||||
configHealthy: true,
|
||||
configConfigured: true,
|
||||
lastSuccessAt: "2026-04-24T10:00:00.000Z",
|
||||
error: null,
|
||||
});
|
||||
expect(body.knownQueues).toEqual([
|
||||
{
|
||||
id: "primary",
|
||||
label: "Primary Queue",
|
||||
description: "Shared SQS queue for standard background jobs.",
|
||||
defaultEnabled: true,
|
||||
},
|
||||
{
|
||||
id: "track",
|
||||
label: "Track Replay Queue",
|
||||
description:
|
||||
"Dedicated async track replay queue used during fail-open recovery.",
|
||||
defaultEnabled: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("GET handles an unconfigured empty config", async () => {
|
||||
mockState.config = { queues: {} };
|
||||
mockState.status = {
|
||||
healthy: false,
|
||||
configured: false,
|
||||
lastSuccessAt: null,
|
||||
error: "missing s3 object",
|
||||
};
|
||||
|
||||
const app = buildApp();
|
||||
const response = await app.request("http://localhost/admin/job-queue-config");
|
||||
const body = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body).toMatchObject({
|
||||
queues: {},
|
||||
configHealthy: false,
|
||||
configConfigured: false,
|
||||
lastSuccessAt: null,
|
||||
error: "missing s3 object",
|
||||
});
|
||||
expect(body.knownQueues).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("PUT saves a validated config payload", async () => {
|
||||
const app = buildApp();
|
||||
|
||||
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
queues: {
|
||||
primary: { enabled: false },
|
||||
track: { enabled: true },
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({ success: true });
|
||||
expect(mockState.updateCalls).toEqual([
|
||||
{
|
||||
queues: {
|
||||
primary: { enabled: false },
|
||||
track: { enabled: true },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("PUT preserves unknown queues for future config expansion", async () => {
|
||||
const app = buildApp();
|
||||
|
||||
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
queues: {
|
||||
reports: { enabled: false },
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(mockState.updateCalls).toEqual([
|
||||
{
|
||||
queues: {
|
||||
reports: { enabled: false },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("PUT accepts an empty payload and writes the schema default", async () => {
|
||||
const app = buildApp();
|
||||
|
||||
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(mockState.updateCalls).toEqual([{ queues: {} }]);
|
||||
});
|
||||
|
||||
test("PUT rejects invalid queue payloads", async () => {
|
||||
const app = buildApp();
|
||||
|
||||
const response = await app.request("http://localhost/admin/job-queue-config", {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
queues: {
|
||||
track: { enabled: "yes" },
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await response.json();
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(body.code).toBe(ErrCode.InvalidInputs);
|
||||
expect(mockState.updateCalls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
35
server/tests/integration/queue/fixtures/livePollingWorker.ts
Normal file
35
server/tests/integration/queue/fixtures/livePollingWorker.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { SQSClient } from "@aws-sdk/client-sqs";
|
||||
import { startPollingLoop } from "@/queue/initWorkers.js";
|
||||
|
||||
const queueUrl = process.env.TEST_QUEUE_URL;
|
||||
const endpoint = process.env.TEST_SQS_ENDPOINT;
|
||||
const shouldPoll = process.env.TEST_SHOULD_POLL === "true";
|
||||
|
||||
if (!queueUrl || !endpoint) {
|
||||
throw new Error("TEST_QUEUE_URL and TEST_SQS_ENDPOINT are required");
|
||||
}
|
||||
|
||||
const createClient = () =>
|
||||
new SQSClient({
|
||||
region: "us-east-1",
|
||||
endpoint,
|
||||
credentials: {
|
||||
accessKeyId: "x",
|
||||
secretAccessKey: "x",
|
||||
},
|
||||
});
|
||||
|
||||
let client = createClient();
|
||||
|
||||
await startPollingLoop({
|
||||
db: {} as never,
|
||||
queueUrl,
|
||||
isFifo: queueUrl.endsWith(".fifo"),
|
||||
getSqsClientFn: () => client,
|
||||
recreateSqsClientFn: () => {
|
||||
client.destroy();
|
||||
client = createClient();
|
||||
return client;
|
||||
},
|
||||
shouldPoll: () => shouldPoll,
|
||||
});
|
||||
271
server/tests/integration/queue/live-worker-process.test.ts
Normal file
271
server/tests/integration/queue/live-worker-process.test.ts
Normal file
@@ -0,0 +1,271 @@
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
|
||||
import {
|
||||
CreateQueueCommand,
|
||||
DeleteQueueCommand,
|
||||
GetQueueAttributesCommand,
|
||||
SendMessageCommand,
|
||||
SQSClient,
|
||||
} from "@aws-sdk/client-sqs";
|
||||
|
||||
const ELASTICMQ_JAR = `${process.env.HOME}/.autumn-agent/elasticmq/elasticmq.jar`;
|
||||
const WORKER_FIXTURE_PATH = new URL(
|
||||
"./fixtures/livePollingWorker.ts",
|
||||
import.meta.url,
|
||||
).pathname;
|
||||
|
||||
const children: ChildProcess[] = [];
|
||||
const queueUrls: string[] = [];
|
||||
let elasticmq: ChildProcess | null = null;
|
||||
let tempDir: string | null = null;
|
||||
let sqsEndpoint = "";
|
||||
let sqs: SQSClient;
|
||||
|
||||
const waitForExit = async ({
|
||||
child,
|
||||
forceKillAfterMs = 1_000,
|
||||
}: {
|
||||
child: ChildProcess;
|
||||
forceKillAfterMs?: number;
|
||||
}) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
}, forceKillAfterMs);
|
||||
|
||||
child.once("exit", () => {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const waitFor = async ({
|
||||
check,
|
||||
timeoutMs = 8_000,
|
||||
intervalMs = 200,
|
||||
}: {
|
||||
check: () => Promise<boolean>;
|
||||
timeoutMs?: number;
|
||||
intervalMs?: number;
|
||||
}) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (await check()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
|
||||
throw new Error(`Condition not met within ${timeoutMs}ms`);
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
const port = 19_000 + Math.floor(Math.random() * 1_000);
|
||||
const statsPort = port + 1;
|
||||
tempDir = await mkdtemp(join(tmpdir(), "elasticmq-"));
|
||||
const configPath = join(tempDir, "elasticmq.conf");
|
||||
sqsEndpoint = `http://127.0.0.1:${port}`;
|
||||
|
||||
await writeFile(
|
||||
configPath,
|
||||
`include classpath("application.conf")
|
||||
node-address {
|
||||
protocol = http
|
||||
host = "127.0.0.1"
|
||||
port = ${port}
|
||||
context-path = ""
|
||||
}
|
||||
rest-sqs {
|
||||
enabled = true
|
||||
bind-port = ${port}
|
||||
bind-hostname = "127.0.0.1"
|
||||
sqs-limits = strict
|
||||
}
|
||||
generate-node-address = false
|
||||
rest-stats {
|
||||
enabled = true
|
||||
bind-port = ${statsPort}
|
||||
bind-hostname = "127.0.0.1"
|
||||
}
|
||||
queues {}
|
||||
`,
|
||||
);
|
||||
|
||||
elasticmq = spawn(
|
||||
"java",
|
||||
[`-Dconfig.file=${configPath}`, "-jar", ELASTICMQ_JAR],
|
||||
{
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
|
||||
sqs = new SQSClient({
|
||||
region: "us-east-1",
|
||||
endpoint: sqsEndpoint,
|
||||
credentials: {
|
||||
accessKeyId: "x",
|
||||
secretAccessKey: "x",
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor({
|
||||
timeoutMs: 15_000,
|
||||
check: async () => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${sqsEndpoint}/?Action=ListQueues&Version=2012-11-05`,
|
||||
);
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (elasticmq) {
|
||||
elasticmq.kill("SIGTERM");
|
||||
await waitForExit({ child: elasticmq });
|
||||
}
|
||||
|
||||
if (tempDir) {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
const createTestQueue = async () => {
|
||||
const name = `autumn-live-${randomUUID()}.fifo`;
|
||||
const response = await sqs.send(
|
||||
new CreateQueueCommand({
|
||||
QueueName: name,
|
||||
Attributes: {
|
||||
FifoQueue: "true",
|
||||
ContentBasedDeduplication: "true",
|
||||
VisibilityTimeout: "5",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
if (!response.QueueUrl) {
|
||||
throw new Error("Failed to create test queue");
|
||||
}
|
||||
|
||||
queueUrls.push(response.QueueUrl);
|
||||
return response.QueueUrl;
|
||||
};
|
||||
|
||||
const getQueueCounts = async ({ queueUrl }: { queueUrl: string }) => {
|
||||
const response = await sqs.send(
|
||||
new GetQueueAttributesCommand({
|
||||
QueueUrl: queueUrl,
|
||||
AttributeNames: [
|
||||
"ApproximateNumberOfMessages",
|
||||
"ApproximateNumberOfMessagesNotVisible",
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
visible: Number.parseInt(
|
||||
response.Attributes?.ApproximateNumberOfMessages ?? "0",
|
||||
10,
|
||||
),
|
||||
notVisible: Number.parseInt(
|
||||
response.Attributes?.ApproximateNumberOfMessagesNotVisible ?? "0",
|
||||
10,
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
const startWorker = ({
|
||||
queueUrl,
|
||||
shouldPoll,
|
||||
}: {
|
||||
queueUrl: string;
|
||||
shouldPoll: boolean;
|
||||
}) => {
|
||||
const child = spawn(
|
||||
"bun",
|
||||
[WORKER_FIXTURE_PATH],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
TEST_QUEUE_URL: queueUrl,
|
||||
TEST_SQS_ENDPOINT: sqsEndpoint,
|
||||
TEST_SHOULD_POLL: shouldPoll ? "true" : "false",
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
|
||||
children.push(child);
|
||||
return child;
|
||||
};
|
||||
|
||||
const sendTestMessage = async ({ queueUrl }: { queueUrl: string }) => {
|
||||
await sqs.send(
|
||||
new SendMessageCommand({
|
||||
QueueUrl: queueUrl,
|
||||
MessageBody: JSON.stringify({
|
||||
name: "integration-test-job",
|
||||
data: {},
|
||||
}),
|
||||
MessageGroupId: "test",
|
||||
MessageDeduplicationId: randomUUID(),
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
for (const child of children.splice(0)) {
|
||||
child.kill("SIGTERM");
|
||||
await waitForExit({ child });
|
||||
}
|
||||
|
||||
for (const queueUrl of queueUrls.splice(0)) {
|
||||
await sqs.send(
|
||||
new DeleteQueueCommand({
|
||||
QueueUrl: queueUrl,
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
describe("live worker process queue polling", () => {
|
||||
test("enabled worker consumes a live SQS message", async () => {
|
||||
const queueUrl = await createTestQueue();
|
||||
|
||||
startWorker({ queueUrl, shouldPoll: true });
|
||||
await sendTestMessage({ queueUrl });
|
||||
|
||||
await waitFor({
|
||||
check: async () => {
|
||||
const counts = await getQueueCounts({ queueUrl });
|
||||
return counts.visible === 0 && counts.notVisible === 0;
|
||||
},
|
||||
});
|
||||
}, 15_000);
|
||||
|
||||
test("disabled worker leaves the live SQS message untouched", async () => {
|
||||
const queueUrl = await createTestQueue();
|
||||
|
||||
startWorker({ queueUrl, shouldPoll: false });
|
||||
await sendTestMessage({ queueUrl });
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
|
||||
const counts = await getQueueCounts({ queueUrl });
|
||||
expect(counts.visible).toBe(1);
|
||||
expect(counts.notVisible).toBe(0);
|
||||
}, 15_000);
|
||||
});
|
||||
89
server/tests/integration/queue/start-polling-loop.test.ts
Normal file
89
server/tests/integration/queue/start-polling-loop.test.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import { startPollingLoop } from "@/queue/initWorkers.js";
|
||||
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
|
||||
const makeAbortError = () => {
|
||||
const error = new Error("aborted") as Error & { name: string };
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
};
|
||||
|
||||
describe("startPollingLoop", () => {
|
||||
beforeEach(() => {
|
||||
globalThis.setTimeout = (((callback: TimerHandler) => {
|
||||
if (typeof callback === "function") {
|
||||
callback();
|
||||
}
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
}) as unknown) as typeof setTimeout;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
});
|
||||
|
||||
test("does not poll while the queue is disabled", async () => {
|
||||
let shouldPollCalls = 0;
|
||||
let sendCalls = 0;
|
||||
|
||||
await startPollingLoop({
|
||||
db: {} as never,
|
||||
queueUrl: "https://sqs.eu-west-1.amazonaws.com/123/track.fifo",
|
||||
isFifo: true,
|
||||
getSqsClientFn: () =>
|
||||
({
|
||||
send: async () => {
|
||||
sendCalls++;
|
||||
throw makeAbortError();
|
||||
},
|
||||
}) as never,
|
||||
recreateSqsClientFn: () =>
|
||||
({
|
||||
send: async () => {
|
||||
sendCalls++;
|
||||
throw makeAbortError();
|
||||
},
|
||||
}) as never,
|
||||
shouldPoll: () => {
|
||||
shouldPollCalls++;
|
||||
return shouldPollCalls > 1;
|
||||
},
|
||||
});
|
||||
|
||||
expect(shouldPollCalls).toBeGreaterThan(1);
|
||||
expect(sendCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("polls immediately when the queue is enabled", async () => {
|
||||
let shouldPollCalls = 0;
|
||||
let sendCalls = 0;
|
||||
|
||||
await startPollingLoop({
|
||||
db: {} as never,
|
||||
queueUrl: "https://sqs.eu-west-1.amazonaws.com/123/primary.fifo",
|
||||
isFifo: true,
|
||||
getSqsClientFn: () =>
|
||||
({
|
||||
send: async () => {
|
||||
sendCalls++;
|
||||
throw makeAbortError();
|
||||
},
|
||||
}) as never,
|
||||
recreateSqsClientFn: () =>
|
||||
({
|
||||
send: async () => {
|
||||
sendCalls++;
|
||||
throw makeAbortError();
|
||||
},
|
||||
}) as never,
|
||||
shouldPoll: () => {
|
||||
shouldPollCalls++;
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
expect(shouldPollCalls).toBe(1);
|
||||
expect(sendCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -7,21 +7,9 @@ import {
|
||||
} from "@/internal/balances/utils/types/redisDeductionError.js";
|
||||
|
||||
const mockState = {
|
||||
queueCalls: [] as Record<string, unknown>[],
|
||||
postgresCalls: [] as Record<string, unknown>[],
|
||||
};
|
||||
|
||||
mock.module("@/internal/balances/track/utils/queueTrack.js", () => ({
|
||||
queueTrack: async (args: Record<string, unknown>) => {
|
||||
mockState.queueCalls.push(args);
|
||||
return {
|
||||
customer_id: "cus_123",
|
||||
feature_id: "messages",
|
||||
balance: null,
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module(
|
||||
"@/internal/balances/track/v3/runPostgresTrackV3.js",
|
||||
() => ({
|
||||
@@ -45,31 +33,29 @@ const ctx = {
|
||||
|
||||
describe("handleRedisTrackErrorV3", () => {
|
||||
beforeEach(() => {
|
||||
mockState.queueCalls = [];
|
||||
mockState.postgresCalls = [];
|
||||
});
|
||||
|
||||
test("queues track instead of falling back to Postgres when Redis is unavailable", async () => {
|
||||
const response = await handleRedisTrackErrorV3({
|
||||
ctx,
|
||||
error: new RedisDeductionError({
|
||||
message: "Redis not ready for deduction",
|
||||
code: RedisDeductionErrorCode.RedisUnavailable,
|
||||
}),
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
feature_id: "messages",
|
||||
value: 1,
|
||||
},
|
||||
fullSubject: {} as never,
|
||||
featureDeductions: [],
|
||||
test("rethrows when Redis is unavailable", async () => {
|
||||
const error = new RedisDeductionError({
|
||||
message: "Redis not ready for deduction",
|
||||
code: RedisDeductionErrorCode.RedisUnavailable,
|
||||
});
|
||||
|
||||
expect(mockState.queueCalls).toHaveLength(1);
|
||||
await expect(
|
||||
handleRedisTrackErrorV3({
|
||||
ctx,
|
||||
error,
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
feature_id: "messages",
|
||||
value: 1,
|
||||
},
|
||||
fullSubject: {} as never,
|
||||
featureDeductions: [],
|
||||
}),
|
||||
).rejects.toBe(error);
|
||||
|
||||
expect(mockState.postgresCalls).toHaveLength(0);
|
||||
expect(response).toMatchObject({
|
||||
customer_id: "cus_123",
|
||||
balance: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -42,6 +42,10 @@ mock.module("@/internal/balances/track/utils/queueTrack.js", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module("@/external/redis/initUtils/redisV2Availability.js", () => ({
|
||||
shouldUseRedisV2: () => true,
|
||||
}));
|
||||
|
||||
import { runTrackWithRollout } from "@/internal/balances/track/runTrackWithRollout.js";
|
||||
|
||||
const ctx = {
|
||||
|
||||
75
server/tests/unit/balances/track/queueTrack.test.ts
Normal file
75
server/tests/unit/balances/track/queueTrack.test.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import { ApiVersion, ApiVersionClass, AppEnv } from "@autumn/shared";
|
||||
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||
|
||||
const mockState = {
|
||||
queueCalls: [] as Record<string, unknown>[],
|
||||
};
|
||||
|
||||
mock.module("@/queue/queueUtils.js", () => ({
|
||||
addTaskToQueue: async (args: Record<string, unknown>) => {
|
||||
mockState.queueCalls.push(args);
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module("@/internal/balances/track/utils/getQueuedTrackResponse.js", () => ({
|
||||
getQueuedTrackResponse: () => ({
|
||||
customer_id: "cus_123",
|
||||
value: 2,
|
||||
balance: null,
|
||||
}),
|
||||
}));
|
||||
|
||||
import { queueTrack } from "@/internal/balances/track/utils/queueTrack.js";
|
||||
|
||||
describe("queueTrack", () => {
|
||||
const originalTrackQueueUrl = process.env.TRACK_SQS_QUEUE_URL;
|
||||
|
||||
beforeEach(() => {
|
||||
mockState.queueCalls = [];
|
||||
process.env.TRACK_SQS_QUEUE_URL =
|
||||
"https://sqs.eu-west-1.amazonaws.com/123456789012/track-dev.fifo";
|
||||
});
|
||||
|
||||
test("queues track with request identity and entity-scoped grouping", async () => {
|
||||
const ctx = {
|
||||
id: "req_123",
|
||||
org: { id: "org_123" },
|
||||
env: AppEnv.Sandbox,
|
||||
apiVersion: new ApiVersionClass(ApiVersion.V2_1),
|
||||
logger: {
|
||||
warn: mock(() => {}),
|
||||
},
|
||||
} as unknown as AutumnContext;
|
||||
|
||||
await queueTrack({
|
||||
ctx,
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
entity_id: "ent_123",
|
||||
feature_id: "messages",
|
||||
value: 2,
|
||||
},
|
||||
});
|
||||
|
||||
expect(mockState.queueCalls).toHaveLength(1);
|
||||
expect(mockState.queueCalls[0]).toMatchObject({
|
||||
queueUrl:
|
||||
"https://sqs.eu-west-1.amazonaws.com/123456789012/track-dev.fifo",
|
||||
messageGroupId: "org_123:sandbox:cus_123:ent_123",
|
||||
messageDeduplicationId: "req_123",
|
||||
payload: {
|
||||
orgId: "org_123",
|
||||
env: AppEnv.Sandbox,
|
||||
customerId: "cus_123",
|
||||
entityId: "ent_123",
|
||||
requestId: "req_123",
|
||||
apiVersion: ApiVersion.V2_1,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.TRACK_SQS_QUEUE_URL = originalTrackQueueUrl;
|
||||
});
|
||||
});
|
||||
88
server/tests/unit/balances/track/runQueuedTrack.test.ts
Normal file
88
server/tests/unit/balances/track/runQueuedTrack.test.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import { ApiVersion, ApiVersionClass, AppEnv, ErrCode, RecaseError } from "@autumn/shared";
|
||||
import type { AutumnContext } from "@/honoUtils/HonoEnv.js";
|
||||
|
||||
const mockState = {
|
||||
runTrackV3Calls: [] as Record<string, unknown>[],
|
||||
getFeatureDeductionCalls: [] as Record<string, unknown>[],
|
||||
runTrackV3Error: null as unknown,
|
||||
};
|
||||
|
||||
mock.module("@/internal/balances/track/utils/getFeatureDeductions.js", () => ({
|
||||
getTrackFeatureDeductionsForBody: (args: Record<string, unknown>) => {
|
||||
mockState.getFeatureDeductionCalls.push(args);
|
||||
return [];
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module("@/internal/balances/track/v3/runTrackV3.js", () => ({
|
||||
runTrackV3: async (args: Record<string, unknown>) => {
|
||||
mockState.runTrackV3Calls.push(args);
|
||||
if (mockState.runTrackV3Error) throw mockState.runTrackV3Error;
|
||||
return { customer_id: "cus_123", balance: null };
|
||||
},
|
||||
}));
|
||||
|
||||
import { runQueuedTrack } from "@/internal/balances/track/runQueuedTrack.js";
|
||||
|
||||
const ctx = {
|
||||
id: "req_123",
|
||||
env: AppEnv.Sandbox,
|
||||
org: { id: "org_123" },
|
||||
apiVersion: new ApiVersionClass(ApiVersion.V2_1),
|
||||
logger: {
|
||||
info: mock(() => {}),
|
||||
},
|
||||
} as unknown as AutumnContext;
|
||||
|
||||
describe("runQueuedTrack", () => {
|
||||
beforeEach(() => {
|
||||
mockState.runTrackV3Calls = [];
|
||||
mockState.getFeatureDeductionCalls = [];
|
||||
mockState.runTrackV3Error = null;
|
||||
});
|
||||
|
||||
test("replays queued track through runTrackV3", async () => {
|
||||
await runQueuedTrack({
|
||||
ctx,
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
feature_id: "messages",
|
||||
value: 1,
|
||||
},
|
||||
apiVersion: ApiVersion.V2_1,
|
||||
});
|
||||
|
||||
expect(mockState.getFeatureDeductionCalls).toHaveLength(1);
|
||||
expect(mockState.runTrackV3Calls).toHaveLength(1);
|
||||
expect(mockState.runTrackV3Calls[0]).toMatchObject({
|
||||
ctx,
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
feature_id: "messages",
|
||||
},
|
||||
featureDeductions: [],
|
||||
apiVersion: ApiVersion.V2_1,
|
||||
});
|
||||
});
|
||||
|
||||
test("treats duplicate idempotency as already applied", async () => {
|
||||
mockState.runTrackV3Error = new RecaseError({
|
||||
message: "duplicate",
|
||||
code: ErrCode.DuplicateIdempotencyKey,
|
||||
statusCode: 409,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runQueuedTrack({
|
||||
ctx,
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
feature_id: "messages",
|
||||
value: 1,
|
||||
},
|
||||
apiVersion: ApiVersion.V2_1,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,20 +1,14 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import { ApiVersion, AppEnv } from "@autumn/shared";
|
||||
import type { SQSClient } from "@aws-sdk/client-sqs";
|
||||
|
||||
const mockState = {
|
||||
commands: [] as Record<string, unknown>[],
|
||||
originalSend: null as null | SQSClient["send"],
|
||||
};
|
||||
|
||||
mock.module("@/queue/initSqs.js", () => ({
|
||||
QUEUE_URL: "https://sqs.eu-west-1.amazonaws.com/123456789012/primary.fifo",
|
||||
getSqsClient: () => ({
|
||||
send: async (command: { input: Record<string, unknown> }) => {
|
||||
mockState.commands.push(command.input);
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
import { JobName } from "@/queue/JobName.js";
|
||||
import { getSqsClient } from "@/queue/initSqs.js";
|
||||
import { addTaskToQueue } from "@/queue/queueUtils.js";
|
||||
|
||||
describe("addTaskToQueue queue override", () => {
|
||||
@@ -23,11 +17,21 @@ describe("addTaskToQueue queue override", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
mockState.commands = [];
|
||||
const sqsClient = getSqsClient();
|
||||
mockState.originalSend = sqsClient.send.bind(sqsClient);
|
||||
sqsClient.send = (async (command: { input: Record<string, unknown> }) => {
|
||||
mockState.commands.push(command.input);
|
||||
return {};
|
||||
}) as typeof sqsClient.send;
|
||||
delete process.env.SQS_QUEUE_URL;
|
||||
delete process.env.QUEUE_URL;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (mockState.originalSend) {
|
||||
const sqsClient = getSqsClient();
|
||||
sqsClient.send = mockState.originalSend as typeof sqsClient.send;
|
||||
}
|
||||
process.env.SQS_QUEUE_URL = originalSqsQueueUrl;
|
||||
process.env.QUEUE_URL = originalQueueUrl;
|
||||
});
|
||||
@@ -42,6 +46,8 @@ describe("addTaskToQueue queue override", () => {
|
||||
payload: {
|
||||
orgId: "org_123",
|
||||
env: AppEnv.Sandbox,
|
||||
customerId: "cus_123",
|
||||
requestId: "req_123",
|
||||
apiVersion: ApiVersion.V2_1,
|
||||
body: {
|
||||
customer_id: "cus_123",
|
||||
|
||||
@@ -5,6 +5,7 @@ import { useAxiosInstance } from "@/services/useAxiosInstance";
|
||||
import { CustomerBlockDialog } from "./CustomerBlockDialog";
|
||||
import { EdgeConfigDialog } from "./EdgeConfigDialog";
|
||||
import { FeatureFlagsDialog } from "./FeatureFlagsDialog";
|
||||
import { JobQueuesDialog } from "./JobQueuesDialog";
|
||||
import { OrgLimitsDialog } from "./OrgLimitsDialog";
|
||||
import { RawEdgeConfigDialog } from "./RawEdgeConfigDialog";
|
||||
import { RedisV2CacheDialog } from "./RedisV2CacheDialog";
|
||||
@@ -27,6 +28,7 @@ export function EdgeConfigTab() {
|
||||
const [featureFlagsOpen, setFeatureFlagsOpen] = useState(false);
|
||||
const [customerBlockOpen, setCustomerBlockOpen] = useState(false);
|
||||
const [orgLimitsOpen, setOrgLimitsOpen] = useState(false);
|
||||
const [jobQueuesOpen, setJobQueuesOpen] = useState(false);
|
||||
const [stripeSyncOpen, setStripeSyncOpen] = useState(false);
|
||||
const [redisV2CacheOpen, setRedisV2CacheOpen] = useState(false);
|
||||
|
||||
@@ -156,6 +158,23 @@ export function EdgeConfigTab() {
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between border-t border-border p-4 last:border-b-0">
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<div className="text-sm font-medium text-t1">Job Queues</div>
|
||||
<div className="text-xs text-t3">
|
||||
Pause or resume worker consumption for shared and dedicated SQS
|
||||
queues.
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => setJobQueuesOpen(true)}
|
||||
>
|
||||
Edit
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between border-t border-border p-4 last:border-b-0">
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<div className="text-sm font-medium text-t1">Stripe Sync</div>
|
||||
@@ -215,6 +234,11 @@ export function EdgeConfigTab() {
|
||||
|
||||
<OrgLimitsDialog open={orgLimitsOpen} onOpenChange={setOrgLimitsOpen} />
|
||||
|
||||
<JobQueuesDialog
|
||||
open={jobQueuesOpen}
|
||||
onOpenChange={setJobQueuesOpen}
|
||||
/>
|
||||
|
||||
<StripeSyncDialog
|
||||
open={stripeSyncOpen}
|
||||
onOpenChange={setStripeSyncOpen}
|
||||
|
||||
235
vite/src/views/admin/components/JobQueuesDialog.tsx
Normal file
235
vite/src/views/admin/components/JobQueuesDialog.tsx
Normal file
@@ -0,0 +1,235 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { Badge } from "@/components/v2/badges/Badge";
|
||||
import { Button } from "@/components/v2/buttons/Button";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/v2/dialogs/Dialog";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { useAxiosInstance } from "@/services/useAxiosInstance";
|
||||
import { getBackendErr } from "@/utils/genUtils";
|
||||
|
||||
type QueueEntry = {
|
||||
enabled: boolean;
|
||||
};
|
||||
|
||||
type KnownQueue = {
|
||||
id: string;
|
||||
label: string;
|
||||
description: string;
|
||||
defaultEnabled: boolean;
|
||||
};
|
||||
|
||||
type JobQueueConfig = {
|
||||
queues: Record<string, QueueEntry>;
|
||||
knownQueues: KnownQueue[];
|
||||
configHealthy: boolean;
|
||||
configConfigured: boolean;
|
||||
lastSuccessAt: string | null;
|
||||
error: string | null;
|
||||
};
|
||||
|
||||
const DEFAULT_CONFIG: JobQueueConfig = {
|
||||
queues: {},
|
||||
knownQueues: [],
|
||||
configHealthy: false,
|
||||
configConfigured: false,
|
||||
lastSuccessAt: null,
|
||||
error: null,
|
||||
};
|
||||
|
||||
export function JobQueuesDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
}: {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}) {
|
||||
const axiosInstance = useAxiosInstance();
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [config, setConfig] = useState<JobQueueConfig>(DEFAULT_CONFIG);
|
||||
const [initialEnabledByQueue, setInitialEnabledByQueue] = useState<
|
||||
Record<string, boolean>
|
||||
>({});
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
|
||||
let cancelled = false;
|
||||
setLoading(true);
|
||||
|
||||
void axiosInstance
|
||||
.get<JobQueueConfig>("/admin/job-queue-config")
|
||||
.then(({ data }) => {
|
||||
if (cancelled) return;
|
||||
const nextConfig = { ...DEFAULT_CONFIG, ...data };
|
||||
setConfig(nextConfig);
|
||||
setInitialEnabledByQueue(
|
||||
Object.fromEntries(
|
||||
nextConfig.knownQueues.map((queue) => [
|
||||
queue.id,
|
||||
nextConfig.queues[queue.id]?.enabled ?? queue.defaultEnabled,
|
||||
]),
|
||||
),
|
||||
);
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!cancelled) {
|
||||
toast.error(getBackendErr(error, "Failed to load job queue config"));
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoading(false);
|
||||
});
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [axiosInstance, open]);
|
||||
|
||||
const effectiveQueues = useMemo(
|
||||
() =>
|
||||
config.knownQueues.map((queue) => ({
|
||||
...queue,
|
||||
enabled:
|
||||
config.queues[queue.id]?.enabled ?? queue.defaultEnabled,
|
||||
})),
|
||||
[config],
|
||||
);
|
||||
|
||||
const toggleQueue = ({
|
||||
queueId,
|
||||
enabled,
|
||||
}: {
|
||||
queueId: string;
|
||||
enabled: boolean;
|
||||
}) => {
|
||||
setConfig((current) => ({
|
||||
...current,
|
||||
queues: {
|
||||
...current.queues,
|
||||
[queueId]: { enabled },
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
const dirty = effectiveQueues.some(
|
||||
(queue) => queue.enabled !== initialEnabledByQueue[queue.id],
|
||||
);
|
||||
|
||||
const handleSave = async () => {
|
||||
setSaving(true);
|
||||
try {
|
||||
await axiosInstance.put("/admin/job-queue-config", {
|
||||
queues: Object.fromEntries(
|
||||
effectiveQueues.map((queue) => [
|
||||
queue.id,
|
||||
{ enabled: queue.enabled },
|
||||
]),
|
||||
),
|
||||
});
|
||||
toast.success("Job queue config saved");
|
||||
onOpenChange(false);
|
||||
} catch (error) {
|
||||
toast.error(getBackendErr(error, "Failed to save job queue config"));
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-2xl bg-card">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Job Queues</DialogTitle>
|
||||
<DialogDescription>
|
||||
Control which SQS queues workers actively poll. Disabling a queue
|
||||
pauses consumption without changing producers.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
{loading ? (
|
||||
<div className="py-8 text-center text-sm text-t3">Loading...</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-col gap-3">
|
||||
{effectiveQueues.map((queue) => (
|
||||
<div
|
||||
key={queue.id}
|
||||
className="flex items-center justify-between rounded-lg border border-border p-3"
|
||||
>
|
||||
<div className="flex flex-col gap-0.5 pr-4">
|
||||
<div className="text-sm font-medium text-t1">
|
||||
{queue.label}
|
||||
</div>
|
||||
<div className="text-xs text-t3">
|
||||
{queue.description}
|
||||
</div>
|
||||
<div className="text-[11px] text-t3">
|
||||
Default: {queue.defaultEnabled ? "enabled" : "disabled"}
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={queue.enabled}
|
||||
onCheckedChange={(enabled) =>
|
||||
toggleQueue({ queueId: queue.id, enabled })
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border border-border p-3 text-xs text-t3">
|
||||
<div className="mb-2 flex items-center gap-2">
|
||||
<Badge
|
||||
variant="muted"
|
||||
className={
|
||||
config.configHealthy
|
||||
? "border-emerald-200 bg-emerald-50 text-emerald-700"
|
||||
: "border-amber-200 bg-amber-50 text-amber-700"
|
||||
}
|
||||
>
|
||||
{config.configHealthy
|
||||
? "Config healthy"
|
||||
: "Config unavailable"}
|
||||
</Badge>
|
||||
{config.lastSuccessAt && (
|
||||
<span>
|
||||
Last refresh:{" "}
|
||||
{new Date(config.lastSuccessAt).toLocaleString()}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
{config.configConfigured === false
|
||||
? "S3 job queue config is not configured."
|
||||
: config.error ||
|
||||
"Queue polling changes propagate to workers within ~60 seconds."}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="secondary" onClick={() => onOpenChange(false)}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={handleSave}
|
||||
isLoading={saving}
|
||||
disabled={loading || !dirty}
|
||||
>
|
||||
Save
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user