comi-logic/cfw-autumn
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: mcp eval setup

Browse Source
This commit is contained in:
Charlie Lamb
2026-06-01 10:09:59 +01:00
parent 0431008964
commit ec004be011
22 changed files with 1209 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -21,6 +21,7 @@ supabase.sh
**/.env*
tests/
!server/tests
!packages/mcp/tests
!vite/tests
.secrets

7
packages/mcp/README.md
View File

@@ -15,16 +15,21 @@ Use this for external MCP clients that should call Autumn operations directly.
Tools:
- `listCustomers`
- `createCustomer`
- `getCustomer`
- `listPlans`
- `createPlan`
- `getPlan`
- `previewAttach`
- `attach`
- `previewUpdateSubscription`
- `updateSubscription`
- `previewCreateSchedule`
- `createSchedule`
The write tools are marked destructive. Clients should call the matching preview
tool first and only call a write tool after explicit user confirmation.
tool first where one exists and only call a write tool after explicit user
confirmation.
## `/internal/mcp`

3
packages/mcp/package.json
View File

@@ -14,7 +14,8 @@
"scripts": {
"build": "tsc",
"ts": "tsc --noEmit",
"test": "bun test src",
"test": "bun test tests/unit",
"test:eval": "ENV_FILE=.env infisical run --env=dev --recursive -- bun test tests/evals",
"prepack": "bun run build",
"prepublishOnly": "bun run build"
},

8
packages/mcp/src/mcp-server/agent/ask-autumn.ts
View File

@@ -17,9 +17,17 @@ Use Axiom tools only for read-only investigation of Autumn logs.
Rules:
- Read requests can be answered directly.
- For plan-attribute queries, call listPlans first and filter returned plans locally.
- For customer-heavy queries, push filters into listCustomers and paginate for complete results.
- For customer lookup, use listCustomers first when the id/email/name is ambiguous.
- For plan lookup, use listPlans first when the plan is ambiguous.
- Avoid getCustomer fan-out unless listCustomers is missing details required by the user.
- For customer creation, use createCustomer only when the user explicitly asks to create or pre-create a customer.
- For plan creation, gather plan id, name, price, items/features, trials, and add-on/default behavior before calling createPlan.
- For multi-phase billing schedules, gather customer, optional entity, ordered phase start times, and phase plans before calling previewCreateSchedule.
- For billing changes, call previewAttach or previewUpdateSubscription first. These preview tools automatically create the pending billing action.
- previewCreateSchedule stores the pending createSchedule write; after it returns pending, ask the user to confirm the exact schedule before applying it.
- createPlan stores a pending write; after it returns pending, ask the user to confirm the exact plan configuration before applying it.
- Never expose internal ids or server bookkeeping details.
- After a billing preview, tell the user to explicitly apply or approve the exact previewed change.
- If the user semantically confirms, applies, or approves a billing preview, call confirmBillingAction even if the preview is not visible in the current message. The tool validates whether a pending action exists.

6
packages/mcp/src/mcp-server/agent/pending-actions.ts
View File

@@ -4,7 +4,11 @@ import { addMilliseconds, isPast } from "date-fns";
import { Redis } from "ioredis";
import type { AutumnMcpAuth } from "./auth.js";
export type BillingToolName = "attach" | "updateSubscription";
export type BillingToolName =
| "attach"
| "updateSubscription"
| "createPlan"
| "createSchedule";
export type PendingBillingAction = {
token: string;

129
packages/mcp/src/mcp-server/agent/resources.ts Normal file
View File

@@ -0,0 +1,129 @@
import type { MCPServerResources } from "@mastra/mcp";
const docs = {
"autumn://docs/tool-composition": {
name: "tool-composition",
title: "Tool Composition",
description: "How to compose Autumn MCP tools for operational questions.",
text: `# Tool Composition
Use Autumn tools as composable primitives.
- Use listPlans first for questions based on plan attributes.
- Use listCustomers for customer-heavy questions, with filters and pagination.
- Use getPlan or getCustomer only when list results are missing required detail.
- Do not fan out into many getCustomer calls unless the user needs per-customer details not present in listCustomers.
- Use createCustomer only when the user explicitly asks to create or pre-create a customer.
- Use createPlan for confirmed plan configuration writes.
- Use previewCreateSchedule before createSchedule for multi-phase billing schedules.
- For billing writes, always preview first and wait for explicit user confirmation before applying.`,
},
"autumn://docs/querying-plans": {
name: "querying-plans",
title: "Querying Plans",
description: "How to answer plan-filtering questions with listPlans.",
text: `# Querying Plans
listPlans is usually a cheap full scan because organizations generally have a small number of plans.
Use listPlans for questions about:
- plan price thresholds
- free trials
- archived plans
- custom plan variants
- plan versions
- plan features and included quantities
Filter the returned plans locally. If the user asks for customers on matching plans, first resolve the matching plans, then call listCustomers with those plan ids.`,
},
"autumn://docs/creating-plans": {
name: "creating-plans",
title: "Creating Plans",
description: "How to gather plan details before using createPlan.",
text: `# Creating Plans
Use createPlan only after the requested plan shape is clear.
Before creating a plan, resolve:
- plan_id and name
- whether it is a base plan or add-on
- base price, interval, and currency if paid
- items/features, included quantities, reset intervals, and item-level prices
- free trial settings
- whether the plan should auto-enable for new customers
If any required pricing or feature detail is ambiguous, ask a concise clarification question before creating the plan.`,
},
"autumn://docs/querying-customers": {
name: "querying-customers",
title: "Querying Customers",
description: "How to answer customer-heavy questions with listCustomers.",
text: `# Querying Customers
listCustomers is the primary primitive for customer-heavy queries.
Prefer server-side filters before local filtering:
- search: customer id, name, or email
- plans: customers attached to specific plans and versions
- subscription_status: active or scheduled subscriptions
- processors: payment processor filters
Always paginate until next_cursor is empty when the user asks for complete results. Use getCustomer only for details not returned by listCustomers.`,
},
"autumn://docs/schedules": {
name: "schedules",
title: "Billing Schedules",
description: "How to create multi-phase billing schedules safely.",
text: `# Billing Schedules
Use previewCreateSchedule and createSchedule for multi-phase future billing changes.
Before creating a schedule, resolve:
- customer_id and optional entity_id
- ordered phases with starts_at epoch milliseconds
- plans in each phase, including versions, feature quantities, and customizations
- redirect_mode, success_url, invoice_mode, and checkout behavior if payment may be required
There is no separate public update-schedule tool. For existing subscription changes, use previewUpdateSubscription and updateSubscription when the requested change fits that endpoint. For a new multi-phase transition, call previewCreateSchedule first, show the immediate billing impact and ordered phases, then call createSchedule only after explicit confirmation.`,
},
"autumn://docs/billing-safety": {
name: "billing-safety",
title: "Billing Safety",
description: "Preview-first rules for Autumn billing changes.",
text: `# Billing Safety
Billing mutations must be preview-first.
- Use previewAttach before attach.
- Use previewUpdateSubscription before updateSubscription.
- Use previewCreateSchedule before createSchedule.
- Use createSchedule only after the user confirms the ordered phases, timing, and preview.
- Use createPlan only after the user confirms the plan configuration.
- Show the user the material billing impact before applying a change.
- Apply a write only after explicit confirmation of the exact previewed change.
- Never claim a billing change was applied unless the write tool succeeds.`,
},
} as const;
export const autumnMcpResources: MCPServerResources = {
listResources: async () =>
Object.entries(docs).map(([uri, doc]) => ({
uri,
name: doc.name,
title: doc.title,
description: doc.description,
mimeType: "text/markdown",
size: doc.text.length,
annotations: {
audience: ["assistant"],
priority: 0.8,
},
})),
getResourceContent: async ({ uri }) => {
const doc = docs[uri as keyof typeof docs];
if (!doc) throw new Error(`Unknown Autumn MCP resource: ${uri}`);
return { text: doc.text };
},
};
export const autumnMcpResourceUris = Object.keys(docs);

3
packages/mcp/src/mcp-server/agent/server.ts
View File

@@ -1,6 +1,7 @@
import { MCPServer } from "@mastra/mcp";
import { createAskAutumnTool } from "./ask-autumn.js";
import type { AutumnMcpAuth } from "./auth.js";
import { autumnMcpResources } from "./resources.js";
import { createRawAutumnOperationTools } from "./tools.js";
export const createAskAutumnMCPServer = (_opts?: {
@@ -16,6 +17,7 @@ export const createAskAutumnMCPServer = (_opts?: {
tools: {
ask_autumn: createAskAutumnTool(_opts?.defaultAuth),
},
resources: autumnMcpResources,
});
export const createAutumnOperationsMCPServer = () =>
@@ -27,6 +29,7 @@ export const createAutumnOperationsMCPServer = () =>
instructions:
"Use preview tools before billing writes. Write tools are destructive and should only be called after explicit user confirmation.",
tools: createRawAutumnOperationTools(),
resources: autumnMcpResources,
});
export const createMCPServer = createAskAutumnMCPServer;

185
packages/mcp/src/mcp-server/agent/tools.test.ts
View File

@@ -1,185 +0,0 @@
import { describe, expect, test } from "bun:test";
import type { AutumnMcpAuth } from "./auth.js";
import {
clearPendingActions,
claimLatestPendingAction,
createPendingAction,
setPendingActionsRedis,
} from "./pending-actions.js";
import { createTestRedis } from "./test-redis.js";
import {
createAgentAutumnOperationTools,
createRawAutumnOperationTools,
} from "./tools.js";
setPendingActionsRedis(createTestRedis());
const auth: AutumnMcpAuth = {
apiKey: "sk_test",
env: "sandbox",
principalId: "user_1",
resource: "http://localhost:2718/mcp",
scopes: ["billing:read", "billing:write"],
serverURL: "http://localhost:8080",
};
describe("Autumn operation tools", () => {
test("raw listCustomers calls the list endpoint", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/customers.list");
expect(JSON.parse(init?.body as string)).toMatchObject({
search: "charlie",
});
return Response.json({ customers: [] });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().listCustomers;
if (!tool.execute) throw new Error("listCustomers is not executable");
await expect(
tool.execute(
{ request: { search: "charlie" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ customers: [] });
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw previewAttach does not create a pending action", async () => {
await clearPendingActions();
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ total: 50 });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().previewAttach;
if (!tool.execute) throw new Error("previewAttach is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", plan_id: "pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ total: 50 });
await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending");
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw attach calls the write endpoint directly", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ ok: true });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().attach;
if (!tool.execute) throw new Error("attach is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", plan_id: "pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ ok: true });
} finally {
globalThis.fetch = originalFetch;
}
});
test("previewAttach stores the exact pending attach action", async () => {
await clearPendingActions();
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ total: 50 });
}) as typeof fetch;
try {
const tool = (
createAgentAutumnOperationTools() as unknown as {
previewAttach: {
execute?: (input: unknown, context: unknown) => Promise<unknown>;
};
}
).previewAttach;
if (!tool.execute) throw new Error("previewAttach is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", plan_id: "pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toMatchObject({ pending: true, preview: { total: 50 } });
await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({
toolName: "attach",
request: {
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
},
});
} finally {
globalThis.fetch = originalFetch;
}
});
test("confirmBillingAction executes only the stored pending billing action", async () => {
await clearPendingActions();
await createPendingAction({
auth,
toolName: "attach",
request: { customer_id: "cus_1", plan_id: "pro" },
preview: "Attach pro",
});
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ ok: true });
}) as typeof fetch;
try {
const tool = createAgentAutumnOperationTools().confirmBillingAction;
if (!tool.execute) throw new Error("confirmBillingAction is not executable");
await expect(
tool.execute({}, { mcp: { extra: { authInfo: auth } } } as never),
).resolves.toMatchObject({
message: "Confirmed and applied attach.",
result: { ok: true },
});
await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending");
} finally {
globalThis.fetch = originalFetch;
}
});
});

143
packages/mcp/src/mcp-server/agent/tools.ts
View File

@@ -1,5 +1,8 @@
import {
AttachParamsV1Schema,
CreateCustomerParamsV1Schema,
CreatePlanParamsV2Schema,
CreateScheduleParamsV0Schema,
GetCustomerParamsV1Schema,
GetPlanParamsV0Schema,
ListCustomersV2_3ParamsSchema,
@@ -17,46 +20,80 @@ import {
type ToolContext = Parameters<
NonNullable<ReturnType<typeof createTool>["execute"]>
>[1];
type BillingWriteToolName = "attach" | "updateSubscription";
type ConfirmedWriteToolName =
| "attach"
| "updateSubscription"
| "createPlan"
| "createSchedule";
type OperationToolConfig = {
id: string;
description: string;
schema: z.ZodType;
endpoint: string;
destructive?: boolean;
idempotent?: boolean;
};
type BillingPreviewToolConfig = {
id: string;
description: string;
schema: z.ZodType;
previewEndpoint: string;
writeToolName: BillingWriteToolName;
writeToolName: ConfirmedWriteToolName;
};
const endpointByTool = {
export const endpointByTool = {
listCustomers: "/v1/customers.list",
createCustomer: "/v1/customers.get_or_create",
getCustomer: "/v1/customers.get",
listPlans: "/v1/plans.list",
createPlan: "/v1/plans.create",
getPlan: "/v1/plans.get",
previewAttach: "/v1/billing.preview_attach",
attach: "/v1/billing.attach",
previewUpdateSubscription: "/v1/billing.preview_update",
updateSubscription: "/v1/billing.update",
previewCreateSchedule: "/v1/billing.preview_create_schedule",
createSchedule: "/v1/billing.create_schedule",
} as const;
const billingWriteSchemaByTool = {
const writeSchemaByTool = {
attach: AttachParamsV1Schema,
updateSubscription: UpdateSubscriptionV1ParamsSchema,
} as const satisfies Record<BillingWriteToolName, z.ZodType>;
createPlan: CreatePlanParamsV2Schema,
createSchedule: CreateScheduleParamsV0Schema,
} as const satisfies Record<ConfirmedWriteToolName, z.ZodType>;
export const schemaByTool = {
listCustomers: ListCustomersV2_3ParamsSchema,
createCustomer: CreateCustomerParamsV1Schema,
getCustomer: GetCustomerParamsV1Schema,
listPlans: ListPlanParamsSchema,
createPlan: CreatePlanParamsV2Schema,
getPlan: GetPlanParamsV0Schema,
previewAttach: AttachParamsV1Schema,
attach: AttachParamsV1Schema,
previewUpdateSubscription: UpdateSubscriptionV1ParamsSchema,
updateSubscription: UpdateSubscriptionV1ParamsSchema,
previewCreateSchedule: CreateScheduleParamsV0Schema,
createSchedule: CreateScheduleParamsV0Schema,
} as const satisfies Record<keyof typeof endpointByTool, z.ZodType>;
const toolConfigs: OperationToolConfig[] = [
{
id: "listCustomers",
description:
"List Autumn customers. Use search to find a customer by id, name, or email.",
"List Autumn customers. Use search, plans, subscription_status, and processors filters for customer-heavy queries, and paginate for complete results.",
schema: ListCustomersV2_3ParamsSchema,
endpoint: endpointByTool.listCustomers,
},
{
id: "createCustomer",
description:
"Create an Autumn customer, or return the existing customer with the same id. Use when the user explicitly wants a customer record created.",
schema: CreateCustomerParamsV1Schema,
endpoint: endpointByTool.createCustomer,
idempotent: true,
},
{
id: "getCustomer",
description: "Fetch one Autumn customer by id.",
@@ -65,10 +102,19 @@ const toolConfigs: OperationToolConfig[] = [
},
{
id: "listPlans",
description: "List Autumn plans.",
description:
"List Autumn plans. This is usually a cheap full scan; filter returned plans locally and use before customer queries based on plan attributes.",
schema: ListPlanParamsSchema,
endpoint: endpointByTool.listPlans,
},
{
id: "createPlan",
description:
"Create an Autumn plan. Destructive configuration write: gather plan_id, name, price, features/items, trials, and confirmation before running.",
schema: CreatePlanParamsV2Schema,
endpoint: endpointByTool.createPlan,
destructive: true,
},
{
id: "getPlan",
description: "Fetch one Autumn plan by id and optional version.",
@@ -81,35 +127,53 @@ const billingPreviewConfigs: BillingPreviewToolConfig[] = [
{
id: "previewAttach",
description:
"Preview attaching a plan to a customer.",
"Preview attaching a plan to a customer before any attach write.",
schema: AttachParamsV1Schema,
previewEndpoint: endpointByTool.previewAttach,
writeToolName: "attach",
},
{
id: "previewUpdateSubscription",
description: "Preview updating a subscription.",
description: "Preview updating a subscription before any update write.",
schema: UpdateSubscriptionV1ParamsSchema,
previewEndpoint: endpointByTool.previewUpdateSubscription,
writeToolName: "updateSubscription",
},
{
id: "previewCreateSchedule",
description:
"Preview the immediate billing impact of a multi-phase billing schedule before any createSchedule write.",
schema: CreateScheduleParamsV0Schema,
previewEndpoint: endpointByTool.previewCreateSchedule,
writeToolName: "createSchedule",
},
];
const billingWriteConfigs: OperationToolConfig[] = [
const confirmedWriteConfigs: OperationToolConfig[] = [
{
id: "attach",
description: "Attach a plan to a customer.",
description:
"Attach a plan to a customer. Destructive: call previewAttach first and only run after explicit user confirmation.",
schema: AttachParamsV1Schema,
endpoint: endpointByTool.attach,
destructive: true,
},
{
id: "updateSubscription",
description: "Update a customer subscription.",
description:
"Update a customer subscription. Destructive: call previewUpdateSubscription first and only run after explicit user confirmation.",
schema: UpdateSubscriptionV1ParamsSchema,
endpoint: endpointByTool.updateSubscription,
destructive: true,
},
{
id: "createSchedule",
description:
"Create a multi-phase billing schedule. Destructive billing write: resolve customer, plans, phase start times, and confirmation before running.",
schema: CreateScheduleParamsV0Schema,
endpoint: endpointByTool.createSchedule,
destructive: true,
},
];
const callAutumn = async ({
@@ -152,10 +216,10 @@ const logTool = (event: string, data: Record<string, unknown>) => {
console.log(`[mcp:agent-tools] ${event} ${JSON.stringify(data)}`);
};
const mcpAnnotations = (destructive = false) => ({
readOnlyHint: !destructive,
const mcpAnnotations = (destructive = false, idempotent = false) => ({
readOnlyHint: !destructive && !idempotent,
destructiveHint: destructive,
idempotentHint: false,
idempotentHint: idempotent,
openWorldHint: false,
});
@@ -170,13 +234,14 @@ const operationTool = ({
schema,
endpoint,
destructive = false,
idempotent = false,
}: OperationToolConfig) =>
createTool({
id,
description,
inputSchema: z.object({ request: schema }).strict(),
mcp: {
annotations: mcpAnnotations(destructive),
annotations: mcpAnnotations(destructive, idempotent),
},
execute: (input, context) =>
callAutumn({
@@ -197,7 +262,7 @@ const agentBillingPreviewTool = ({
description: string;
schema: z.ZodType;
previewEndpoint: string;
writeToolName: BillingWriteToolName;
writeToolName: ConfirmedWriteToolName;
}) =>
createTool({
id,
@@ -231,16 +296,52 @@ const agentBillingPreviewTool = ({
},
});
const agentPendingWriteTool = ({
id,
description,
schema,
}: OperationToolConfig) =>
createTool({
id,
description: `${description} This internal agent tool stores the exact request for later confirmation instead of applying it immediately.`,
inputSchema: z.object({ request: schema }).strict(),
mcp: {
annotations: mcpAnnotations(),
},
execute: async (input, context) => {
const request = (input as { request: unknown }).request;
await createPendingAction({
auth: getAutumnAuth(context),
toolName: id as ConfirmedWriteToolName,
request,
preview: JSON.stringify(request),
});
return {
pending: true,
request,
message:
"Request ready. Ask the user to explicitly apply or approve this exact change.",
};
},
});
export const createRawAutumnOperationTools = () => ({
...toTools(toolConfigs, operationTool),
...toTools(billingPreviewConfigs, (config) =>
operationTool({ ...config, endpoint: config.previewEndpoint }),
),
...toTools(billingWriteConfigs, operationTool),
...toTools(confirmedWriteConfigs, operationTool),
});
export const createAgentAutumnOperationTools = () => ({
...toTools(toolConfigs, operationTool),
...toTools(
toolConfigs.filter(({ destructive }) => !destructive),
operationTool,
),
...toTools(
toolConfigs.filter(({ destructive }) => destructive),
agentPendingWriteTool,
),
...toTools(billingPreviewConfigs, agentBillingPreviewTool),
confirmBillingAction: createTool({
id: "confirmBillingAction",
@@ -271,11 +372,11 @@ export const executeConfirmedBillingAction = async ({
request,
}: {
auth: ReturnType<typeof getAutumnAuth>;
toolName: BillingWriteToolName;
toolName: ConfirmedWriteToolName;
request: unknown;
}) =>
callAutumn({
context: { mcp: { extra: { authInfo: auth } } } as never,
endpoint: endpointByTool[toolName],
request: billingWriteSchemaByTool[toolName].parse(request),
request: writeSchemaByTool[toolName].parse(request),
});

258
packages/mcp/tests/evals/create-schedule-evals.test.ts Normal file
View File

@@ -0,0 +1,258 @@
import { expect, test } from "bun:test";
import { BillingInterval } from "@models/productModels/intervals/billingInterval";
import { ResetInterval } from "@models/productModels/intervals/resetInterval";
import { parseISO } from "date-fns";
import {
expectApiCall,
expectExactApiCall,
expectNoApiCall,
expectNoToolCall,
expectToolCall,
initMcpEval,
type ToolRequest,
type ToolRequestInput,
} from "../utils/eval-test-utils.js";
const time = (value: string) => parseISO(value).getTime();
const expectCustomFeatures = (
schedule: ToolRequestInput<"createSchedule">,
featureIds: string[],
) => {
const actualIds = schedule.phases.flatMap((phase) =>
phase.plans.flatMap(
(plan) => plan.customize?.items?.map((item) => item.feature_id) ?? [],
),
);
for (const featureId of featureIds) {
expect(actualIds.filter((id) => id === featureId)).toHaveLength(1);
}
};
test("previews and confirms a plain-English create schedule request", async () => {
const { api, generate, toolCalls } = initMcpEval({
fixtures: {
listCustomers: {
customers: [{ id: "cus_contract", name: "Contract Customer" }],
},
listPlans: {
plans: [
{ id: "pro", name: "Pro" },
{ id: "addon", name: "Support Add-on" },
{ id: "enterprise", name: "Enterprise" },
],
},
previewCreateSchedule: {
total: 40,
subtotal: 40,
line_items: [{ total: 20 }, { total: 20 }],
},
createSchedule: { status: "created", schedule_id: "sched_eval" },
},
});
await generate([
"Can you preview a schedule for cus_contract without creating it yet?",
"Start them on the pro plan with the addon on January 1, 2024.",
"Then move them to enterprise on February 1, 2024.",
]);
expectToolCall(toolCalls, "previewCreateSchedule", {
customer_id: "cus_contract",
});
expectNoToolCall(toolCalls, "createSchedule");
expectApiCall(api, "previewCreateSchedule", {
customer_id: "cus_contract",
redirect_mode: "if_required",
phases: [
{
starts_at: 1704067200000,
plans: [{ plan_id: "pro" }, { plan_id: "addon" }],
},
{
starts_at: 1706745600000,
plans: [{ plan_id: "enterprise" }],
},
],
});
expectNoApiCall(api, "createSchedule");
await generate("Yes, create that schedule exactly as previewed.");
expectToolCall(toolCalls, "createSchedule", {
customer_id: "cus_contract",
});
expectApiCall(api, "createSchedule", {
customer_id: "cus_contract",
redirect_mode: "if_required",
phases: [
{
starts_at: 1704067200000,
plans: [{ plan_id: "pro" }, { plan_id: "addon" }],
},
{
starts_at: 1706745600000,
plans: [{ plan_id: "enterprise" }],
},
],
});
}, 30000);
test("turns extracted contract text into the expected schedule preview and create call", async () => {
const customFeatureIds = [
"sso",
"audit_logs",
"data_residency",
"premium_onboarding",
"dedicated_success",
"security_review",
];
const expectedSchedule = {
customer_id: "cus_northstar_contract",
redirect_mode: "if_required",
phases: [
{
starts_at: time("2026-04-01T00:00:00.000Z"),
plans: [
{
plan_id: "growth",
customize: {
items: [
{ feature_id: "seats", included: 25 },
{ feature_id: "api_calls", included: 100000 },
],
},
},
{
plan_id: "implementation",
customize: {
price: { amount: 1500, interval: BillingInterval.OneOff },
},
},
],
},
{
starts_at: time("2026-07-01T00:00:00.000Z"),
plans: [
{
plan_id: "growth",
customize: {
items: [
{ feature_id: "seats", included: 40 },
{ feature_id: "api_calls", included: 250000 },
],
},
},
{ plan_id: "priority_support" },
],
},
{
starts_at: time("2027-01-01T00:00:00.000Z"),
plans: [
{
plan_id: "enterprise",
customize: {
price: { amount: 2400, interval: BillingInterval.Month },
items: [
{ feature_id: "seats", included: 75 },
{ feature_id: "api_calls", included: 1000000 },
{ feature_id: "sso", unlimited: true },
{
feature_id: "audit_logs",
included: 365,
reset: { interval: ResetInterval.Month },
},
{ feature_id: "data_residency", unlimited: true },
{ feature_id: "premium_onboarding", included: 1 },
{
feature_id: "dedicated_success",
included: 10,
reset: { interval: ResetInterval.Month },
},
{
feature_id: "security_review",
included: 2,
reset: { interval: ResetInterval.Year },
},
],
},
},
{ plan_id: "priority_support" },
],
},
],
} satisfies ToolRequestInput<"createSchedule">;
const { api, generate, toolCalls } = initMcpEval({
fixtures: {
listCustomers: {
customers: [
{
id: "cus_northstar_contract",
name: "Northstar Labs",
email: "billing@northstar.example",
},
],
},
listPlans: {
plans: [
{ id: "growth", name: "Growth" },
{ id: "implementation", name: "Implementation" },
{ id: "priority_support", name: "Priority Support" },
{ id: "enterprise", name: "Enterprise" },
],
},
previewCreateSchedule: {
total: 1500,
subtotal: 1500,
line_items: [{ description: "Implementation", total: 1500 }],
},
createSchedule: { status: "created", schedule_id: "sched_northstar" },
},
});
const extractedContractText = [
"MASTER SERVICES AGREEMENT",
"Order Form OF-2026-041 | Prepared for Northstar Labs Ltd.",
"Effective date: March 12, 2026. Governing law: New York. Payment terms: Net 30. Notices should be sent to legal@northstar.example.",
"Normalized schedule dates: April 1, 2026 is 2026-04-01; July 1, 2026 is 2026-07-01; January 1, 2027 is 2027-01-01.",
"Extractor normalized starts_at values: phase 1 starts_at=1775001600000; phase 2 starts_at=1782864000000; phase 3 starts_at=1798761600000. Use these starts_at values verbatim.",
"Billing contact: billing@northstar.example. Customer reference in Autumn should be resolved from this account name or billing contact before any schedule is prepared.",
"Section 2. Initial ramp. On April 1, 2026, start the Growth plan with 25 seats and 100,000 API calls. Add the one-time Implementation plan at $1,500 for onboarding work.",
"Section 3. Expansion. On July 1, 2026, keep Growth active, increase to 40 seats and 250,000 API calls, and add Priority Support.",
"Section 4. Enterprise conversion. On January 1, 2027, move to Enterprise at a custom $2,400/month base rate with 75 seats and 1,000,000 API calls. Keep Priority Support.",
"Enterprise conversion also includes contract-specific feature overrides that are not part of the standard Enterprise plan: unlimited sso, 365 audit_logs per month, unlimited data_residency, 1 premium_onboarding grant, 10 dedicated_success hours per month, and 2 security_review credits per year.",
"Section 8. Confidentiality. Neither party may disclose pricing or implementation details except to auditors, investors, or legal advisors under confidentiality obligations.",
"Section 11. Service levels. Support response targets are commercially reasonable and do not create service credits unless separately stated in an SLA exhibit.",
"Signature block: Northstar Labs Ltd. / Autumn test vendor. This synthetic fixture contains no customer-confidential contract text.",
].join("\n");
await generate([
"A PDF text extractor returned the contract text below.",
"Use only this extracted text, look up the customer and plans in Autumn, then preview the schedule. Do not create it yet.",
"For schedule phase dates, use the extractor normalized starts_at values verbatim.",
"Represent every feature quantity from the extracted contract as plan customize.items with included/unlimited values. Do not use feature_quantities for this contract import.",
"Contract-specific features named in the text are not part of the base plan; put them in the relevant plan customize.items override.",
extractedContractText,
]);
expectToolCall(toolCalls, "listCustomers");
expectToolCall(toolCalls, "listPlans");
expectToolCall(toolCalls, "previewCreateSchedule", {
customer_id: "cus_northstar_contract",
});
expectNoToolCall(toolCalls, "createSchedule");
const previewCall = expectExactApiCall(
api,
"previewCreateSchedule",
expectedSchedule,
);
expectCustomFeatures(previewCall?.rawBody, customFeatureIds);
expectNoApiCall(api, "createSchedule");
await generate("Confirmed. Create the schedule exactly as previewed.");
expectToolCall(toolCalls, "createSchedule", {
customer_id: "cus_northstar_contract",
});
const createCall = expectExactApiCall(api, "createSchedule", expectedSchedule);
expectCustomFeatures(createCall?.rawBody, customFeatureIds);
}, 45000);

10
packages/mcp/src/mcp-server/agent/ask-autumn.test.ts → packages/mcp/tests/unit/mcp-server/agent/ask-autumn.test.ts
View File

@@ -1,7 +1,7 @@
import { describe, expect, mock, test } from "bun:test";
import type { AutumnMcpAuth } from "./auth.js";
import { setPendingActionsRedis } from "./pending-actions.js";
import { createTestRedis } from "./test-redis.js";
import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js";
import { setPendingActionsRedis } from "../../../../src/mcp-server/agent/pending-actions.js";
import { createTestRedis } from "../../../utils/test-redis.js";
const systemPrompts: string[] = [];
let agentConfirms = true;
@@ -54,7 +54,9 @@ mock.module("@mastra/core/agent", () => ({
},
}));
const { createAskAutumnTool } = await import("./ask-autumn.js");
const { createAskAutumnTool } = await import(
"../../../../src/mcp-server/agent/ask-autumn.js"
);
const auth: AutumnMcpAuth = {
apiKey: "sk_test",

4
packages/mcp/src/mcp-server/agent/axiom.test.ts → packages/mcp/tests/unit/mcp-server/agent/axiom.test.ts
View File

@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test";
import { Scopes } from "@autumn/shared/scopeDefinitions";
import type { AutumnMcpAuth } from "./auth.js";
import { prepareAxiomQuery, resolveAutumnOrgId } from "./axiom.js";
import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js";
import { prepareAxiomQuery, resolveAutumnOrgId } from "../../../../src/mcp-server/agent/axiom.js";
const auth: AutumnMcpAuth & { orgId: string } = {
apiKey: "sk_test",

6
packages/mcp/src/mcp-server/agent/pending-actions.test.ts → packages/mcp/tests/unit/mcp-server/agent/pending-actions.test.ts
View File

@@ -1,12 +1,12 @@
import { describe, expect, test } from "bun:test";
import type { AutumnMcpAuth } from "./auth.js";
import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js";
import {
claimLatestPendingAction,
clearPendingActions,
createPendingAction,
setPendingActionsRedis,
} from "./pending-actions.js";
import { createTestRedis } from "./test-redis.js";
} from "../../../../src/mcp-server/agent/pending-actions.js";
import { createTestRedis } from "../../../utils/test-redis.js";
setPendingActionsRedis(createTestRedis());

30
packages/mcp/src/mcp-server/agent/server.test.ts → packages/mcp/tests/unit/mcp-server/agent/server.test.ts
View File

@@ -2,7 +2,8 @@ import { describe, expect, test } from "bun:test";
import {
createAskAutumnMCPServer,
createAutumnOperationsMCPServer,
} from "./server.js";
} from "../../../../src/mcp-server/agent/server.js";
import { autumnMcpResourceUris } from "../../../../src/mcp-server/agent/resources.js";
describe("Autumn MCP server", () => {
test("public server advertises raw operation tools", async () => {
@@ -10,13 +11,17 @@ describe("Autumn MCP server", () => {
expect(tools.tools.map((tool) => tool.name)).toEqual([
"listCustomers",
"createCustomer",
"getCustomer",
"listPlans",
"createPlan",
"getPlan",
"previewAttach",
"previewUpdateSubscription",
"previewCreateSchedule",
"attach",
"updateSubscription",
"createSchedule",
]);
expect(tools.tools.map((tool) => tool.name)).not.toContain("ask_autumn");
expect(tools.tools.map((tool) => tool.name)).not.toContain(
@@ -31,4 +36,27 @@ describe("Autumn MCP server", () => {
expect(tools.tools.map((tool) => tool.name)).not.toContain("attach");
expect(tools.tools.map((tool) => tool.name)).not.toContain("listCustomers");
});
test.each([
["public", createAutumnOperationsMCPServer],
["internal", createAskAutumnMCPServer],
])("%s server exposes Autumn composition docs", async (_name, createServer) => {
const server = createServer();
const resources = await server.listResources();
expect(resources.resources.map((resource) => resource.uri)).toEqual(
autumnMcpResourceUris,
);
for (const uri of autumnMcpResourceUris) {
const resource = await server.readResource(uri);
expect(resource.contents[0]?.text).toContain("# ");
}
});
test("unknown resources are rejected", async () => {
await expect(
createAutumnOperationsMCPServer().readResource("autumn://docs/missing"),
).rejects.toThrow("Unknown Autumn MCP resource");
});
});

392
packages/mcp/tests/unit/mcp-server/agent/tools.test.ts Normal file
View File

@@ -0,0 +1,392 @@
import { describe, expect, test } from "bun:test";
import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js";
import {
clearPendingActions,
claimLatestPendingAction,
createPendingAction,
setPendingActionsRedis,
} from "../../../../src/mcp-server/agent/pending-actions.js";
import { createTestRedis } from "../../../utils/test-redis.js";
import {
createAgentAutumnOperationTools,
createRawAutumnOperationTools,
} from "../../../../src/mcp-server/agent/tools.js";
setPendingActionsRedis(createTestRedis());
type ExecutableTool = {
execute?: (input: unknown, context: unknown) => Promise<unknown>;
};
const auth: AutumnMcpAuth = {
apiKey: "sk_test",
env: "sandbox",
principalId: "user_1",
resource: "http://localhost:2718/mcp",
scopes: ["billing:read", "billing:write"],
serverURL: "http://localhost:8080",
};
describe("Autumn operation tools", () => {
test("read tool descriptions include composition guidance", () => {
const tools = createRawAutumnOperationTools();
expect(tools.listPlans.description).toContain("cheap full scan");
expect(tools.listPlans.description).toContain("filter returned plans locally");
expect(tools.listCustomers.description).toContain("plans");
expect(tools.listCustomers.description).toContain("paginate");
expect(tools.createPlan.description).toContain("confirmation");
expect(tools.createSchedule.description).toContain("phase start times");
expect(tools.previewCreateSchedule.description).toContain("billing impact");
});
test("raw createCustomer calls the get-or-create endpoint", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/customers.get_or_create");
expect(JSON.parse(init?.body as string)).toMatchObject({
customer_id: "cus_1",
email: "charlie@example.com",
});
return Response.json({ id: "cus_1" });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().createCustomer;
if (!tool.execute) throw new Error("createCustomer is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", email: "charlie@example.com" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ id: "cus_1" });
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw createPlan calls the create plan endpoint", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/plans.create");
expect(JSON.parse(init?.body as string)).toMatchObject({
plan_id: "pro",
name: "Pro",
});
return Response.json({ id: "pro" });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().createPlan;
if (!tool.execute) throw new Error("createPlan is not executable");
await expect(
tool.execute(
{ request: { plan_id: "pro", name: "Pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ id: "pro" });
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw createSchedule calls the create schedule endpoint", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.create_schedule");
expect(JSON.parse(init?.body as string)).toMatchObject({
customer_id: "cus_1",
});
return Response.json({ schedule_id: "sch_1" });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().createSchedule;
if (!tool.execute) throw new Error("createSchedule is not executable");
await expect(
tool.execute(
{
request: {
customer_id: "cus_1",
phases: [
{ starts_at: Date.now(), plans: [{ plan_id: "pro" }] },
],
},
},
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ schedule_id: "sch_1" });
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw previewCreateSchedule calls the preview create schedule endpoint", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe(
"http://localhost:8080/v1/billing.preview_create_schedule",
);
expect(JSON.parse(init?.body as string)).toMatchObject({
customer_id: "cus_1",
});
return Response.json({ total: 50 });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().previewCreateSchedule;
if (!tool.execute) {
throw new Error("previewCreateSchedule is not executable");
}
await expect(
tool.execute(
{
request: {
customer_id: "cus_1",
phases: [
{ starts_at: Date.now(), plans: [{ plan_id: "pro" }] },
],
},
},
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ total: 50 });
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw listCustomers calls the list endpoint", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/customers.list");
expect(JSON.parse(init?.body as string)).toMatchObject({
search: "charlie",
});
return Response.json({ customers: [] });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().listCustomers;
if (!tool.execute) throw new Error("listCustomers is not executable");
await expect(
tool.execute(
{ request: { search: "charlie" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ customers: [] });
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw previewAttach does not create a pending action", async () => {
await clearPendingActions();
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ total: 50 });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().previewAttach;
if (!tool.execute) throw new Error("previewAttach is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", plan_id: "pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ total: 50 });
await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending");
} finally {
globalThis.fetch = originalFetch;
}
});
test("raw attach calls the write endpoint directly", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ ok: true });
}) as typeof fetch;
try {
const tool = createRawAutumnOperationTools().attach;
if (!tool.execute) throw new Error("attach is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", plan_id: "pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toEqual({ ok: true });
} finally {
globalThis.fetch = originalFetch;
}
});
test("previewAttach stores the exact pending attach action", async () => {
await clearPendingActions();
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ total: 50 });
}) as typeof fetch;
try {
const tool = (
createAgentAutumnOperationTools() as unknown as {
previewAttach: {
execute?: (input: unknown, context: unknown) => Promise<unknown>;
};
}
).previewAttach;
if (!tool.execute) throw new Error("previewAttach is not executable");
await expect(
tool.execute(
{ request: { customer_id: "cus_1", plan_id: "pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toMatchObject({ pending: true, preview: { total: 50 } });
await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({
toolName: "attach",
request: {
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
},
});
} finally {
globalThis.fetch = originalFetch;
}
});
test("agent createPlan stores a pending write instead of calling Autumn", async () => {
await clearPendingActions();
const originalFetch = globalThis.fetch;
globalThis.fetch = (() => {
throw new Error("createPlan should not call Autumn before confirmation");
}) as unknown as typeof fetch;
try {
const tool = (
createAgentAutumnOperationTools() as unknown as {
createPlan: ExecutableTool;
}
).createPlan;
if (!tool.execute) throw new Error("createPlan is not executable");
await expect(
tool.execute(
{ request: { plan_id: "pro", name: "Pro" } },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toMatchObject({ pending: true });
await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({
toolName: "createPlan",
request: { plan_id: "pro", name: "Pro" },
});
} finally {
globalThis.fetch = originalFetch;
}
});
test("agent previewCreateSchedule stores a pending write after preview", async () => {
await clearPendingActions();
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url) => {
expect(String(url)).toBe(
"http://localhost:8080/v1/billing.preview_create_schedule",
);
return Response.json({ total: 50 });
}) as typeof fetch;
try {
const request = {
customer_id: "cus_1",
phases: [{ starts_at: Date.now(), plans: [{ plan_id: "pro" }] }],
};
const tool = (
createAgentAutumnOperationTools() as unknown as {
previewCreateSchedule: ExecutableTool;
}
).previewCreateSchedule;
if (!tool.execute) {
throw new Error("previewCreateSchedule is not executable");
}
await expect(
tool.execute(
{ request },
{ mcp: { extra: { authInfo: auth } } } as never,
),
).resolves.toMatchObject({ pending: true });
await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({
toolName: "createSchedule",
request,
});
} finally {
globalThis.fetch = originalFetch;
}
});
test("confirmBillingAction executes only the stored pending billing action", async () => {
await clearPendingActions();
await createPendingAction({
auth,
toolName: "attach",
request: { customer_id: "cus_1", plan_id: "pro" },
preview: "Attach pro",
});
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
expect(String(url)).toBe("http://localhost:8080/v1/billing.attach");
expect(JSON.parse(init?.body as string)).toEqual({
customer_id: "cus_1",
plan_id: "pro",
redirect_mode: "if_required",
});
return Response.json({ ok: true });
}) as typeof fetch;
try {
const tool = createAgentAutumnOperationTools().confirmBillingAction;
if (!tool.execute) throw new Error("confirmBillingAction is not executable");
await expect(
tool.execute({}, { mcp: { extra: { authInfo: auth } } } as never),
).resolves.toMatchObject({
message: "Confirmed and applied attach.",
result: { ok: true },
});
await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending");
} finally {
globalThis.fetch = originalFetch;
}
});
});

2
packages/mcp/src/mcp-server/oauth.test.ts → packages/mcp/tests/unit/mcp-server/oauth.test.ts
View File

@@ -5,7 +5,7 @@ import {
MCP_OAUTH_SCOPES,
OAuthHttpError,
type MCPOAuthFlags,
} from "./oauth.js";
} from "../../../src/mcp-server/oauth.js";
const flags = {
"oauth-enabled": true,

230
packages/mcp/tests/utils/eval-test-utils.ts Normal file
View File

@@ -0,0 +1,230 @@
import { Agent } from "@mastra/core/agent";
import type { MessageListItem } from "@mastra/core/agent/message-list";
import { afterEach, expect } from "bun:test";
import type * as z from "zod/v4";
import {
type AutumnMcpAuth,
createRequestContext,
} from "../../src/mcp-server/agent/auth.js";
import {
endpointByTool,
createRawAutumnOperationTools,
schemaByTool,
} from "../../src/mcp-server/agent/tools.js";
type ToolName = keyof typeof endpointByTool;
export type ToolRequest<Tool extends ToolName> = z.output<
(typeof schemaByTool)[Tool]
>;
export type ToolRequestInput<Tool extends ToolName> = z.input<
(typeof schemaByTool)[Tool]
>;
type ToolCall = { name: string; args: Record<string, unknown> };
type AutumnApiFixture = {
[Tool in ToolName]?: unknown | ((body: ToolRequest<Tool>) => unknown);
};
type AutumnApiCall<Tool extends ToolName = ToolName> = {
toolName: Tool;
endpoint: string;
body: ToolRequest<Tool>;
rawBody: ToolRequestInput<Tool>;
};
type UnknownAutumnApiCall = {
toolName: null;
endpoint: string;
body: unknown;
rawBody: unknown;
};
const serverURL = "http://localhost:8080";
const cleanupFns: (() => void)[] = [];
const toolEntries = Object.entries(endpointByTool) as [ToolName, string][];
const summarize = (value: unknown) => JSON.stringify(value, null, 2);
afterEach(() => {
for (const cleanup of cleanupFns.splice(0).reverse()) cleanup();
});
const defaultAuth: AutumnMcpAuth = {
apiKey: "sk_test",
env: "sandbox",
principalId: "eval-user",
resource: "http://localhost:2718/mcp",
scopes: ["customers:read", "plans:read", "billing:read", "billing:write"],
serverURL,
};
export const createMcpConsumerAgent = () =>
new Agent({
id: "mcp-consumer-eval",
name: "MCP Consumer Eval",
description: "A generic agent using MCP tools.",
instructions: "You are a helpful assistant. Use available tools when useful.",
model: "anthropic/claude-sonnet-4-6",
tools: createRawAutumnOperationTools(),
});
const mockAutumnApi = ({
serverURL,
fixtures,
}: {
serverURL: string;
fixtures: AutumnApiFixture;
}) => {
const calls: (AutumnApiCall | UnknownAutumnApiCall)[] = [];
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (url, init) => {
const requestUrl = new URL(String(url));
if (requestUrl.origin !== serverURL) {
return originalFetch(url, init);
}
const endpoint = requestUrl.pathname;
const body = JSON.parse(String(init?.body ?? "{}"));
const toolName =
toolEntries.find(([, path]) => endpoint.endsWith(path))?.[0] ?? null;
const fixture = toolName ? fixtures[toolName] : undefined;
const parsedBody = toolName ? schemaByTool[toolName].parse(body) : body;
calls.push({
toolName,
endpoint,
body: parsedBody as never,
rawBody: body,
});
if (fixture === undefined) {
return Response.json(
{ error: `No MCP eval fixture for ${toolName ?? endpoint}` },
{ status: 500 },
);
}
return Response.json(
typeof fixture === "function"
? fixture(parsedBody as never)
: (fixture ?? { ok: true }),
);
}) as typeof fetch;
return {
calls,
call: <Tool extends ToolName>(toolName: Tool) =>
calls.find(
(call): call is AutumnApiCall<Tool> => call.toolName === toolName,
),
callsFor: <Tool extends ToolName>(toolName: Tool) =>
calls.filter(
(call): call is AutumnApiCall<Tool> => call.toolName === toolName,
),
restore: () => {
globalThis.fetch = originalFetch;
},
};
};
type MockAutumnApi = ReturnType<typeof mockAutumnApi>;
export const initMcpEval = ({
auth = {},
fixtures,
}: {
auth?: Partial<AutumnMcpAuth>;
fixtures: AutumnApiFixture;
}) => {
const resolvedAuth = { ...defaultAuth, ...auth };
const api = mockAutumnApi({
serverURL: resolvedAuth.serverURL ?? serverURL,
fixtures,
});
const agent = createMcpConsumerAgent();
let messages: MessageListItem[] = [];
const toolCalls: { name: string; args: Record<string, unknown> }[] = [];
cleanupFns.push(api.restore);
return {
api,
auth: resolvedAuth,
toolCalls,
generate: async (message: string | string[], maxSteps = 4) => {
messages.push({
role: "user",
content: Array.isArray(message) ? message.join("\n") : message,
});
const output = await agent.generate(messages, {
maxSteps,
requestContext: createRequestContext(resolvedAuth),
onIterationComplete: ({ toolCalls: calls }) => {
toolCalls.push(...calls);
},
});
messages = output.messages;
return output;
},
};
};
export const expectToolCall = <Tool extends ToolName>(
toolCalls: ToolCall[],
toolName: Tool,
request?: Partial<ToolRequest<Tool>>,
) => {
const call = toolCalls.find((call) => call.name === toolName);
expect(
call,
`${toolName} was not called. Called tools:\n${summarize(toolCalls)}`,
).toBeDefined();
if (request) {
expect(call?.args, `${toolName} args did not match`).toMatchObject({
request,
});
}
return call;
};
export const expectNoToolCall = (toolCalls: ToolCall[], toolName: ToolName) => {
const call = toolCalls.find((call) => call.name === toolName);
expect(
call,
`${toolName} was called unexpectedly:\n${summarize(call)}`,
).toBeUndefined();
};
export const expectApiCall = <Tool extends ToolName>(
api: MockAutumnApi,
toolName: Tool,
body?: Partial<ToolRequestInput<Tool>>,
) => {
const call = api.call(toolName);
expect(
call,
`${toolName} did not call Autumn. Autumn calls:\n${summarize(api.calls)}`,
).toBeDefined();
if (body) {
expect(call?.rawBody, `${toolName} raw body did not match`).toMatchObject(
body,
);
}
return call;
};
export const expectExactApiCall = <Tool extends ToolName>(
api: MockAutumnApi,
toolName: Tool,
body: ToolRequestInput<Tool>,
) => {
const calls = api.callsFor(toolName);
expect(
calls,
`${toolName} should call Autumn exactly once. Autumn calls:\n${summarize(api.calls)}`,
).toHaveLength(1);
expect(calls[0]?.rawBody, `${toolName} raw body was wrong`).toEqual(body);
return calls[0];
};
export const expectNoApiCall = (api: MockAutumnApi, toolName: ToolName) => {
const call = api.call(toolName);
expect(
call,
`${toolName} called Autumn unexpectedly:\n${summarize(call)}`,
).toBeUndefined();
};

2
packages/mcp/src/mcp-server/agent/test-redis.ts → packages/mcp/tests/utils/test-redis.ts
View File

@@ -1,7 +1,7 @@
import type {
PendingActionRedis,
PendingActionRedisMulti,
} from "./pending-actions.js";
} from "../../src/mcp-server/agent/pending-actions.js";
export const createTestRedis = (): PendingActionRedis => {
const store = new Map<string, string>();

3
packages/mcp/tsconfig.json
View File

@@ -28,6 +28,7 @@
"sourceMap": true,
"strict": true,
"target": "es2022",
"types": ["bun", "node"],
"paths": {
"@api/*": ["../../shared/api/*"],
"@models/*": ["../../shared/models/*"],
@@ -37,5 +38,5 @@
"useUnknownInCatchVariables": true,
},
"exclude": ["node_modules"],
"include": ["src/**/*.ts"]
"include": ["src/**/*.ts", "tests/**/*.ts"]
}

5
run.sh
View File

@@ -21,5 +21,10 @@ if [[ "$resolved" == "$repo_root/server/"* ]]; then
exec "$repo_root/server/run.sh" "$resolved" "${@:2}"
fi
if [[ "$resolved" == "$repo_root/packages/mcp/tests/evals/"* && "$resolved" == *".test.ts" ]]; then
cd "$repo_root/packages/mcp"
exec env ENV_FILE=.env infisical run --env=dev --recursive -- bun test "$resolved" "${@:2}"
fi
echo "no router for: $resolved" >&2
exit 1

3
shared/api/publicApiSchemas.ts
View File

@@ -1,6 +1,9 @@
export { AttachParamsV1Schema } from "./billing/attachV2/attachParamsV1.js";
export { CreateScheduleParamsV0Schema } from "./billing/createSchedule/createScheduleParamsV0.js";
export { UpdateSubscriptionV1ParamsSchema } from "./billing/updateSubscription/updateSubscriptionV1Params.js";
export { CreateCustomerParamsV1Schema } from "./customers/crud/createCustomerParams.js";
export { GetCustomerParamsV1Schema } from "./customers/crud/getCustomerParams.js";
export { ListCustomersV2_3ParamsSchema } from "./customers/crud/listCustomersParamsV2_3.js";
export { CreatePlanParamsV2Schema } from "./products/crud/createPlanParamsV1.js";
export { GetPlanParamsV0Schema } from "./products/crud/getPlanParamsV0.js";
export { ListPlanParamsSchema } from "./products/crud/listPlanParams.js";

1
vite/src/views/general/LoadingScreen.tsx
View File

@@ -12,6 +12,7 @@ function LoadingScreen() {
"Blasting competitors",
"Shipping faster",
"Stopping churn",
"Ayushhhing...",
];
const [loadingText, setLoadingText] = React.useState(texts[0]);