From ec004be0112e07eb9a5a09f99da74fd7d0219cd6 Mon Sep 17 00:00:00 2001 From: Charlie Lamb Date: Mon, 1 Jun 2026 10:09:59 +0100 Subject: [PATCH] chore: mcp eval setup --- .gitignore | 1 + packages/mcp/README.md | 7 +- packages/mcp/package.json | 3 +- .../mcp/src/mcp-server/agent/ask-autumn.ts | 8 + .../src/mcp-server/agent/pending-actions.ts | 6 +- .../mcp/src/mcp-server/agent/resources.ts | 129 ++++++ packages/mcp/src/mcp-server/agent/server.ts | 3 + .../mcp/src/mcp-server/agent/tools.test.ts | 185 --------- packages/mcp/src/mcp-server/agent/tools.ts | 143 ++++++- .../tests/evals/create-schedule-evals.test.ts | 258 ++++++++++++ .../unit}/mcp-server/agent/ask-autumn.test.ts | 10 +- .../unit}/mcp-server/agent/axiom.test.ts | 4 +- .../mcp-server/agent/pending-actions.test.ts | 6 +- .../unit}/mcp-server/agent/server.test.ts | 30 +- .../tests/unit/mcp-server/agent/tools.test.ts | 392 ++++++++++++++++++ .../unit}/mcp-server/oauth.test.ts | 2 +- packages/mcp/tests/utils/eval-test-utils.ts | 230 ++++++++++ .../agent => tests/utils}/test-redis.ts | 2 +- packages/mcp/tsconfig.json | 3 +- run.sh | 5 + shared/api/publicApiSchemas.ts | 3 + vite/src/views/general/LoadingScreen.tsx | 1 + 22 files changed, 1209 insertions(+), 222 deletions(-) create mode 100644 packages/mcp/src/mcp-server/agent/resources.ts delete mode 100644 packages/mcp/src/mcp-server/agent/tools.test.ts create mode 100644 packages/mcp/tests/evals/create-schedule-evals.test.ts rename packages/mcp/{src => tests/unit}/mcp-server/agent/ask-autumn.test.ts (95%) rename packages/mcp/{src => tests/unit}/mcp-server/agent/axiom.test.ts (93%) rename packages/mcp/{src => tests/unit}/mcp-server/agent/pending-actions.test.ts (91%) rename packages/mcp/{src => tests/unit}/mcp-server/agent/server.test.ts (52%) create mode 100644 packages/mcp/tests/unit/mcp-server/agent/tools.test.ts rename packages/mcp/{src => tests/unit}/mcp-server/oauth.test.ts (98%) create mode 100644 packages/mcp/tests/utils/eval-test-utils.ts rename packages/mcp/{src/mcp-server/agent => tests/utils}/test-redis.ts (93%) diff --git a/.gitignore b/.gitignore index 7840e7c55..ce500faac 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ supabase.sh **/.env* tests/ !server/tests +!packages/mcp/tests !vite/tests .secrets diff --git a/packages/mcp/README.md b/packages/mcp/README.md index ec219be1f..feeec3403 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -15,16 +15,21 @@ Use this for external MCP clients that should call Autumn operations directly. Tools: - `listCustomers` +- `createCustomer` - `getCustomer` - `listPlans` +- `createPlan` - `getPlan` - `previewAttach` - `attach` - `previewUpdateSubscription` - `updateSubscription` +- `previewCreateSchedule` +- `createSchedule` The write tools are marked destructive. Clients should call the matching preview -tool first and only call a write tool after explicit user confirmation. +tool first where one exists and only call a write tool after explicit user +confirmation. ## `/internal/mcp` diff --git a/packages/mcp/package.json b/packages/mcp/package.json index e6ba4f841..4976ab7f5 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -14,7 +14,8 @@ "scripts": { "build": "tsc", "ts": "tsc --noEmit", - "test": "bun test src", + "test": "bun test tests/unit", + "test:eval": "ENV_FILE=.env infisical run --env=dev --recursive -- bun test tests/evals", "prepack": "bun run build", "prepublishOnly": "bun run build" }, diff --git a/packages/mcp/src/mcp-server/agent/ask-autumn.ts b/packages/mcp/src/mcp-server/agent/ask-autumn.ts index 179e10cb0..f8d277e95 100644 --- a/packages/mcp/src/mcp-server/agent/ask-autumn.ts +++ b/packages/mcp/src/mcp-server/agent/ask-autumn.ts @@ -17,9 +17,17 @@ Use Axiom tools only for read-only investigation of Autumn logs. Rules: - Read requests can be answered directly. +- For plan-attribute queries, call listPlans first and filter returned plans locally. +- For customer-heavy queries, push filters into listCustomers and paginate for complete results. - For customer lookup, use listCustomers first when the id/email/name is ambiguous. - For plan lookup, use listPlans first when the plan is ambiguous. +- Avoid getCustomer fan-out unless listCustomers is missing details required by the user. +- For customer creation, use createCustomer only when the user explicitly asks to create or pre-create a customer. +- For plan creation, gather plan id, name, price, items/features, trials, and add-on/default behavior before calling createPlan. +- For multi-phase billing schedules, gather customer, optional entity, ordered phase start times, and phase plans before calling previewCreateSchedule. - For billing changes, call previewAttach or previewUpdateSubscription first. These preview tools automatically create the pending billing action. +- previewCreateSchedule stores the pending createSchedule write; after it returns pending, ask the user to confirm the exact schedule before applying it. +- createPlan stores a pending write; after it returns pending, ask the user to confirm the exact plan configuration before applying it. - Never expose internal ids or server bookkeeping details. - After a billing preview, tell the user to explicitly apply or approve the exact previewed change. - If the user semantically confirms, applies, or approves a billing preview, call confirmBillingAction even if the preview is not visible in the current message. The tool validates whether a pending action exists. diff --git a/packages/mcp/src/mcp-server/agent/pending-actions.ts b/packages/mcp/src/mcp-server/agent/pending-actions.ts index a19116eb5..ff2cb825c 100644 --- a/packages/mcp/src/mcp-server/agent/pending-actions.ts +++ b/packages/mcp/src/mcp-server/agent/pending-actions.ts @@ -4,7 +4,11 @@ import { addMilliseconds, isPast } from "date-fns"; import { Redis } from "ioredis"; import type { AutumnMcpAuth } from "./auth.js"; -export type BillingToolName = "attach" | "updateSubscription"; +export type BillingToolName = + | "attach" + | "updateSubscription" + | "createPlan" + | "createSchedule"; export type PendingBillingAction = { token: string; diff --git a/packages/mcp/src/mcp-server/agent/resources.ts b/packages/mcp/src/mcp-server/agent/resources.ts new file mode 100644 index 000000000..cf6f2e5a0 --- /dev/null +++ b/packages/mcp/src/mcp-server/agent/resources.ts @@ -0,0 +1,129 @@ +import type { MCPServerResources } from "@mastra/mcp"; + +const docs = { + "autumn://docs/tool-composition": { + name: "tool-composition", + title: "Tool Composition", + description: "How to compose Autumn MCP tools for operational questions.", + text: `# Tool Composition + +Use Autumn tools as composable primitives. + +- Use listPlans first for questions based on plan attributes. +- Use listCustomers for customer-heavy questions, with filters and pagination. +- Use getPlan or getCustomer only when list results are missing required detail. +- Do not fan out into many getCustomer calls unless the user needs per-customer details not present in listCustomers. +- Use createCustomer only when the user explicitly asks to create or pre-create a customer. +- Use createPlan for confirmed plan configuration writes. +- Use previewCreateSchedule before createSchedule for multi-phase billing schedules. +- For billing writes, always preview first and wait for explicit user confirmation before applying.`, + }, + "autumn://docs/querying-plans": { + name: "querying-plans", + title: "Querying Plans", + description: "How to answer plan-filtering questions with listPlans.", + text: `# Querying Plans + +listPlans is usually a cheap full scan because organizations generally have a small number of plans. + +Use listPlans for questions about: +- plan price thresholds +- free trials +- archived plans +- custom plan variants +- plan versions +- plan features and included quantities + +Filter the returned plans locally. If the user asks for customers on matching plans, first resolve the matching plans, then call listCustomers with those plan ids.`, + }, + "autumn://docs/creating-plans": { + name: "creating-plans", + title: "Creating Plans", + description: "How to gather plan details before using createPlan.", + text: `# Creating Plans + +Use createPlan only after the requested plan shape is clear. + +Before creating a plan, resolve: +- plan_id and name +- whether it is a base plan or add-on +- base price, interval, and currency if paid +- items/features, included quantities, reset intervals, and item-level prices +- free trial settings +- whether the plan should auto-enable for new customers + +If any required pricing or feature detail is ambiguous, ask a concise clarification question before creating the plan.`, + }, + "autumn://docs/querying-customers": { + name: "querying-customers", + title: "Querying Customers", + description: "How to answer customer-heavy questions with listCustomers.", + text: `# Querying Customers + +listCustomers is the primary primitive for customer-heavy queries. + +Prefer server-side filters before local filtering: +- search: customer id, name, or email +- plans: customers attached to specific plans and versions +- subscription_status: active or scheduled subscriptions +- processors: payment processor filters + +Always paginate until next_cursor is empty when the user asks for complete results. Use getCustomer only for details not returned by listCustomers.`, + }, + "autumn://docs/schedules": { + name: "schedules", + title: "Billing Schedules", + description: "How to create multi-phase billing schedules safely.", + text: `# Billing Schedules + +Use previewCreateSchedule and createSchedule for multi-phase future billing changes. + +Before creating a schedule, resolve: +- customer_id and optional entity_id +- ordered phases with starts_at epoch milliseconds +- plans in each phase, including versions, feature quantities, and customizations +- redirect_mode, success_url, invoice_mode, and checkout behavior if payment may be required + +There is no separate public update-schedule tool. For existing subscription changes, use previewUpdateSubscription and updateSubscription when the requested change fits that endpoint. For a new multi-phase transition, call previewCreateSchedule first, show the immediate billing impact and ordered phases, then call createSchedule only after explicit confirmation.`, + }, + "autumn://docs/billing-safety": { + name: "billing-safety", + title: "Billing Safety", + description: "Preview-first rules for Autumn billing changes.", + text: `# Billing Safety + +Billing mutations must be preview-first. + +- Use previewAttach before attach. +- Use previewUpdateSubscription before updateSubscription. +- Use previewCreateSchedule before createSchedule. +- Use createSchedule only after the user confirms the ordered phases, timing, and preview. +- Use createPlan only after the user confirms the plan configuration. +- Show the user the material billing impact before applying a change. +- Apply a write only after explicit confirmation of the exact previewed change. +- Never claim a billing change was applied unless the write tool succeeds.`, + }, +} as const; + +export const autumnMcpResources: MCPServerResources = { + listResources: async () => + Object.entries(docs).map(([uri, doc]) => ({ + uri, + name: doc.name, + title: doc.title, + description: doc.description, + mimeType: "text/markdown", + size: doc.text.length, + annotations: { + audience: ["assistant"], + priority: 0.8, + }, + })), + getResourceContent: async ({ uri }) => { + const doc = docs[uri as keyof typeof docs]; + if (!doc) throw new Error(`Unknown Autumn MCP resource: ${uri}`); + return { text: doc.text }; + }, +}; + +export const autumnMcpResourceUris = Object.keys(docs); diff --git a/packages/mcp/src/mcp-server/agent/server.ts b/packages/mcp/src/mcp-server/agent/server.ts index 27956e205..2093c9a5c 100644 --- a/packages/mcp/src/mcp-server/agent/server.ts +++ b/packages/mcp/src/mcp-server/agent/server.ts @@ -1,6 +1,7 @@ import { MCPServer } from "@mastra/mcp"; import { createAskAutumnTool } from "./ask-autumn.js"; import type { AutumnMcpAuth } from "./auth.js"; +import { autumnMcpResources } from "./resources.js"; import { createRawAutumnOperationTools } from "./tools.js"; export const createAskAutumnMCPServer = (_opts?: { @@ -16,6 +17,7 @@ export const createAskAutumnMCPServer = (_opts?: { tools: { ask_autumn: createAskAutumnTool(_opts?.defaultAuth), }, + resources: autumnMcpResources, }); export const createAutumnOperationsMCPServer = () => @@ -27,6 +29,7 @@ export const createAutumnOperationsMCPServer = () => instructions: "Use preview tools before billing writes. Write tools are destructive and should only be called after explicit user confirmation.", tools: createRawAutumnOperationTools(), + resources: autumnMcpResources, }); export const createMCPServer = createAskAutumnMCPServer; diff --git a/packages/mcp/src/mcp-server/agent/tools.test.ts b/packages/mcp/src/mcp-server/agent/tools.test.ts deleted file mode 100644 index a8a183bc8..000000000 --- a/packages/mcp/src/mcp-server/agent/tools.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { AutumnMcpAuth } from "./auth.js"; -import { - clearPendingActions, - claimLatestPendingAction, - createPendingAction, - setPendingActionsRedis, -} from "./pending-actions.js"; -import { createTestRedis } from "./test-redis.js"; -import { - createAgentAutumnOperationTools, - createRawAutumnOperationTools, -} from "./tools.js"; - -setPendingActionsRedis(createTestRedis()); - -const auth: AutumnMcpAuth = { - apiKey: "sk_test", - env: "sandbox", - principalId: "user_1", - resource: "http://localhost:2718/mcp", - scopes: ["billing:read", "billing:write"], - serverURL: "http://localhost:8080", -}; - -describe("Autumn operation tools", () => { - test("raw listCustomers calls the list endpoint", async () => { - const originalFetch = globalThis.fetch; - globalThis.fetch = (async (url, init) => { - expect(String(url)).toBe("http://localhost:8080/v1/customers.list"); - expect(JSON.parse(init?.body as string)).toMatchObject({ - search: "charlie", - }); - return Response.json({ customers: [] }); - }) as typeof fetch; - - try { - const tool = createRawAutumnOperationTools().listCustomers; - if (!tool.execute) throw new Error("listCustomers is not executable"); - - await expect( - tool.execute( - { request: { search: "charlie" } }, - { mcp: { extra: { authInfo: auth } } } as never, - ), - ).resolves.toEqual({ customers: [] }); - } finally { - globalThis.fetch = originalFetch; - } - }); - - test("raw previewAttach does not create a pending action", async () => { - await clearPendingActions(); - const originalFetch = globalThis.fetch; - globalThis.fetch = (async (url, init) => { - expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach"); - expect(JSON.parse(init?.body as string)).toEqual({ - customer_id: "cus_1", - plan_id: "pro", - redirect_mode: "if_required", - }); - return Response.json({ total: 50 }); - }) as typeof fetch; - - try { - const tool = createRawAutumnOperationTools().previewAttach; - if (!tool.execute) throw new Error("previewAttach is not executable"); - - await expect( - tool.execute( - { request: { customer_id: "cus_1", plan_id: "pro" } }, - { mcp: { extra: { authInfo: auth } } } as never, - ), - ).resolves.toEqual({ total: 50 }); - await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending"); - } finally { - globalThis.fetch = originalFetch; - } - }); - - test("raw attach calls the write endpoint directly", async () => { - const originalFetch = globalThis.fetch; - globalThis.fetch = (async (url, init) => { - expect(String(url)).toBe("http://localhost:8080/v1/billing.attach"); - expect(JSON.parse(init?.body as string)).toEqual({ - customer_id: "cus_1", - plan_id: "pro", - redirect_mode: "if_required", - }); - return Response.json({ ok: true }); - }) as typeof fetch; - - try { - const tool = createRawAutumnOperationTools().attach; - if (!tool.execute) throw new Error("attach is not executable"); - - await expect( - tool.execute( - { request: { customer_id: "cus_1", plan_id: "pro" } }, - { mcp: { extra: { authInfo: auth } } } as never, - ), - ).resolves.toEqual({ ok: true }); - } finally { - globalThis.fetch = originalFetch; - } - }); - - test("previewAttach stores the exact pending attach action", async () => { - await clearPendingActions(); - const originalFetch = globalThis.fetch; - globalThis.fetch = (async (url, init) => { - expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach"); - expect(JSON.parse(init?.body as string)).toEqual({ - customer_id: "cus_1", - plan_id: "pro", - redirect_mode: "if_required", - }); - return Response.json({ total: 50 }); - }) as typeof fetch; - - try { - const tool = ( - createAgentAutumnOperationTools() as unknown as { - previewAttach: { - execute?: (input: unknown, context: unknown) => Promise; - }; - } - ).previewAttach; - if (!tool.execute) throw new Error("previewAttach is not executable"); - - await expect( - tool.execute( - { request: { customer_id: "cus_1", plan_id: "pro" } }, - { mcp: { extra: { authInfo: auth } } } as never, - ), - ).resolves.toMatchObject({ pending: true, preview: { total: 50 } }); - - await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({ - toolName: "attach", - request: { - customer_id: "cus_1", - plan_id: "pro", - redirect_mode: "if_required", - }, - }); - } finally { - globalThis.fetch = originalFetch; - } - }); - - test("confirmBillingAction executes only the stored pending billing action", async () => { - await clearPendingActions(); - await createPendingAction({ - auth, - toolName: "attach", - request: { customer_id: "cus_1", plan_id: "pro" }, - preview: "Attach pro", - }); - const originalFetch = globalThis.fetch; - globalThis.fetch = (async (url, init) => { - expect(String(url)).toBe("http://localhost:8080/v1/billing.attach"); - expect(JSON.parse(init?.body as string)).toEqual({ - customer_id: "cus_1", - plan_id: "pro", - redirect_mode: "if_required", - }); - return Response.json({ ok: true }); - }) as typeof fetch; - - try { - const tool = createAgentAutumnOperationTools().confirmBillingAction; - if (!tool.execute) throw new Error("confirmBillingAction is not executable"); - - await expect( - tool.execute({}, { mcp: { extra: { authInfo: auth } } } as never), - ).resolves.toMatchObject({ - message: "Confirmed and applied attach.", - result: { ok: true }, - }); - await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending"); - } finally { - globalThis.fetch = originalFetch; - } - }); -}); diff --git a/packages/mcp/src/mcp-server/agent/tools.ts b/packages/mcp/src/mcp-server/agent/tools.ts index ad94824df..0831932fd 100644 --- a/packages/mcp/src/mcp-server/agent/tools.ts +++ b/packages/mcp/src/mcp-server/agent/tools.ts @@ -1,5 +1,8 @@ import { AttachParamsV1Schema, + CreateCustomerParamsV1Schema, + CreatePlanParamsV2Schema, + CreateScheduleParamsV0Schema, GetCustomerParamsV1Schema, GetPlanParamsV0Schema, ListCustomersV2_3ParamsSchema, @@ -17,46 +20,80 @@ import { type ToolContext = Parameters< NonNullable["execute"]> >[1]; -type BillingWriteToolName = "attach" | "updateSubscription"; +type ConfirmedWriteToolName = + | "attach" + | "updateSubscription" + | "createPlan" + | "createSchedule"; type OperationToolConfig = { id: string; description: string; schema: z.ZodType; endpoint: string; destructive?: boolean; + idempotent?: boolean; }; type BillingPreviewToolConfig = { id: string; description: string; schema: z.ZodType; previewEndpoint: string; - writeToolName: BillingWriteToolName; + writeToolName: ConfirmedWriteToolName; }; -const endpointByTool = { +export const endpointByTool = { listCustomers: "/v1/customers.list", + createCustomer: "/v1/customers.get_or_create", getCustomer: "/v1/customers.get", listPlans: "/v1/plans.list", + createPlan: "/v1/plans.create", getPlan: "/v1/plans.get", previewAttach: "/v1/billing.preview_attach", attach: "/v1/billing.attach", previewUpdateSubscription: "/v1/billing.preview_update", updateSubscription: "/v1/billing.update", + previewCreateSchedule: "/v1/billing.preview_create_schedule", + createSchedule: "/v1/billing.create_schedule", } as const; -const billingWriteSchemaByTool = { +const writeSchemaByTool = { attach: AttachParamsV1Schema, updateSubscription: UpdateSubscriptionV1ParamsSchema, -} as const satisfies Record; + createPlan: CreatePlanParamsV2Schema, + createSchedule: CreateScheduleParamsV0Schema, +} as const satisfies Record; + +export const schemaByTool = { + listCustomers: ListCustomersV2_3ParamsSchema, + createCustomer: CreateCustomerParamsV1Schema, + getCustomer: GetCustomerParamsV1Schema, + listPlans: ListPlanParamsSchema, + createPlan: CreatePlanParamsV2Schema, + getPlan: GetPlanParamsV0Schema, + previewAttach: AttachParamsV1Schema, + attach: AttachParamsV1Schema, + previewUpdateSubscription: UpdateSubscriptionV1ParamsSchema, + updateSubscription: UpdateSubscriptionV1ParamsSchema, + previewCreateSchedule: CreateScheduleParamsV0Schema, + createSchedule: CreateScheduleParamsV0Schema, +} as const satisfies Record; const toolConfigs: OperationToolConfig[] = [ { id: "listCustomers", description: - "List Autumn customers. Use search to find a customer by id, name, or email.", + "List Autumn customers. Use search, plans, subscription_status, and processors filters for customer-heavy queries, and paginate for complete results.", schema: ListCustomersV2_3ParamsSchema, endpoint: endpointByTool.listCustomers, }, + { + id: "createCustomer", + description: + "Create an Autumn customer, or return the existing customer with the same id. Use when the user explicitly wants a customer record created.", + schema: CreateCustomerParamsV1Schema, + endpoint: endpointByTool.createCustomer, + idempotent: true, + }, { id: "getCustomer", description: "Fetch one Autumn customer by id.", @@ -65,10 +102,19 @@ const toolConfigs: OperationToolConfig[] = [ }, { id: "listPlans", - description: "List Autumn plans.", + description: + "List Autumn plans. This is usually a cheap full scan; filter returned plans locally and use before customer queries based on plan attributes.", schema: ListPlanParamsSchema, endpoint: endpointByTool.listPlans, }, + { + id: "createPlan", + description: + "Create an Autumn plan. Destructive configuration write: gather plan_id, name, price, features/items, trials, and confirmation before running.", + schema: CreatePlanParamsV2Schema, + endpoint: endpointByTool.createPlan, + destructive: true, + }, { id: "getPlan", description: "Fetch one Autumn plan by id and optional version.", @@ -81,35 +127,53 @@ const billingPreviewConfigs: BillingPreviewToolConfig[] = [ { id: "previewAttach", description: - "Preview attaching a plan to a customer.", + "Preview attaching a plan to a customer before any attach write.", schema: AttachParamsV1Schema, previewEndpoint: endpointByTool.previewAttach, writeToolName: "attach", }, { id: "previewUpdateSubscription", - description: "Preview updating a subscription.", + description: "Preview updating a subscription before any update write.", schema: UpdateSubscriptionV1ParamsSchema, previewEndpoint: endpointByTool.previewUpdateSubscription, writeToolName: "updateSubscription", }, + { + id: "previewCreateSchedule", + description: + "Preview the immediate billing impact of a multi-phase billing schedule before any createSchedule write.", + schema: CreateScheduleParamsV0Schema, + previewEndpoint: endpointByTool.previewCreateSchedule, + writeToolName: "createSchedule", + }, ]; -const billingWriteConfigs: OperationToolConfig[] = [ +const confirmedWriteConfigs: OperationToolConfig[] = [ { id: "attach", - description: "Attach a plan to a customer.", + description: + "Attach a plan to a customer. Destructive: call previewAttach first and only run after explicit user confirmation.", schema: AttachParamsV1Schema, endpoint: endpointByTool.attach, destructive: true, }, { id: "updateSubscription", - description: "Update a customer subscription.", + description: + "Update a customer subscription. Destructive: call previewUpdateSubscription first and only run after explicit user confirmation.", schema: UpdateSubscriptionV1ParamsSchema, endpoint: endpointByTool.updateSubscription, destructive: true, }, + { + id: "createSchedule", + description: + "Create a multi-phase billing schedule. Destructive billing write: resolve customer, plans, phase start times, and confirmation before running.", + schema: CreateScheduleParamsV0Schema, + endpoint: endpointByTool.createSchedule, + destructive: true, + }, ]; const callAutumn = async ({ @@ -152,10 +216,10 @@ const logTool = (event: string, data: Record) => { console.log(`[mcp:agent-tools] ${event} ${JSON.stringify(data)}`); }; -const mcpAnnotations = (destructive = false) => ({ - readOnlyHint: !destructive, +const mcpAnnotations = (destructive = false, idempotent = false) => ({ + readOnlyHint: !destructive && !idempotent, destructiveHint: destructive, - idempotentHint: false, + idempotentHint: idempotent, openWorldHint: false, }); @@ -170,13 +234,14 @@ const operationTool = ({ schema, endpoint, destructive = false, + idempotent = false, }: OperationToolConfig) => createTool({ id, description, inputSchema: z.object({ request: schema }).strict(), mcp: { - annotations: mcpAnnotations(destructive), + annotations: mcpAnnotations(destructive, idempotent), }, execute: (input, context) => callAutumn({ @@ -197,7 +262,7 @@ const agentBillingPreviewTool = ({ description: string; schema: z.ZodType; previewEndpoint: string; - writeToolName: BillingWriteToolName; + writeToolName: ConfirmedWriteToolName; }) => createTool({ id, @@ -231,16 +296,52 @@ const agentBillingPreviewTool = ({ }, }); +const agentPendingWriteTool = ({ + id, + description, + schema, +}: OperationToolConfig) => + createTool({ + id, + description: `${description} This internal agent tool stores the exact request for later confirmation instead of applying it immediately.`, + inputSchema: z.object({ request: schema }).strict(), + mcp: { + annotations: mcpAnnotations(), + }, + execute: async (input, context) => { + const request = (input as { request: unknown }).request; + await createPendingAction({ + auth: getAutumnAuth(context), + toolName: id as ConfirmedWriteToolName, + request, + preview: JSON.stringify(request), + }); + return { + pending: true, + request, + message: + "Request ready. Ask the user to explicitly apply or approve this exact change.", + }; + }, + }); + export const createRawAutumnOperationTools = () => ({ ...toTools(toolConfigs, operationTool), ...toTools(billingPreviewConfigs, (config) => operationTool({ ...config, endpoint: config.previewEndpoint }), ), - ...toTools(billingWriteConfigs, operationTool), + ...toTools(confirmedWriteConfigs, operationTool), }); export const createAgentAutumnOperationTools = () => ({ - ...toTools(toolConfigs, operationTool), + ...toTools( + toolConfigs.filter(({ destructive }) => !destructive), + operationTool, + ), + ...toTools( + toolConfigs.filter(({ destructive }) => destructive), + agentPendingWriteTool, + ), ...toTools(billingPreviewConfigs, agentBillingPreviewTool), confirmBillingAction: createTool({ id: "confirmBillingAction", @@ -271,11 +372,11 @@ export const executeConfirmedBillingAction = async ({ request, }: { auth: ReturnType; - toolName: BillingWriteToolName; + toolName: ConfirmedWriteToolName; request: unknown; }) => callAutumn({ context: { mcp: { extra: { authInfo: auth } } } as never, endpoint: endpointByTool[toolName], - request: billingWriteSchemaByTool[toolName].parse(request), + request: writeSchemaByTool[toolName].parse(request), }); diff --git a/packages/mcp/tests/evals/create-schedule-evals.test.ts b/packages/mcp/tests/evals/create-schedule-evals.test.ts new file mode 100644 index 000000000..d438aa619 --- /dev/null +++ b/packages/mcp/tests/evals/create-schedule-evals.test.ts @@ -0,0 +1,258 @@ +import { expect, test } from "bun:test"; +import { BillingInterval } from "@models/productModels/intervals/billingInterval"; +import { ResetInterval } from "@models/productModels/intervals/resetInterval"; +import { parseISO } from "date-fns"; +import { + expectApiCall, + expectExactApiCall, + expectNoApiCall, + expectNoToolCall, + expectToolCall, + initMcpEval, + type ToolRequest, + type ToolRequestInput, +} from "../utils/eval-test-utils.js"; + +const time = (value: string) => parseISO(value).getTime(); +const expectCustomFeatures = ( + schedule: ToolRequestInput<"createSchedule">, + featureIds: string[], +) => { + const actualIds = schedule.phases.flatMap((phase) => + phase.plans.flatMap( + (plan) => plan.customize?.items?.map((item) => item.feature_id) ?? [], + ), + ); + for (const featureId of featureIds) { + expect(actualIds.filter((id) => id === featureId)).toHaveLength(1); + } +}; + +test("previews and confirms a plain-English create schedule request", async () => { + const { api, generate, toolCalls } = initMcpEval({ + fixtures: { + listCustomers: { + customers: [{ id: "cus_contract", name: "Contract Customer" }], + }, + listPlans: { + plans: [ + { id: "pro", name: "Pro" }, + { id: "addon", name: "Support Add-on" }, + { id: "enterprise", name: "Enterprise" }, + ], + }, + previewCreateSchedule: { + total: 40, + subtotal: 40, + line_items: [{ total: 20 }, { total: 20 }], + }, + createSchedule: { status: "created", schedule_id: "sched_eval" }, + }, + }); + + await generate([ + "Can you preview a schedule for cus_contract without creating it yet?", + "Start them on the pro plan with the addon on January 1, 2024.", + "Then move them to enterprise on February 1, 2024.", + ]); + + expectToolCall(toolCalls, "previewCreateSchedule", { + customer_id: "cus_contract", + }); + expectNoToolCall(toolCalls, "createSchedule"); + + expectApiCall(api, "previewCreateSchedule", { + customer_id: "cus_contract", + redirect_mode: "if_required", + phases: [ + { + starts_at: 1704067200000, + plans: [{ plan_id: "pro" }, { plan_id: "addon" }], + }, + { + starts_at: 1706745600000, + plans: [{ plan_id: "enterprise" }], + }, + ], + }); + expectNoApiCall(api, "createSchedule"); + + await generate("Yes, create that schedule exactly as previewed."); + + expectToolCall(toolCalls, "createSchedule", { + customer_id: "cus_contract", + }); + expectApiCall(api, "createSchedule", { + customer_id: "cus_contract", + redirect_mode: "if_required", + phases: [ + { + starts_at: 1704067200000, + plans: [{ plan_id: "pro" }, { plan_id: "addon" }], + }, + { + starts_at: 1706745600000, + plans: [{ plan_id: "enterprise" }], + }, + ], + }); +}, 30000); + +test("turns extracted contract text into the expected schedule preview and create call", async () => { + const customFeatureIds = [ + "sso", + "audit_logs", + "data_residency", + "premium_onboarding", + "dedicated_success", + "security_review", + ]; + const expectedSchedule = { + customer_id: "cus_northstar_contract", + redirect_mode: "if_required", + phases: [ + { + starts_at: time("2026-04-01T00:00:00.000Z"), + plans: [ + { + plan_id: "growth", + customize: { + items: [ + { feature_id: "seats", included: 25 }, + { feature_id: "api_calls", included: 100000 }, + ], + }, + }, + { + plan_id: "implementation", + customize: { + price: { amount: 1500, interval: BillingInterval.OneOff }, + }, + }, + ], + }, + { + starts_at: time("2026-07-01T00:00:00.000Z"), + plans: [ + { + plan_id: "growth", + customize: { + items: [ + { feature_id: "seats", included: 40 }, + { feature_id: "api_calls", included: 250000 }, + ], + }, + }, + { plan_id: "priority_support" }, + ], + }, + { + starts_at: time("2027-01-01T00:00:00.000Z"), + plans: [ + { + plan_id: "enterprise", + customize: { + price: { amount: 2400, interval: BillingInterval.Month }, + items: [ + { feature_id: "seats", included: 75 }, + { feature_id: "api_calls", included: 1000000 }, + { feature_id: "sso", unlimited: true }, + { + feature_id: "audit_logs", + included: 365, + reset: { interval: ResetInterval.Month }, + }, + { feature_id: "data_residency", unlimited: true }, + { feature_id: "premium_onboarding", included: 1 }, + { + feature_id: "dedicated_success", + included: 10, + reset: { interval: ResetInterval.Month }, + }, + { + feature_id: "security_review", + included: 2, + reset: { interval: ResetInterval.Year }, + }, + ], + }, + }, + { plan_id: "priority_support" }, + ], + }, + ], + } satisfies ToolRequestInput<"createSchedule">; + const { api, generate, toolCalls } = initMcpEval({ + fixtures: { + listCustomers: { + customers: [ + { + id: "cus_northstar_contract", + name: "Northstar Labs", + email: "billing@northstar.example", + }, + ], + }, + listPlans: { + plans: [ + { id: "growth", name: "Growth" }, + { id: "implementation", name: "Implementation" }, + { id: "priority_support", name: "Priority Support" }, + { id: "enterprise", name: "Enterprise" }, + ], + }, + previewCreateSchedule: { + total: 1500, + subtotal: 1500, + line_items: [{ description: "Implementation", total: 1500 }], + }, + createSchedule: { status: "created", schedule_id: "sched_northstar" }, + }, + }); + const extractedContractText = [ + "MASTER SERVICES AGREEMENT", + "Order Form OF-2026-041 | Prepared for Northstar Labs Ltd.", + "Effective date: March 12, 2026. Governing law: New York. Payment terms: Net 30. Notices should be sent to legal@northstar.example.", + "Normalized schedule dates: April 1, 2026 is 2026-04-01; July 1, 2026 is 2026-07-01; January 1, 2027 is 2027-01-01.", + "Extractor normalized starts_at values: phase 1 starts_at=1775001600000; phase 2 starts_at=1782864000000; phase 3 starts_at=1798761600000. Use these starts_at values verbatim.", + "Billing contact: billing@northstar.example. Customer reference in Autumn should be resolved from this account name or billing contact before any schedule is prepared.", + "Section 2. Initial ramp. On April 1, 2026, start the Growth plan with 25 seats and 100,000 API calls. Add the one-time Implementation plan at $1,500 for onboarding work.", + "Section 3. Expansion. On July 1, 2026, keep Growth active, increase to 40 seats and 250,000 API calls, and add Priority Support.", + "Section 4. Enterprise conversion. On January 1, 2027, move to Enterprise at a custom $2,400/month base rate with 75 seats and 1,000,000 API calls. Keep Priority Support.", + "Enterprise conversion also includes contract-specific feature overrides that are not part of the standard Enterprise plan: unlimited sso, 365 audit_logs per month, unlimited data_residency, 1 premium_onboarding grant, 10 dedicated_success hours per month, and 2 security_review credits per year.", + "Section 8. Confidentiality. Neither party may disclose pricing or implementation details except to auditors, investors, or legal advisors under confidentiality obligations.", + "Section 11. Service levels. Support response targets are commercially reasonable and do not create service credits unless separately stated in an SLA exhibit.", + "Signature block: Northstar Labs Ltd. / Autumn test vendor. This synthetic fixture contains no customer-confidential contract text.", + ].join("\n"); + + await generate([ + "A PDF text extractor returned the contract text below.", + "Use only this extracted text, look up the customer and plans in Autumn, then preview the schedule. Do not create it yet.", + "For schedule phase dates, use the extractor normalized starts_at values verbatim.", + "Represent every feature quantity from the extracted contract as plan customize.items with included/unlimited values. Do not use feature_quantities for this contract import.", + "Contract-specific features named in the text are not part of the base plan; put them in the relevant plan customize.items override.", + extractedContractText, + ]); + + expectToolCall(toolCalls, "listCustomers"); + expectToolCall(toolCalls, "listPlans"); + expectToolCall(toolCalls, "previewCreateSchedule", { + customer_id: "cus_northstar_contract", + }); + expectNoToolCall(toolCalls, "createSchedule"); + const previewCall = expectExactApiCall( + api, + "previewCreateSchedule", + expectedSchedule, + ); + expectCustomFeatures(previewCall?.rawBody, customFeatureIds); + expectNoApiCall(api, "createSchedule"); + + await generate("Confirmed. Create the schedule exactly as previewed."); + + expectToolCall(toolCalls, "createSchedule", { + customer_id: "cus_northstar_contract", + }); + const createCall = expectExactApiCall(api, "createSchedule", expectedSchedule); + expectCustomFeatures(createCall?.rawBody, customFeatureIds); +}, 45000); diff --git a/packages/mcp/src/mcp-server/agent/ask-autumn.test.ts b/packages/mcp/tests/unit/mcp-server/agent/ask-autumn.test.ts similarity index 95% rename from packages/mcp/src/mcp-server/agent/ask-autumn.test.ts rename to packages/mcp/tests/unit/mcp-server/agent/ask-autumn.test.ts index 468afef42..e3639d404 100644 --- a/packages/mcp/src/mcp-server/agent/ask-autumn.test.ts +++ b/packages/mcp/tests/unit/mcp-server/agent/ask-autumn.test.ts @@ -1,7 +1,7 @@ import { describe, expect, mock, test } from "bun:test"; -import type { AutumnMcpAuth } from "./auth.js"; -import { setPendingActionsRedis } from "./pending-actions.js"; -import { createTestRedis } from "./test-redis.js"; +import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js"; +import { setPendingActionsRedis } from "../../../../src/mcp-server/agent/pending-actions.js"; +import { createTestRedis } from "../../../utils/test-redis.js"; const systemPrompts: string[] = []; let agentConfirms = true; @@ -54,7 +54,9 @@ mock.module("@mastra/core/agent", () => ({ }, })); -const { createAskAutumnTool } = await import("./ask-autumn.js"); +const { createAskAutumnTool } = await import( + "../../../../src/mcp-server/agent/ask-autumn.js" +); const auth: AutumnMcpAuth = { apiKey: "sk_test", diff --git a/packages/mcp/src/mcp-server/agent/axiom.test.ts b/packages/mcp/tests/unit/mcp-server/agent/axiom.test.ts similarity index 93% rename from packages/mcp/src/mcp-server/agent/axiom.test.ts rename to packages/mcp/tests/unit/mcp-server/agent/axiom.test.ts index 935fd0c6a..d5c9c2a4c 100644 --- a/packages/mcp/src/mcp-server/agent/axiom.test.ts +++ b/packages/mcp/tests/unit/mcp-server/agent/axiom.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { Scopes } from "@autumn/shared/scopeDefinitions"; -import type { AutumnMcpAuth } from "./auth.js"; -import { prepareAxiomQuery, resolveAutumnOrgId } from "./axiom.js"; +import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js"; +import { prepareAxiomQuery, resolveAutumnOrgId } from "../../../../src/mcp-server/agent/axiom.js"; const auth: AutumnMcpAuth & { orgId: string } = { apiKey: "sk_test", diff --git a/packages/mcp/src/mcp-server/agent/pending-actions.test.ts b/packages/mcp/tests/unit/mcp-server/agent/pending-actions.test.ts similarity index 91% rename from packages/mcp/src/mcp-server/agent/pending-actions.test.ts rename to packages/mcp/tests/unit/mcp-server/agent/pending-actions.test.ts index 308d5c764..2b1ccbf27 100644 --- a/packages/mcp/src/mcp-server/agent/pending-actions.test.ts +++ b/packages/mcp/tests/unit/mcp-server/agent/pending-actions.test.ts @@ -1,12 +1,12 @@ import { describe, expect, test } from "bun:test"; -import type { AutumnMcpAuth } from "./auth.js"; +import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js"; import { claimLatestPendingAction, clearPendingActions, createPendingAction, setPendingActionsRedis, -} from "./pending-actions.js"; -import { createTestRedis } from "./test-redis.js"; +} from "../../../../src/mcp-server/agent/pending-actions.js"; +import { createTestRedis } from "../../../utils/test-redis.js"; setPendingActionsRedis(createTestRedis()); diff --git a/packages/mcp/src/mcp-server/agent/server.test.ts b/packages/mcp/tests/unit/mcp-server/agent/server.test.ts similarity index 52% rename from packages/mcp/src/mcp-server/agent/server.test.ts rename to packages/mcp/tests/unit/mcp-server/agent/server.test.ts index 713fbdb54..76835192f 100644 --- a/packages/mcp/src/mcp-server/agent/server.test.ts +++ b/packages/mcp/tests/unit/mcp-server/agent/server.test.ts @@ -2,7 +2,8 @@ import { describe, expect, test } from "bun:test"; import { createAskAutumnMCPServer, createAutumnOperationsMCPServer, -} from "./server.js"; +} from "../../../../src/mcp-server/agent/server.js"; +import { autumnMcpResourceUris } from "../../../../src/mcp-server/agent/resources.js"; describe("Autumn MCP server", () => { test("public server advertises raw operation tools", async () => { @@ -10,13 +11,17 @@ describe("Autumn MCP server", () => { expect(tools.tools.map((tool) => tool.name)).toEqual([ "listCustomers", + "createCustomer", "getCustomer", "listPlans", + "createPlan", "getPlan", "previewAttach", "previewUpdateSubscription", + "previewCreateSchedule", "attach", "updateSubscription", + "createSchedule", ]); expect(tools.tools.map((tool) => tool.name)).not.toContain("ask_autumn"); expect(tools.tools.map((tool) => tool.name)).not.toContain( @@ -31,4 +36,27 @@ describe("Autumn MCP server", () => { expect(tools.tools.map((tool) => tool.name)).not.toContain("attach"); expect(tools.tools.map((tool) => tool.name)).not.toContain("listCustomers"); }); + + test.each([ + ["public", createAutumnOperationsMCPServer], + ["internal", createAskAutumnMCPServer], + ])("%s server exposes Autumn composition docs", async (_name, createServer) => { + const server = createServer(); + const resources = await server.listResources(); + + expect(resources.resources.map((resource) => resource.uri)).toEqual( + autumnMcpResourceUris, + ); + + for (const uri of autumnMcpResourceUris) { + const resource = await server.readResource(uri); + expect(resource.contents[0]?.text).toContain("# "); + } + }); + + test("unknown resources are rejected", async () => { + await expect( + createAutumnOperationsMCPServer().readResource("autumn://docs/missing"), + ).rejects.toThrow("Unknown Autumn MCP resource"); + }); }); diff --git a/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts b/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts new file mode 100644 index 000000000..6c38b4724 --- /dev/null +++ b/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts @@ -0,0 +1,392 @@ +import { describe, expect, test } from "bun:test"; +import type { AutumnMcpAuth } from "../../../../src/mcp-server/agent/auth.js"; +import { + clearPendingActions, + claimLatestPendingAction, + createPendingAction, + setPendingActionsRedis, +} from "../../../../src/mcp-server/agent/pending-actions.js"; +import { createTestRedis } from "../../../utils/test-redis.js"; +import { + createAgentAutumnOperationTools, + createRawAutumnOperationTools, +} from "../../../../src/mcp-server/agent/tools.js"; + +setPendingActionsRedis(createTestRedis()); + +type ExecutableTool = { + execute?: (input: unknown, context: unknown) => Promise; +}; + +const auth: AutumnMcpAuth = { + apiKey: "sk_test", + env: "sandbox", + principalId: "user_1", + resource: "http://localhost:2718/mcp", + scopes: ["billing:read", "billing:write"], + serverURL: "http://localhost:8080", +}; + +describe("Autumn operation tools", () => { + test("read tool descriptions include composition guidance", () => { + const tools = createRawAutumnOperationTools(); + + expect(tools.listPlans.description).toContain("cheap full scan"); + expect(tools.listPlans.description).toContain("filter returned plans locally"); + expect(tools.listCustomers.description).toContain("plans"); + expect(tools.listCustomers.description).toContain("paginate"); + expect(tools.createPlan.description).toContain("confirmation"); + expect(tools.createSchedule.description).toContain("phase start times"); + expect(tools.previewCreateSchedule.description).toContain("billing impact"); + }); + + test("raw createCustomer calls the get-or-create endpoint", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/customers.get_or_create"); + expect(JSON.parse(init?.body as string)).toMatchObject({ + customer_id: "cus_1", + email: "charlie@example.com", + }); + return Response.json({ id: "cus_1" }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().createCustomer; + if (!tool.execute) throw new Error("createCustomer is not executable"); + + await expect( + tool.execute( + { request: { customer_id: "cus_1", email: "charlie@example.com" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ id: "cus_1" }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("raw createPlan calls the create plan endpoint", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/plans.create"); + expect(JSON.parse(init?.body as string)).toMatchObject({ + plan_id: "pro", + name: "Pro", + }); + return Response.json({ id: "pro" }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().createPlan; + if (!tool.execute) throw new Error("createPlan is not executable"); + + await expect( + tool.execute( + { request: { plan_id: "pro", name: "Pro" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ id: "pro" }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("raw createSchedule calls the create schedule endpoint", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/billing.create_schedule"); + expect(JSON.parse(init?.body as string)).toMatchObject({ + customer_id: "cus_1", + }); + return Response.json({ schedule_id: "sch_1" }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().createSchedule; + if (!tool.execute) throw new Error("createSchedule is not executable"); + + await expect( + tool.execute( + { + request: { + customer_id: "cus_1", + phases: [ + { starts_at: Date.now(), plans: [{ plan_id: "pro" }] }, + ], + }, + }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ schedule_id: "sch_1" }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("raw previewCreateSchedule calls the preview create schedule endpoint", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe( + "http://localhost:8080/v1/billing.preview_create_schedule", + ); + expect(JSON.parse(init?.body as string)).toMatchObject({ + customer_id: "cus_1", + }); + return Response.json({ total: 50 }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().previewCreateSchedule; + if (!tool.execute) { + throw new Error("previewCreateSchedule is not executable"); + } + + await expect( + tool.execute( + { + request: { + customer_id: "cus_1", + phases: [ + { starts_at: Date.now(), plans: [{ plan_id: "pro" }] }, + ], + }, + }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ total: 50 }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("raw listCustomers calls the list endpoint", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/customers.list"); + expect(JSON.parse(init?.body as string)).toMatchObject({ + search: "charlie", + }); + return Response.json({ customers: [] }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().listCustomers; + if (!tool.execute) throw new Error("listCustomers is not executable"); + + await expect( + tool.execute( + { request: { search: "charlie" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ customers: [] }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("raw previewAttach does not create a pending action", async () => { + await clearPendingActions(); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach"); + expect(JSON.parse(init?.body as string)).toEqual({ + customer_id: "cus_1", + plan_id: "pro", + redirect_mode: "if_required", + }); + return Response.json({ total: 50 }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().previewAttach; + if (!tool.execute) throw new Error("previewAttach is not executable"); + + await expect( + tool.execute( + { request: { customer_id: "cus_1", plan_id: "pro" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ total: 50 }); + await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending"); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("raw attach calls the write endpoint directly", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/billing.attach"); + expect(JSON.parse(init?.body as string)).toEqual({ + customer_id: "cus_1", + plan_id: "pro", + redirect_mode: "if_required", + }); + return Response.json({ ok: true }); + }) as typeof fetch; + + try { + const tool = createRawAutumnOperationTools().attach; + if (!tool.execute) throw new Error("attach is not executable"); + + await expect( + tool.execute( + { request: { customer_id: "cus_1", plan_id: "pro" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toEqual({ ok: true }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("previewAttach stores the exact pending attach action", async () => { + await clearPendingActions(); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/billing.preview_attach"); + expect(JSON.parse(init?.body as string)).toEqual({ + customer_id: "cus_1", + plan_id: "pro", + redirect_mode: "if_required", + }); + return Response.json({ total: 50 }); + }) as typeof fetch; + + try { + const tool = ( + createAgentAutumnOperationTools() as unknown as { + previewAttach: { + execute?: (input: unknown, context: unknown) => Promise; + }; + } + ).previewAttach; + if (!tool.execute) throw new Error("previewAttach is not executable"); + + await expect( + tool.execute( + { request: { customer_id: "cus_1", plan_id: "pro" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toMatchObject({ pending: true, preview: { total: 50 } }); + + await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({ + toolName: "attach", + request: { + customer_id: "cus_1", + plan_id: "pro", + redirect_mode: "if_required", + }, + }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("agent createPlan stores a pending write instead of calling Autumn", async () => { + await clearPendingActions(); + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { + throw new Error("createPlan should not call Autumn before confirmation"); + }) as unknown as typeof fetch; + + try { + const tool = ( + createAgentAutumnOperationTools() as unknown as { + createPlan: ExecutableTool; + } + ).createPlan; + if (!tool.execute) throw new Error("createPlan is not executable"); + + await expect( + tool.execute( + { request: { plan_id: "pro", name: "Pro" } }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toMatchObject({ pending: true }); + + await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({ + toolName: "createPlan", + request: { plan_id: "pro", name: "Pro" }, + }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("agent previewCreateSchedule stores a pending write after preview", async () => { + await clearPendingActions(); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url) => { + expect(String(url)).toBe( + "http://localhost:8080/v1/billing.preview_create_schedule", + ); + return Response.json({ total: 50 }); + }) as typeof fetch; + + try { + const request = { + customer_id: "cus_1", + phases: [{ starts_at: Date.now(), plans: [{ plan_id: "pro" }] }], + }; + const tool = ( + createAgentAutumnOperationTools() as unknown as { + previewCreateSchedule: ExecutableTool; + } + ).previewCreateSchedule; + if (!tool.execute) { + throw new Error("previewCreateSchedule is not executable"); + } + + await expect( + tool.execute( + { request }, + { mcp: { extra: { authInfo: auth } } } as never, + ), + ).resolves.toMatchObject({ pending: true }); + + await expect(claimLatestPendingAction(auth)).resolves.toMatchObject({ + toolName: "createSchedule", + request, + }); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("confirmBillingAction executes only the stored pending billing action", async () => { + await clearPendingActions(); + await createPendingAction({ + auth, + toolName: "attach", + request: { customer_id: "cus_1", plan_id: "pro" }, + preview: "Attach pro", + }); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (url, init) => { + expect(String(url)).toBe("http://localhost:8080/v1/billing.attach"); + expect(JSON.parse(init?.body as string)).toEqual({ + customer_id: "cus_1", + plan_id: "pro", + redirect_mode: "if_required", + }); + return Response.json({ ok: true }); + }) as typeof fetch; + + try { + const tool = createAgentAutumnOperationTools().confirmBillingAction; + if (!tool.execute) throw new Error("confirmBillingAction is not executable"); + + await expect( + tool.execute({}, { mcp: { extra: { authInfo: auth } } } as never), + ).resolves.toMatchObject({ + message: "Confirmed and applied attach.", + result: { ok: true }, + }); + await expect(claimLatestPendingAction(auth)).rejects.toThrow("No pending"); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/packages/mcp/src/mcp-server/oauth.test.ts b/packages/mcp/tests/unit/mcp-server/oauth.test.ts similarity index 98% rename from packages/mcp/src/mcp-server/oauth.test.ts rename to packages/mcp/tests/unit/mcp-server/oauth.test.ts index d6f0ed092..7424ccd90 100644 --- a/packages/mcp/src/mcp-server/oauth.test.ts +++ b/packages/mcp/tests/unit/mcp-server/oauth.test.ts @@ -5,7 +5,7 @@ import { MCP_OAUTH_SCOPES, OAuthHttpError, type MCPOAuthFlags, -} from "./oauth.js"; +} from "../../../src/mcp-server/oauth.js"; const flags = { "oauth-enabled": true, diff --git a/packages/mcp/tests/utils/eval-test-utils.ts b/packages/mcp/tests/utils/eval-test-utils.ts new file mode 100644 index 000000000..2e9c1a642 --- /dev/null +++ b/packages/mcp/tests/utils/eval-test-utils.ts @@ -0,0 +1,230 @@ +import { Agent } from "@mastra/core/agent"; +import type { MessageListItem } from "@mastra/core/agent/message-list"; +import { afterEach, expect } from "bun:test"; +import type * as z from "zod/v4"; +import { + type AutumnMcpAuth, + createRequestContext, +} from "../../src/mcp-server/agent/auth.js"; +import { + endpointByTool, + createRawAutumnOperationTools, + schemaByTool, +} from "../../src/mcp-server/agent/tools.js"; + +type ToolName = keyof typeof endpointByTool; +export type ToolRequest = z.output< + (typeof schemaByTool)[Tool] +>; +export type ToolRequestInput = z.input< + (typeof schemaByTool)[Tool] +>; +type ToolCall = { name: string; args: Record }; +type AutumnApiFixture = { + [Tool in ToolName]?: unknown | ((body: ToolRequest) => unknown); +}; +type AutumnApiCall = { + toolName: Tool; + endpoint: string; + body: ToolRequest; + rawBody: ToolRequestInput; +}; +type UnknownAutumnApiCall = { + toolName: null; + endpoint: string; + body: unknown; + rawBody: unknown; +}; + +const serverURL = "http://localhost:8080"; +const cleanupFns: (() => void)[] = []; +const toolEntries = Object.entries(endpointByTool) as [ToolName, string][]; +const summarize = (value: unknown) => JSON.stringify(value, null, 2); + +afterEach(() => { + for (const cleanup of cleanupFns.splice(0).reverse()) cleanup(); +}); + +const defaultAuth: AutumnMcpAuth = { + apiKey: "sk_test", + env: "sandbox", + principalId: "eval-user", + resource: "http://localhost:2718/mcp", + scopes: ["customers:read", "plans:read", "billing:read", "billing:write"], + serverURL, +}; + +export const createMcpConsumerAgent = () => + new Agent({ + id: "mcp-consumer-eval", + name: "MCP Consumer Eval", + description: "A generic agent using MCP tools.", + instructions: "You are a helpful assistant. Use available tools when useful.", + model: "anthropic/claude-sonnet-4-6", + tools: createRawAutumnOperationTools(), + }); + +const mockAutumnApi = ({ + serverURL, + fixtures, +}: { + serverURL: string; + fixtures: AutumnApiFixture; +}) => { + const calls: (AutumnApiCall | UnknownAutumnApiCall)[] = []; + const originalFetch = globalThis.fetch; + + globalThis.fetch = (async (url, init) => { + const requestUrl = new URL(String(url)); + if (requestUrl.origin !== serverURL) { + return originalFetch(url, init); + } + + const endpoint = requestUrl.pathname; + const body = JSON.parse(String(init?.body ?? "{}")); + const toolName = + toolEntries.find(([, path]) => endpoint.endsWith(path))?.[0] ?? null; + const fixture = toolName ? fixtures[toolName] : undefined; + const parsedBody = toolName ? schemaByTool[toolName].parse(body) : body; + calls.push({ + toolName, + endpoint, + body: parsedBody as never, + rawBody: body, + }); + + if (fixture === undefined) { + return Response.json( + { error: `No MCP eval fixture for ${toolName ?? endpoint}` }, + { status: 500 }, + ); + } + + return Response.json( + typeof fixture === "function" + ? fixture(parsedBody as never) + : (fixture ?? { ok: true }), + ); + }) as typeof fetch; + + return { + calls, + call: (toolName: Tool) => + calls.find( + (call): call is AutumnApiCall => call.toolName === toolName, + ), + callsFor: (toolName: Tool) => + calls.filter( + (call): call is AutumnApiCall => call.toolName === toolName, + ), + restore: () => { + globalThis.fetch = originalFetch; + }, + }; +}; +type MockAutumnApi = ReturnType; + +export const initMcpEval = ({ + auth = {}, + fixtures, +}: { + auth?: Partial; + fixtures: AutumnApiFixture; +}) => { + const resolvedAuth = { ...defaultAuth, ...auth }; + const api = mockAutumnApi({ + serverURL: resolvedAuth.serverURL ?? serverURL, + fixtures, + }); + const agent = createMcpConsumerAgent(); + let messages: MessageListItem[] = []; + const toolCalls: { name: string; args: Record }[] = []; + cleanupFns.push(api.restore); + + return { + api, + auth: resolvedAuth, + toolCalls, + generate: async (message: string | string[], maxSteps = 4) => { + messages.push({ + role: "user", + content: Array.isArray(message) ? message.join("\n") : message, + }); + const output = await agent.generate(messages, { + maxSteps, + requestContext: createRequestContext(resolvedAuth), + onIterationComplete: ({ toolCalls: calls }) => { + toolCalls.push(...calls); + }, + }); + messages = output.messages; + return output; + }, + }; +}; + +export const expectToolCall = ( + toolCalls: ToolCall[], + toolName: Tool, + request?: Partial>, +) => { + const call = toolCalls.find((call) => call.name === toolName); + expect( + call, + `${toolName} was not called. Called tools:\n${summarize(toolCalls)}`, + ).toBeDefined(); + if (request) { + expect(call?.args, `${toolName} args did not match`).toMatchObject({ + request, + }); + } + return call; +}; + +export const expectNoToolCall = (toolCalls: ToolCall[], toolName: ToolName) => { + const call = toolCalls.find((call) => call.name === toolName); + expect( + call, + `${toolName} was called unexpectedly:\n${summarize(call)}`, + ).toBeUndefined(); +}; + +export const expectApiCall = ( + api: MockAutumnApi, + toolName: Tool, + body?: Partial>, +) => { + const call = api.call(toolName); + expect( + call, + `${toolName} did not call Autumn. Autumn calls:\n${summarize(api.calls)}`, + ).toBeDefined(); + if (body) { + expect(call?.rawBody, `${toolName} raw body did not match`).toMatchObject( + body, + ); + } + return call; +}; + +export const expectExactApiCall = ( + api: MockAutumnApi, + toolName: Tool, + body: ToolRequestInput, +) => { + const calls = api.callsFor(toolName); + expect( + calls, + `${toolName} should call Autumn exactly once. Autumn calls:\n${summarize(api.calls)}`, + ).toHaveLength(1); + expect(calls[0]?.rawBody, `${toolName} raw body was wrong`).toEqual(body); + return calls[0]; +}; + +export const expectNoApiCall = (api: MockAutumnApi, toolName: ToolName) => { + const call = api.call(toolName); + expect( + call, + `${toolName} called Autumn unexpectedly:\n${summarize(call)}`, + ).toBeUndefined(); +}; diff --git a/packages/mcp/src/mcp-server/agent/test-redis.ts b/packages/mcp/tests/utils/test-redis.ts similarity index 93% rename from packages/mcp/src/mcp-server/agent/test-redis.ts rename to packages/mcp/tests/utils/test-redis.ts index 6299bf447..6d69c3e80 100644 --- a/packages/mcp/src/mcp-server/agent/test-redis.ts +++ b/packages/mcp/tests/utils/test-redis.ts @@ -1,7 +1,7 @@ import type { PendingActionRedis, PendingActionRedisMulti, -} from "./pending-actions.js"; +} from "../../src/mcp-server/agent/pending-actions.js"; export const createTestRedis = (): PendingActionRedis => { const store = new Map(); diff --git a/packages/mcp/tsconfig.json b/packages/mcp/tsconfig.json index 1435df961..d9d31a185 100644 --- a/packages/mcp/tsconfig.json +++ b/packages/mcp/tsconfig.json @@ -28,6 +28,7 @@ "sourceMap": true, "strict": true, "target": "es2022", + "types": ["bun", "node"], "paths": { "@api/*": ["../../shared/api/*"], "@models/*": ["../../shared/models/*"], @@ -37,5 +38,5 @@ "useUnknownInCatchVariables": true, }, "exclude": ["node_modules"], - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "tests/**/*.ts"] } diff --git a/run.sh b/run.sh index 066ead246..9eb7b254e 100755 --- a/run.sh +++ b/run.sh @@ -21,5 +21,10 @@ if [[ "$resolved" == "$repo_root/server/"* ]]; then exec "$repo_root/server/run.sh" "$resolved" "${@:2}" fi +if [[ "$resolved" == "$repo_root/packages/mcp/tests/evals/"* && "$resolved" == *".test.ts" ]]; then + cd "$repo_root/packages/mcp" + exec env ENV_FILE=.env infisical run --env=dev --recursive -- bun test "$resolved" "${@:2}" +fi + echo "no router for: $resolved" >&2 exit 1 diff --git a/shared/api/publicApiSchemas.ts b/shared/api/publicApiSchemas.ts index ff281e85d..5f9d7409d 100644 --- a/shared/api/publicApiSchemas.ts +++ b/shared/api/publicApiSchemas.ts @@ -1,6 +1,9 @@ export { AttachParamsV1Schema } from "./billing/attachV2/attachParamsV1.js"; +export { CreateScheduleParamsV0Schema } from "./billing/createSchedule/createScheduleParamsV0.js"; export { UpdateSubscriptionV1ParamsSchema } from "./billing/updateSubscription/updateSubscriptionV1Params.js"; +export { CreateCustomerParamsV1Schema } from "./customers/crud/createCustomerParams.js"; export { GetCustomerParamsV1Schema } from "./customers/crud/getCustomerParams.js"; export { ListCustomersV2_3ParamsSchema } from "./customers/crud/listCustomersParamsV2_3.js"; +export { CreatePlanParamsV2Schema } from "./products/crud/createPlanParamsV1.js"; export { GetPlanParamsV0Schema } from "./products/crud/getPlanParamsV0.js"; export { ListPlanParamsSchema } from "./products/crud/listPlanParams.js"; diff --git a/vite/src/views/general/LoadingScreen.tsx b/vite/src/views/general/LoadingScreen.tsx index 4701a5bfb..478e7daf4 100644 --- a/vite/src/views/general/LoadingScreen.tsx +++ b/vite/src/views/general/LoadingScreen.tsx @@ -12,6 +12,7 @@ function LoadingScreen() { "Blasting competitors", "Shipping faster", "Stopping churn", + "Ayushhhing...", ]; const [loadingText, setLoadingText] = React.useState(texts[0]);