diff --git a/ai b/ai index 0e52f71fb..a84bc7418 160000 --- a/ai +++ b/ai @@ -1 +1 @@ -Subproject commit 0e52f71fbd69e7a4a58b63863a8c4929bfd9ebf8 +Subproject commit a84bc7418cd985f3b831b24a5e49f447b0eff0d4 diff --git a/apps/leaf/src/agent/agent.ts b/apps/leaf/src/agent/agent.ts index bdf0b8d80..7bde85c7f 100644 --- a/apps/leaf/src/agent/agent.ts +++ b/apps/leaf/src/agent/agent.ts @@ -15,34 +15,9 @@ import { createFirecrawlTools } from "./firecrawl.js"; import { createAutumnMcpClient, getAutumnMcpTools } from "./mcp.js"; import { sandboxConfig } from "./sandbox/config.js"; import { createSandboxTools } from "./sandbox/createSandboxTools.js"; +import { agentDocUris, createAutumnChatAgent } from "./chatAgent.js"; -export const agentDocUris = [ - "autumn://docs/tool-composition", - "autumn://docs/feature-catalog", - "autumn://docs/querying-plans", - "autumn://docs/querying-customers", - "autumn://docs/schedules", - "autumn://docs/balances", - "autumn://docs/billing-safety", - "autumn://docs/request-logs", - "autumn://docs/request-log-customers", - "autumn://docs/request-log-balances", - "autumn://docs/request-log-billing", - "autumn://docs/request-log-stripe-webhooks", - "autumn://docs/request-log-analytics", -]; - -const instructions = `You are Autumn Chat. -Use Autumn MCP tools for customer, plan, balance, schedule, and billing work. -Use web search only for current or external web context. Never use web search for Autumn customer, plan, billing, balance, or schedule state. -When web content influences the answer, cite the source URLs. -Prefer searchWeb first, then scrapeUrl only for the most relevant result. -Use listFeatures only when creating/customizing plan items or setting non-zero prepaid feature quantities and feature ids/types are not already known; never invent feature ids. -Use the sandbox only for short parsing, calculation, transformation, and file-analysis tasks. Never send secrets to the sandbox, never use it for Autumn writes, and treat sandbox output as advisory. -Preview billing-impacting changes first, summarize the preview in short Slack-friendly bullets, then call the matching write tool with the same request args. -When Autumn responses include epoch millisecond timestamps, use epochMillisecondsToDate before explaining those timestamps to a user. -Treat Slack PDFs and images attached to the latest message as part of the user's request. If an attachment was skipped or unavailable, say so briefly instead of pretending to have read it. -The runtime pauses destructive tools for approval before execution, so do not ask for confirmation in plain text.`; +export { agentDocUris, createAutumnChatAgent } from "./chatAgent.js"; const envSelectionSchema = z.strictObject({ env: z.nativeEnum(AppEnv), @@ -212,10 +187,9 @@ export const runChatAgent = async ({ }); } await onAction?.("Reasoning over the request"); - const agent = new Agent({ - id: "autumn-chat", - name: "Autumn Chat", - instructions: `${instructions}\n\nCurrent Autumn environment: ${env}.\n\n${docsText}`, + const agent = createAutumnChatAgent({ + docsText, + env, model: chatEnv.CHAT_MODEL, tools: { ...tools, ...firecrawlTools, ...sandboxTools }, }); diff --git a/apps/leaf/src/agent/chatAgent.ts b/apps/leaf/src/agent/chatAgent.ts new file mode 100644 index 000000000..0cdee8281 --- /dev/null +++ b/apps/leaf/src/agent/chatAgent.ts @@ -0,0 +1,50 @@ +import type { AppEnv } from "@autumn/shared"; +import { Agent } from "@mastra/core/agent"; +import type { ToolsInput } from "@mastra/core/agent"; + +export const agentDocUris = [ + "autumn://docs/tool-composition", + "autumn://docs/feature-catalog", + "autumn://docs/querying-plans", + "autumn://docs/querying-customers", + "autumn://docs/schedules", + "autumn://docs/balances", + "autumn://docs/billing-safety", + "autumn://docs/request-logs", + "autumn://docs/request-log-customers", + "autumn://docs/request-log-balances", + "autumn://docs/request-log-billing", + "autumn://docs/request-log-stripe-webhooks", + "autumn://docs/request-log-analytics", +]; + +export const autumnChatInstructions = `You are Autumn Chat. +Use Autumn MCP tools for customer, plan, balance, schedule, and billing work. +Use web search only for current or external web context. Never use web search for Autumn customer, plan, billing, balance, or schedule state. +When web content influences the answer, cite the source URLs. +Prefer searchWeb first, then scrapeUrl only for the most relevant result. +Use listFeatures only when creating/customizing plan items or setting non-zero prepaid feature quantities and feature ids/types are not already known; never invent feature ids. +Use the sandbox only for short parsing, calculation, transformation, and file-analysis tasks. Never send secrets to the sandbox, never use it for Autumn writes, and treat sandbox output as advisory. +Preview billing-impacting changes first, summarize the preview in short Slack-friendly bullets, then call the matching write tool with the same request args. +When Autumn responses include epoch millisecond timestamps, use epochMillisecondsToDate before explaining those timestamps to a user. +Treat Slack PDFs and images attached to the latest message as part of the user's request. If an attachment was skipped or unavailable, say so briefly instead of pretending to have read it. +The runtime pauses destructive tools for approval before execution, so do not ask for confirmation in plain text.`; + +export const createAutumnChatAgent = ({ + docsText, + env, + model, + tools, +}: { + docsText: string; + env: AppEnv; + model: string; + tools: ToolsInput; +}) => + new Agent({ + id: "autumn-chat", + name: "Autumn Chat", + instructions: `${autumnChatInstructions}\n\nCurrent Autumn environment: ${env}.\n\n${docsText}`, + model, + tools, + }); diff --git a/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts b/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts index 318f50c32..7a121cf21 100644 --- a/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts +++ b/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts @@ -3,13 +3,18 @@ import { SpanType } from "@mastra/core/observability"; import { Observability, SamplingStrategyType } from "@mastra/observability"; import { currentSpan } from "braintrust"; import { braintrustConfig } from "./config.js"; +import { createBraintrustLogger } from "./createBraintrustLogger.js"; export const createMastraBraintrustObservability = ({ apiKey = process.env.BRAINTRUST_API_KEY, - braintrustLogger, enabled = braintrustConfig.enabled, projectName = braintrustConfig.projectName, serviceName = braintrustConfig.serviceName, + braintrustLogger = createBraintrustLogger({ + apiKey, + enabled, + projectName, + }), }: { apiKey?: string; braintrustLogger?: unknown; diff --git a/apps/leaf/src/ui/blocks.ts b/apps/leaf/src/ui/blocks.ts index 0a730c1e1..513ca5fa1 100644 --- a/apps/leaf/src/ui/blocks.ts +++ b/apps/leaf/src/ui/blocks.ts @@ -13,7 +13,13 @@ const getRequest = (args?: Record) => : args) as Record | undefined; const getFieldValue = (value: unknown) => - typeof value === "string" || typeof value === "number" ? String(value) : null; + typeof value === "string" || typeof value === "number" + ? String(value) + : typeof value === "boolean" + ? value + ? "Yes" + : "No" + : null; const getRecord = (value: unknown) => value && typeof value === "object" ? (value as Record) : {}; @@ -26,6 +32,32 @@ const formatPrice = (request: Record) => { return amount ? `$${amount}${interval ? `/${interval}` : ""}` : null; }; +const formatInvoiceMode = (value: unknown) => { + if (typeof value === "boolean") return value ? "enabled" : "disabled"; + const invoiceMode = getRecord(value); + if (!Object.keys(invoiceMode).length) return null; + + return [ + invoiceMode.enabled === true + ? "enabled" + : invoiceMode.enabled === false + ? "disabled" + : null, + invoiceMode.finalize === false + ? "draft invoice" + : invoiceMode.finalize === true + ? "finalize invoice" + : null, + invoiceMode.enable_plan_immediately === true + ? "enable immediately" + : invoiceMode.enable_plan_immediately === false + ? "access waits" + : null, + ] + .filter((part): part is string => Boolean(part)) + .join(", "); +}; + const envLabel = (env?: AppEnv) => env === "live" ? "Live" : env === "sandbox" ? "Sandbox" : null; @@ -59,7 +91,8 @@ const requestFields = ({ ["Entity", request.entity_id], ["Subscription", request.subscription_id], ["Price", formatPrice(request)], - ["Invoice mode", request.invoice_mode], + ["Enable immediately", request.enable_plan_immediately], + ["Invoice mode", formatInvoiceMode(request.invoice_mode)], ["Proration", request.proration_behavior], ["Redirect", request.redirect_mode], ].flatMap(([label, value]) => { diff --git a/apps/leaf/tests/evals/agent/billing/attach/custom-base-price.eval.ts b/apps/leaf/tests/evals/agent/billing/attach/custom-base-price.eval.ts new file mode 100644 index 000000000..8aaae1b89 --- /dev/null +++ b/apps/leaf/tests/evals/agent/billing/attach/custom-base-price.eval.ts @@ -0,0 +1,95 @@ +import { + billing, + response, + tools, +} from "../../../fixtures/expectations/index.js"; +import { withCustomers } from "../../../fixtures/createSetup.js"; +import { orgSetups } from "../../../fixtures/orgSetups.js"; +import { approve, initEval, user } from "../../../harness/index.js"; +import { billingAttachScores } from "../../../utils/scorers.js"; + +type EvalMetadata = { + domain: "billing"; + flow: "attach"; +}; + +const experimentName = "attach-custom-price"; +const customPrice = 49; + +const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers }) => ({ + account: customers.base({ + email: "billing@northstar.example", + id: "cus_attach_custom_price", + name: "Northstar Labs", + }), + }), +}); +const customer = setup.refs.customers.account; +const enterprisePlan = setup.refs.plans.enterprise; + +const expectedAttachRequest = { + customer_id: customer.id, + customize: { + price: { + amount: customPrice, + interval: "month", + }, + }, + enable_plan_immediately: true, + invoice_mode: { + enable_plan_immediately: true, + enabled: true, + finalize: false, + }, + plan_id: enterprisePlan.id, + redirect_mode: "if_required", +}; + +initEval({ + experimentName, + setup, + metadata: { + domain: "billing", + flow: "attach", + }, + scores: billingAttachScores(), + cases: [ + { + name: "custom monthly price with draft invoice", + conversation: [ + user({ + message: + "Please attach the Enterprise plan to Northstar Labs with a custom base price of $49/month.", + }), + user({ message: "Looks good, attach it." }), + approve(), + ], + expect: [ + tools.called({ + toolNames: ["listCustomers", "listPlans"], + }), + billing.previewBeforeWrite({ + preview: { + body: expectedAttachRequest, + toolName: "previewAttach", + }, + write: { + body: expectedAttachRequest, + toolName: "attach", + }, + }), + response.mentions({ + phrases: [ + "Northstar Labs", + "Enterprise", + "$49", + "invoice", + "immediately", + ], + }), + ], + }, + ], +}); diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts b/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts index 1135f4153..f2f734502 100644 --- a/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts +++ b/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts @@ -7,6 +7,7 @@ type CustomerArgs = Parameters[0]; /** Customer presets for common eval scenarios; compose subscriptions explicitly. */ export const customers = { + base: (args?: CustomerArgs): BaseApiCustomerV5 => baseCustomer(args), active: (args?: CustomerArgs): BaseApiCustomerV5 => baseCustomer(args), withPlan: ({ plan, diff --git a/apps/leaf/tests/evals/fixtures/expectations/api.ts b/apps/leaf/tests/evals/fixtures/expectations/api.ts new file mode 100644 index 000000000..03b4e7404 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/expectations/api.ts @@ -0,0 +1,34 @@ +import type { + ApiCalledExpectation, + ApiCalledInOrderExpectation, + ExpectedApiCall, +} from "./types.js"; + +export const api = { + call: ({ + body, + toolName, + }: { + body?: Record; + toolName: ExpectedApiCall["toolName"]; + }): ExpectedApiCall => ({ + ...(body ? { body } : {}), + toolName, + }), + called: ({ + calls, + }: { + calls: ExpectedApiCall[]; + }): ApiCalledExpectation => ({ + calls, + type: "api.called", + }), + calledInOrder: ({ + calls, + }: { + calls: ExpectedApiCall[]; + }): ApiCalledInOrderExpectation => ({ + calls, + type: "api.calledInOrder", + }), +}; diff --git a/apps/leaf/tests/evals/fixtures/expectations/billing.ts b/apps/leaf/tests/evals/fixtures/expectations/billing.ts new file mode 100644 index 000000000..74bf380f3 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/expectations/billing.ts @@ -0,0 +1,16 @@ +import { api } from "./api.js"; +import type { + ApiCalledInOrderExpectation, + ExpectedApiCall, +} from "./types.js"; + +export const billing = { + previewBeforeWrite: ({ + preview, + write, + }: { + preview: ExpectedApiCall; + write: ExpectedApiCall; + }): ApiCalledInOrderExpectation => + api.calledInOrder({ calls: [preview, write] }), +}; diff --git a/apps/leaf/tests/evals/fixtures/expectations/index.ts b/apps/leaf/tests/evals/fixtures/expectations/index.ts new file mode 100644 index 000000000..146061556 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/expectations/index.ts @@ -0,0 +1,14 @@ +export { api } from "./api.js"; +export { billing } from "./billing.js"; +export { response } from "./response.js"; +export { tools } from "./tools.js"; +export type { + ApiCalledExpectation, + ApiCalledInOrderExpectation, + EvalExpectation, + EvalExpected, + ExpectedApiCall, + LegacyEvalExpected, + ResponseMentionsExpectation, + ToolsCalledExpectation, +} from "./types.js"; diff --git a/apps/leaf/tests/evals/fixtures/expectations/response.ts b/apps/leaf/tests/evals/fixtures/expectations/response.ts new file mode 100644 index 000000000..d014be036 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/expectations/response.ts @@ -0,0 +1,12 @@ +import type { ResponseMentionsExpectation } from "./types.js"; + +export const response = { + mentions: ({ + phrases, + }: { + phrases: string[]; + }): ResponseMentionsExpectation => ({ + phrases, + type: "response.mentions", + }), +}; diff --git a/apps/leaf/tests/evals/fixtures/expectations/tools.ts b/apps/leaf/tests/evals/fixtures/expectations/tools.ts new file mode 100644 index 000000000..f6a554e5f --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/expectations/tools.ts @@ -0,0 +1,12 @@ +import type { ToolsCalledExpectation } from "./types.js"; + +export const tools = { + called: ({ + toolNames, + }: { + toolNames: ToolsCalledExpectation["toolNames"]; + }): ToolsCalledExpectation => ({ + toolNames, + type: "tools.called", + }), +}; diff --git a/apps/leaf/tests/evals/fixtures/expectations/types.ts b/apps/leaf/tests/evals/fixtures/expectations/types.ts new file mode 100644 index 000000000..ec208ab46 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/expectations/types.ts @@ -0,0 +1,42 @@ +import type { AutumnEvalToolName } from "../../harness/context/types.js"; + +export type ExpectedApiCall = { + body?: Record; + toolName: AutumnEvalToolName; +}; + +export type LegacyEvalExpected = { + apiCalls?: ExpectedApiCall[]; + finalTextIncludes?: string[]; + toolCalls?: AutumnEvalToolName[]; +}; + +export type ToolsCalledExpectation = { + toolNames: AutumnEvalToolName[]; + type: "tools.called"; +}; + +export type ApiCalledExpectation = { + calls: ExpectedApiCall[]; + type: "api.called"; +}; + +export type ApiCalledInOrderExpectation = { + calls: ExpectedApiCall[]; + type: "api.calledInOrder"; +}; + +export type ResponseMentionsExpectation = { + phrases: string[]; + type: "response.mentions"; +}; + +export type EvalExpectation = + | ApiCalledExpectation + | ApiCalledInOrderExpectation + | ResponseMentionsExpectation + | ToolsCalledExpectation; + +export type EvalExpected = + | LegacyEvalExpected + | (EvalExpectation[] & LegacyEvalExpected); diff --git a/apps/leaf/tests/evals/fixtures/responses.ts b/apps/leaf/tests/evals/fixtures/responses.ts index aade090c5..a01c14323 100644 --- a/apps/leaf/tests/evals/fixtures/responses.ts +++ b/apps/leaf/tests/evals/fixtures/responses.ts @@ -2,6 +2,18 @@ import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; const planAmount = (plan: ApiPlanV1) => plan.price?.amount ?? 0; +const schedulePhases = (phases: unknown) => + Array.isArray(phases) + ? phases.map((phase, index) => { + const record = phase as Record; + return { + customer_product_ids: [`cp_schedule_${index + 1}`], + phase_id: `phase_${index + 1}`, + starts_at: + typeof record.starts_at === "number" ? record.starts_at : null, + }; + }) + : []; export const responses = { attachPreview: ({ @@ -30,4 +42,37 @@ export const responses = { plan_id: plan.id, status: "created", }), + createSchedulePreview: ({ + customerId, + phases, + }: { + customerId: string; + phases: unknown; + }) => ({ + customer_id: customerId, + currency: "usd", + line_items: schedulePhases(phases).map((phase, index) => ({ + description: `Schedule phase ${index + 1}`, + starts_at: phase.starts_at, + total: 0, + })), + total: 0, + }), + createScheduleSuccess: ({ + customerId, + entityId = null, + phases, + }: { + customerId: string; + entityId?: string | null; + phases: unknown; + }) => ({ + customer_id: customerId, + entity_id: entityId, + invoice: null, + payment_url: null, + phases: schedulePhases(phases), + schedule_id: `sched_${customerId}`, + status: "created", + }), }; diff --git a/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts b/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts index 31a4b585b..c7fff50e5 100644 --- a/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts +++ b/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts @@ -8,7 +8,9 @@ const serverURL = "http://localhost:8080"; const endpointToTool = { "/v1/balances.create": "createBalance", "/v1/billing.attach": "attach", + "/v1/billing.create_schedule": "createSchedule", "/v1/billing.preview_attach": "previewAttach", + "/v1/billing.preview_create_schedule": "previewCreateSchedule", "/v1/customers.get": "getCustomer", "/v1/customers.get_or_create": "getOrCreateCustomer", "/v1/customers.list": "listCustomers", @@ -51,6 +53,18 @@ const defaultHandlers = { return responses.attachSuccess({ customer, plan }); }, createBalance: () => ({ status: "created" }), + createSchedule: ({ body, setup }) => { + const customerId = getString(body, "customer_id"); + const customer = setup.customers.find( + (customer) => customer.id === customerId, + ); + if (!customer) return { error: "customer not found" }; + return responses.createScheduleSuccess({ + customerId, + entityId: getString(body, "entity_id") || null, + phases: body.phases, + }); + }, getCustomer: ({ body, setup }) => { const customer = setup.customers.find( (customer) => customer.id === getString(body, "customer_id"), @@ -108,6 +122,17 @@ const defaultHandlers = { if (!customer || !plan) return { error: "missing customer or plan" }; return responses.attachPreview({ customer, plan }); }, + previewCreateSchedule: ({ body, setup }) => { + const customerId = getString(body, "customer_id"); + const customer = setup.customers.find( + (customer) => customer.id === customerId, + ); + if (!customer) return { error: "customer not found" }; + return responses.createSchedulePreview({ + customerId, + phases: body.phases, + }); + }, updateCustomer: ({ body, setup }) => { const customer = setup.customers.find( (customer) => customer.id === getString(body, "customer_id"), diff --git a/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts b/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts index ada3dcaa9..ca46b4b06 100644 --- a/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts +++ b/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts @@ -1,13 +1,22 @@ import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { Socket } from "node:net"; import { MCPServer } from "@mastra/mcp"; import { setAnalyticsSink } from "../../../../../../packages/mcp/src/analytics/analyticsSink.js"; import type { AutumnMcpAuth } from "../../../../../../packages/mcp/src/server/auth/auth.js"; import { createRawAutumnOperationTools } from "../../../../../../packages/mcp/src/tools/index.js"; import type { EvalMcpServer } from "./types.js"; -const closeServer = (server: Server) => +const closeServer = ({ + server, + sockets, +}: { + server: Server; + sockets: Set; +}) => new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); + server.closeAllConnections?.(); + for (const socket of sockets) socket.destroy(); }); const createEvalMcpServer = () => @@ -24,6 +33,7 @@ const createEvalMcpServer = () => export const createAutumnMcpServer = (auth: AutumnMcpAuth) => new Promise((resolve) => { setAnalyticsSink(null); + const sockets = new Set(); const server = createServer(async (req, res) => { const url = new URL(req.url ?? "/mcp", `http://${req.headers.host}`); if (url.pathname !== "/mcp") { @@ -40,13 +50,17 @@ export const createAutumnMcpServer = (auth: AutumnMcpAuth) => url, }); }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); server.listen(0, "127.0.0.1", () => { const address = server.address(); if (!address || typeof address === "string") { throw new Error("MCP eval server did not bind to a TCP port."); } resolve({ - close: () => closeServer(server), + close: () => closeServer({ server, sockets }), url: new URL(`http://127.0.0.1:${address.port}/mcp`), }); }); diff --git a/apps/leaf/tests/evals/harness/context/types.ts b/apps/leaf/tests/evals/harness/context/types.ts index 5bded396f..0606ae22a 100644 --- a/apps/leaf/tests/evals/harness/context/types.ts +++ b/apps/leaf/tests/evals/harness/context/types.ts @@ -4,6 +4,7 @@ import type { EvalSetup } from "../../fixtures/types.js"; export type AutumnEvalToolName = | "attach" | "createBalance" + | "createSchedule" | "getCustomer" | "getOrCreateCustomer" | "getPlan" @@ -11,6 +12,7 @@ export type AutumnEvalToolName = | "listFeatures" | "listPlans" | "previewAttach" + | "previewCreateSchedule" | "updateCustomer"; export type AutumnApiCall = { diff --git a/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts b/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts index f725c5dbf..99d3a4c94 100644 --- a/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts +++ b/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts @@ -162,6 +162,7 @@ export const createGenericMcpAgentDriver = ({ return { text: output.text }; }, cleanup: async () => { + await mastra.shutdown(); await mcpClient.disconnect(); }, getToolCalls: () => [...toolCalls], diff --git a/apps/leaf/tests/evals/harness/drivers/leafAgent.ts b/apps/leaf/tests/evals/harness/drivers/leafAgent.ts new file mode 100644 index 000000000..d24540750 --- /dev/null +++ b/apps/leaf/tests/evals/harness/drivers/leafAgent.ts @@ -0,0 +1,200 @@ +import { AppEnv } from "@autumn/shared"; +import type { MessageListItem } from "@mastra/core/agent/message-list"; +import type { ToolsInput } from "@mastra/core/agent"; +import { Mastra } from "@mastra/core/mastra"; +import { InMemoryStore } from "@mastra/core/storage"; +import { MCPClient } from "@mastra/mcp"; +import { createRequestContext } from "../../../../../../packages/mcp/src/server/auth/auth.js"; +import { + agentDocUris, + createAutumnChatAgent, +} from "../../../../src/agent/chatAgent.js"; +import { createLeafTracingOptions } from "../../../../src/internal/observability/leafTracingOptions.js"; +import { createMastraBraintrustObservability } from "../../../../src/providers/braintrust/index.js"; +import { defaultGenericMcpAgentConfig } from "../configs/genericMcpAgentConfig.js"; +import type { + EvalAgentDriver, + EvalDriverStartInput, + EvalToolCall, +} from "./types.js"; + +type LeafAgentDriverConfig = { + maxSteps?: number; + model?: string; +}; + +type ToolWithApproval = { + execute?: unknown; + mcp?: { annotations?: { destructiveHint?: boolean } }; + needsApprovalFn?: unknown; + requireApproval?: unknown; +}; + +const applyToolApprovalPolicy = (tools: Record) => { + for (const tool of Object.values(tools)) { + const requiresApproval = tool.mcp?.annotations?.destructiveHint === true; + tool.requireApproval = requiresApproval; + if (!requiresApproval) tool.needsApprovalFn = undefined; + } +}; + +const instrumentToolCalls = ({ + tools, + toolCalls, + trace, +}: { + tools: Record; + toolCalls: EvalToolCall[]; + trace: EvalDriverStartInput["trace"]; +}) => { + for (const [name, tool] of Object.entries(tools)) { + if (typeof tool.execute !== "function") continue; + const execute = tool.execute.bind(tool) as ( + args: Record, + ...rest: unknown[] + ) => Promise; + tool.execute = async ( + args: Record, + ...rest: unknown[] + ) => { + const call = { args, name }; + toolCalls.push(call); + trace.event({ call, type: "tool_call" }); + return execute(args, ...rest); + }; + } +}; + +const readDocs = async (mcpClient: MCPClient) => { + const resources = await Promise.allSettled( + agentDocUris.map((uri) => mcpClient.resources.read("autumn", uri)), + ); + return resources + .flatMap((result) => + result.status === "fulfilled" + ? result.value.contents.flatMap((content) => + "text" in content ? [content.text] : [], + ) + : [], + ) + .join("\n\n"); +}; + +export const createLeafAgentDriver = ({ + maxSteps = defaultGenericMcpAgentConfig.maxSteps, + model = defaultGenericMcpAgentConfig.model, +}: LeafAgentDriverConfig = {}): EvalAgentDriver => ({ + name: "leaf-agent", + start: async ({ context, setup, today, trace }: EvalDriverStartInput) => { + const mcpClient = new MCPClient({ + id: `leaf-agent-eval-${crypto.randomUUID()}`, + servers: { + autumn: { + requireToolApproval: ({ annotations }) => + annotations?.destructiveHint === true, + url: context.mcpServer.url, + }, + }, + }); + const [{ toolsets, errors }, docsText] = await Promise.all([ + mcpClient.listToolsetsWithErrors(), + readDocs(mcpClient), + ]); + if (Object.keys(errors).length) { + throw new Error(`MCP tool discovery failed: ${JSON.stringify(errors)}`); + } + + const env = + context.auth.env === AppEnv.Live ? AppEnv.Live : AppEnv.Sandbox; + const tools = (toolsets.autumn ?? {}) as Record; + applyToolApprovalPolicy(tools); + const toolCalls: EvalToolCall[] = []; + instrumentToolCalls({ toolCalls, tools, trace }); + + const agent = createAutumnChatAgent({ + docsText, + env, + model, + tools: tools as ToolsInput, + }); + const mastra = new Mastra({ + agents: { chat: agent }, + logger: false, + observability: createMastraBraintrustObservability(), + storage: new InMemoryStore({ + id: `leaf-agent-eval-${crypto.randomUUID()}`, + }), + }); + const evalAgent = mastra.getAgent("chat"); + let messages: MessageListItem[] = []; + let pendingApproval: { runId: string; toolCallId?: string } | null = null; + + const options = (stepLimit?: number) => ({ + context: today + ? [ + { + content: `Current date: ${today.toISOString()}.`, + role: "system" as const, + }, + ] + : undefined, + maxSteps: stepLimit ?? maxSteps, + requestContext: createRequestContext(context.auth), + tracingOptions: createLeafTracingOptions({ + env, + orgId: context.auth.orgId, + setup: setup.tag, + source: "eval", + }), + }); + + const rememberApproval = (output: { + finishReason?: string; + runId?: string; + suspendPayload?: { toolCallId?: string }; + }) => { + pendingApproval = + output.finishReason === "suspended" && output.runId + ? { + runId: output.runId, + toolCallId: output.suspendPayload?.toolCallId, + } + : null; + if (pendingApproval) trace.event({ type: "approval_pending" }); + }; + + return { + approve: async ({ maxSteps: stepLimit } = {}) => { + if (!pendingApproval) { + throw new Error("No pending approval to approve."); + } + trace.event({ type: "approval_approved" }); + const output = await evalAgent.approveToolCallGenerate({ + ...options(stepLimit), + runId: pendingApproval.runId, + toolCallId: pendingApproval.toolCallId, + }); + messages = output.messages; + rememberApproval(output); + trace.event({ text: output.text ?? "", type: "agent_text" }); + return { text: output.text }; + }, + cleanup: async () => { + await mastra.shutdown(); + await mcpClient.disconnect(); + }, + getToolCalls: () => [...toolCalls], + hasPendingApproval: () => pendingApproval !== null, + send: async (message, { maxSteps: stepLimit } = {}) => { + messages.push({ content: message, role: "user" }); + const output = await evalAgent.generate(messages, options(stepLimit)); + messages = output.messages; + rememberApproval(output); + trace.event({ text: output.text ?? "", type: "agent_text" }); + return { text: output.text }; + }, + }; + }, +}); + +export type { LeafAgentDriverConfig }; diff --git a/apps/leaf/tests/evals/harness/index.ts b/apps/leaf/tests/evals/harness/index.ts index c59c919c9..679de99ac 100644 --- a/apps/leaf/tests/evals/harness/index.ts +++ b/apps/leaf/tests/evals/harness/index.ts @@ -22,12 +22,17 @@ export type { } from "./createEvalContext.js"; export { createEvalContext } from "./createEvalContext.js"; export { createGenericMcpAgentDriver } from "./drivers/genericMcpAgent.js"; +export { + createLeafAgentDriver, + type LeafAgentDriverConfig, +} from "./drivers/leafAgent.js"; export type { EvalAgentDriver, EvalAgentOutput, EvalToolCall, RunningEvalDriver, } from "./drivers/types.js"; +export { approve, initEval, user } from "./initEval.js"; export { createEvalTrace } from "./tracing/createEvalTrace.js"; export type { EvalTrace, diff --git a/apps/leaf/tests/evals/harness/initEval.ts b/apps/leaf/tests/evals/harness/initEval.ts new file mode 100644 index 000000000..878bbef58 --- /dev/null +++ b/apps/leaf/tests/evals/harness/initEval.ts @@ -0,0 +1,117 @@ +import { Eval } from "braintrust"; +import type { AutumnMcpAuth } from "../../../../../packages/mcp/src/server/auth/auth.js"; +import type { EvalSetup } from "../fixtures/types.js"; +import { + standardEvalScores, + type EvalExpected, + type EvalScorer, +} from "../utils/scorers.js"; +import { + createEvalContext, + type EvalRunResult, + type EvalTurn, +} from "./createEvalContext.js"; +import type { AutumnApiMockOverrides } from "./context/types.js"; +import { createLeafAgentDriver } from "./drivers/leafAgent.js"; +import type { EvalAgentDriver } from "./drivers/types.js"; +import type { EvalTraceLevel } from "./tracing/types.js"; + +type EvalCaseMetadata = Record; + +type InitEvalCase = { + conversation: EvalTurn[]; + expect?: EvalExpected; + metadata?: Partial; + name?: string; +}; + +type InitEvalInput = { + conversation: EvalTurn[]; +}; + +type InitEvalOptions = { + auth?: Partial; + autumnApiOverrides?: AutumnApiMockOverrides; + cases: InitEvalCase[]; + driver?: EvalAgentDriver; + experimentName: string; + metadata: Metadata; + scores?: EvalScorer[]; + setup: EvalSetup; + timeout?: number; + today?: Date; + trace?: { level?: EvalTraceLevel }; +}; + +export const user = ({ + maxSteps, + message, +}: { + maxSteps?: number; + message: string; +}): EvalTurn => ({ + ...(maxSteps === undefined ? {} : { maxSteps }), + message, + type: "user", +}); + +export const approve = ({ + maxSteps, + optional = true, +}: { + maxSteps?: number; + optional?: boolean; +} = {}): EvalTurn => ({ + ...(maxSteps === undefined ? {} : { maxSteps }), + optional, + type: "approve", +}); + +export const initEval = ({ + auth, + autumnApiOverrides, + cases, + driver = createLeafAgentDriver(), + experimentName, + metadata, + scores = standardEvalScores(), + setup, + timeout = 45_000, + today, + trace, +}: InitEvalOptions) => + Eval( + "leaf", + { + experimentName, + data: cases.map((testCase) => ({ + expected: testCase.expect ?? {}, + input: { conversation: testCase.conversation }, + metadata: { + ...metadata, + ...testCase.metadata, + ...(testCase.name ? { caseName: testCase.name } : {}), + setup: setup.tag, + }, + })), + scores, + task: async (input) => { + const context = await createEvalContext({ + auth, + autumnApiOverrides, + driver, + name: experimentName, + setup, + today, + trace, + }); + try { + return await context.runConversation(input.conversation); + } finally { + await context.cleanup(); + } + }, + timeout, + }, + { noSendLogs: !process.env.BRAINTRUST_API_KEY }, + ); diff --git a/apps/leaf/tests/evals/harness/tracing/formatTrace.ts b/apps/leaf/tests/evals/harness/tracing/formatTrace.ts index 830cff26b..35e472480 100644 --- a/apps/leaf/tests/evals/harness/tracing/formatTrace.ts +++ b/apps/leaf/tests/evals/harness/tracing/formatTrace.ts @@ -10,6 +10,76 @@ const bodyOf = (value: Record) => ? (value.request as Record) : value; +const billingToolNames = new Set([ + "attach", + "createSchedule", + "previewAttach", + "previewCreateSchedule", +]); + +const monthNames = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; + +const looksLikeEpochMsField = (key: string, value: number) => + (value >= 946_684_800_000 && + value <= 4_102_444_800_000 && + (key.endsWith("_at") || + key.endsWith("_time") || + key === "timestamp" || + key === "date")) || + false; + +const formatEpochMs = (value: number) => { + const date = new Date(value); + const day = date.getUTCDate(); + const month = monthNames[date.getUTCMonth()]; + const year = date.getUTCFullYear(); + const hour = String(date.getUTCHours()).padStart(2, "0"); + const minute = String(date.getUTCMinutes()).padStart(2, "0"); + return `${day} ${month} ${year} ${hour}:${minute} UTC (${value})`; +}; + +const humanizeEpochMs = (value: unknown, key = ""): unknown => { + if (typeof value === "number" && looksLikeEpochMsField(key, value)) { + return formatEpochMs(value); + } + if (Array.isArray(value)) { + return value.map((item) => humanizeEpochMs(item)); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([entryKey, entryValue]) => [ + entryKey, + humanizeEpochMs(entryValue, entryKey), + ]), + ); + } + return value; +}; + +const formatJsonBody = ({ + body, + label, +}: { + body: Record; + label: string; +}) => { + const json = JSON.stringify(humanizeEpochMs(body), null, 2); + return json ? `\n[${label}]\n${json}` : ""; +}; + const compactFields = (body: Record) => [ ["customer", body.customer_id], @@ -31,13 +101,20 @@ const compactFields = (body: Record) => .join(" "); const formatToolCall = (call: EvalToolCall) => { - const fields = compactFields(bodyOf(call.args)); - return `[tool] ${call.name}${fields ? ` ${fields}` : ""}`; + const body = bodyOf(call.args); + const fields = compactFields(body); + const details = billingToolNames.has(call.name) + ? formatJsonBody({ body, label: "tool:body" }) + : ""; + return `[tool] ${call.name}${fields ? ` ${fields}` : ""}${details}`; }; const formatApiCall = (call: AutumnApiCall) => { const fields = compactFields(call.body); - return `[api] POST ${call.endpoint}${fields ? ` ${fields}` : ""}`; + const details = call.endpoint.startsWith("/v1/billing.") + ? formatJsonBody({ body: call.body, label: "api:body" }) + : ""; + return `[api] POST ${call.endpoint}${fields ? ` ${fields}` : ""}${details}`; }; const summarizeRecord = (record: Record) => diff --git a/apps/leaf/tests/evals/mcp/billing/multi-year-schedule.eval.ts b/apps/leaf/tests/evals/mcp/billing/multi-year-schedule.eval.ts new file mode 100644 index 000000000..6305f5f59 --- /dev/null +++ b/apps/leaf/tests/evals/mcp/billing/multi-year-schedule.eval.ts @@ -0,0 +1,329 @@ +import { Eval } from "braintrust"; +import { withCustomers } from "../../fixtures/createSetup.js"; +import { orgSetups } from "../../fixtures/orgSetups.js"; +import { + createEvalContext, + createGenericMcpAgentDriver, +} from "../../harness/index.js"; +import { + type EvalExpected, + type EvalOutput, + expectedApiCalls, + expectedToolCalls, + finalTextIncludes, + noCreateScheduleBeforePreview, + noScheduleCalls, +} from "../../utils/scorers.js"; + +type EvalInput = { + approval?: string; + details?: string; + prompt: string; +}; + +type EvalMetadata = { + domain: "billing"; + scenario: "multi-year-sales-led-schedule"; + setup: string; + source: "customer-slack-scenario-mining"; +}; + +type EvalScoreArgs = { + expected?: EvalExpected; + output: EvalOutput; +}; + +const experimentName = "multi-year-schedule"; +const evalToday = new Date("2026-06-08T00:00:00.000Z"); +const addUtcYears = ({ + date, + years, +}: { + date: Date; + years: number; +}) => + new Date( + Date.UTC( + date.getUTCFullYear() + years, + date.getUTCMonth(), + date.getUTCDate(), + date.getUTCHours(), + date.getUTCMinutes(), + date.getUTCSeconds(), + date.getUTCMilliseconds(), + ), + ); +const phaseStart = (yearsFromToday: number) => + addUtcYears({ date: evalToday, years: yearsFromToday }).getTime(); + +const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers, plans, subscriptions }) => ({ + account: customers.active({ + email: "finance@northwind.example", + id: "cus_sales_led_schedule", + name: "Northwind Labs", + subscriptions: [ + subscriptions.active({ + currentPeriodEnd: addUtcYears({ date: evalToday, years: 1 }), + currentPeriodStart: evalToday, + id: "sub_sales_led_enterprise", + plan: plans.enterprise, + }), + ], + }), + }), +}); +const customer = setup.refs.customers.account; +const enterprisePlan = setup.refs.plans.enterprise; + +const expectedPhases = [ + { + plans: [ + { + customize: { + price: { amount: 100_000, interval: "year" }, + }, + plan_id: enterprisePlan.id, + }, + ], + starts_at: phaseStart(0), + }, + { + plans: [ + { + customize: { + price: { amount: 125_000, interval: "year" }, + }, + plan_id: enterprisePlan.id, + }, + ], + starts_at: phaseStart(1), + }, + { + plans: [ + { + customize: { + price: { amount: 150_000, interval: "year" }, + }, + plan_id: enterprisePlan.id, + }, + ], + starts_at: phaseStart(2), + }, +]; + +const usesOnlyPriceOverrides = ({ output }: { output: EvalOutput }) => { + const scheduleCalls = output.apiCalls.filter( + (call) => + call.toolName === "previewCreateSchedule" || + call.toolName === "createSchedule", + ); + if (!scheduleCalls.length) return 0; + + return scheduleCalls.every((call) => + Array.isArray(call.body.phases) + ? call.body.phases.every((phase) => { + const phaseRecord = phase as Record; + return Array.isArray(phaseRecord.plans) + ? phaseRecord.plans.every((plan) => { + const planRecord = plan as Record; + const customize = planRecord.customize as + | Record + | undefined; + return ( + planRecord.feature_quantities === undefined && + customize?.items === undefined && + customize?.price !== undefined + ); + }) + : false; + }) + : false, + ) + ? 1 + : 0; +}; + +Eval( + "leaf", + { + experimentName, + data: [ + { + expected: { + apiCalls: [ + { toolName: "listCustomers" }, + { toolName: "listPlans" }, + { toolName: "listFeatures" }, + { + body: { customer_id: customer.id }, + toolName: "getCustomer", + }, + { + body: { + customer_id: customer.id, + phases: expectedPhases, + redirect_mode: "if_required", + }, + toolName: "previewCreateSchedule", + }, + { + body: { + customer_id: customer.id, + phases: expectedPhases, + redirect_mode: "if_required", + }, + toolName: "createSchedule", + }, + ], + finalTextIncludes: [ + "Northwind Labs", + "Enterprise", + "2026", + "2027", + "2028", + "credits", + "unchanged", + ], + toolCalls: [ + "listCustomers", + "listPlans", + "listFeatures", + "getCustomer", + "previewCreateSchedule", + "createSchedule", + ], + }, + input: { + approval: "Looks good, create the schedule.", + details: [ + "Use customer Northwind Labs, customer id cus_sales_led_schedule.", + "Use the Enterprise plan at customer level.", + "Contract starts today, June 8, 2026.", + "Year 1 is $100,000/year starting June 8, 2026.", + "Year 2 is $125,000/year starting one year from today, June 8, 2027.", + "Year 3 is $150,000/year starting two years from today, June 8, 2028.", + "Credits and feature access stay unchanged in every year.", + "Do not send an invoice or checkout now; preview the schedule first.", + ].join(" "), + prompt: + "Northwind Labs has a three-year sales-led Enterprise schedule. Please provision it in Autumn; the annual price changes each year, but credits do not change.", + }, + metadata: { + domain: "billing", + scenario: "multi-year-sales-led-schedule", + setup: setup.tag, + source: "customer-slack-scenario-mining", + }, + }, + { + expected: { + finalTextIncludes: ["Northwind Labs", "now", "past"], + }, + input: { + prompt: [ + "Please provision a three-year Enterprise schedule for Northwind Labs, customer id cus_sales_led_schedule.", + "Year 1 is $100,000/year, year 2 is $125,000/year, and year 3 is $150,000/year.", + "Credits and feature access stay unchanged in every year.", + ].join(" "), + }, + metadata: { + domain: "billing", + scenario: "multi-year-sales-led-schedule", + setup: setup.tag, + source: "customer-slack-scenario-mining", + }, + }, + ], + scores: [ + (args: EvalScoreArgs) => ({ + name: "Expected tool calls", + score: expectedToolCalls(args), + }), + (args: EvalScoreArgs) => ({ + name: "Expected API calls", + score: expectedApiCalls(args), + }), + (args: EvalScoreArgs) => ({ + name: "Final text includes", + score: finalTextIncludes(args), + }), + (args: EvalScoreArgs) => ({ + name: "Preview before create schedule", + score: noCreateScheduleBeforePreview(args), + }), + (args: EvalScoreArgs) => ({ + name: "Only price overrides", + score: + args.expected?.apiCalls?.some( + (call) => call.toolName === "previewCreateSchedule", + ) || + args.expected?.apiCalls?.some( + (call) => call.toolName === "createSchedule", + ) + ? usesOnlyPriceOverrides(args) + : 1, + }), + (args: EvalScoreArgs) => ({ + name: "No schedule calls without start clarification", + score: + args.expected?.apiCalls || args.expected?.toolCalls + ? 1 + : noScheduleCalls(args), + }), + ], + task: async (input: EvalInput) => { + const context = await createEvalContext({ + autumnApiOverrides: { + createSchedule: ({ body }) => ({ + customer_id: body.customer_id, + entity_id: null, + invoice: null, + payment_url: null, + phases: expectedPhases.map((phase, index) => ({ + customer_product_ids: [`cp_schedule_${index + 1}`], + phase_id: `phase_schedule_${index + 1}`, + starts_at: phase.starts_at, + })), + schedule_id: "sched_sales_led_multiyear", + status: "created", + }), + previewCreateSchedule: ({ body }) => ({ + currency: "usd", + customer_id: body.customer_id, + line_items: expectedPhases.map((phase, index) => ({ + description: `Enterprise year ${index + 1}`, + starts_at: phase.starts_at, + total: phase.plans[0].customize.price.amount, + })), + total: 375_000, + }), + }, + driver: createGenericMcpAgentDriver(), + name: experimentName, + setup, + today: evalToday, + }); + try { + const turns = [ + { message: input.prompt, type: "user" as const }, + ...(input.details + ? [{ message: input.details, type: "user" as const }] + : []), + ...(input.approval + ? [ + { message: input.approval, type: "user" as const }, + { optional: true, type: "approve" as const }, + ] + : []), + ]; + return await context.runConversation(turns); + } finally { + await context.cleanup(); + } + }, + timeout: 60_000, + }, + { noSendLogs: !process.env.BRAINTRUST_API_KEY }, +); diff --git a/apps/leaf/tests/evals/utils/scorers.ts b/apps/leaf/tests/evals/utils/scorers.ts index fa5906401..23468de7c 100644 --- a/apps/leaf/tests/evals/utils/scorers.ts +++ b/apps/leaf/tests/evals/utils/scorers.ts @@ -1,7 +1,17 @@ +import type { AutumnApiCall } from "../harness/context/types.js"; import type { - AutumnApiCall, - AutumnEvalToolName, -} from "../harness/context/types.js"; + EvalExpected, + EvalExpectation, + ExpectedApiCall, + LegacyEvalExpected, +} from "../fixtures/expectations/types.js"; + +export type { + EvalExpected, + EvalExpectation, + ExpectedApiCall, + LegacyEvalExpected, +} from "../fixtures/expectations/types.js"; export type EvalOutput = { apiCalls: AutumnApiCall[]; @@ -9,13 +19,14 @@ export type EvalOutput = { toolCalls: Array<{ name: string; args: Record }>; }; -export type EvalExpected = { - apiCalls?: Array<{ - body?: Record; - toolName: AutumnEvalToolName; - }>; - finalTextIncludes?: string[]; - toolCalls?: AutumnEvalToolName[]; +export type EvalScoreArgs = { + expected?: EvalExpected; + output: EvalOutput; +}; + +export type EvalScorer = (args: EvalScoreArgs) => { + name: string; + score: number; }; const includesObject = ( @@ -28,34 +39,101 @@ const includesObject = ( : actual[key] === value, ); +const isExpectationList = ( + expected?: EvalExpected, +): expected is EvalExpectation[] => Array.isArray(expected); + +const getLegacyExpected = ( + expected?: EvalExpected, +): LegacyEvalExpected | undefined => + isExpectationList(expected) ? undefined : expected; + +const getExpectationList = (expected?: EvalExpected): EvalExpectation[] => + isExpectationList(expected) ? expected : []; + +const getExpectedToolNames = (expected?: EvalExpected) => [ + ...(getLegacyExpected(expected)?.toolCalls ?? []), + ...getExpectationList(expected).flatMap((expectation) => + expectation.type === "tools.called" ? expectation.toolNames : [], + ), +]; + +const getExpectedApiCalls = (expected?: EvalExpected) => [ + ...(getLegacyExpected(expected)?.apiCalls ?? []), + ...getExpectationList(expected).flatMap((expectation) => + expectation.type === "api.called" || + expectation.type === "api.calledInOrder" + ? expectation.calls + : [], + ), +]; + +const getExpectedApiCallOrder = (expected?: EvalExpected) => + getExpectationList(expected).flatMap((expectation) => + expectation.type === "api.calledInOrder" ? [expectation.calls] : [], + ); + +const getExpectedResponsePhrases = (expected?: EvalExpected) => [ + ...(getLegacyExpected(expected)?.finalTextIncludes ?? []), + ...getExpectationList(expected).flatMap((expectation) => + expectation.type === "response.mentions" ? expectation.phrases : [], + ), +]; + +const matchesApiCall = ({ + actual, + expected, +}: { + actual: AutumnApiCall; + expected: ExpectedApiCall; +}) => + actual.toolName === expected.toolName && + (!expected.body || includesObject(actual.body, expected.body)); + export const expectedApiCalls = ({ expected, output, -}: { - expected?: EvalExpected; - output: EvalOutput; -}) => { - const expectedCalls = expected?.apiCalls ?? []; +}: EvalScoreArgs) => { + const expectedCalls = getExpectedApiCalls(expected); if (!expectedCalls.length) return 1; return expectedCalls.every((expectedCall) => - output.apiCalls.some( - (call) => - call.toolName === expectedCall.toolName && - (!expectedCall.body || includesObject(call.body, expectedCall.body)), + output.apiCalls.some((call) => + matchesApiCall({ actual: call, expected: expectedCall }), ), ) ? 1 : 0; }; +export const expectedApiCallsInOrder = ({ + expected, + output, +}: EvalScoreArgs) => { + const expectedCallGroups = getExpectedApiCallOrder(expected); + if (!expectedCallGroups.length) return 1; + + return expectedCallGroups.every((expectedCalls) => { + let startIndex = 0; + for (const expectedCall of expectedCalls) { + const foundIndex = output.apiCalls.findIndex( + (call, index) => + index >= startIndex && + matchesApiCall({ actual: call, expected: expectedCall }), + ); + if (foundIndex === -1) return false; + startIndex = foundIndex + 1; + } + return true; + }) + ? 1 + : 0; +}; + export const expectedToolCalls = ({ expected, output, -}: { - expected?: EvalExpected; - output: EvalOutput; -}) => { - const expectedTools = expected?.toolCalls ?? []; +}: EvalScoreArgs) => { + const expectedTools = getExpectedToolNames(expected); if (!expectedTools.length) return 1; return expectedTools.every((toolName) => output.toolCalls.some((call) => call.name === toolName), @@ -67,11 +145,8 @@ export const expectedToolCalls = ({ export const finalTextIncludes = ({ expected, output, -}: { - expected?: EvalExpected; - output: EvalOutput; -}) => { - const phrases = expected?.finalTextIncludes ?? []; +}: EvalScoreArgs) => { + const phrases = getExpectedResponsePhrases(expected); if (!phrases.length) return 1; const text = output.finalText.toLowerCase(); return phrases.every((phrase) => text.includes(phrase.toLowerCase())) ? 1 : 0; @@ -89,3 +164,62 @@ export const noAttachBeforePreview = ({ output }: { output: EvalOutput }) => { ? 1 : 0; }; + +export const noCreateScheduleBeforePreview = ({ + output, +}: { + output: EvalOutput; +}) => { + const createIndex = output.apiCalls.findIndex( + (call) => call.toolName === "createSchedule", + ); + const previewIndex = output.apiCalls.findIndex( + (call) => call.toolName === "previewCreateSchedule", + ); + return createIndex === -1 || + (previewIndex !== -1 && previewIndex < createIndex) + ? 1 + : 0; +}; + +export const noScheduleCalls = ({ output }: { output: EvalOutput }) => + output.apiCalls.every( + (call) => + call.toolName !== "previewCreateSchedule" && + call.toolName !== "createSchedule", + ) && + output.toolCalls.every( + (call) => + call.name !== "previewCreateSchedule" && call.name !== "createSchedule", + ) + ? 1 + : 0; + +export const standardEvalScores = (): EvalScorer[] => [ + (args) => ({ + name: "Expected tool calls", + score: expectedToolCalls(args), + }), + (args) => ({ + name: "Expected API calls", + score: expectedApiCalls(args), + }), + (args) => ({ + name: "Expected API call order", + score: expectedApiCallsInOrder(args), + }), + (args) => ({ + name: "Final text includes", + score: finalTextIncludes(args), + }), +]; + +export const billingAttachScores = (): EvalScorer[] => standardEvalScores(); + +export const billingScheduleScores = (): EvalScorer[] => [ + ...standardEvalScores(), + (args) => ({ + name: "Preview before create schedule", + score: noCreateScheduleBeforePreview(args), + }), +]; diff --git a/bun.lock b/bun.lock index d7a9cf06b..901f47543 100644 --- a/bun.lock +++ b/bun.lock @@ -251,6 +251,9 @@ "packages/auth": { "name": "@autumn/auth", "version": "0.0.1", + "dependencies": { + "@autumn/shared": "workspace:*", + }, "devDependencies": { "@types/bun": "^1.2.13", "@types/node": "^18.19.3", diff --git a/packages/mcp/src/resources/billing/billing-safety.md b/packages/mcp/src/resources/billing/billing-safety.md index b5b3db0f7..fce2bb107 100644 --- a/packages/mcp/src/resources/billing/billing-safety.md +++ b/packages/mcp/src/resources/billing/billing-safety.md @@ -16,7 +16,7 @@ Billing mutations must be preview-first. - Use previewCreateSchedule before createSchedule. - Use previewCreateBalance before createBalance. - Use createSchedule only after the user confirms the ordered phases, timing, and preview. -- When using invoice_mode, usually set enable_plan_immediately true unless the user explicitly mentions otherwise. +- Default paid billing changes should use a draft invoice: set enable_plan_immediately true and invoice_mode enabled true, enable_plan_immediately true, finalize false. Only change if the user specifies otherwise. - invoice_mode requires customer email; if missing, ask for it and call updateCustomer with customer_id and email before billing. - Use listFeatures only when customizing plan items or passing non-zero prepaid feature_quantities and the required feature ids/types are not already known. - Use previewAttach before attach, including feature_quantities, custom prices/items, reset intervals, discounts, and checkout behavior. diff --git a/packages/mcp/src/resources/billing/schedules.md b/packages/mcp/src/resources/billing/schedules.md index eb1803d9a..9b76684c3 100644 --- a/packages/mcp/src/resources/billing/schedules.md +++ b/packages/mcp/src/resources/billing/schedules.md @@ -17,7 +17,7 @@ Before creating a schedule, resolve: - plans in each phase, including versions, feature quantities, and customizations - redirect_mode, success_url, invoice_mode, and checkout behavior if payment may be required -When using invoice_mode, usually set enable_plan_immediately true unless the user explicitly wants access to wait for payment. +Default paid schedule billing should use a draft invoice: set enable_plan_immediately true and invoice_mode enabled true, enable_plan_immediately true, finalize false. invoice_mode requires customer email; if missing, ask for it and call updateCustomer with customer_id and email before billing. Use listFeatures only when a phase customizes plan items or sets non-zero prepaid feature_quantities and the exact feature ids or types are not already known. Scheduling an existing plan as-is does not need feature lookup. diff --git a/packages/mcp/src/tools/billing.ts b/packages/mcp/src/tools/billing.ts index 559f79a2f..59238235b 100644 --- a/packages/mcp/src/tools/billing.ts +++ b/packages/mcp/src/tools/billing.ts @@ -49,38 +49,80 @@ const domain = { billingPreviews: [ billingPreview({ id: "previewAttach", - description: - "Preview attaching a plan before attach. Include feature_quantities and custom items/prices; map recurring custom grants like 'per month/year' to reset.interval. When using invoice_mode, usually set enable_plan_immediately true unless the user explicitly wants access to wait for payment. invoice_mode requires customer email; if missing, ask for it and call updateCustomer with customer_id and email before billing.", + description: ` +- Preview attaching a plan before attach. +- Include feature_quantities and custom items/prices. +- Map recurring custom grants like 'per month/year' to reset.interval. +- Default paid attach billing: set enable_plan_immediately true and invoice_mode enabled true, enable_plan_immediately true, finalize false. +- Only change the default invoice mode if the user asks for checkout, immediate finalization/payment, no invoice, or delayed access. +- invoice_mode requires customer email; if missing, call updateCustomer first. +`.trim(), writeToolName: "attach", }), billingPreview({ id: "previewUpdateSubscription", - description: - "Preview updating a subscription before updateSubscription. Include quantity/custom item changes; recurring custom grants need reset.interval.", + description: ` +- Preview updating a subscription before updateSubscription. +- Include quantity and custom item changes. +- Recurring custom grants need reset.interval. +`.trim(), writeToolName: "updateSubscription", }), billingPreview({ id: "previewCreateSchedule", - description: - "Preview billing impact of a multi-phase schedule before createSchedule. starts_at accepts epoch milliseconds or ISO/date strings; preserve exact calendar dates from the user or contract. Use redirect_mode if_required unless the user explicitly asks to disable checkout/redirects. If using invoice_mode, usually set enable_plan_immediately true unless the user explicitly wants access to wait for payment. invoice_mode requires customer email; if missing, ask for it and call updateCustomer with customer_id and email before billing. If changing an existing/customer contract schedule, inspect the customer first. For schedules, put phase-specific feature quantities and contract feature limits/overrides in plan.customize.items, not feature_quantities; map 'per month/year' to reset.interval month/year. If the user says year 1 is already paid or should have no billing changes, do not add a year-1 phase; start phases at the first future billing change.", + description: ` +- Preview billing impact of a multi-phase schedule before createSchedule. +- First phase starts_at must be explicit: now or a past/backdated date. +- Do not infer first starts_at from 'year 1' or use a future first phase. +- Ask before previewing if first phase start is unclear. +- Preserve exact user/contract dates for later phases. +- Use redirect_mode if_required unless user asks otherwise. +- Default paid schedule billing: set enable_plan_immediately true and invoice_mode enabled true, enable_plan_immediately true, finalize false. +- Only change the default invoice mode if the user asks for checkout, immediate finalization/payment, no invoice, or delayed access. +- invoice_mode requires customer email; if missing, call updateCustomer first. +- Inspect customer first when changing an existing/customer contract schedule. +- Put schedule feature overrides in plan.customize.items, not feature_quantities. +- Map recurring grants like 'per month/year' to reset.interval month/year. +- If year 1 is already paid/no billing changes, omit it. +`.trim(), writeToolName: "createSchedule", }), ], confirmedWrites: [ confirmedWrite({ id: "attach", - description: - "Attach a plan to a customer. Destructive: preview first; preserve feature_quantities, custom prices/items, reset intervals, discounts, and checkout behavior. When using invoice_mode, usually set enable_plan_immediately true unless the user explicitly wants access to wait for payment. invoice_mode requires customer email; if missing, ask for it and call updateCustomer with customer_id and email before billing.", + description: ` +- Attach a plan to a customer. +- Destructive: preview first. +- Preserve feature_quantities, custom prices/items, reset intervals, discounts, and checkout behavior. +- Preserve the previewed billing mode. Default paid attach billing uses enable_plan_immediately true and invoice_mode enabled true, enable_plan_immediately true, finalize false. +- invoice_mode requires customer email; if missing, call updateCustomer first. +`.trim(), }), confirmedWrite({ id: "updateSubscription", - description: - "Update a subscription. Destructive: preview first; preserve quantity/custom item changes and reset intervals from the previewed request.", + description: ` +- Update a subscription. +- Destructive: preview first. +- Preserve quantity/custom item changes and reset intervals from the previewed request. +`.trim(), }), confirmedWrite({ id: "createSchedule", - description: - "Create a multi-phase billing schedule. Destructive: preview first; preserve phase starts_at and redirect_mode values from the previewed request. Use redirect_mode if_required unless the user explicitly asks to disable checkout/redirects. When using invoice_mode, usually set enable_plan_immediately true unless the user explicitly wants access to wait for payment. invoice_mode requires customer email; if missing, ask for it and call updateCustomer with customer_id and email before billing. If changing an existing/customer contract schedule, inspect the customer first. For schedules, put phase-specific feature quantities and contract feature limits/overrides in plan.customize.items, not feature_quantities. If year 1 is already paid/no billing changes, do not add a year-1 phase; start at the first future billing change.", + description: ` +- Create a multi-phase billing schedule. +- Destructive: preview first. +- Preserve phase starts_at and redirect_mode values from the previewed request. +- First phase starts_at must be explicit: now or a past/backdated date. +- Do not infer first starts_at from 'year 1' or use a future first phase. +- Ask before creating if first phase start is unclear. +- Use redirect_mode if_required unless user asks otherwise. +- Preserve the previewed billing mode. Default paid schedule billing uses enable_plan_immediately true and invoice_mode enabled true, enable_plan_immediately true, finalize false. +- invoice_mode requires customer email; if missing, call updateCustomer first. +- Inspect customer first when changing an existing/customer contract schedule. +- Put schedule feature overrides in plan.customize.items, not feature_quantities. +- If year 1 is already paid/no billing changes, omit it. +`.trim(), }), ], } satisfies ToolDomain; diff --git a/packages/mcp/tests/unit/mcp-server/agent/server.test.ts b/packages/mcp/tests/unit/mcp-server/agent/server.test.ts index b82d6f267..3b83b9adb 100644 --- a/packages/mcp/tests/unit/mcp-server/agent/server.test.ts +++ b/packages/mcp/tests/unit/mcp-server/agent/server.test.ts @@ -87,12 +87,14 @@ describe("Autumn MCP server", () => { expect(billingSafety.contents[0]?.text).toContain( "invoice_mode requires customer email", ); + expect(billingSafety.contents[0]?.text).toContain("finalize false"); expect(billingSafety.contents[0]?.text).toContain("updateCustomer"); const schedules = await server.readResource("autumn://docs/schedules"); expect(schedules.contents[0]?.text).toContain( "invoice_mode requires customer email", ); + expect(schedules.contents[0]?.text).toContain("finalize false"); expect(schedules.contents[0]?.text).toContain("updateCustomer"); for (const uri of logResourceUris) { diff --git a/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts b/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts index fc7c32782..a0ae2f4b6 100644 --- a/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts +++ b/packages/mcp/tests/unit/mcp-server/agent/tools.test.ts @@ -54,22 +54,26 @@ describe("Autumn operation tools", () => { expect(tools.previewAttach.description).toContain( "enable_plan_immediately", ); + expect(tools.previewAttach.description).toContain("finalize false"); expect(tools.previewAttach.description).toContain( "invoice_mode requires customer email", ); expect(tools.attach.description).toContain("enable_plan_immediately"); + expect(tools.attach.description).toContain("finalize false"); expect(tools.attach.description).toContain( "invoice_mode requires customer email", ); expect(tools.previewCreateSchedule.description).toContain( "enable_plan_immediately", ); + expect(tools.previewCreateSchedule.description).toContain("finalize false"); expect(tools.previewCreateSchedule.description).toContain( "invoice_mode requires customer email", ); expect(tools.createSchedule.description).toContain( "enable_plan_immediately", ); + expect(tools.createSchedule.description).toContain("finalize false"); expect(tools.createSchedule.description).toContain( "invoice_mode requires customer email", );