diff --git a/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts b/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts index a78f5353f..49adc55c7 100644 --- a/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts +++ b/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts @@ -245,7 +245,9 @@ export const claudeManagedEngine: AgentEngine = { conversationSpan.end(); span.log({ metadata: { - finish_reason: result.suspended ? "suspended" : "stop", + finish_reason: result.suspendedQueue?.length + ? "suspended" + : "stop", }, metrics: { completion_tokens: result.usage.outputTokens, @@ -269,7 +271,8 @@ export const claudeManagedEngine: AgentEngine = { logger, text: outcome.textParts.join("\n\n"), }); - if (outcome.errorMessage && !finalText && !outcome.suspended) { + const suspended = outcome.suspendedQueue?.[0]; + if (outcome.errorMessage && !finalText && !suspended) { throw new Error(`Claude Managed agent failed: ${outcome.errorMessage}`); } @@ -278,7 +281,7 @@ export const claudeManagedEngine: AgentEngine = { context: { env }, data: { cost_tokens: outcome.usage.inputTokens + outcome.usage.outputTokens, - finish_reason: outcome.suspended ? "suspended" : "stop", + finish_reason: suspended ? "suspended" : "stop", resumed: !newSession, run_id: activeSessionId, }, @@ -286,14 +289,14 @@ export const claudeManagedEngine: AgentEngine = { return { env, - finishReason: outcome.suspended ? "suspended" : "stop", + finishReason: suspended ? "suspended" : "stop", previewApproval: previewCapture.captured as PreviewApproval | undefined, runId: activeSessionId, - suspendPayload: outcome.suspended + suspendPayload: suspended ? { - args: outcome.suspended.args, - toolCallId: outcome.suspended.toolCallId, - toolName: outcome.suspended.toolName, + args: suspended.args, + toolCallId: suspended.toolCallId, + toolName: suspended.toolName, } : undefined, text: finalText, diff --git a/apps/leaf/src/harness/claudeManaged/session/driveSessionTurn.ts b/apps/leaf/src/harness/claudeManaged/session/driveSessionTurn.ts index ec55709fa..5fe967dba 100644 --- a/apps/leaf/src/harness/claudeManaged/session/driveSessionTurn.ts +++ b/apps/leaf/src/harness/claudeManaged/session/driveSessionTurn.ts @@ -7,13 +7,16 @@ export type SessionTurnUsage = { outputTokens: number; }; +export type SuspendedToolCall = { + args: Record; + toolCallId: string; + toolName: string; +}; + export type SessionTurnOutcome = { errorMessage?: string; - suspended?: { - args: Record; - toolCallId: string; - toolName: string; - }; + /** All confirmations the turn is waiting on. */ + suspendedQueue?: SuspendedToolCall[]; textParts: string[]; usage: SessionTurnUsage; }; @@ -106,16 +109,18 @@ export const driveSessionTurn = async ({ break; } else if (event.type === "session.status_idle") { if (event.stop_reason.type === "requires_action") { - const id = - event.stop_reason.event_ids.find((e) => pendingAsk.has(e)) ?? - event.stop_reason.event_ids[0]; - const call = id ? pendingAsk.get(id) : undefined; - if (id && call) { - outcome.suspended = { - args: call.input, - toolCallId: id, - toolName: call.name, + // Awaited ids can reference tool calls streamed in an earlier + // turn; surface them even without local metadata. + const queue = event.stop_reason.event_ids.map((eventId) => { + const call = pendingAsk.get(eventId); + return { + args: call?.input ?? {}, + toolCallId: eventId, + toolName: call?.name ?? "unknown", }; + }); + if (queue.length > 0) { + outcome.suspendedQueue = queue; } } // requires_action, end_turn, and retries_exhausted are all turn-terminal. diff --git a/apps/leaf/tests/evals/claudeManaged/multi-entity-contract.eval.ts b/apps/leaf/tests/evals/claudeManaged/multi-entity-contract.eval.ts new file mode 100644 index 000000000..750901272 --- /dev/null +++ b/apps/leaf/tests/evals/claudeManaged/multi-entity-contract.eval.ts @@ -0,0 +1,189 @@ +// Multi-entity provisioning: one signed order form, two workspaces under the +// same customer, a different package and price attached per entity. +import { withCustomers } from "../fixtures/createSetup.js"; +import { + api, + billing, + response, + tools, +} from "../fixtures/expectations/index.js"; +import { orgSetups } from "../fixtures/orgSetups.js"; +import { + approve, + contractAttachment, + createClaudeManagedLiveDriver, + initEval, + user, +} from "../harness/index.js"; +import { billingAttachScores } from "../utils/scorers.js"; + +type EvalMetadata = { + domain: "billing"; + flow: "attach"; +}; + +const experimentName = "attach-multi-entity"; + +const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers }) => ({ + silvercrest: customers.base({ + email: "ap@silvercrestmedia.example", + id: "silvercrest-media", + name: "Silvercrest Media, Inc.", + }), + }), + entities: ({ customers, entities, features }) => ({ + newsroom: entities.base({ + customer: customers.silvercrest, + feature: features.workspaces, + id: "silvercrest-newsroom", + name: "Newsroom", + }), + archive: entities.base({ + customer: customers.silvercrest, + feature: features.workspaces, + id: "silvercrest-archive", + name: "Archive", + }), + }), +}); + +const customer = setup.refs.customers.silvercrest; + +const invoiceMode = { + enable_plan_immediately: true, + enabled: true, + finalize: false, + net_terms_days: 30, +}; + +const expectedNewsroomAttach = { + customer_id: customer.id, + customize: { + add_items: [ + { + feature_id: setup.refs.features.hosted_solution.id, + unlimited: true, + }, + ], + price: { amount: 1_150, interval: "month" }, + remove_items: [{ feature_id: setup.refs.features.revision_history.id }], + }, + entity_id: setup.refs.entities.newsroom.id, + invoice_mode: invoiceMode, + plan_id: setup.refs.plans.enterprise.id, +}; + +const expectedArchiveAttach = { + customer_id: customer.id, + customize: { + price: { amount: 650, interval: "month" }, + remove_items: [{ feature_id: setup.refs.features.compliance_controls.id }], + }, + entity_id: setup.refs.entities.archive.id, + invoice_mode: invoiceMode, + plan_id: setup.refs.plans.scale.id, +}; + +initEval({ + experimentName, + setup, + metadata: { + domain: "billing", + flow: "attach", + }, + driver: createClaudeManagedLiveDriver(), + scores: billingAttachScores(), + timeout: 300_000, + cases: [ + { + name: "order form with two workspace packages attaches each entity separately", + conversation: [ + user({ + attachments: [ + contractAttachment({ fixtureId: "multi-workspace-order" }), + ], + message: + "I uploaded the signed order form for Silvercrest Media. Please provision both workspaces in Autumn.", + }), + user({ message: "Looks good, attach both." }), + approve(), + // Optional: the agent may batch both writes under one approval. + approve({ optional: true }), + ], + expect: [ + tools.called({ + toolNames: [ + "getAgentRules", + "listPlans", + "listFeatures", + "listCustomers", + "listEntities", + "previewAttach", + "attach", + ], + }), + // Both entities exist; discover them, don't mint duplicates. + api.calledTimes({ call: { toolName: "createEntity" }, count: 0 }), + ...billing.previewThenWrite({ + body: expectedNewsroomAttach, + write: "attach", + }), + ...billing.previewThenWrite({ + body: expectedArchiveAttach, + write: "attach", + }), + api.calledTimes({ + call: { body: expectedNewsroomAttach, toolName: "attach" }, + count: 1, + }), + api.calledTimes({ + call: { body: expectedArchiveAttach, toolName: "attach" }, + count: 1, + }), + api.calledTimes({ call: { toolName: "attach" }, count: 2 }), + // Preview totals are not pinned; error-recovery re-previews are fine. + api.calledTimes({ + call: { body: expectedNewsroomAttach, toolName: "previewAttach" }, + count: 1, + }), + api.calledTimes({ + call: { body: expectedArchiveAttach, toolName: "previewAttach" }, + count: 1, + }), + // Single-term order form effective on provisioning: attach, not a schedule. + api.calledTimes({ call: { toolName: "createSchedule" }, count: 0 }), + api.calledTimes({ + call: { toolName: "previewCreateSchedule" }, + count: 0, + }), + // No backdating to signature dates, items PUT, trial, or prepaid quantities. + ...(["previewAttach", "attach"] as const).map((toolName) => + api.bodyExcludes({ + fields: [ + "starts_at", + "ends_at", + "feature_quantities", + "no_billing_changes", + "entity_data", + "customize.items", + "customize.free_trial", + ], + toolName, + }), + ), + response.mentions({ + phrases: [ + "Silvercrest", + "Newsroom", + "Archive", + "Enterprise", + "Scale", + "Hosted Solution", + ], + }), + ], + }, + ], +}); diff --git a/apps/leaf/tests/evals/fixtures/expectations/api.ts b/apps/leaf/tests/evals/fixtures/expectations/api.ts index dfcdc781c..ec42e11a6 100644 --- a/apps/leaf/tests/evals/fixtures/expectations/api.ts +++ b/apps/leaf/tests/evals/fixtures/expectations/api.ts @@ -4,6 +4,7 @@ import type { ApiCalledAfterApprovalExpectation, ApiCalledExpectation, ApiCalledInOrderExpectation, + ApiCalledTimesExpectation, ExpectedApiCall, } from "./types.js"; @@ -38,6 +39,18 @@ export const api = { call, type: "api.calledAfterApproval", }), + /** Calls matching `call` must occur exactly `count` times; 0 forbids a tool. */ + calledTimes: ({ + call, + count, + }: { + call: ExpectedApiCall; + count: number; + }): ApiCalledTimesExpectation => ({ + call, + count, + type: "api.calledTimes", + }), bodyExcludes: ({ fields, toolName, diff --git a/apps/leaf/tests/evals/fixtures/expectations/index.ts b/apps/leaf/tests/evals/fixtures/expectations/index.ts index 503301880..b561f72e0 100644 --- a/apps/leaf/tests/evals/fixtures/expectations/index.ts +++ b/apps/leaf/tests/evals/fixtures/expectations/index.ts @@ -8,6 +8,7 @@ export type { ApiCalledAfterApprovalExpectation, ApiCalledExpectation, ApiCalledInOrderExpectation, + ApiCalledTimesExpectation, EvalExpectation, EvalExpected, ExpectedApiCall, diff --git a/apps/leaf/tests/evals/fixtures/expectations/types.ts b/apps/leaf/tests/evals/fixtures/expectations/types.ts index dd610a9d9..c0ba11e35 100644 --- a/apps/leaf/tests/evals/fixtures/expectations/types.ts +++ b/apps/leaf/tests/evals/fixtures/expectations/types.ts @@ -31,6 +31,12 @@ export type ApiCalledAfterApprovalExpectation = { type: "api.calledAfterApproval"; }; +export type ApiCalledTimesExpectation = { + call: ExpectedApiCall; + count: number; + type: "api.calledTimes"; +}; + export type ApiBodyExcludesExpectation = { fields: string[]; toolName: AutumnEvalToolName; @@ -72,6 +78,7 @@ export type EvalExpectation = | ApiCalledAfterApprovalExpectation | ApiCalledExpectation | ApiCalledInOrderExpectation + | ApiCalledTimesExpectation | ResponseAskedExpectation | ResponseAskedBeforeToolExpectation | ResponseConciseExpectation diff --git a/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts b/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts index 46a3200eb..212469e69 100644 --- a/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts +++ b/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts @@ -2,6 +2,7 @@ import { mergeAgentRules, type PartialAgentRules } from "@autumn/shared"; import { customers } from "../../fixtures/customers/index.js"; import { entities } from "../../fixtures/entities/index.js"; import { responses } from "../../fixtures/responses.js"; +import type { EvalSetup } from "../../fixtures/types.js"; import type { EvalTrace } from "../tracing/types.js"; import type { AutumnApiMock, AutumnApiMockOverrides } from "./types.js"; @@ -96,6 +97,24 @@ const normalizeScheduleBody = (body: Record) => ({ : body.phases, }); +// The real API rejects entity ids the customer does not have. +const findAttachEntityError = ({ + body, + customerId, + setup, +}: { + body: Record; + customerId: string | null; + setup: EvalSetup; +}) => { + const entityId = getString(body, "entity_id"); + if (!entityId) return null; + const entity = setup.entities.find( + (entity) => entity.id === entityId && entity.customer_id === customerId, + ); + return entity ? null : { error: `entity ${entityId} not found for customer` }; +}; + const defaultHandlers = { attach: ({ body, setup }) => { const customer = setup.customers.find( @@ -105,6 +124,12 @@ const defaultHandlers = { (plan) => plan.id === getString(body, "plan_id"), ); if (!customer || !plan) return { error: "missing customer or plan" }; + const entityError = findAttachEntityError({ + body, + customerId: customer.id, + setup, + }); + if (entityError) return entityError; customer.subscriptions = [ ...customer.subscriptions, { @@ -246,6 +271,12 @@ const defaultHandlers = { (plan) => plan.id === getString(body, "plan_id"), ); if (!customer || !plan) return { error: "missing customer or plan" }; + const entityError = findAttachEntityError({ + body, + customerId: customer.id, + setup, + }); + if (entityError) return entityError; return responses.attachPreview({ customer, plan, request: body }); }, previewCreateSchedule: ({ body, setup }) => { diff --git a/apps/leaf/tests/evals/harness/drivers/claudeManagedLive/sessionDriver.ts b/apps/leaf/tests/evals/harness/drivers/claudeManagedLive/sessionDriver.ts index efeace000..1624a5e2d 100644 --- a/apps/leaf/tests/evals/harness/drivers/claudeManagedLive/sessionDriver.ts +++ b/apps/leaf/tests/evals/harness/drivers/claudeManagedLive/sessionDriver.ts @@ -21,7 +21,7 @@ export const createLiveSessionDriver = ({ trace: EvalDriverStartInput["trace"]; }) => { const toolCalls: EvalToolCall[] = []; - let pendingToolUseId: string | undefined; + let pendingToolUseIds: string[] = []; const runTurn = async ({ input, @@ -30,7 +30,7 @@ export const createLiveSessionDriver = ({ input?: string; kickoff: () => Promise; }) => { - pendingToolUseId = undefined; + pendingToolUseIds = []; const turnSpan = currentSpan().startSpan({ name: input ? "user-turn" : "approval-turn", type: "llm", @@ -67,14 +67,16 @@ export const createLiveSessionDriver = ({ console.error("[cma-live] turn failed:", error); throw error; }); - if (outcome.suspended) { - pendingToolUseId = outcome.suspended.toolCallId; + if (outcome.suspendedQueue?.length) { + pendingToolUseIds = outcome.suspendedQueue.map( + (call) => call.toolCallId, + ); trace.event({ type: "approval_pending" }); } const text = outcome.textParts.join("\n\n"); turnSpan.log({ output: text }); turnSpan.end(); - if (outcome.errorMessage && !text && !outcome.suspended) { + if (outcome.errorMessage && !text && !outcome.suspendedQueue?.length) { throw new Error(`CMA live eval turn failed: ${outcome.errorMessage}`); } trace.event({ text, type: "agent_text" }); @@ -83,9 +85,9 @@ export const createLiveSessionDriver = ({ return { approve: async () => { - if (!pendingToolUseId) throw new Error("No pending approval to approve."); + const [toolUseId] = pendingToolUseIds; + if (!toolUseId) throw new Error("No pending approval to approve."); trace.event({ type: "approval_approved" }); - const toolUseId = pendingToolUseId; return runTurn({ kickoff: () => client.beta.sessions.events.send(sessionId, { @@ -100,7 +102,7 @@ export const createLiveSessionDriver = ({ }); }, getToolCalls: () => [...toolCalls], - hasPendingApproval: () => pendingToolUseId !== undefined, + hasPendingApproval: () => pendingToolUseIds.length > 0, send: async ({ attachments, text, @@ -108,21 +110,21 @@ export const createLiveSessionDriver = ({ attachments?: Attachment[]; text: string; }) => { - if (pendingToolUseId) { - const toDeny = pendingToolUseId; - pendingToolUseId = undefined; + // Denying can surface further queued confirmations; drain them all + // before the session will accept a plain user message. + while (pendingToolUseIds.length > 0) { + const toDeny = [...pendingToolUseIds]; + pendingToolUseIds = []; await runTurn({ kickoff: () => client.beta.sessions.events.send(sessionId, { - events: [ - { - deny_message: - "Preview and wait for explicit user confirmation before writing.", - result: "deny", - tool_use_id: toDeny, - type: "user.tool_confirmation", - }, - ], + events: toDeny.map((toolUseId) => ({ + deny_message: + "Preview and wait for explicit user confirmation before writing.", + result: "deny", + tool_use_id: toolUseId, + type: "user.tool_confirmation", + })), }), }); } diff --git a/apps/leaf/tests/evals/utils/scorers.ts b/apps/leaf/tests/evals/utils/scorers.ts index a1162f10c..84e683abc 100644 --- a/apps/leaf/tests/evals/utils/scorers.ts +++ b/apps/leaf/tests/evals/utils/scorers.ts @@ -133,6 +133,11 @@ const getExpectedApiBodyExclusions = (expected?: EvalExpected) => expectation.type === "api.bodyExcludes" ? [expectation] : [], ); +const getExpectedApiCallTimes = (expected?: EvalExpected) => + getExpectationList(expected).flatMap((expectation) => + expectation.type === "api.calledTimes" ? [expectation] : [], + ); + const getExpectedApiBodyNumberFields = (expected?: EvalExpected) => getExpectationList(expected).flatMap((expectation) => expectation.type === "api.bodyNumberFields" ? [expectation] : [], @@ -275,6 +280,19 @@ export const expectedApiCallsAfterApproval = ({ : 0; }; +export const expectedApiCallTimes = ({ expected, output }: EvalScoreArgs) => { + const expectations = getExpectedApiCallTimes(expected); + if (!expectations.length) return 1; + return expectations.every( + (expectation) => + output.apiCalls.filter((call) => + matchesApiCall({ actual: call, expected: expectation.call }), + ).length === expectation.count, + ) + ? 1 + : 0; +}; + export const expectedToolCalls = ({ expected, output }: EvalScoreArgs) => { const expectedTools = getExpectedToolNames(expected); if (!expectedTools.length) return 1; @@ -295,7 +313,13 @@ export const expectedApiBodyExclusions = ({ output.apiCalls .filter((call) => call.toolName === exclusion.toolName) .every((call) => - exclusion.fields.every((field) => !(field in call.body)), + exclusion.fields.every((field) => + field.includes(".") + ? valuesAtPath({ path: field, value: call.body }).every( + (value: unknown) => value === undefined, + ) + : !(field in call.body), + ), ), ) ? 1 @@ -332,8 +356,7 @@ export const expectedApiBodyNumberFields = ({ export const finalTextIncludes = ({ expected, output }: EvalScoreArgs) => { const phrases = getExpectedResponsePhrases(expected); if (!phrases.length) return 1; - const text = output.finalText.toLowerCase(); - return phrases.every((phrase) => text.includes(phrase.toLowerCase())) ? 1 : 0; + return textMatches({ phrases, text: output.finalText }) ? 1 : 0; }; export const askedClarification = ({ expected, output }: EvalScoreArgs) => { @@ -518,6 +541,10 @@ const scorersByExpectationType: Record = { name: "Expected API calls after approval", score: expectedApiCallsAfterApproval, }), + "api.calledTimes": namedScorer({ + name: "Expected API call counts", + score: expectedApiCallTimes, + }), "api.bodyExcludes": namedScorer({ name: "Expected API body exclusions", score: expectedApiBodyExclusions, diff --git a/packages/mcp/src/resources/billing/billing-safety.md b/packages/mcp/src/resources/billing/billing-safety.md index 325464ba2..92063288b 100644 --- a/packages/mcp/src/resources/billing/billing-safety.md +++ b/packages/mcp/src/resources/billing/billing-safety.md @@ -28,7 +28,13 @@ Billing mutations must be preview-first and must carry the exact intended plan c - Use createPlan only after the user confirms the plan configuration. - Never claim a billing change was applied unless the write tool succeeds. +Entity-scoped billing (agent rules attach_to_entities): +- Resolve the target entity with listEntities before previewing; match by name when the user or a document names a workspace, seat, or team. +- Entity ids come from listEntities, createEntity, or the user — never from document reference codes, SKUs, or invented slugs. +- Create an entity only after listEntities for that customer confirms it does not exist. + Custom plan mapping applies to attach, updateSubscription, and createSchedule: +- Monetary amounts are major currency units regardless of contract formatting: $1,150.00 is amount 1150, never 115000. - Keep commercial terms separate from entitlements: selected plan or phase fees go in plan.customize.price; add_items, remove_items, and update_items are only for feature entitlements. - Year 1 / Year 2 fees in a 24-month order form are annual phase prices unless the contract says otherwise; do not ask for billing cadence. - Matching the plan name is not enough when the contract lists fees, limits, or features that define the purchased package. diff --git a/packages/mcp/src/tools/entities.ts b/packages/mcp/src/tools/entities.ts index 8914d28c0..012b416b3 100644 --- a/packages/mcp/src/tools/entities.ts +++ b/packages/mcp/src/tools/entities.ts @@ -26,8 +26,8 @@ const domain = { id: "createEntity", description: ` - Create an entity under a customer. -- Use when the user provides customer_id, entity_id, and entity name. -- For entity-scoped attach, create missing entities before previewAttach. +- Create only after listEntities for the customer confirms the entity does not exist. +- Follow Billing Safety entity rules for entity-scoped billing. `.trim(), idempotent: true, }), @@ -36,7 +36,7 @@ const domain = { description: ` - List entities across the current org. - Pass customer_id to list entities for one customer. -- Use before entity-scoped billing or balance work when entity ids are unknown. +- Use before entity-scoped billing or balance work to resolve or verify entity ids. `.trim(), }), operation({