chore: add multi entity attach eval

apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts

@@ -245,7 +245,9 @@ export const claudeManagedEngine: AgentEngine = {
 						conversationSpan.end();
 						span.log({
 							metadata: {
-								finish_reason: result.suspended ? "suspended" : "stop",
+								finish_reason: result.suspendedQueue?.length
+									? "suspended"
+									: "stop",
 							},
 							metrics: {
 								completion_tokens: result.usage.outputTokens,
@@ -269,7 +271,8 @@ export const claudeManagedEngine: AgentEngine = {
 			logger,
 			text: outcome.textParts.join("\n\n"),
 		});
-		if (outcome.errorMessage && !finalText && !outcome.suspended) {
+		const suspended = outcome.suspendedQueue?.[0];
+		if (outcome.errorMessage && !finalText && !suspended) {
 			throw new Error(`Claude Managed agent failed: ${outcome.errorMessage}`);
 		}
 
@@ -278,7 +281,7 @@ export const claudeManagedEngine: AgentEngine = {
 			context: { env },
 			data: {
 				cost_tokens: outcome.usage.inputTokens + outcome.usage.outputTokens,
-				finish_reason: outcome.suspended ? "suspended" : "stop",
+				finish_reason: suspended ? "suspended" : "stop",
 				resumed: !newSession,
 				run_id: activeSessionId,
 			},
@@ -286,14 +289,14 @@ export const claudeManagedEngine: AgentEngine = {
 
 		return {
 			env,
-			finishReason: outcome.suspended ? "suspended" : "stop",
+			finishReason: suspended ? "suspended" : "stop",
 			previewApproval: previewCapture.captured as PreviewApproval | undefined,
 			runId: activeSessionId,
-			suspendPayload: outcome.suspended
+			suspendPayload: suspended
 				? {
-						args: outcome.suspended.args,
-						toolCallId: outcome.suspended.toolCallId,
-						toolName: outcome.suspended.toolName,
+						args: suspended.args,
+						toolCallId: suspended.toolCallId,
+						toolName: suspended.toolName,
 					}
 				: undefined,
 			text: finalText,

apps/leaf/src/harness/claudeManaged/session/driveSessionTurn.ts

@@ -7,13 +7,16 @@ export type SessionTurnUsage = {
 	outputTokens: number;
 };
 
+export type SuspendedToolCall = {
+	args: Record<string, unknown>;
+	toolCallId: string;
+	toolName: string;
+};
+
 export type SessionTurnOutcome = {
 	errorMessage?: string;
-	suspended?: {
-		args: Record<string, unknown>;
-		toolCallId: string;
-		toolName: string;
-	};
+	/** All confirmations the turn is waiting on. */
+	suspendedQueue?: SuspendedToolCall[];
 	textParts: string[];
 	usage: SessionTurnUsage;
 };
@@ -106,16 +109,18 @@ export const driveSessionTurn = async ({
 			break;
 		} else if (event.type === "session.status_idle") {
 			if (event.stop_reason.type === "requires_action") {
-				const id =
-					event.stop_reason.event_ids.find((e) => pendingAsk.has(e)) ??
-					event.stop_reason.event_ids[0];
-				const call = id ? pendingAsk.get(id) : undefined;
-				if (id && call) {
-					outcome.suspended = {
-						args: call.input,
-						toolCallId: id,
-						toolName: call.name,
+				// Awaited ids can reference tool calls streamed in an earlier
+				// turn; surface them even without local metadata.
+				const queue = event.stop_reason.event_ids.map((eventId) => {
+					const call = pendingAsk.get(eventId);
+					return {
+						args: call?.input ?? {},
+						toolCallId: eventId,
+						toolName: call?.name ?? "unknown",
 					};
+				});
+				if (queue.length > 0) {
+					outcome.suspendedQueue = queue;
 				}
 			}
 			// requires_action, end_turn, and retries_exhausted are all turn-terminal.

apps/leaf/tests/evals/claudeManaged/multi-entity-contract.eval.ts

@@ -0,0 +1,189 @@
+// Multi-entity provisioning: one signed order form, two workspaces under the
+// same customer, a different package and price attached per entity.
+import { withCustomers } from "../fixtures/createSetup.js";
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../fixtures/expectations/index.js";
+import { orgSetups } from "../fixtures/orgSetups.js";
+import {
+	approve,
+	contractAttachment,
+	createClaudeManagedLiveDriver,
+	initEval,
+	user,
+} from "../harness/index.js";
+import { billingAttachScores } from "../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "attach";
+};
+
+const experimentName = "attach-multi-entity";
+
+const setup = withCustomers({
+	setup: orgSetups.knowledgePlatform(),
+	customers: ({ customers }) => ({
+		silvercrest: customers.base({
+			email: "ap@silvercrestmedia.example",
+			id: "silvercrest-media",
+			name: "Silvercrest Media, Inc.",
+		}),
+	}),
+	entities: ({ customers, entities, features }) => ({
+		newsroom: entities.base({
+			customer: customers.silvercrest,
+			feature: features.workspaces,
+			id: "silvercrest-newsroom",
+			name: "Newsroom",
+		}),
+		archive: entities.base({
+			customer: customers.silvercrest,
+			feature: features.workspaces,
+			id: "silvercrest-archive",
+			name: "Archive",
+		}),
+	}),
+});
+
+const customer = setup.refs.customers.silvercrest;
+
+const invoiceMode = {
+	enable_plan_immediately: true,
+	enabled: true,
+	finalize: false,
+	net_terms_days: 30,
+};
+
+const expectedNewsroomAttach = {
+	customer_id: customer.id,
+	customize: {
+		add_items: [
+			{
+				feature_id: setup.refs.features.hosted_solution.id,
+				unlimited: true,
+			},
+		],
+		price: { amount: 1_150, interval: "month" },
+		remove_items: [{ feature_id: setup.refs.features.revision_history.id }],
+	},
+	entity_id: setup.refs.entities.newsroom.id,
+	invoice_mode: invoiceMode,
+	plan_id: setup.refs.plans.enterprise.id,
+};
+
+const expectedArchiveAttach = {
+	customer_id: customer.id,
+	customize: {
+		price: { amount: 650, interval: "month" },
+		remove_items: [{ feature_id: setup.refs.features.compliance_controls.id }],
+	},
+	entity_id: setup.refs.entities.archive.id,
+	invoice_mode: invoiceMode,
+	plan_id: setup.refs.plans.scale.id,
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	metadata: {
+		domain: "billing",
+		flow: "attach",
+	},
+	driver: createClaudeManagedLiveDriver(),
+	scores: billingAttachScores(),
+	timeout: 300_000,
+	cases: [
+		{
+			name: "order form with two workspace packages attaches each entity separately",
+			conversation: [
+				user({
+					attachments: [
+						contractAttachment({ fixtureId: "multi-workspace-order" }),
+					],
+					message:
+						"I uploaded the signed order form for Silvercrest Media. Please provision both workspaces in Autumn.",
+				}),
+				user({ message: "Looks good, attach both." }),
+				approve(),
+				// Optional: the agent may batch both writes under one approval.
+				approve({ optional: true }),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listPlans",
+						"listFeatures",
+						"listCustomers",
+						"listEntities",
+						"previewAttach",
+						"attach",
+					],
+				}),
+				// Both entities exist; discover them, don't mint duplicates.
+				api.calledTimes({ call: { toolName: "createEntity" }, count: 0 }),
+				...billing.previewThenWrite({
+					body: expectedNewsroomAttach,
+					write: "attach",
+				}),
+				...billing.previewThenWrite({
+					body: expectedArchiveAttach,
+					write: "attach",
+				}),
+				api.calledTimes({
+					call: { body: expectedNewsroomAttach, toolName: "attach" },
+					count: 1,
+				}),
+				api.calledTimes({
+					call: { body: expectedArchiveAttach, toolName: "attach" },
+					count: 1,
+				}),
+				api.calledTimes({ call: { toolName: "attach" }, count: 2 }),
+				// Preview totals are not pinned; error-recovery re-previews are fine.
+				api.calledTimes({
+					call: { body: expectedNewsroomAttach, toolName: "previewAttach" },
+					count: 1,
+				}),
+				api.calledTimes({
+					call: { body: expectedArchiveAttach, toolName: "previewAttach" },
+					count: 1,
+				}),
+				// Single-term order form effective on provisioning: attach, not a schedule.
+				api.calledTimes({ call: { toolName: "createSchedule" }, count: 0 }),
+				api.calledTimes({
+					call: { toolName: "previewCreateSchedule" },
+					count: 0,
+				}),
+				// No backdating to signature dates, items PUT, trial, or prepaid quantities.
+				...(["previewAttach", "attach"] as const).map((toolName) =>
+					api.bodyExcludes({
+						fields: [
+							"starts_at",
+							"ends_at",
+							"feature_quantities",
+							"no_billing_changes",
+							"entity_data",
+							"customize.items",
+							"customize.free_trial",
+						],
+						toolName,
+					}),
+				),
+				response.mentions({
+					phrases: [
+						"Silvercrest",
+						"Newsroom",
+						"Archive",
+						"Enterprise",
+						"Scale",
+						"Hosted Solution",
+					],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/fixtures/expectations/api.ts

@@ -4,6 +4,7 @@ import type {
 	ApiCalledAfterApprovalExpectation,
 	ApiCalledExpectation,
 	ApiCalledInOrderExpectation,
+	ApiCalledTimesExpectation,
 	ExpectedApiCall,
 } from "./types.js";
 
@@ -38,6 +39,18 @@ export const api = {
 		call,
 		type: "api.calledAfterApproval",
 	}),
+	/** Calls matching `call` must occur exactly `count` times; 0 forbids a tool. */
+	calledTimes: ({
+		call,
+		count,
+	}: {
+		call: ExpectedApiCall;
+		count: number;
+	}): ApiCalledTimesExpectation => ({
+		call,
+		count,
+		type: "api.calledTimes",
+	}),
 	bodyExcludes: ({
 		fields,
 		toolName,

apps/leaf/tests/evals/fixtures/expectations/index.ts

@@ -8,6 +8,7 @@ export type {
 	ApiCalledAfterApprovalExpectation,
 	ApiCalledExpectation,
 	ApiCalledInOrderExpectation,
+	ApiCalledTimesExpectation,
 	EvalExpectation,
 	EvalExpected,
 	ExpectedApiCall,

apps/leaf/tests/evals/fixtures/expectations/types.ts

@@ -31,6 +31,12 @@ export type ApiCalledAfterApprovalExpectation = {
 	type: "api.calledAfterApproval";
 };
 
+export type ApiCalledTimesExpectation = {
+	call: ExpectedApiCall;
+	count: number;
+	type: "api.calledTimes";
+};
+
 export type ApiBodyExcludesExpectation = {
 	fields: string[];
 	toolName: AutumnEvalToolName;
@@ -72,6 +78,7 @@ export type EvalExpectation =
 	| ApiCalledAfterApprovalExpectation
 	| ApiCalledExpectation
 	| ApiCalledInOrderExpectation
+	| ApiCalledTimesExpectation
 	| ResponseAskedExpectation
 	| ResponseAskedBeforeToolExpectation
 	| ResponseConciseExpectation

apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts

@@ -2,6 +2,7 @@ import { mergeAgentRules, type PartialAgentRules } from "@autumn/shared";
 import { customers } from "../../fixtures/customers/index.js";
 import { entities } from "../../fixtures/entities/index.js";
 import { responses } from "../../fixtures/responses.js";
+import type { EvalSetup } from "../../fixtures/types.js";
 import type { EvalTrace } from "../tracing/types.js";
 import type { AutumnApiMock, AutumnApiMockOverrides } from "./types.js";
 
@@ -96,6 +97,24 @@ const normalizeScheduleBody = (body: Record<string, unknown>) => ({
 		: body.phases,
 });
 
+// The real API rejects entity ids the customer does not have.
+const findAttachEntityError = ({
+	body,
+	customerId,
+	setup,
+}: {
+	body: Record<string, unknown>;
+	customerId: string | null;
+	setup: EvalSetup;
+}) => {
+	const entityId = getString(body, "entity_id");
+	if (!entityId) return null;
+	const entity = setup.entities.find(
+		(entity) => entity.id === entityId && entity.customer_id === customerId,
+	);
+	return entity ? null : { error: `entity ${entityId} not found for customer` };
+};
+
 const defaultHandlers = {
 	attach: ({ body, setup }) => {
 		const customer = setup.customers.find(
@@ -105,6 +124,12 @@ const defaultHandlers = {
 			(plan) => plan.id === getString(body, "plan_id"),
 		);
 		if (!customer || !plan) return { error: "missing customer or plan" };
+		const entityError = findAttachEntityError({
+			body,
+			customerId: customer.id,
+			setup,
+		});
+		if (entityError) return entityError;
 		customer.subscriptions = [
 			...customer.subscriptions,
 			{
@@ -246,6 +271,12 @@ const defaultHandlers = {
 			(plan) => plan.id === getString(body, "plan_id"),
 		);
 		if (!customer || !plan) return { error: "missing customer or plan" };
+		const entityError = findAttachEntityError({
+			body,
+			customerId: customer.id,
+			setup,
+		});
+		if (entityError) return entityError;
 		return responses.attachPreview({ customer, plan, request: body });
 	},
 	previewCreateSchedule: ({ body, setup }) => {

apps/leaf/tests/evals/harness/drivers/claudeManagedLive/sessionDriver.ts

@@ -21,7 +21,7 @@ export const createLiveSessionDriver = ({
 	trace: EvalDriverStartInput["trace"];
 }) => {
 	const toolCalls: EvalToolCall[] = [];
-	let pendingToolUseId: string | undefined;
+	let pendingToolUseIds: string[] = [];
 
 	const runTurn = async ({
 		input,
@@ -30,7 +30,7 @@ export const createLiveSessionDriver = ({
 		input?: string;
 		kickoff: () => Promise<unknown>;
 	}) => {
-		pendingToolUseId = undefined;
+		pendingToolUseIds = [];
 		const turnSpan = currentSpan().startSpan({
 			name: input ? "user-turn" : "approval-turn",
 			type: "llm",
@@ -67,14 +67,16 @@ export const createLiveSessionDriver = ({
 			console.error("[cma-live] turn failed:", error);
 			throw error;
 		});
-		if (outcome.suspended) {
-			pendingToolUseId = outcome.suspended.toolCallId;
+		if (outcome.suspendedQueue?.length) {
+			pendingToolUseIds = outcome.suspendedQueue.map(
+				(call) => call.toolCallId,
+			);
 			trace.event({ type: "approval_pending" });
 		}
 		const text = outcome.textParts.join("\n\n");
 		turnSpan.log({ output: text });
 		turnSpan.end();
-		if (outcome.errorMessage && !text && !outcome.suspended) {
+		if (outcome.errorMessage && !text && !outcome.suspendedQueue?.length) {
 			throw new Error(`CMA live eval turn failed: ${outcome.errorMessage}`);
 		}
 		trace.event({ text, type: "agent_text" });
@@ -83,9 +85,9 @@ export const createLiveSessionDriver = ({
 
 	return {
 		approve: async () => {
-			if (!pendingToolUseId) throw new Error("No pending approval to approve.");
+			const [toolUseId] = pendingToolUseIds;
+			if (!toolUseId) throw new Error("No pending approval to approve.");
 			trace.event({ type: "approval_approved" });
-			const toolUseId = pendingToolUseId;
 			return runTurn({
 				kickoff: () =>
 					client.beta.sessions.events.send(sessionId, {
@@ -100,7 +102,7 @@ export const createLiveSessionDriver = ({
 			});
 		},
 		getToolCalls: () => [...toolCalls],
-		hasPendingApproval: () => pendingToolUseId !== undefined,
+		hasPendingApproval: () => pendingToolUseIds.length > 0,
 		send: async ({
 			attachments,
 			text,
@@ -108,21 +110,21 @@ export const createLiveSessionDriver = ({
 			attachments?: Attachment[];
 			text: string;
 		}) => {
-			if (pendingToolUseId) {
-				const toDeny = pendingToolUseId;
-				pendingToolUseId = undefined;
+			// Denying can surface further queued confirmations; drain them all
+			// before the session will accept a plain user message.
+			while (pendingToolUseIds.length > 0) {
+				const toDeny = [...pendingToolUseIds];
+				pendingToolUseIds = [];
 				await runTurn({
 					kickoff: () =>
 						client.beta.sessions.events.send(sessionId, {
-							events: [
-								{
-									deny_message:
-										"Preview and wait for explicit user confirmation before writing.",
-									result: "deny",
-									tool_use_id: toDeny,
-									type: "user.tool_confirmation",
-								},
-							],
+							events: toDeny.map((toolUseId) => ({
+								deny_message:
+									"Preview and wait for explicit user confirmation before writing.",
+								result: "deny",
+								tool_use_id: toolUseId,
+								type: "user.tool_confirmation",
+							})),
 						}),
 				});
 			}

apps/leaf/tests/evals/utils/scorers.ts

@@ -133,6 +133,11 @@ const getExpectedApiBodyExclusions = (expected?: EvalExpected) =>
 		expectation.type === "api.bodyExcludes" ? [expectation] : [],
 	);
 
+const getExpectedApiCallTimes = (expected?: EvalExpected) =>
+	getExpectationList(expected).flatMap((expectation) =>
+		expectation.type === "api.calledTimes" ? [expectation] : [],
+	);
+
 const getExpectedApiBodyNumberFields = (expected?: EvalExpected) =>
 	getExpectationList(expected).flatMap((expectation) =>
 		expectation.type === "api.bodyNumberFields" ? [expectation] : [],
@@ -275,6 +280,19 @@ export const expectedApiCallsAfterApproval = ({
 		: 0;
 };
 
+export const expectedApiCallTimes = ({ expected, output }: EvalScoreArgs) => {
+	const expectations = getExpectedApiCallTimes(expected);
+	if (!expectations.length) return 1;
+	return expectations.every(
+		(expectation) =>
+			output.apiCalls.filter((call) =>
+				matchesApiCall({ actual: call, expected: expectation.call }),
+			).length === expectation.count,
+	)
+		? 1
+		: 0;
+};
+
 export const expectedToolCalls = ({ expected, output }: EvalScoreArgs) => {
 	const expectedTools = getExpectedToolNames(expected);
 	if (!expectedTools.length) return 1;
@@ -295,7 +313,13 @@ export const expectedApiBodyExclusions = ({
 		output.apiCalls
 			.filter((call) => call.toolName === exclusion.toolName)
 			.every((call) =>
-				exclusion.fields.every((field) => !(field in call.body)),
+				exclusion.fields.every((field) =>
+					field.includes(".")
+						? valuesAtPath({ path: field, value: call.body }).every(
+								(value: unknown) => value === undefined,
+							)
+						: !(field in call.body),
+				),
 			),
 	)
 		? 1
@@ -332,8 +356,7 @@ export const expectedApiBodyNumberFields = ({
 export const finalTextIncludes = ({ expected, output }: EvalScoreArgs) => {
 	const phrases = getExpectedResponsePhrases(expected);
 	if (!phrases.length) return 1;
-	const text = output.finalText.toLowerCase();
-	return phrases.every((phrase) => text.includes(phrase.toLowerCase())) ? 1 : 0;
+	return textMatches({ phrases, text: output.finalText }) ? 1 : 0;
 };
 
 export const askedClarification = ({ expected, output }: EvalScoreArgs) => {
@@ -518,6 +541,10 @@ const scorersByExpectationType: Record<EvalExpectation["type"], EvalScorer> = {
 		name: "Expected API calls after approval",
 		score: expectedApiCallsAfterApproval,
 	}),
+	"api.calledTimes": namedScorer({
+		name: "Expected API call counts",
+		score: expectedApiCallTimes,
+	}),
 	"api.bodyExcludes": namedScorer({
 		name: "Expected API body exclusions",
 		score: expectedApiBodyExclusions,

packages/mcp/src/resources/billing/billing-safety.md

@@ -28,7 +28,13 @@ Billing mutations must be preview-first and must carry the exact intended plan c
 - Use createPlan only after the user confirms the plan configuration.
 - Never claim a billing change was applied unless the write tool succeeds.
 
+Entity-scoped billing (agent rules attach_to_entities):
+- Resolve the target entity with listEntities before previewing; match by name when the user or a document names a workspace, seat, or team.
+- Entity ids come from listEntities, createEntity, or the user — never from document reference codes, SKUs, or invented slugs.
+- Create an entity only after listEntities for that customer confirms it does not exist.
+
 Custom plan mapping applies to attach, updateSubscription, and createSchedule:
+- Monetary amounts are major currency units regardless of contract formatting: $1,150.00 is amount 1150, never 115000.
 - Keep commercial terms separate from entitlements: selected plan or phase fees go in plan.customize.price; add_items, remove_items, and update_items are only for feature entitlements.
 - Year 1 / Year 2 fees in a 24-month order form are annual phase prices unless the contract says otherwise; do not ask for billing cadence.
 - Matching the plan name is not enough when the contract lists fees, limits, or features that define the purchased package.

packages/mcp/src/tools/entities.ts

@@ -26,8 +26,8 @@ const domain = {
 			id: "createEntity",
 			description: `
 - Create an entity under a customer.
-- Use when the user provides customer_id, entity_id, and entity name.
-- For entity-scoped attach, create missing entities before previewAttach.
+- Create only after listEntities for the customer confirms the entity does not exist.
+- Follow Billing Safety entity rules for entity-scoped billing.
 			`.trim(),
 			idempotent: true,
 		}),
@@ -36,7 +36,7 @@ const domain = {
 			description: `
 - List entities across the current org.
 - Pass customer_id to list entities for one customer.
-- Use before entity-scoped billing or balance work when entity ids are unknown.
+- Use before entity-scoped billing or balance work to resolve or verify entity ids.
 			`.trim(),
 		}),
 		operation({