Merge pull request #1904 from useautumn/charlie/agent3-evals-2

chore: agent slack evals stuff

apps/leaf/session-trace.ts

@@ -0,0 +1,27 @@
+import Anthropic from "@anthropic-ai/sdk";
+
+const sessionId = process.argv[2];
+if (!sessionId) throw new Error("usage: bun session-trace.ts <sessionId>");
+const client = new Anthropic();
+
+const events: any[] = [];
+for await (const event of client.beta.sessions.events.list(sessionId)) {
+	events.push(event);
+}
+
+let prev: number | null = null;
+for (const e of events) {
+	const ts = e.processed_at ? Date.parse(e.processed_at) : null;
+	const gap = ts && prev ? `+${((ts - prev) / 1000).toFixed(1)}s` : "";
+	prev = ts ?? prev;
+	const time = ts ? new Date(ts).toISOString().slice(11, 23) : "??";
+	let detail = "";
+	if (e.type === "agent.mcp_tool_use") detail = `${e.tool_name ?? e.name} ${JSON.stringify(e.input ?? {}).slice(0, 120)}`;
+	else if (e.type === "agent.mcp_tool_result") detail = `${e.tool_name ?? ""} ${JSON.stringify(e.content ?? e.result ?? {}).slice(0, 120)}`;
+	else if (e.type === "agent.message") detail = JSON.stringify(e.content ?? e.text ?? "").slice(0, 150);
+	else if (e.type?.startsWith("span.")) detail = JSON.stringify({ name: e.name, model: e.model, usage: e.usage }).slice(0, 150);
+	else if (e.type === "user.message") detail = JSON.stringify(e.content ?? "").slice(0, 100);
+	else detail = JSON.stringify(e).slice(0, 120);
+	console.log(`${time} ${gap.padStart(8)} ${e.type}  ${detail}`);
+}
+console.log(`\ntotal events: ${events.length}`);

apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts

@@ -5,8 +5,14 @@ import { claudeManagedConfig } from "../../../harness/claudeManaged/config.js";
 import { ensureLeafResources } from "../../../harness/claudeManaged/ensureLeafResources.js";
 import { ensureMemoryStore } from "../../../harness/claudeManaged/memory/ensureMemoryStore.js";
 import { cmaRepo } from "../../../harness/claudeManaged/repos/claudeManagedRepo.js";
-import { driveSessionTurn } from "../../../harness/claudeManaged/session/driveSessionTurn.js";
-import { buildUserMessageContent } from "../../../harness/claudeManaged/session/userMessage.js";
+import {
+	driveSessionTurn,
+	type SessionTurnOutcome,
+} from "../../../harness/claudeManaged/session/driveSessionTurn.js";
+import {
+	buildUserMessageContent,
+	type UserMessageContentBlock,
+} from "../../../harness/claudeManaged/session/userMessage.js";
 import { ensureAutumnVault } from "../../../harness/claudeManaged/vaults/ensureAutumnVault.js";
 import { containsSecret } from "../../../internal/sandbox/tool/guardrails.js";
 import { db } from "../../../lib/db.js";
@@ -14,6 +20,7 @@ import { createBraintrustLogger } from "../../../providers/braintrust/index.js";
 import { formatToolAction } from "../../tools/autumnMcp.js";
 import {
 	createPreviewCapture,
+	isSilentTool,
 	type PreviewApproval,
 } from "../../tools/toolPolicy.js";
 import type { AgentEngine, MessageContext, MessageParams } from "../types.js";
@@ -39,6 +46,29 @@ const redactSecrets = ({
 	return "[response withheld: it appeared to contain a credential]";
 };
 
+// Approval cards are executed by confirming a suspended session tool, so a
+// preview-only turn must be nudged into a real write-tool suspension.
+const buildNudgeText = ({ toolName }: { toolName: string }) =>
+	`Call the ${toolName} tool now with the exact args from your preview. It will pause for user approval automatically — do not ask for confirmation or repeat the summary.`;
+
+const mergeTurnOutcomes = (
+	first: SessionTurnOutcome,
+	second: SessionTurnOutcome,
+): SessionTurnOutcome => ({
+	errorMessage: second.errorMessage ?? first.errorMessage,
+	suspendedQueue: second.suspendedQueue,
+	textParts: [...first.textParts, ...second.textParts],
+	usage: {
+		cacheCreationInputTokens:
+			first.usage.cacheCreationInputTokens +
+			second.usage.cacheCreationInputTokens,
+		cacheReadInputTokens:
+			first.usage.cacheReadInputTokens + second.usage.cacheReadInputTokens,
+		inputTokens: first.usage.inputTokens + second.usage.inputTokens,
+		outputTokens: first.usage.outputTokens + second.usage.outputTokens,
+	},
+});
+
 // The agent's system prompt is env/thread-agnostic (one shared agent), so a new
 // session's first message carries the env + recent thread context.
 const buildMessageText = ({
@@ -153,7 +183,13 @@ export const claudeManagedEngine: AgentEngine = {
 		const previewCapture = createPreviewCapture();
 		const text = buildMessageText({ env, newSession, params });
 
-		const runTurn = ({ span }: { span?: Span }) => {
+		const driveTurn = ({
+			content,
+			span,
+		}: {
+			content: UserMessageContentBlock[];
+			span?: Span;
+		}) => {
 			const openToolSpans = new Map<string, Span>();
 			return driveSessionTurn({
 				autumnMcpServerName: claudeManagedConfig.autumnMcpServerName,
@@ -162,10 +198,7 @@ export const claudeManagedEngine: AgentEngine = {
 					client.beta.sessions.events.send(activeSessionId, {
 						events: [
 							{
-								content: buildUserMessageContent({
-									attachments: params.attachments,
-									text,
-								}),
+								content,
 								type: "user.message",
 							},
 						],
@@ -175,7 +208,9 @@ export const claudeManagedEngine: AgentEngine = {
 						event: "leaf.mcp_tool_called",
 						tool: name,
 					});
-					await onAction?.(formatToolAction({ args: input, toolName: name }));
+					if (!isSilentTool(name)) {
+						await onAction?.(formatToolAction({ args: input, toolName: name }));
+					}
 					previewCapture.onToolCall({ input, name });
 					if (span) {
 						openToolSpans.set(
@@ -197,6 +232,44 @@ export const claudeManagedEngine: AgentEngine = {
 			});
 		};
 
+		const runTurn = async ({ span }: { span?: Span }) => {
+			const first = await driveTurn({
+				content: buildUserMessageContent({
+					attachments: params.attachments,
+					text,
+				}),
+				span,
+			});
+			const captured = previewCapture.captured;
+			if (first.suspendedQueue?.length || first.errorMessage || !captured) {
+				return first;
+			}
+			// Preview-only turn: nudge once so the approval card comes from a real
+			// suspension (tool_use_id) instead of an unexecutable preview capture.
+			logger.info("Nudging Claude Managed agent to call write tool", {
+				event: "leaf.claude_managed_preview_nudge",
+				context: { env, org_id: org.id },
+				tool: captured.toolName,
+			});
+			const nudge = await driveTurn({
+				content: [
+					{
+						text: buildNudgeText({ toolName: captured.toolName }),
+						type: "text",
+					},
+				],
+				span,
+			});
+			if (!nudge.suspendedQueue?.length) {
+				logger.warn("Claude Managed agent did not suspend after nudge", {
+					event: "leaf.claude_managed_preview_nudge_failed",
+					context: { env, org_id: org.id },
+					tool: captured.toolName,
+				});
+			}
+			return mergeTurnOutcomes(first, nudge);
+		};
+
 		// One parent span per turn; child spans per Autumn tool; token usage as
 		// metrics. session_id + thread_id metadata link a thread's turns together.
 		const outcome = braintrustEnabled

apps/leaf/src/agent/tools/autumnMcp.ts

@@ -7,6 +7,7 @@ import { logger as rootLogger } from "../../lib/logger.js";
 import {
 	type createPreviewCapture,
 	getWriteToolForPreview,
+	isSilentTool,
 	toolLabel,
 } from "./toolPolicy.js";
 
@@ -95,10 +96,10 @@ export const formatToolAction = ({
 		args.request && typeof args.request === "object"
 			? (args.request as Record<string, unknown>)
 			: args;
+	// Only human-meaningful values — opaque ids (customer_id, entity_id) bloat
+	// the progress line without telling the reader anything.
 	const details = [
-		["customer", request.customer_id],
 		["plan", request.plan_id],
-		["entity", request.entity_id],
 		["search", request.search],
 	].flatMap(([label, value]) =>
 		typeof value === "string" && value ? [`${label}: ${value}`] : [],
@@ -145,7 +146,9 @@ export const getAutumnMcpTools = async ({
 					event: "leaf.mcp_tool_called",
 					tool: toolName,
 				});
-				await options.onToolCall?.(formatToolAction({ toolName, args }));
+				if (!isSilentTool(toolName)) {
+					await options.onToolCall?.(formatToolAction({ toolName, args }));
+				}
 				const result = await execute(args, ...rest);
 				if (getWriteToolForPreview(toolName)) {
 					logger.info("Captured Autumn MCP preview", {

apps/leaf/src/agent/tools/toolPolicy.ts

@@ -19,6 +19,15 @@ const previewWriteTools: Record<string, string> = {
 export const getWriteToolForPreview = (toolName: string) =>
 	previewWriteTools[toolName.replace(/^autumn_/, "")];
 
+// Pure utility tools the agent calls constantly — not worth a progress line.
+const silentTools = new Set([
+	"dateToEpochMilliseconds",
+	"epochMillisecondsToDate",
+]);
+
+export const isSilentTool = (toolName: string) =>
+	silentTools.has(normalizeToolName(toolName));
+
 export const toolLabel = (toolName: string) =>
 	labels[toolName.replace(/^autumn_/, "")] ??
 	toolName

apps/leaf/src/bot.ts

@@ -135,8 +135,11 @@ const runAndReply = async ({
 			return;
 		}
 
-		loading = await startLoading(target);
-		const logAction = createActionLogger(loading);
+		// Follow-up turns skip the Plan message so the user is only notified once
+		// (for the answer); progress shows via the silent typing status instead.
+		const isFollowUp = recentMessages?.some((m) => m.isBot) ?? false;
+		loading = await startLoading(target, { showPlan: !isFollowUp });
+		const logAction = createActionLogger(loading, target);
 		const rawFiles = getSlackFilesFromRaw({ raw });
 		const botToken = decrypt(installation.bot_access_token);
 		const output = await runMessage({

apps/leaf/src/internal/approvals/actions/approveAndRun.ts

@@ -2,6 +2,7 @@ import Anthropic from "@anthropic-ai/sdk";
 import { claudeManagedConfig } from "../../../harness/claudeManaged/config.js";
 import { driveSessionTurn } from "../../../harness/claudeManaged/session/driveSessionTurn.js";
 import { db } from "../../../lib/db.js";
+import { logger } from "../../../lib/logger.js";
 import { chatApprovalRepo } from "../repos/chatApprovalRepo.js";
 import { approvalErrorResult } from "../utils/approvalErrors.js";
 
@@ -47,6 +48,12 @@ export const approveAndRun = async ({
 		});
 		const text = outcome.textParts.join("\n\n");
 		const failed = Boolean(outcome.errorMessage) && !text;
+		if (failed) {
+			logger.error("[chat] Approval run failed", outcome.errorMessage, {
+				event: "leaf.approval_run_failed",
+				approval_id: approvalId,
+			});
+		}
 		await chatApprovalRepo.finalize({
 			approvalId,
 			db,
@@ -55,6 +62,10 @@ export const approveAndRun = async ({
 		});
 		return failed ? approvalErrorResult(outcome.errorMessage) : { text };
 	} catch (error) {
+		logger.error("[chat] Approval run failed", error, {
+			event: "leaf.approval_run_failed",
+			approval_id: approvalId,
+		});
 		await chatApprovalRepo.finalize({
 			approvalId,
 			db,

apps/leaf/src/internal/approvals/actions/postApprovalRequest.ts

@@ -36,6 +36,17 @@ export const postApprovalRequest = async ({
 	const approval = approvalRequestFromOutput(output);
 	if (!approval) return false;
 
+	// approveAndRun can only confirm a suspended session tool, so a card missing
+	// either id would always fail at approval time — fall back to plain text.
+	if (!approval.runId || !approval.toolCallId) {
+		logger.warn("Skipped unexecutable approval request", {
+			event: "leaf.approval_unexecutable_skipped",
+			context: { env: approval.env, org_id: installation.org_id },
+			tool: approval.toolName,
+		});
+		return false;
+	}
+
 	const approvalId = await chatApprovalRepo.insert({
 		db,
 		data: {

apps/leaf/src/ui/progress.ts

@@ -4,9 +4,15 @@ import { Plan } from "chat";
 export type ReplyTarget = Thread | Channel;
 export type LoadingState = Plan | null;
 
-export const startLoading = async (target: ReplyTarget) => {
+// Posting the Plan notifies the user; follow-up turns skip it and rely on the
+// notification-free typing status line instead.
+export const startLoading = async (
+	target: ReplyTarget,
+	{ showPlan = true }: { showPlan?: boolean } = {},
+) => {
 	try {
 		await target.startTyping("Starting Autumn...");
+		if (!showPlan) return null;
 		const loading = new Plan({ initialMessage: "Starting Autumn..." });
 		await target.post(loading);
 		return loading;
@@ -16,15 +22,22 @@ export const startLoading = async (target: ReplyTarget) => {
 	}
 };
 
-export const createActionLogger = (loading: LoadingState) => {
+export const createActionLogger = (
+	loading: LoadingState,
+	target?: ReplyTarget,
+) => {
 	const seen = new Set<string>();
 	let first = true;
 
 	return async (message: string) => {
-		if (!loading || seen.has(message)) return;
+		if (seen.has(message)) return;
 		seen.add(message);
 
 		try {
+			if (!loading) {
+				await target?.startTyping(message);
+				return;
+			}
 			if (first) {
 				first = false;
 				await loading.reset({ initialMessage: message });
@@ -42,10 +55,7 @@ export const finishLoading = async (
 	loading: LoadingState,
 	message: string,
 ) => {
-	if (!loading) {
-		await target.post({ markdown: message });
-		return;
-	}
+	if (!loading) return;
 
 	try {
 		await loading.complete({ completeMessage: message });

apps/leaf/tests/evals/claudeManaged/missing-email.eval.ts

@@ -0,0 +1,162 @@
+// Missing-email invoice flow: invoice_mode needs a real billing email, and this
+// customer has none. The agent must ask for it and persist it via updateCustomer
+// before any billing call, then run the standard preview-then-attach contract.
+import { withCustomers } from "../fixtures/createSetup.js";
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../fixtures/expectations/index.js";
+import { orgSetups } from "../fixtures/orgSetups.js";
+import {
+	approve,
+	createClaudeManagedLiveDriver,
+	initEval,
+	user,
+} from "../harness/index.js";
+import { billingAttachScores } from "../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "attach";
+};
+
+const experimentName = "missing-email";
+
+const billingEmail = "billing@harborlight.example";
+
+const setup = withCustomers({
+	setup: orgSetups.knowledgePlatform(),
+	customers: ({ customers }) => ({
+		harborlight: customers.base({
+			email: null,
+			id: "harborlight-journal",
+			name: "Harborlight Journal",
+		}),
+	}),
+	entities: ({ customers, entities, features }) => ({
+		editorial: entities.base({
+			customer: customers.harborlight,
+			feature: features.workspaces,
+			id: "harborlight-editorial",
+			name: "Editorial",
+		}),
+	}),
+});
+
+const customer = setup.refs.customers.harborlight;
+
+// No net terms were given, so net_terms_days must stay absent.
+const invoiceMode = {
+	enable_plan_immediately: true,
+	enabled: true,
+	finalize: false,
+};
+
+const expectedAttachRequest = {
+	customer_id: customer.id,
+	entity_id: setup.refs.entities.editorial.id,
+	invoice_mode: invoiceMode,
+	plan_id: setup.refs.plans.scale.id,
+};
+
+const expectedUpdateCustomerRequest = {
+	customer_id: customer.id,
+	email: billingEmail,
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	driver: createClaudeManagedLiveDriver(),
+	metadata: {
+		domain: "billing",
+		flow: "attach",
+	},
+	scores: billingAttachScores(),
+	timeout: 300_000,
+	cases: [
+		{
+			name: "invoice attach waits for the billing email before any billing call",
+			conversation: [
+				user({
+					message:
+						"Attach the Scale plan to Harborlight Journal's Editorial workspace and bill them by invoice.",
+				}),
+				user({ message: `Their billing email is ${billingEmail}.` }),
+				user({ message: "Looks good, attach it." }),
+				approve(),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listPlans",
+						"listCustomers",
+						"listEntities",
+						"updateCustomer",
+						"previewAttach",
+						"attach",
+					],
+				}),
+				response.askedBeforeTool({
+					phrases: ["email"],
+					toolName: "updateCustomer",
+				}),
+				// Email must be saved before billing starts, not just before attach.
+				api.calledInOrder({
+					calls: [
+						{ body: expectedUpdateCustomerRequest, toolName: "updateCustomer" },
+						{ body: expectedAttachRequest, toolName: "previewAttach" },
+						{ body: expectedAttachRequest, toolName: "attach" },
+					],
+				}),
+				...billing.previewThenWrite({
+					body: expectedAttachRequest,
+					write: "attach",
+				}),
+				api.calledTimes({
+					call: {
+						body: expectedUpdateCustomerRequest,
+						toolName: "updateCustomer",
+					},
+					count: 1,
+				}),
+				api.calledTimes({ call: { toolName: "updateCustomer" }, count: 1 }),
+				api.calledTimes({
+					call: { body: expectedAttachRequest, toolName: "attach" },
+					count: 1,
+				}),
+				api.calledTimes({ call: { toolName: "attach" }, count: 1 }),
+				api.calledTimes({
+					call: { body: expectedAttachRequest, toolName: "previewAttach" },
+					count: 1,
+				}),
+				api.calledTimes({ call: { toolName: "createEntity" }, count: 0 }),
+				api.calledTimes({ call: { toolName: "createSchedule" }, count: 0 }),
+				api.calledTimes({
+					call: { toolName: "previewCreateSchedule" },
+					count: 0,
+				}),
+				...(["previewAttach", "attach"] as const).map((toolName) =>
+					api.bodyExcludes({
+						fields: [
+							"invoice_mode.net_terms_days",
+							"customize",
+							"starts_at",
+							"ends_at",
+							"feature_quantities",
+							"no_billing_changes",
+							"entity_data",
+						],
+						toolName,
+					}),
+				),
+				response.mentions({
+					phrases: ["Harborlight", "Scale", "invoice"],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/claudeManaged/payment-failure.eval.ts

@@ -0,0 +1,135 @@
+// Payment-failure surfacing: an approved charge-now attach comes back declined.
+// The agent must report the failure honestly — no success claims, no retries —
+// while the preview-first contract and the approved body stay intact.
+import { withCustomers } from "../fixtures/createSetup.js";
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../fixtures/expectations/index.js";
+import { orgSetups } from "../fixtures/orgSetups.js";
+import { responses } from "../fixtures/responses.js";
+import {
+	approve,
+	createClaudeManagedLiveDriver,
+	initEval,
+	user,
+} from "../harness/index.js";
+import { billingAttachScores } from "../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "attach";
+};
+
+const experimentName = "payment-failure";
+
+const setup = withCustomers({
+	setup: orgSetups.knowledgePlatform(),
+	customers: ({ customers }) => ({
+		tidemark: customers.base({
+			email: "ap@tidemarksystems.example",
+			id: "tidemark-systems",
+			name: "Tidemark Systems",
+		}),
+	}),
+	entities: ({ customers, entities, features }) => ({
+		production: entities.base({
+			customer: customers.tidemark,
+			feature: features.workspaces,
+			id: "tidemark-production",
+			name: "Production",
+		}),
+	}),
+});
+
+// Charge-now with no invoice: invoice_mode must be omitted entirely.
+const expectedAttachRequest = {
+	customer_id: setup.refs.customers.tidemark.id,
+	entity_id: setup.refs.entities.production.id,
+	plan_id: setup.refs.plans.scale.id,
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	autumnApiOverrides: {
+		attach: ({ body }) =>
+			responses.attachPaymentFailure({
+				reason: "The card on file was declined.",
+				request: body,
+			}),
+	},
+	driver: createClaudeManagedLiveDriver(),
+	metadata: {
+		domain: "billing",
+		flow: "attach",
+	},
+	scores: billingAttachScores(),
+	timeout: 300_000,
+	cases: [
+		{
+			name: "declined charge is surfaced as a failure, not a success",
+			conversation: [
+				user({
+					message:
+						"Attach the Scale plan to Tidemark Systems' Production workspace and charge their card on file now — no invoice.",
+				}),
+				user({ message: "Looks good, attach it." }),
+				approve(),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listPlans",
+						"listCustomers",
+						"listEntities",
+						"previewAttach",
+						"attach",
+					],
+				}),
+				...billing.previewThenWrite({
+					body: expectedAttachRequest,
+					write: "attach",
+				}),
+				api.calledTimes({
+					call: { body: expectedAttachRequest, toolName: "attach" },
+					count: 1,
+				}),
+				// A declined card is surfaced, never retried.
+				api.calledTimes({ call: { toolName: "attach" }, count: 1 }),
+				api.calledTimes({
+					call: { body: expectedAttachRequest, toolName: "previewAttach" },
+					count: 1,
+				}),
+				api.calledTimes({ call: { toolName: "createEntity" }, count: 0 }),
+				api.calledTimes({ call: { toolName: "createSchedule" }, count: 0 }),
+				api.calledTimes({
+					call: { toolName: "previewCreateSchedule" },
+					count: 0,
+				}),
+				api.calledTimes({ call: { toolName: "updateCustomer" }, count: 0 }),
+				...(["previewAttach", "attach"] as const).map((toolName) =>
+					api.bodyExcludes({
+						fields: [
+							"invoice_mode",
+							"customize",
+							"starts_at",
+							"ends_at",
+							"feature_quantities",
+							"no_billing_changes",
+							"entity_data",
+						],
+						toolName,
+					}),
+				),
+				response.mentions({
+					notPhrases: ["now active", "all set"],
+					phrases: ["Tidemark", "Scale", "card", "declined"],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/fixtures/expectations/response.ts

@@ -44,10 +44,13 @@ export const response = {
 		type: "response.concise",
 	}),
 	mentions: ({
+		notPhrases,
 		phrases,
 	}: {
 		phrases: string[];
+		notPhrases?: string[];
 	}): ResponseMentionsExpectation => ({
+		...(notPhrases ? { notPhrases } : {}),
 		phrases,
 		type: "response.mentions",
 	}),

apps/leaf/tests/evals/fixtures/expectations/types.ts

@@ -50,6 +50,7 @@ export type ApiBodyNumberFieldsExpectation = {
 };
 
 export type ResponseMentionsExpectation = {
+	notPhrases?: string[];
 	phrases: string[];
 	type: "response.mentions";
 };

apps/leaf/tests/evals/fixtures/responses.ts

@@ -1,3 +1,4 @@
+import type { BillingResponse } from "@api/billing/common/billingResponse.js";
 import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js";
 import type { ApiPlanV1 } from "@api/products/apiPlanV1.js";
 
@@ -87,6 +88,25 @@ export const responses = {
 		plan_id: plan.id,
 		status: "created",
 	}),
+	// Mirrors the real BillingResponse when a charge is declined: no invoice,
+	// payment_url null, required_action carries the failure code and reason.
+	attachPaymentFailure: ({
+		reason,
+		request,
+	}: {
+		reason: string;
+		request?: unknown;
+	}): BillingResponse => {
+		const body = asRecord(request);
+		return {
+			customer_id: typeof body.customer_id === "string" ? body.customer_id : "",
+			...(typeof body.entity_id === "string"
+				? { entity_id: body.entity_id }
+				: {}),
+			payment_url: null,
+			required_action: { code: "payment_failed", reason },
+		};
+	},
 	createSchedulePreview: ({
 		customerId,
 		phases,

apps/leaf/tests/evals/utils/scorers.ts

@@ -143,12 +143,17 @@ const getExpectedApiBodyNumberFields = (expected?: EvalExpected) =>
 		expectation.type === "api.bodyNumberFields" ? [expectation] : [],
 	);
 
-const getExpectedResponsePhrases = (expected?: EvalExpected) => [
-	...(getLegacyExpected(expected)?.finalTextIncludes ?? []),
-	...getExpectationList(expected).flatMap((expectation) =>
-		expectation.type === "response.mentions" ? expectation.phrases : [],
-	),
-];
+const getExpectedResponseMentions = (
+	expected?: EvalExpected,
+): { notPhrases?: string[]; phrases: string[] }[] => {
+	const legacyPhrases = getLegacyExpected(expected)?.finalTextIncludes ?? [];
+	return [
+		...(legacyPhrases.length ? [{ phrases: legacyPhrases }] : []),
+		...getExpectationList(expected).flatMap((expectation) =>
+			expectation.type === "response.mentions" ? [expectation] : [],
+		),
+	];
+};
 
 const getExpectedQuestions = (expected?: EvalExpected) =>
 	getExpectationList(expected).flatMap((expectation) =>
@@ -366,9 +371,17 @@ export const expectedApiBodyNumberFields = ({
 };
 
 export const finalTextIncludes = ({ expected, output }: EvalScoreArgs) => {
-	const phrases = getExpectedResponsePhrases(expected);
-	if (!phrases.length) return 1;
-	return textMatches({ phrases, text: output.finalText }) ? 1 : 0;
+	const mentions = getExpectedResponseMentions(expected);
+	if (!mentions.length) return 1;
+	return mentions.every((mention) =>
+		textMatches({
+			notPhrases: mention.notPhrases,
+			phrases: mention.phrases,
+			text: output.finalText,
+		}),
+	)
+		? 1
+		: 0;
 };
 
 export const askedClarification = ({ expected, output }: EvalScoreArgs) => {
@@ -394,20 +407,26 @@ export const askedClarificationBeforeTool = ({
 }: EvalScoreArgs) => {
 	const questions = getExpectedQuestionsBeforeTool(expected);
 	if (!questions.length) return 1;
-	const turns = output.turns?.filter((turn) => turn.type === "user") ?? [];
-	return questions.every((question) =>
-		turns.some((turn) => {
-			const hasAsked = textMatches({
-				notPhrases: question.notPhrases,
-				phrases: question.phrases,
-				text: turn.text ?? "",
-			});
-			const hasTargetTool =
-				turn.toolCalls?.some((call) => call.name === question.toolName) ??
-				false;
-			return hasAsked && !hasTargetTool;
-		}),
-	)
+	// Per-turn toolCalls are cumulative snapshots, so the asking turn must
+	// predate the tool's first appearance across any snapshot.
+	const turns = output.turns ?? [];
+	return questions.every((question) => {
+		const callsTargetTool = (turn: (typeof turns)[number]) =>
+			turn.toolCalls?.some((call) => call.name === question.toolName) ?? false;
+		const firstToolTurn = turns.findIndex(callsTargetTool);
+		const askTurn = turns.findIndex(
+			(turn) =>
+				turn.type === "user" &&
+				!callsTargetTool(turn) &&
+				textMatches({
+					notPhrases: question.notPhrases,
+					phrases: question.phrases,
+					text: turn.text ?? "",
+				}),
+		);
+		if (askTurn === -1) return false;
+		return firstToolTurn === -1 || askTurn < firstToolTurn;
+	})
 		? 1
 		: 0;
 };