diff --git a/apps/leaf/session-trace.ts b/apps/leaf/session-trace.ts new file mode 100644 index 000000000..2dad8b590 --- /dev/null +++ b/apps/leaf/session-trace.ts @@ -0,0 +1,27 @@ +import Anthropic from "@anthropic-ai/sdk"; + +const sessionId = process.argv[2]; +if (!sessionId) throw new Error("usage: bun session-trace.ts "); +const client = new Anthropic(); + +const events: any[] = []; +for await (const event of client.beta.sessions.events.list(sessionId)) { + events.push(event); +} + +let prev: number | null = null; +for (const e of events) { + const ts = e.processed_at ? Date.parse(e.processed_at) : null; + const gap = ts && prev ? `+${((ts - prev) / 1000).toFixed(1)}s` : ""; + prev = ts ?? prev; + const time = ts ? new Date(ts).toISOString().slice(11, 23) : "??"; + let detail = ""; + if (e.type === "agent.mcp_tool_use") detail = `${e.tool_name ?? e.name} ${JSON.stringify(e.input ?? {}).slice(0, 120)}`; + else if (e.type === "agent.mcp_tool_result") detail = `${e.tool_name ?? ""} ${JSON.stringify(e.content ?? e.result ?? {}).slice(0, 120)}`; + else if (e.type === "agent.message") detail = JSON.stringify(e.content ?? e.text ?? "").slice(0, 150); + else if (e.type?.startsWith("span.")) detail = JSON.stringify({ name: e.name, model: e.model, usage: e.usage }).slice(0, 150); + else if (e.type === "user.message") detail = JSON.stringify(e.content ?? "").slice(0, 100); + else detail = JSON.stringify(e).slice(0, 120); + console.log(`${time} ${gap.padStart(8)} ${e.type} ${detail}`); +} +console.log(`\ntotal events: ${events.length}`); diff --git a/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts b/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts index 49adc55c7..091484233 100644 --- a/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts +++ b/apps/leaf/src/agent/runMessage/engines/claudeManagedEngine.ts @@ -5,8 +5,14 @@ import { claudeManagedConfig } from "../../../harness/claudeManaged/config.js"; import { ensureLeafResources } from "../../../harness/claudeManaged/ensureLeafResources.js"; import { ensureMemoryStore } from "../../../harness/claudeManaged/memory/ensureMemoryStore.js"; import { cmaRepo } from "../../../harness/claudeManaged/repos/claudeManagedRepo.js"; -import { driveSessionTurn } from "../../../harness/claudeManaged/session/driveSessionTurn.js"; -import { buildUserMessageContent } from "../../../harness/claudeManaged/session/userMessage.js"; +import { + driveSessionTurn, + type SessionTurnOutcome, +} from "../../../harness/claudeManaged/session/driveSessionTurn.js"; +import { + buildUserMessageContent, + type UserMessageContentBlock, +} from "../../../harness/claudeManaged/session/userMessage.js"; import { ensureAutumnVault } from "../../../harness/claudeManaged/vaults/ensureAutumnVault.js"; import { containsSecret } from "../../../internal/sandbox/tool/guardrails.js"; import { db } from "../../../lib/db.js"; @@ -14,6 +20,7 @@ import { createBraintrustLogger } from "../../../providers/braintrust/index.js"; import { formatToolAction } from "../../tools/autumnMcp.js"; import { createPreviewCapture, + isSilentTool, type PreviewApproval, } from "../../tools/toolPolicy.js"; import type { AgentEngine, MessageContext, MessageParams } from "../types.js"; @@ -39,6 +46,29 @@ const redactSecrets = ({ return "[response withheld: it appeared to contain a credential]"; }; +// Approval cards are executed by confirming a suspended session tool, so a +// preview-only turn must be nudged into a real write-tool suspension. +const buildNudgeText = ({ toolName }: { toolName: string }) => + `Call the ${toolName} tool now with the exact args from your preview. It will pause for user approval automatically — do not ask for confirmation or repeat the summary.`; + +const mergeTurnOutcomes = ( + first: SessionTurnOutcome, + second: SessionTurnOutcome, +): SessionTurnOutcome => ({ + errorMessage: second.errorMessage ?? first.errorMessage, + suspendedQueue: second.suspendedQueue, + textParts: [...first.textParts, ...second.textParts], + usage: { + cacheCreationInputTokens: + first.usage.cacheCreationInputTokens + + second.usage.cacheCreationInputTokens, + cacheReadInputTokens: + first.usage.cacheReadInputTokens + second.usage.cacheReadInputTokens, + inputTokens: first.usage.inputTokens + second.usage.inputTokens, + outputTokens: first.usage.outputTokens + second.usage.outputTokens, + }, +}); + // The agent's system prompt is env/thread-agnostic (one shared agent), so a new // session's first message carries the env + recent thread context. const buildMessageText = ({ @@ -153,7 +183,13 @@ export const claudeManagedEngine: AgentEngine = { const previewCapture = createPreviewCapture(); const text = buildMessageText({ env, newSession, params }); - const runTurn = ({ span }: { span?: Span }) => { + const driveTurn = ({ + content, + span, + }: { + content: UserMessageContentBlock[]; + span?: Span; + }) => { const openToolSpans = new Map(); return driveSessionTurn({ autumnMcpServerName: claudeManagedConfig.autumnMcpServerName, @@ -162,10 +198,7 @@ export const claudeManagedEngine: AgentEngine = { client.beta.sessions.events.send(activeSessionId, { events: [ { - content: buildUserMessageContent({ - attachments: params.attachments, - text, - }), + content, type: "user.message", }, ], @@ -175,7 +208,9 @@ export const claudeManagedEngine: AgentEngine = { event: "leaf.mcp_tool_called", tool: name, }); - await onAction?.(formatToolAction({ args: input, toolName: name })); + if (!isSilentTool(name)) { + await onAction?.(formatToolAction({ args: input, toolName: name })); + } previewCapture.onToolCall({ input, name }); if (span) { openToolSpans.set( @@ -197,6 +232,44 @@ export const claudeManagedEngine: AgentEngine = { }); }; + const runTurn = async ({ span }: { span?: Span }) => { + const first = await driveTurn({ + content: buildUserMessageContent({ + attachments: params.attachments, + text, + }), + span, + }); + const captured = previewCapture.captured; + if (first.suspendedQueue?.length || first.errorMessage || !captured) { + return first; + } + // Preview-only turn: nudge once so the approval card comes from a real + // suspension (tool_use_id) instead of an unexecutable preview capture. + logger.info("Nudging Claude Managed agent to call write tool", { + event: "leaf.claude_managed_preview_nudge", + context: { env, org_id: org.id }, + tool: captured.toolName, + }); + const nudge = await driveTurn({ + content: [ + { + text: buildNudgeText({ toolName: captured.toolName }), + type: "text", + }, + ], + span, + }); + if (!nudge.suspendedQueue?.length) { + logger.warn("Claude Managed agent did not suspend after nudge", { + event: "leaf.claude_managed_preview_nudge_failed", + context: { env, org_id: org.id }, + tool: captured.toolName, + }); + } + return mergeTurnOutcomes(first, nudge); + }; + // One parent span per turn; child spans per Autumn tool; token usage as // metrics. session_id + thread_id metadata link a thread's turns together. const outcome = braintrustEnabled diff --git a/apps/leaf/src/agent/tools/autumnMcp.ts b/apps/leaf/src/agent/tools/autumnMcp.ts index f479864b5..52950abb8 100644 --- a/apps/leaf/src/agent/tools/autumnMcp.ts +++ b/apps/leaf/src/agent/tools/autumnMcp.ts @@ -7,6 +7,7 @@ import { logger as rootLogger } from "../../lib/logger.js"; import { type createPreviewCapture, getWriteToolForPreview, + isSilentTool, toolLabel, } from "./toolPolicy.js"; @@ -95,10 +96,10 @@ export const formatToolAction = ({ args.request && typeof args.request === "object" ? (args.request as Record) : args; + // Only human-meaningful values — opaque ids (customer_id, entity_id) bloat + // the progress line without telling the reader anything. const details = [ - ["customer", request.customer_id], ["plan", request.plan_id], - ["entity", request.entity_id], ["search", request.search], ].flatMap(([label, value]) => typeof value === "string" && value ? [`${label}: ${value}`] : [], @@ -145,7 +146,9 @@ export const getAutumnMcpTools = async ({ event: "leaf.mcp_tool_called", tool: toolName, }); - await options.onToolCall?.(formatToolAction({ toolName, args })); + if (!isSilentTool(toolName)) { + await options.onToolCall?.(formatToolAction({ toolName, args })); + } const result = await execute(args, ...rest); if (getWriteToolForPreview(toolName)) { logger.info("Captured Autumn MCP preview", { diff --git a/apps/leaf/src/agent/tools/toolPolicy.ts b/apps/leaf/src/agent/tools/toolPolicy.ts index daf34d2d3..457ece743 100644 --- a/apps/leaf/src/agent/tools/toolPolicy.ts +++ b/apps/leaf/src/agent/tools/toolPolicy.ts @@ -19,6 +19,15 @@ const previewWriteTools: Record = { export const getWriteToolForPreview = (toolName: string) => previewWriteTools[toolName.replace(/^autumn_/, "")]; +// Pure utility tools the agent calls constantly — not worth a progress line. +const silentTools = new Set([ + "dateToEpochMilliseconds", + "epochMillisecondsToDate", +]); + +export const isSilentTool = (toolName: string) => + silentTools.has(normalizeToolName(toolName)); + export const toolLabel = (toolName: string) => labels[toolName.replace(/^autumn_/, "")] ?? toolName diff --git a/apps/leaf/src/bot.ts b/apps/leaf/src/bot.ts index d476b4921..5f0540bc1 100644 --- a/apps/leaf/src/bot.ts +++ b/apps/leaf/src/bot.ts @@ -135,8 +135,11 @@ const runAndReply = async ({ return; } - loading = await startLoading(target); - const logAction = createActionLogger(loading); + // Follow-up turns skip the Plan message so the user is only notified once + // (for the answer); progress shows via the silent typing status instead. + const isFollowUp = recentMessages?.some((m) => m.isBot) ?? false; + loading = await startLoading(target, { showPlan: !isFollowUp }); + const logAction = createActionLogger(loading, target); const rawFiles = getSlackFilesFromRaw({ raw }); const botToken = decrypt(installation.bot_access_token); const output = await runMessage({ diff --git a/apps/leaf/src/internal/approvals/actions/approveAndRun.ts b/apps/leaf/src/internal/approvals/actions/approveAndRun.ts index be4afdbd7..e622344f9 100644 --- a/apps/leaf/src/internal/approvals/actions/approveAndRun.ts +++ b/apps/leaf/src/internal/approvals/actions/approveAndRun.ts @@ -2,6 +2,7 @@ import Anthropic from "@anthropic-ai/sdk"; import { claudeManagedConfig } from "../../../harness/claudeManaged/config.js"; import { driveSessionTurn } from "../../../harness/claudeManaged/session/driveSessionTurn.js"; import { db } from "../../../lib/db.js"; +import { logger } from "../../../lib/logger.js"; import { chatApprovalRepo } from "../repos/chatApprovalRepo.js"; import { approvalErrorResult } from "../utils/approvalErrors.js"; @@ -47,6 +48,12 @@ export const approveAndRun = async ({ }); const text = outcome.textParts.join("\n\n"); const failed = Boolean(outcome.errorMessage) && !text; + if (failed) { + logger.error("[chat] Approval run failed", outcome.errorMessage, { + event: "leaf.approval_run_failed", + approval_id: approvalId, + }); + } await chatApprovalRepo.finalize({ approvalId, db, @@ -55,6 +62,10 @@ export const approveAndRun = async ({ }); return failed ? approvalErrorResult(outcome.errorMessage) : { text }; } catch (error) { + logger.error("[chat] Approval run failed", error, { + event: "leaf.approval_run_failed", + approval_id: approvalId, + }); await chatApprovalRepo.finalize({ approvalId, db, diff --git a/apps/leaf/src/internal/approvals/actions/postApprovalRequest.ts b/apps/leaf/src/internal/approvals/actions/postApprovalRequest.ts index 150529fc5..4c3c2fdfe 100644 --- a/apps/leaf/src/internal/approvals/actions/postApprovalRequest.ts +++ b/apps/leaf/src/internal/approvals/actions/postApprovalRequest.ts @@ -36,6 +36,17 @@ export const postApprovalRequest = async ({ const approval = approvalRequestFromOutput(output); if (!approval) return false; + // approveAndRun can only confirm a suspended session tool, so a card missing + // either id would always fail at approval time — fall back to plain text. + if (!approval.runId || !approval.toolCallId) { + logger.warn("Skipped unexecutable approval request", { + event: "leaf.approval_unexecutable_skipped", + context: { env: approval.env, org_id: installation.org_id }, + tool: approval.toolName, + }); + return false; + } + const approvalId = await chatApprovalRepo.insert({ db, data: { diff --git a/apps/leaf/src/ui/progress.ts b/apps/leaf/src/ui/progress.ts index 8625f4193..36ae103df 100644 --- a/apps/leaf/src/ui/progress.ts +++ b/apps/leaf/src/ui/progress.ts @@ -4,9 +4,15 @@ import { Plan } from "chat"; export type ReplyTarget = Thread | Channel; export type LoadingState = Plan | null; -export const startLoading = async (target: ReplyTarget) => { +// Posting the Plan notifies the user; follow-up turns skip it and rely on the +// notification-free typing status line instead. +export const startLoading = async ( + target: ReplyTarget, + { showPlan = true }: { showPlan?: boolean } = {}, +) => { try { await target.startTyping("Starting Autumn..."); + if (!showPlan) return null; const loading = new Plan({ initialMessage: "Starting Autumn..." }); await target.post(loading); return loading; @@ -16,15 +22,22 @@ export const startLoading = async (target: ReplyTarget) => { } }; -export const createActionLogger = (loading: LoadingState) => { +export const createActionLogger = ( + loading: LoadingState, + target?: ReplyTarget, +) => { const seen = new Set(); let first = true; return async (message: string) => { - if (!loading || seen.has(message)) return; + if (seen.has(message)) return; seen.add(message); try { + if (!loading) { + await target?.startTyping(message); + return; + } if (first) { first = false; await loading.reset({ initialMessage: message }); @@ -42,10 +55,7 @@ export const finishLoading = async ( loading: LoadingState, message: string, ) => { - if (!loading) { - await target.post({ markdown: message }); - return; - } + if (!loading) return; try { await loading.complete({ completeMessage: message }); diff --git a/apps/leaf/tests/evals/claudeManaged/missing-email.eval.ts b/apps/leaf/tests/evals/claudeManaged/missing-email.eval.ts new file mode 100644 index 000000000..8ffd0d5b8 --- /dev/null +++ b/apps/leaf/tests/evals/claudeManaged/missing-email.eval.ts @@ -0,0 +1,162 @@ +// Missing-email invoice flow: invoice_mode needs a real billing email, and this +// customer has none. The agent must ask for it and persist it via updateCustomer +// before any billing call, then run the standard preview-then-attach contract. +import { withCustomers } from "../fixtures/createSetup.js"; +import { + api, + billing, + response, + tools, +} from "../fixtures/expectations/index.js"; +import { orgSetups } from "../fixtures/orgSetups.js"; +import { + approve, + createClaudeManagedLiveDriver, + initEval, + user, +} from "../harness/index.js"; +import { billingAttachScores } from "../utils/scorers.js"; + +type EvalMetadata = { + domain: "billing"; + flow: "attach"; +}; + +const experimentName = "missing-email"; + +const billingEmail = "billing@harborlight.example"; + +const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers }) => ({ + harborlight: customers.base({ + email: null, + id: "harborlight-journal", + name: "Harborlight Journal", + }), + }), + entities: ({ customers, entities, features }) => ({ + editorial: entities.base({ + customer: customers.harborlight, + feature: features.workspaces, + id: "harborlight-editorial", + name: "Editorial", + }), + }), +}); + +const customer = setup.refs.customers.harborlight; + +// No net terms were given, so net_terms_days must stay absent. +const invoiceMode = { + enable_plan_immediately: true, + enabled: true, + finalize: false, +}; + +const expectedAttachRequest = { + customer_id: customer.id, + entity_id: setup.refs.entities.editorial.id, + invoice_mode: invoiceMode, + plan_id: setup.refs.plans.scale.id, +}; + +const expectedUpdateCustomerRequest = { + customer_id: customer.id, + email: billingEmail, +}; + +initEval({ + experimentName, + setup, + driver: createClaudeManagedLiveDriver(), + metadata: { + domain: "billing", + flow: "attach", + }, + scores: billingAttachScores(), + timeout: 300_000, + cases: [ + { + name: "invoice attach waits for the billing email before any billing call", + conversation: [ + user({ + message: + "Attach the Scale plan to Harborlight Journal's Editorial workspace and bill them by invoice.", + }), + user({ message: `Their billing email is ${billingEmail}.` }), + user({ message: "Looks good, attach it." }), + approve(), + ], + expect: [ + tools.called({ + toolNames: [ + "getAgentRules", + "listPlans", + "listCustomers", + "listEntities", + "updateCustomer", + "previewAttach", + "attach", + ], + }), + response.askedBeforeTool({ + phrases: ["email"], + toolName: "updateCustomer", + }), + // Email must be saved before billing starts, not just before attach. + api.calledInOrder({ + calls: [ + { body: expectedUpdateCustomerRequest, toolName: "updateCustomer" }, + { body: expectedAttachRequest, toolName: "previewAttach" }, + { body: expectedAttachRequest, toolName: "attach" }, + ], + }), + ...billing.previewThenWrite({ + body: expectedAttachRequest, + write: "attach", + }), + api.calledTimes({ + call: { + body: expectedUpdateCustomerRequest, + toolName: "updateCustomer", + }, + count: 1, + }), + api.calledTimes({ call: { toolName: "updateCustomer" }, count: 1 }), + api.calledTimes({ + call: { body: expectedAttachRequest, toolName: "attach" }, + count: 1, + }), + api.calledTimes({ call: { toolName: "attach" }, count: 1 }), + api.calledTimes({ + call: { body: expectedAttachRequest, toolName: "previewAttach" }, + count: 1, + }), + api.calledTimes({ call: { toolName: "createEntity" }, count: 0 }), + api.calledTimes({ call: { toolName: "createSchedule" }, count: 0 }), + api.calledTimes({ + call: { toolName: "previewCreateSchedule" }, + count: 0, + }), + ...(["previewAttach", "attach"] as const).map((toolName) => + api.bodyExcludes({ + fields: [ + "invoice_mode.net_terms_days", + "customize", + "starts_at", + "ends_at", + "feature_quantities", + "no_billing_changes", + "entity_data", + ], + toolName, + }), + ), + response.mentions({ + phrases: ["Harborlight", "Scale", "invoice"], + }), + ], + }, + ], +}); diff --git a/apps/leaf/tests/evals/claudeManaged/payment-failure.eval.ts b/apps/leaf/tests/evals/claudeManaged/payment-failure.eval.ts new file mode 100644 index 000000000..1da983757 --- /dev/null +++ b/apps/leaf/tests/evals/claudeManaged/payment-failure.eval.ts @@ -0,0 +1,135 @@ +// Payment-failure surfacing: an approved charge-now attach comes back declined. +// The agent must report the failure honestly — no success claims, no retries — +// while the preview-first contract and the approved body stay intact. +import { withCustomers } from "../fixtures/createSetup.js"; +import { + api, + billing, + response, + tools, +} from "../fixtures/expectations/index.js"; +import { orgSetups } from "../fixtures/orgSetups.js"; +import { responses } from "../fixtures/responses.js"; +import { + approve, + createClaudeManagedLiveDriver, + initEval, + user, +} from "../harness/index.js"; +import { billingAttachScores } from "../utils/scorers.js"; + +type EvalMetadata = { + domain: "billing"; + flow: "attach"; +}; + +const experimentName = "payment-failure"; + +const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers }) => ({ + tidemark: customers.base({ + email: "ap@tidemarksystems.example", + id: "tidemark-systems", + name: "Tidemark Systems", + }), + }), + entities: ({ customers, entities, features }) => ({ + production: entities.base({ + customer: customers.tidemark, + feature: features.workspaces, + id: "tidemark-production", + name: "Production", + }), + }), +}); + +// Charge-now with no invoice: invoice_mode must be omitted entirely. +const expectedAttachRequest = { + customer_id: setup.refs.customers.tidemark.id, + entity_id: setup.refs.entities.production.id, + plan_id: setup.refs.plans.scale.id, +}; + +initEval({ + experimentName, + setup, + autumnApiOverrides: { + attach: ({ body }) => + responses.attachPaymentFailure({ + reason: "The card on file was declined.", + request: body, + }), + }, + driver: createClaudeManagedLiveDriver(), + metadata: { + domain: "billing", + flow: "attach", + }, + scores: billingAttachScores(), + timeout: 300_000, + cases: [ + { + name: "declined charge is surfaced as a failure, not a success", + conversation: [ + user({ + message: + "Attach the Scale plan to Tidemark Systems' Production workspace and charge their card on file now — no invoice.", + }), + user({ message: "Looks good, attach it." }), + approve(), + ], + expect: [ + tools.called({ + toolNames: [ + "getAgentRules", + "listPlans", + "listCustomers", + "listEntities", + "previewAttach", + "attach", + ], + }), + ...billing.previewThenWrite({ + body: expectedAttachRequest, + write: "attach", + }), + api.calledTimes({ + call: { body: expectedAttachRequest, toolName: "attach" }, + count: 1, + }), + // A declined card is surfaced, never retried. + api.calledTimes({ call: { toolName: "attach" }, count: 1 }), + api.calledTimes({ + call: { body: expectedAttachRequest, toolName: "previewAttach" }, + count: 1, + }), + api.calledTimes({ call: { toolName: "createEntity" }, count: 0 }), + api.calledTimes({ call: { toolName: "createSchedule" }, count: 0 }), + api.calledTimes({ + call: { toolName: "previewCreateSchedule" }, + count: 0, + }), + api.calledTimes({ call: { toolName: "updateCustomer" }, count: 0 }), + ...(["previewAttach", "attach"] as const).map((toolName) => + api.bodyExcludes({ + fields: [ + "invoice_mode", + "customize", + "starts_at", + "ends_at", + "feature_quantities", + "no_billing_changes", + "entity_data", + ], + toolName, + }), + ), + response.mentions({ + notPhrases: ["now active", "all set"], + phrases: ["Tidemark", "Scale", "card", "declined"], + }), + ], + }, + ], +}); diff --git a/apps/leaf/tests/evals/fixtures/expectations/response.ts b/apps/leaf/tests/evals/fixtures/expectations/response.ts index c7a144208..8491fffbe 100644 --- a/apps/leaf/tests/evals/fixtures/expectations/response.ts +++ b/apps/leaf/tests/evals/fixtures/expectations/response.ts @@ -44,10 +44,13 @@ export const response = { type: "response.concise", }), mentions: ({ + notPhrases, phrases, }: { phrases: string[]; + notPhrases?: string[]; }): ResponseMentionsExpectation => ({ + ...(notPhrases ? { notPhrases } : {}), phrases, type: "response.mentions", }), diff --git a/apps/leaf/tests/evals/fixtures/expectations/types.ts b/apps/leaf/tests/evals/fixtures/expectations/types.ts index c0ba11e35..012232e3f 100644 --- a/apps/leaf/tests/evals/fixtures/expectations/types.ts +++ b/apps/leaf/tests/evals/fixtures/expectations/types.ts @@ -50,6 +50,7 @@ export type ApiBodyNumberFieldsExpectation = { }; export type ResponseMentionsExpectation = { + notPhrases?: string[]; phrases: string[]; type: "response.mentions"; }; diff --git a/apps/leaf/tests/evals/fixtures/responses.ts b/apps/leaf/tests/evals/fixtures/responses.ts index f4f9233b9..768e462e6 100644 --- a/apps/leaf/tests/evals/fixtures/responses.ts +++ b/apps/leaf/tests/evals/fixtures/responses.ts @@ -1,3 +1,4 @@ +import type { BillingResponse } from "@api/billing/common/billingResponse.js"; import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; @@ -87,6 +88,25 @@ export const responses = { plan_id: plan.id, status: "created", }), + // Mirrors the real BillingResponse when a charge is declined: no invoice, + // payment_url null, required_action carries the failure code and reason. + attachPaymentFailure: ({ + reason, + request, + }: { + reason: string; + request?: unknown; + }): BillingResponse => { + const body = asRecord(request); + return { + customer_id: typeof body.customer_id === "string" ? body.customer_id : "", + ...(typeof body.entity_id === "string" + ? { entity_id: body.entity_id } + : {}), + payment_url: null, + required_action: { code: "payment_failed", reason }, + }; + }, createSchedulePreview: ({ customerId, phases, diff --git a/apps/leaf/tests/evals/utils/scorers.ts b/apps/leaf/tests/evals/utils/scorers.ts index fef0a9d08..405c37536 100644 --- a/apps/leaf/tests/evals/utils/scorers.ts +++ b/apps/leaf/tests/evals/utils/scorers.ts @@ -143,12 +143,17 @@ const getExpectedApiBodyNumberFields = (expected?: EvalExpected) => expectation.type === "api.bodyNumberFields" ? [expectation] : [], ); -const getExpectedResponsePhrases = (expected?: EvalExpected) => [ - ...(getLegacyExpected(expected)?.finalTextIncludes ?? []), - ...getExpectationList(expected).flatMap((expectation) => - expectation.type === "response.mentions" ? expectation.phrases : [], - ), -]; +const getExpectedResponseMentions = ( + expected?: EvalExpected, +): { notPhrases?: string[]; phrases: string[] }[] => { + const legacyPhrases = getLegacyExpected(expected)?.finalTextIncludes ?? []; + return [ + ...(legacyPhrases.length ? [{ phrases: legacyPhrases }] : []), + ...getExpectationList(expected).flatMap((expectation) => + expectation.type === "response.mentions" ? [expectation] : [], + ), + ]; +}; const getExpectedQuestions = (expected?: EvalExpected) => getExpectationList(expected).flatMap((expectation) => @@ -366,9 +371,17 @@ export const expectedApiBodyNumberFields = ({ }; export const finalTextIncludes = ({ expected, output }: EvalScoreArgs) => { - const phrases = getExpectedResponsePhrases(expected); - if (!phrases.length) return 1; - return textMatches({ phrases, text: output.finalText }) ? 1 : 0; + const mentions = getExpectedResponseMentions(expected); + if (!mentions.length) return 1; + return mentions.every((mention) => + textMatches({ + notPhrases: mention.notPhrases, + phrases: mention.phrases, + text: output.finalText, + }), + ) + ? 1 + : 0; }; export const askedClarification = ({ expected, output }: EvalScoreArgs) => { @@ -394,20 +407,26 @@ export const askedClarificationBeforeTool = ({ }: EvalScoreArgs) => { const questions = getExpectedQuestionsBeforeTool(expected); if (!questions.length) return 1; - const turns = output.turns?.filter((turn) => turn.type === "user") ?? []; - return questions.every((question) => - turns.some((turn) => { - const hasAsked = textMatches({ - notPhrases: question.notPhrases, - phrases: question.phrases, - text: turn.text ?? "", - }); - const hasTargetTool = - turn.toolCalls?.some((call) => call.name === question.toolName) ?? - false; - return hasAsked && !hasTargetTool; - }), - ) + // Per-turn toolCalls are cumulative snapshots, so the asking turn must + // predate the tool's first appearance across any snapshot. + const turns = output.turns ?? []; + return questions.every((question) => { + const callsTargetTool = (turn: (typeof turns)[number]) => + turn.toolCalls?.some((call) => call.name === question.toolName) ?? false; + const firstToolTurn = turns.findIndex(callsTargetTool); + const askTurn = turns.findIndex( + (turn) => + turn.type === "user" && + !callsTargetTool(turn) && + textMatches({ + notPhrases: question.notPhrases, + phrases: question.phrases, + text: turn.text ?? "", + }), + ); + if (askTurn === -1) return false; + return firstToolTurn === -1 || askTurn < firstToolTurn; + }) ? 1 : 0; };