fix: invoice contract test

apps/leaf/tests/evals/agent/billing/attach/custom-base-price.eval.ts

@@ -1,4 +1,5 @@
 import {
+	api,
 	billing,
 	response,
 	tools,
@@ -25,9 +26,18 @@ const setup = withCustomers({
 			name: "Northstar Labs",
 		}),
 	}),
+	entities: ({ customers, entities, features }) => ({
+		workspace: entities.base({
+			customer: customers.account,
+			feature: features.workspaces,
+			id: "workspace_northstar",
+			name: "Northstar Workspace",
+		}),
+	}),
 });
 const customer = setup.refs.customers.account;
 const enterprisePlan = setup.refs.plans.enterprise;
+const workspace = setup.refs.entities.workspace;
 
 const expectedAttachRequest = {
 	customer_id: customer.id,
@@ -38,13 +48,13 @@ const expectedAttachRequest = {
 		},
 	},
 	enable_plan_immediately: true,
+	entity_id: workspace.id,
 	invoice_mode: {
 		enable_plan_immediately: true,
 		enabled: true,
 		finalize: false,
 	},
 	plan_id: enterprisePlan.id,
-	redirect_mode: "if_required",
 };
 
 initEval<EvalMetadata>({
@@ -61,14 +71,26 @@ initEval<EvalMetadata>({
 			conversation: [
 				user({
 					message:
-						"Please attach the Enterprise plan to Northstar Labs with a custom base price of $49/month.",
+						"Please attach the Enterprise plan to Northstar Labs for Northstar Workspace with a custom base price of $49/month.",
 				}),
 				user({ message: "Looks good, attach it." }),
 				approve(),
 			],
 			expect: [
 				tools.called({
-					toolNames: ["listCustomers", "listPlans"],
+					toolNames: ["listCustomers", "listPlans", "listEntities"],
+				}),
+				api.calledInOrder({
+					calls: [
+						{
+							body: { customer_id: customer.id },
+							toolName: "listEntities",
+						},
+						{
+							body: expectedAttachRequest,
+							toolName: "previewAttach",
+						},
+					],
 				}),
 				billing.previewBeforeWrite({
 					preview: {
@@ -80,9 +102,16 @@ initEval<EvalMetadata>({
 						toolName: "attach",
 					},
 				}),
+				api.calledAfterApproval({
+					call: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
 				response.mentions({
 					phrases: [
 						"Northstar Labs",
+						"Northstar Workspace",
 						"Enterprise",
 						"$49",
 						"invoice",

apps/leaf/tests/evals/agent/billing/attach/entity-rules.eval.ts

@@ -0,0 +1,128 @@
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../../../fixtures/expectations/index.js";
+import { withCustomers } from "../../../fixtures/createSetup.js";
+import { orgSetups } from "../../../fixtures/orgSetups.js";
+import { approve, initEval, user } from "../../../harness/index.js";
+import { billingAttachScores } from "../../../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "attach";
+};
+
+const experimentName = "attach-entity-rules";
+
+const setup = withCustomers({
+	setup: orgSetups.knowledgePlatform(),
+	customers: ({ customers }) => ({
+		account: customers.base({
+			email: "billing@alder.example",
+			id: "cus_attach_entity_rules",
+			name: "Alder Systems",
+		}),
+	}),
+	entities: ({ customers, entities, features }) => ({
+		workspaceAlpha: entities.base({
+			customer: customers.account,
+			feature: features.workspaces,
+			id: "workspace_alpha",
+			name: "Workspace Alpha",
+		}),
+		workspaceBeta: entities.base({
+			customer: customers.account,
+			feature: features.workspaces,
+			id: "workspace_beta",
+			name: "Workspace Beta",
+		}),
+	}),
+});
+const customer = setup.refs.customers.account;
+const scalePlan = setup.refs.plans.scale;
+const workspace = setup.refs.entities.workspaceAlpha;
+
+const expectedAttachRequest = {
+	customer_id: customer.id,
+	enable_plan_immediately: true,
+	entity_id: workspace.id,
+	invoice_mode: {
+		enable_plan_immediately: true,
+		enabled: true,
+		finalize: false,
+	},
+	plan_id: scalePlan.id,
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	metadata: {
+		domain: "billing",
+		flow: "attach",
+	},
+	scores: billingAttachScores(),
+	cases: [
+		{
+			name: "entity attach rules require entity selection before preview",
+			conversation: [
+				user({
+					message: "Please attach the Scale plan to Alder Systems.",
+				}),
+				user({ message: "Use Workspace Alpha." }),
+				user({ message: "Looks good, attach it." }),
+				approve(),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listCustomers",
+						"listPlans",
+						"listEntities",
+						"previewAttach",
+						"attach",
+					],
+				}),
+				api.calledInOrder({
+					calls: [
+						{
+							body: { customer_id: customer.id },
+							toolName: "listEntities",
+						},
+						{
+							body: expectedAttachRequest,
+							toolName: "previewAttach",
+						},
+					],
+				}),
+				billing.previewBeforeWrite({
+					preview: {
+						body: expectedAttachRequest,
+						toolName: "previewAttach",
+					},
+					write: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
+				api.calledAfterApproval({
+					call: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
+				response.mentions({
+					phrases: [
+						"Workspace Alpha",
+						"Workspace Beta",
+						"Alder Systems",
+						"Scale",
+					],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/agent/billing/attach/new-customer.eval.ts

@@ -0,0 +1,135 @@
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../../../fixtures/expectations/index.js";
+import { orgSetups } from "../../../fixtures/orgSetups.js";
+import { approve, initEval, user } from "../../../harness/index.js";
+import { billingAttachScores } from "../../../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "attach";
+};
+
+const experimentName = "attach-new-customer";
+
+const setup = orgSetups.knowledgePlatform();
+const customerId = "cus_attach_new_customer";
+const customerEmail = "billing@cobalt.example";
+const entityId = "workspace_cobalt";
+const entityName = "Cobalt Workspace";
+const scalePlan = setup.refs.plans.scale;
+const entityFeature = setup.refs.features.workspaces;
+
+const expectedCreateRequest = {
+	customer_id: customerId,
+	email: customerEmail,
+};
+
+const expectedCreateEntityRequest = {
+	customer_id: customerId,
+	entity_id: entityId,
+	feature_id: entityFeature.id,
+	name: entityName,
+};
+
+const expectedAttachRequest = {
+	customer_id: customerId,
+	enable_plan_immediately: true,
+	entity_id: entityId,
+	invoice_mode: {
+		enable_plan_immediately: true,
+		enabled: true,
+		finalize: false,
+	},
+	plan_id: scalePlan.id,
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	metadata: {
+		domain: "billing",
+		flow: "attach",
+	},
+	scores: billingAttachScores(),
+	cases: [
+		{
+			name: "create missing customer and entity before attach",
+			conversation: [
+				user({
+					message:
+						"Please attach the Scale plan to a new customer that is not in Autumn yet.",
+				}),
+				user({
+					message:
+						"Use customer id cus_attach_new_customer, email billing@cobalt.example, entity id workspace_cobalt, and entity name Cobalt Workspace.",
+				}),
+				user({ message: "Looks good, attach it." }),
+				approve(),
+			],
+			expect: [
+				response.askedBeforeTool({
+					phrases: ["customer", "id", "email", "entity", "name"],
+					notPhrases: ["deployment"],
+					toolName: "getOrCreateCustomer",
+				}),
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listPlans",
+						"getOrCreateCustomer",
+						"createEntity",
+						"previewAttach",
+						"attach",
+					],
+				}),
+				api.calledInOrder({
+					calls: [
+						{
+							body: expectedCreateRequest,
+							toolName: "getOrCreateCustomer",
+						},
+						{
+							body: expectedCreateEntityRequest,
+							toolName: "createEntity",
+						},
+						{
+							body: expectedAttachRequest,
+							toolName: "previewAttach",
+						},
+					],
+				}),
+				billing.previewBeforeWrite({
+					preview: {
+						body: expectedAttachRequest,
+						toolName: "previewAttach",
+					},
+					write: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
+				api.calledAfterApproval({
+					call: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
+				api.bodyExcludes({
+					fields: ["entity_data"],
+					toolName: "previewAttach",
+				}),
+				api.bodyExcludes({
+					fields: ["entity_data"],
+					toolName: "attach",
+				}),
+				response.mentions({
+					phrases: [entityName, customerId, "Scale"],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/agent/billing/schedules/backdated-schedule.eval.ts

@@ -0,0 +1,225 @@
+import { BillingInterval } from "@models/productModels/intervals/billingInterval";
+import { ResetInterval } from "@models/productModels/intervals/resetInterval";
+import { withCustomers } from "../../../fixtures/createSetup.js";
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../../../fixtures/expectations/index.js";
+import { orgSetups } from "../../../fixtures/orgSetups.js";
+import { approve, initEval, user } from "../../../harness/index.js";
+import { billingScheduleScores } from "../../../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "schedule";
+};
+
+const experimentName = "backdated-schedule";
+const time = (value: string) => new Date(value).getTime();
+
+const setup = withCustomers({
+	setup: orgSetups.knowledgePlatform(),
+	customers: ({ customers }) => ({
+		northstar: customers.base({
+			email: "billing@northstar.example",
+			id: "cus_northstar_contract",
+			name: "Northstar Labs",
+		}),
+	}),
+	entities: ({ customers, entities, features }) => ({
+		workspace: entities.base({
+			customer: customers.northstar,
+			feature: features.workspaces,
+			id: "workspace_northstar_platform",
+			name: "Northstar Platform Workspace",
+		}),
+	}),
+});
+
+const expectedScheduleRequest = {
+	customer_id: setup.refs.customers.northstar.id,
+	enable_plan_immediately: true,
+	entity_id: setup.refs.entities.workspace.id,
+	invoice_mode: {
+		enable_plan_immediately: true,
+		enabled: true,
+		finalize: false,
+	},
+	redirect_mode: "if_required",
+	phases: [
+		{
+			starts_at: time("2027-04-01T00:00:00.000Z"),
+			plans: [
+				{
+					plan_id: setup.refs.plans.launch.id,
+					customize: {
+						items: [
+							{ feature_id: "member_slots", included: 25 },
+							{
+								feature_id: "credits",
+								included: 100_000,
+								reset: { interval: ResetInterval.Month },
+							},
+						],
+					},
+				},
+				{
+					plan_id: setup.refs.plans.automationPack.id,
+				},
+			],
+		},
+		{
+			starts_at: time("2027-07-01T00:00:00.000Z"),
+			plans: [
+				{
+					plan_id: setup.refs.plans.scale.id,
+					customize: {
+						items: [
+							{ feature_id: "member_slots", included: 40 },
+							{
+								feature_id: "credits",
+								included: 250_000,
+								reset: { interval: ResetInterval.Month },
+							},
+						],
+					},
+				},
+				{ plan_id: setup.refs.plans.automationPack.id },
+				{ plan_id: setup.refs.plans.securityPack.id },
+			],
+		},
+		{
+			starts_at: time("2028-01-01T00:00:00.000Z"),
+			plans: [
+				{
+					plan_id: setup.refs.plans.enterprise.id,
+					customize: {
+						price: { amount: 2_400, interval: BillingInterval.Month },
+						items: [
+							{ feature_id: "member_slots", included: 75 },
+							{ feature_id: "project_slots", included: 500 },
+							{
+								feature_id: "credits",
+								included: 1_000_000,
+								reset: { interval: ResetInterval.Month },
+							},
+							{ feature_id: "platform_api", unlimited: true },
+							{ feature_id: "approval_chains", unlimited: true },
+							{ feature_id: "compliance_controls", unlimited: true },
+							{ feature_id: "brand_controls", unlimited: true },
+						],
+					},
+				},
+				{ plan_id: setup.refs.plans.securityPack.id },
+				{ plan_id: setup.refs.plans.whiteLabelPack.id },
+			],
+		},
+	],
+};
+
+const extractedContractText = [
+	"MASTER SERVICES AGREEMENT",
+	"Order Form OF-2027-041 | Prepared for Northstar Labs Ltd.",
+	"Effective date: March 12, 2027. Governing law: New York. Payment terms: Net 30. Notices should be sent to legal@northstar.example.",
+	"Extracted service dates: initial ramp starts 2027-04-01; expansion starts 2027-07-01; enterprise conversion starts 2028-01-01.",
+	"Autumn entity scope: Northstar Platform Workspace.",
+	"Billing contact: billing@northstar.example. Customer reference in Autumn should be resolved from this account name or billing contact before any schedule is prepared.",
+	"This is a backdated schedule: today's eval date is 2027-04-15, but the contract start date is 2027-04-01. Preserve the exact April 1, 2027 start date; do not use now for the first phase.",
+	"Section 2. Initial ramp. On April 1, 2027, start the Launch plan with 25 member slots and 100,000 credits per month. Add the Automation Pack.",
+	"Section 3. Expansion. On July 1, 2027, move to Scale, increase to 40 member slots and 250,000 credits per month, and keep Automation Pack. Add Security Pack.",
+	"Section 4. Enterprise conversion. On January 1, 2028, move to Enterprise at a custom $2,400/month base rate with 75 member slots, 500 project slots, and 1,000,000 credits per month. Keep Security Pack and add White Label Pack.",
+	"Enterprise conversion also includes contract-specific feature overrides that are not part of the standard Enterprise plan: unlimited API access, unlimited approval flows, unlimited compliance cntrls, and unlimited brand controls.",
+	"Section 8. Confidentiality. Neither party may disclose pricing or implementation details except to auditors, investors, or legal advisors under confidentiality obligations.",
+	"Section 11. Service levels. Support response targets are commercially reasonable and do not create service credits unless separately stated in an SLA exhibit.",
+	"Signature block: Northstar Labs Ltd. / Autumn Software Inc.",
+].join("\n");
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	metadata: {
+		domain: "billing",
+		flow: "schedule",
+	},
+	scores: billingScheduleScores(),
+	today: new Date("2027-04-15T00:00:00.000Z"),
+	timeout: 75_000,
+	cases: [
+		{
+			name: "contract text to backdated schedule",
+			conversation: [
+				user({
+					message: [
+						"A PDF text extractor returned the contract text below.",
+						"Please handle this in Autumn using only the extracted text.",
+						"This is a backdated schedule; preserve the contract's original phase dates exactly.",
+						"Make sure all feature limits and overrides from the contract are reflected in the schedule.",
+						"Apply the contract-specific feature overrides to the Enterprise phase.",
+						extractedContractText,
+					].join("\n"),
+				}),
+				user({ message: "Looks good, create the schedule." }),
+				approve(),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listCustomers",
+						"listEntities",
+						"listPlans",
+						"previewCreateSchedule",
+						"createSchedule",
+					],
+				}),
+				billing.previewBeforeWrite({
+					preview: {
+						body: expectedScheduleRequest,
+						toolName: "previewCreateSchedule",
+					},
+					write: {
+						body: expectedScheduleRequest,
+						toolName: "createSchedule",
+					},
+				}),
+				api.calledInOrder({
+					calls: [
+						{
+							body: { customer_id: setup.refs.customers.northstar.id },
+							toolName: "listEntities",
+						},
+						{
+							body: expectedScheduleRequest,
+							toolName: "previewCreateSchedule",
+						},
+					],
+				}),
+				api.calledAfterApproval({
+					call: {
+						body: expectedScheduleRequest,
+						toolName: "createSchedule",
+					},
+				}),
+				api.bodyNumberFields({
+					paths: ["phases.*.starts_at"],
+					toolName: "previewCreateSchedule",
+				}),
+				api.bodyNumberFields({
+					paths: ["phases.*.starts_at"],
+					toolName: "createSchedule",
+				}),
+				response.mentions({
+					phrases: [
+						"cus_northstar_contract",
+						"workspace_northstar_platform",
+						"Launch",
+						"Scale",
+						"Enterprise",
+					],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/agent/billing/schedules/custom-boolean-schedule.eval.ts

@@ -0,0 +1,170 @@
+import { BillingInterval } from "@models/productModels/intervals/billingInterval";
+import { withCustomers } from "../../../fixtures/createSetup.js";
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../../../fixtures/expectations/index.js";
+import { orgSetups } from "../../../fixtures/orgSetups.js";
+import {
+	approve,
+	contractAttachment,
+	initEval,
+	user,
+} from "../../../harness/index.js";
+import { billingScheduleScores } from "../../../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "billing";
+	flow: "schedule";
+};
+
+const experimentName = "custom-boolean-schedule";
+const now = new Date("2027-04-01T00:00:00.000Z");
+const time = (value: string) => new Date(value).getTime();
+
+const setup = withCustomers({
+	setup: orgSetups.knowledgePlatform(),
+	customers: ({ customers }) => ({
+		northstar: customers.base({
+			email: "ap@northstarlabs.example",
+			id: "northstar-labs",
+			name: "Northstar Labs, Inc.",
+		}),
+	}),
+	entities: ({ customers, entities, features }) => ({
+		workspace: entities.base({
+			customer: customers.northstar,
+			feature: features.workspaces,
+			id: "northstar-labs-production",
+			name: "Production",
+		}),
+	}),
+});
+
+const enterpriseCustomize = (amount: number) => ({
+	price: { amount, interval: BillingInterval.Year },
+	remove_items: [{ feature_id: setup.refs.features.revision_history.id }],
+	add_items: [
+		{
+			feature_id: setup.refs.features.hosted_solution.id,
+			unlimited: true,
+		},
+		{
+			feature_id: setup.refs.features.unlimited_seats.id,
+			unlimited: true,
+		},
+	],
+});
+
+const enterprisePhase = ({
+	amount,
+	startsAt,
+}: {
+	amount: number;
+	startsAt: number;
+}) => ({
+	starts_at: startsAt,
+	plans: [
+		{
+			plan_id: setup.refs.plans.enterprise.id,
+			customize: enterpriseCustomize(amount),
+		},
+	],
+});
+
+const expectedScheduleRequest = {
+	customer_id: setup.refs.customers.northstar.id,
+	enable_plan_immediately: true,
+	entity_id: setup.refs.entities.workspace.id,
+	invoice_mode: {
+		enable_plan_immediately: true,
+		enabled: true,
+		finalize: false,
+		net_terms_days: 30,
+	},
+	redirect_mode: "if_required",
+	phases: [
+		enterprisePhase({ amount: 7_500, startsAt: now.getTime() }),
+		enterprisePhase({
+			amount: 20_000,
+			startsAt: time("2028-04-01T00:00:00.000Z"),
+		}),
+	],
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	metadata: {
+		domain: "billing",
+		flow: "schedule",
+	},
+	scores: billingScheduleScores(),
+	today: now,
+	timeout: 150_000,
+	cases: [
+		{
+			name: "slack pdf contract to custom enterprise schedule",
+			conversation: [
+				user({
+					attachments: [
+						contractAttachment({ fixtureId: "custom-booleans-schedule" }),
+					],
+					message:
+						"I uploaded the signed order form for Northstar Labs. Please provision it in Autumn.",
+				}),
+				user({
+					message:
+						"Use customer_id northstar-labs and entity_id northstar-labs-production.",
+				}),
+				user({ message: "Looks good. Create the schedule." }),
+				approve({ optional: false }),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listPlans",
+						"listFeatures",
+						"previewCreateSchedule",
+						"createSchedule",
+					],
+				}),
+				billing.previewBeforeWrite({
+					preview: {
+						body: expectedScheduleRequest,
+						toolName: "previewCreateSchedule",
+					},
+					write: {
+						body: expectedScheduleRequest,
+						toolName: "createSchedule",
+					},
+				}),
+				api.calledAfterApproval({
+					call: {
+						body: expectedScheduleRequest,
+						toolName: "createSchedule",
+					},
+				}),
+				api.bodyNumberFields({
+					paths: ["phases.*.starts_at"],
+					toolName: "previewCreateSchedule",
+				}),
+				api.bodyNumberFields({
+					paths: ["phases.*.starts_at"],
+					toolName: "createSchedule",
+				}),
+				response.mentions({
+					phrases: [
+						"Northstar Labs",
+						"Enterprise",
+						"Hosted Solution",
+						"Unlimited Seats",
+					],
+				}),
+			],
+		},
+	],
+});

apps/leaf/tests/evals/agent/mcp/attach-approval.eval.ts

@@ -0,0 +1,104 @@
+import {
+	api,
+	billing,
+	response,
+	tools,
+} from "../../fixtures/expectations/index.js";
+import { createSetup } from "../../fixtures/createSetup.js";
+import { approve, initEval, user } from "../../harness/index.js";
+import { billingAttachScores } from "../../utils/scorers.js";
+
+type EvalMetadata = {
+	domain: "mcp";
+	flow: "approval";
+};
+
+const experimentName = "mcp-attach-approval";
+
+const setup = createSetup({
+	tag: "mcp-attach-approval",
+	features: ({ features }) => ({
+		dashboard: features.boolean({ featureId: "dashboard" }),
+	}),
+	plans: ({ basePrice, features, items, plan }) => ({
+		pro: plan.monthly({
+			basePrice: basePrice.monthly({ amount: 79 }),
+			items: [items.boolean({ feature: features.dashboard })],
+			planId: "pro",
+		}),
+	}),
+	customers: ({ customers }) => ({
+		account: customers.base({
+			email: "billing@atlas.example",
+			id: "cus_mcp_attach_approval",
+			name: "Atlas Labs",
+		}),
+	}),
+});
+
+const customer = setup.refs.customers.account;
+const proPlan = setup.refs.plans.pro;
+
+const expectedAttachRequest = {
+	customer_id: customer.id,
+	enable_plan_immediately: true,
+	invoice_mode: {
+		enable_plan_immediately: true,
+		enabled: true,
+		finalize: false,
+	},
+	plan_id: proPlan.id,
+};
+
+initEval<EvalMetadata>({
+	experimentName,
+	setup,
+	metadata: {
+		domain: "mcp",
+		flow: "approval",
+	},
+	scores: billingAttachScores(),
+	cases: [
+		{
+			name: "destructive attach waits for approval",
+			conversation: [
+				user({
+					message:
+						"Please attach the Pro plan to Atlas Labs. Preview it first, then attach it after approval.",
+				}),
+				user({ message: "Looks good, attach it." }),
+				approve(),
+			],
+			expect: [
+				tools.called({
+					toolNames: [
+						"getAgentRules",
+						"listCustomers",
+						"listPlans",
+						"previewAttach",
+						"attach",
+					],
+				}),
+				billing.previewBeforeWrite({
+					preview: {
+						body: expectedAttachRequest,
+						toolName: "previewAttach",
+					},
+					write: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
+				api.calledAfterApproval({
+					call: {
+						body: expectedAttachRequest,
+						toolName: "attach",
+					},
+				}),
+				response.mentions({
+					phrases: ["Atlas Labs", "Pro"],
+				}),
+			],
+		},
+	],
+});