From c2992c8c688cb11a96f1d82ebaa0709c495ac0ed Mon Sep 17 00:00:00 2001 From: johnyeo Date: Mon, 8 Jun 2026 13:41:06 +0100 Subject: [PATCH] added observability and evals --- .bt/config.json | 6 + .vscode/tasks.json | 6 +- apps/leaf/package.json | 6 + apps/leaf/src/agent/agent.ts | 29 +- apps/leaf/src/agent/messages.ts | 3 + apps/leaf/src/bot.ts | 9 +- .../observability/leafTracingOptions.ts | 49 +++ apps/leaf/src/providers/braintrust/config.ts | 8 + .../braintrust/createBraintrustLogger.ts | 15 + .../createMastraBraintrustObservability.ts | 44 ++ apps/leaf/src/providers/braintrust/index.ts | 3 + .../leaf/src/providers/slack/installations.ts | 32 ++ apps/leaf/src/types.ts | 7 +- apps/leaf/tests/evals/README.md | 39 ++ apps/leaf/tests/evals/fixtures/createSetup.ts | 230 +++++++++++ .../fixtures/customers/base/baseBalance.ts | 60 +++ .../fixtures/customers/base/baseCustomer.ts | 39 ++ .../fixtures/customers/base/baseSchedule.ts | 35 ++ .../customers/base/baseSubscription.ts | 39 ++ .../evals/fixtures/customers/base/index.ts | 4 + .../tests/evals/fixtures/customers/index.ts | 13 + .../fixtures/customers/presets/balances.ts | 6 + .../customers/presets/customerList.ts | 34 ++ .../fixtures/customers/presets/customers.ts | 19 + .../evals/fixtures/customers/presets/index.ts | 5 + .../fixtures/customers/presets/schedules.ts | 9 + .../customers/presets/subscriptions.ts | 10 + apps/leaf/tests/evals/fixtures/orgSetups.ts | 7 + .../tests/evals/fixtures/plans/featureList.ts | 19 + .../tests/evals/fixtures/plans/features.ts | 94 +++++ apps/leaf/tests/evals/fixtures/plans/index.ts | 6 + .../tests/evals/fixtures/plans/itemList.ts | 17 + apps/leaf/tests/evals/fixtures/plans/items.ts | 176 ++++++++ .../tests/evals/fixtures/plans/planList.ts | 57 +++ apps/leaf/tests/evals/fixtures/plans/plans.ts | 219 ++++++++++ apps/leaf/tests/evals/fixtures/responses.ts | 33 ++ .../fixtures/setups/knowledgePlatformSetup.ts | 167 ++++++++ apps/leaf/tests/evals/fixtures/types.ts | 79 ++++ .../harness/configs/genericMcpAgentConfig.ts | 12 + .../harness/context/createAutumnApiMock.ts | 175 ++++++++ .../harness/context/createAutumnMcpServer.ts | 53 +++ .../context/createEvalRuntimeContext.ts | 52 +++ .../leaf/tests/evals/harness/context/types.ts | 51 +++ .../tests/evals/harness/createEvalContext.ts | 107 +++++ .../evals/harness/drivers/genericMcpAgent.ts | 179 ++++++++ .../leaf/tests/evals/harness/drivers/types.ts | 36 ++ apps/leaf/tests/evals/harness/index.ts | 36 ++ .../evals/harness/tracing/createEvalTrace.ts | 26 ++ .../evals/harness/tracing/formatTrace.ts | 108 +++++ .../leaf/tests/evals/harness/tracing/types.ts | 21 + .../mcp/billing/customer-plan-summary.eval.ts | 106 +++++ apps/leaf/tests/evals/utils/scorers.ts | 91 +++++ .../unit/evals/mock-autumn-server.test.ts | 381 ++++++++++++++++++ bun.lock | 268 +++++++++--- packages/mcp/src/index.ts | 1 + run.sh | 37 +- scripts/testScripts/getDescribeAtCursor.ts | 18 +- scripts/testScripts/testDispatcher.ts | 89 ++++ 58 files changed, 3399 insertions(+), 81 deletions(-) create mode 100644 .bt/config.json create mode 100644 apps/leaf/src/internal/observability/leafTracingOptions.ts create mode 100644 apps/leaf/src/providers/braintrust/config.ts create mode 100644 apps/leaf/src/providers/braintrust/createBraintrustLogger.ts create mode 100644 apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts create mode 100644 apps/leaf/src/providers/braintrust/index.ts create mode 100644 apps/leaf/tests/evals/README.md create mode 100644 apps/leaf/tests/evals/fixtures/createSetup.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/base/baseBalance.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/base/baseCustomer.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/base/baseSchedule.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/base/baseSubscription.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/base/index.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/index.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/presets/balances.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/presets/customerList.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/presets/customers.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/presets/index.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/presets/schedules.ts create mode 100644 apps/leaf/tests/evals/fixtures/customers/presets/subscriptions.ts create mode 100644 apps/leaf/tests/evals/fixtures/orgSetups.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/featureList.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/features.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/index.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/itemList.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/items.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/planList.ts create mode 100644 apps/leaf/tests/evals/fixtures/plans/plans.ts create mode 100644 apps/leaf/tests/evals/fixtures/responses.ts create mode 100644 apps/leaf/tests/evals/fixtures/setups/knowledgePlatformSetup.ts create mode 100644 apps/leaf/tests/evals/fixtures/types.ts create mode 100644 apps/leaf/tests/evals/harness/configs/genericMcpAgentConfig.ts create mode 100644 apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts create mode 100644 apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts create mode 100644 apps/leaf/tests/evals/harness/context/createEvalRuntimeContext.ts create mode 100644 apps/leaf/tests/evals/harness/context/types.ts create mode 100644 apps/leaf/tests/evals/harness/createEvalContext.ts create mode 100644 apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts create mode 100644 apps/leaf/tests/evals/harness/drivers/types.ts create mode 100644 apps/leaf/tests/evals/harness/index.ts create mode 100644 apps/leaf/tests/evals/harness/tracing/createEvalTrace.ts create mode 100644 apps/leaf/tests/evals/harness/tracing/formatTrace.ts create mode 100644 apps/leaf/tests/evals/harness/tracing/types.ts create mode 100644 apps/leaf/tests/evals/mcp/billing/customer-plan-summary.eval.ts create mode 100644 apps/leaf/tests/evals/utils/scorers.ts create mode 100644 apps/leaf/tests/unit/evals/mock-autumn-server.test.ts diff --git a/.bt/config.json b/.bt/config.json new file mode 100644 index 000000000..6b6278e63 --- /dev/null +++ b/.bt/config.json @@ -0,0 +1,6 @@ +{ + "profile": null, + "org": "autumn", + "project": "leaf", + "project_id": "b5592c45-a906-4b43-93b7-bd05a8172b0b" +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 5e93cff6a..3c02be40f 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,21 +4,21 @@ { "label": "Run Test Pattern", "type": "shell", - "command": "infisical run --env=dev --recursive -- bun test ${relativeFile} -t \"${input:testPattern}\"", + "command": "./run.sh \"${file}\" -t \"${input:testPattern}\"", "options": { "env": { "NODE_ENV": "development" } }, "problemMatcher": [] }, { "label": "Run Describe at Cursor", "type": "shell", - "command": "infisical run --env=dev --recursive -- bun test ${relativeFile} --timeout 0 -t \"$(bun scripts/testScripts/getDescribeAtCursor.ts ${file} ${lineNumber})\"", + "command": "./run.sh \"${file}\" ${lineNumber}", "options": { "env": { "NODE_ENV": "development" } }, "problemMatcher": [] }, { "label": "Run Current Test File", "type": "shell", - "command": "infisical run --env=dev --recursive -- bun test ${relativeFile}", + "command": "./run.sh \"${file}\"", "options": { "env": { "NODE_ENV": "development" } }, "problemMatcher": [] } diff --git a/apps/leaf/package.json b/apps/leaf/package.json index adcb8cced..cd18b7f4c 100644 --- a/apps/leaf/package.json +++ b/apps/leaf/package.json @@ -5,6 +5,8 @@ "type": "module", "scripts": { "dev": "bun --watch src/index.ts", + "eval": "zsh -lc 'braintrust eval tests/evals/**/*.eval.ts --external-packages @mastra/mcp @mastra/core @mastra/braintrust @mastra/observability \"$@\"' --", + "eval:mcp": "zsh -lc 'braintrust eval tests/evals/mcp/**/*.eval.ts --external-packages @mastra/mcp @mastra/core @mastra/braintrust @mastra/observability \"$@\"' --", "start": "bun src/index.ts", "test": "bun test tests/unit", "ts": "tsc --noEmit" @@ -17,9 +19,13 @@ "@chat-adapter/slack": "^4.29.0", "@chat-adapter/state-pg": "^4.29.0", "@hono/node-server": "^1.19.5", + "@mastra/braintrust": "^1.1.3", "@mastra/core": "^1.36.0", "@mastra/mcp": "^1.8.0", + "@mastra/observability": "^1.14.1", "@mendable/firecrawl-js": "^4.25.1", + "autoevals": "^0.0.132", + "braintrust": "^3.14.0", "chat": "^4.29.0", "date-fns": "^4.1.0", "drizzle-orm": "catalog:", diff --git a/apps/leaf/src/agent/agent.ts b/apps/leaf/src/agent/agent.ts index e354a5f43..bdf0b8d80 100644 --- a/apps/leaf/src/agent/agent.ts +++ b/apps/leaf/src/agent/agent.ts @@ -2,9 +2,13 @@ import type { AutumnLogger } from "@autumn/logging"; import { AppEnv } from "@autumn/shared"; import { Agent } from "@mastra/core/agent"; import type { MessageListInput } from "@mastra/core/agent/message-list"; +import { Mastra } from "@mastra/core/mastra"; +import { InMemoryStore } from "@mastra/core/storage"; import { z } from "zod"; +import { createLeafTracingOptions } from "../internal/observability/leafTracingOptions.js"; import { env as chatEnv } from "../lib/env.js"; import { logger as rootLogger } from "../lib/logger.js"; +import { createMastraBraintrustObservability } from "../providers/braintrust/index.js"; import { createE2bSandboxProvider } from "../providers/e2b/e2bSandboxProvider.js"; import type { ChatContextMessage } from "../types.js"; import { createFirecrawlTools } from "./firecrawl.js"; @@ -123,6 +127,8 @@ export const runChatAgent = async ({ provider, workspaceId, recentMessages, + agentRunId, + orgSlug, }: { token: string; env: AppEnv; @@ -134,6 +140,8 @@ export const runChatAgent = async ({ resourceId: string; provider: string; workspaceId: string; + agentRunId?: string; + orgSlug?: string | null; recentMessages?: ChatContextMessage[]; }) => { const mcp = createAutumnMcpClient({ @@ -211,8 +219,16 @@ export const runChatAgent = async ({ model: chatEnv.CHAT_MODEL, tools: { ...tools, ...firecrawlTools, ...sandboxTools }, }); + const mastra = new Mastra({ + agents: { chat: agent }, + environment: process.env.NODE_ENV, + logger: false, + observability: createMastraBraintrustObservability(), + storage: new InMemoryStore({ id: `leaf-chat-${crypto.randomUUID()}` }), + }); + const chatAgent = mastra.getAgent("chat"); - const output = await agent.generate(message, { + const output = await chatAgent.generate(message, { maxSteps: 8, context: [ { @@ -226,6 +242,17 @@ export const runChatAgent = async ({ }, ...recentMessageContext(recentMessages), ], + tracingOptions: createLeafTracingOptions({ + agentRunId, + channelId, + env, + orgId: resourceId, + orgSlug, + provider, + source: "prod", + threadId, + workspaceId, + }), }); logger.info("Completed chat agent", { event: "leaf.agent_completed", diff --git a/apps/leaf/src/agent/messages.ts b/apps/leaf/src/agent/messages.ts index cafe96f03..43f73f8ad 100644 --- a/apps/leaf/src/agent/messages.ts +++ b/apps/leaf/src/agent/messages.ts @@ -14,6 +14,7 @@ const withTimeout = (promise: Promise, ms: number) => }); export const runMessage = async ({ + agentRunId, attachmentFetchFallback, attachments, installation, @@ -58,7 +59,9 @@ export const runMessage = async ({ onAction, channelId, threadId, + agentRunId, resourceId: installation.org_id, + orgSlug: installation.org_slug, provider: installation.provider, workspaceId: installation.workspace_id, recentMessages, diff --git a/apps/leaf/src/bot.ts b/apps/leaf/src/bot.ts index 90b71c7a8..25bc18215 100644 --- a/apps/leaf/src/bot.ts +++ b/apps/leaf/src/bot.ts @@ -16,7 +16,7 @@ import { fetchSlackAttachmentFallback, getSlackFilesFromRaw, } from "./providers/slack/files.js"; -import { findInstallation } from "./providers/slack/installations.js"; +import { findInstallationWithOrg } from "./providers/slack/installations.js"; import { getRecentMessages } from "./providers/slack/threadContext.js"; import type { ChatContextMessage } from "./types.js"; import { @@ -38,8 +38,8 @@ const findSlackInstallationForWorkspace = async ({ workspaceId: string; }) => { return ( - (await findInstallation(getSlackAdminProvider(), workspaceId)) ?? - (await findInstallation("slack", workspaceId)) + (await findInstallationWithOrg(getSlackAdminProvider(), workspaceId)) ?? + (await findInstallationWithOrg("slack", workspaceId)) ); }; @@ -116,6 +116,8 @@ const runAndReply = async ({ logger = addLeafContext(rootLogger, { ...session.context, agent_run_id: session.agentRunId, + org_id: installation.org_id, + org_slug: installation.org_slug, }); logger.info("Received Slack message", { event: "leaf.slack_message_received", @@ -137,6 +139,7 @@ const runAndReply = async ({ const rawFiles = getSlackFilesFromRaw({ raw }); const botToken = decrypt(installation.bot_access_token); const output = await runMessage({ + agentRunId: session.agentRunId, attachmentFetchFallback: ({ attachment }) => fetchSlackAttachmentFallback({ attachment, diff --git a/apps/leaf/src/internal/observability/leafTracingOptions.ts b/apps/leaf/src/internal/observability/leafTracingOptions.ts new file mode 100644 index 000000000..184014027 --- /dev/null +++ b/apps/leaf/src/internal/observability/leafTracingOptions.ts @@ -0,0 +1,49 @@ +import type { AppEnv } from "@autumn/shared"; +import type { TracingOptions } from "@mastra/core/observability"; + +const compact = (values: Array) => + values.filter((value): value is string => Boolean(value)); + +export const createLeafTracingOptions = ({ + agentRunId, + channelId, + env, + orgId, + orgSlug, + provider, + source, + threadId, + workspaceId, + setup, +}: { + agentRunId?: string; + channelId?: string; + env?: AppEnv | string; + orgId?: string; + orgSlug?: string | null; + provider?: string; + source: "eval" | "prod"; + threadId?: string; + workspaceId?: string; + setup?: string; +}): TracingOptions => ({ + metadata: { + agent_run_id: agentRunId, + autumn_env: env, + org_id: orgId, + org_slug: orgSlug, + provider, + setup, + slack_channel_id: channelId, + slack_thread_id: threadId, + slack_workspace_id: workspaceId, + source, + }, + tags: compact([ + source, + env ? `autumn:${env}` : undefined, + orgSlug ? `org:${orgSlug}` : undefined, + provider ? `provider:${provider}` : undefined, + setup ? `setup:${setup}` : undefined, + ]), +}); diff --git a/apps/leaf/src/providers/braintrust/config.ts b/apps/leaf/src/providers/braintrust/config.ts new file mode 100644 index 000000000..886e07d2f --- /dev/null +++ b/apps/leaf/src/providers/braintrust/config.ts @@ -0,0 +1,8 @@ +export const braintrustConfig = { + enabled: + process.env.LEAF_BRAINTRUST_ENABLED === "true" || + (process.env.NODE_ENV !== "production" && + process.env.LEAF_BRAINTRUST_ENABLED !== "false"), + projectName: process.env.LEAF_BRAINTRUST_PROJECT ?? "leaf", + serviceName: process.env.LEAF_BRAINTRUST_SERVICE ?? "leaf", +}; diff --git a/apps/leaf/src/providers/braintrust/createBraintrustLogger.ts b/apps/leaf/src/providers/braintrust/createBraintrustLogger.ts new file mode 100644 index 000000000..5e2398f10 --- /dev/null +++ b/apps/leaf/src/providers/braintrust/createBraintrustLogger.ts @@ -0,0 +1,15 @@ +import { initLogger, type Logger } from "braintrust"; +import { braintrustConfig } from "./config.js"; + +export const createBraintrustLogger = ({ + apiKey = process.env.BRAINTRUST_API_KEY, + enabled = braintrustConfig.enabled, + projectName = braintrustConfig.projectName, +}: { + apiKey?: string; + enabled?: boolean; + projectName?: string; +} = {}): Logger | undefined => { + if (!enabled || !apiKey) return undefined; + return initLogger({ apiKey, projectName }); +}; diff --git a/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts b/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts new file mode 100644 index 000000000..318f50c32 --- /dev/null +++ b/apps/leaf/src/providers/braintrust/createMastraBraintrustObservability.ts @@ -0,0 +1,44 @@ +import { BraintrustExporter } from "@mastra/braintrust"; +import { SpanType } from "@mastra/core/observability"; +import { Observability, SamplingStrategyType } from "@mastra/observability"; +import { currentSpan } from "braintrust"; +import { braintrustConfig } from "./config.js"; + +export const createMastraBraintrustObservability = ({ + apiKey = process.env.BRAINTRUST_API_KEY, + braintrustLogger, + enabled = braintrustConfig.enabled, + projectName = braintrustConfig.projectName, + serviceName = braintrustConfig.serviceName, +}: { + apiKey?: string; + braintrustLogger?: unknown; + enabled?: boolean; + projectName?: string; + serviceName?: string; +} = {}): Observability | undefined => { + if (!enabled) return undefined; + const exporterConfig = { + apiKey, + braintrustLogger, + currentSpan: () => currentSpan(), + projectName, + } as unknown as ConstructorParameters[0]; + + return new Observability({ + configs: { + braintrust: { + excludeSpanTypes: [SpanType.MODEL_CHUNK], + exporters: [new BraintrustExporter(exporterConfig)], + sampling: { type: SamplingStrategyType.ALWAYS }, + serializationOptions: { + maxArrayLength: 50, + maxDepth: 6, + maxObjectKeys: 80, + maxStringLength: 8_000, + }, + serviceName, + }, + }, + }); +}; diff --git a/apps/leaf/src/providers/braintrust/index.ts b/apps/leaf/src/providers/braintrust/index.ts new file mode 100644 index 000000000..5be05d1ed --- /dev/null +++ b/apps/leaf/src/providers/braintrust/index.ts @@ -0,0 +1,3 @@ +export { braintrustConfig } from "./config.js"; +export { createBraintrustLogger } from "./createBraintrustLogger.js"; +export { createMastraBraintrustObservability } from "./createMastraBraintrustObservability.js"; diff --git a/apps/leaf/src/providers/slack/installations.ts b/apps/leaf/src/providers/slack/installations.ts index 80737fe0d..cda550b31 100644 --- a/apps/leaf/src/providers/slack/installations.ts +++ b/apps/leaf/src/providers/slack/installations.ts @@ -6,6 +6,7 @@ import { type ChatInstallState, type ChatProvider, chatInstallations, + organizations, } from "@autumn/shared"; import { and, eq, or } from "drizzle-orm"; import { replaceInstallationOAuthCredentials } from "../../internal/installations/actions/replaceInstallationOAuthCredentials.js"; @@ -25,6 +26,37 @@ export const findInstallation = (provider: ChatProvider, workspaceId: string) => ), }); +export type ChatInstallationWithOrg = ChatInstallation & { + org_slug?: string; +}; + +export const findInstallationWithOrg = async ( + provider: ChatProvider, + workspaceId: string, +): Promise => { + const [row] = await db + .select({ + installation: chatInstallations, + orgSlug: organizations.slug, + }) + .from(chatInstallations) + .innerJoin(organizations, eq(organizations.id, chatInstallations.org_id)) + .where( + and( + eq(chatInstallations.provider, provider), + eq(chatInstallations.workspace_id, workspaceId), + ), + ) + .limit(1); + + return row + ? { + ...row.installation, + org_slug: row.orgSlug, + } + : undefined; +}; + export const getInstallationKey = ( installation: ChatInstallation, env: AppEnv, diff --git a/apps/leaf/src/types.ts b/apps/leaf/src/types.ts index e96b2eef6..c8be65f68 100644 --- a/apps/leaf/src/types.ts +++ b/apps/leaf/src/types.ts @@ -3,6 +3,10 @@ import { AppEnv, type ChatInstallation } from "@autumn/shared"; import type { Attachment } from "chat"; import { z } from "zod"; +export type LeafChatInstallation = ChatInstallation & { + org_slug?: string; +}; + export const agentOutputSchema = z.preprocess( (value) => { const payload = @@ -63,11 +67,12 @@ export type SignatureArgs = { }; export type BotMessage = { + agentRunId?: string; attachmentFetchFallback?: (params: { attachment: Attachment; }) => Promise; attachments?: Attachment[]; - installation: ChatInstallation; + installation: LeafChatInstallation; logger?: AutumnLogger; onAction?: (message: string) => Promise | void; recentMessages?: ChatContextMessage[]; diff --git a/apps/leaf/tests/evals/README.md b/apps/leaf/tests/evals/README.md new file mode 100644 index 000000000..4aad0b115 --- /dev/null +++ b/apps/leaf/tests/evals/README.md @@ -0,0 +1,39 @@ +# Leaf Evals + +Braintrust evals for Leaf and Autumn MCP behavior. + +## Run + +```sh +bun -F @autumn/leaf eval:mcp +``` + +Eval files set `noSendLogs` when `BRAINTRUST_API_KEY` is absent, but agent +model calls still need the normal model provider environment. + +## Pattern + +- Build setup state with `fixtures/*` builders. +- Create runtime state with `context/createEvalContext`. +- Use a driver factory, usually `createGenericMcpAgentDriver`, to exercise the runtime. +- Assert behavior with deterministic scorers before adding LLM judges. +- Keep real customer/org names out of fixtures; use setup tags like + `invoice-mode-customer-missing-email`. +- Keep `trace.event(...)` terminal-only. Braintrust spans should come from the + Mastra observability exporter unless a test explicitly needs custom spans. + +## Context + +The eval context is intentionally split by responsibility: + +- `harness/context` owns the mock Autumn API and local MCP server. +- `harness/configs` owns defaults and reusable eval configuration objects. +- `harness/drivers` owns agent/client variants that talk to the MCP server. +- `harness/tracing` owns local terminal visibility for user turns, tool calls, + API calls, and approvals. + +Eval files should read like scenarios: choose a fixture setup, create a context, +run conversation turns, return scorer output. + +The old MCP evals under `packages/mcp/tests/evals` are intentionally left in +place while this structure is proven out. diff --git a/apps/leaf/tests/evals/fixtures/createSetup.ts b/apps/leaf/tests/evals/fixtures/createSetup.ts new file mode 100644 index 000000000..f20657f3c --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/createSetup.ts @@ -0,0 +1,230 @@ +import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; +import type { ApiCustomerSchedule } from "@api/customers/components/apiCustomerSchedule"; +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; +import { + balances as balanceFixtures, + customers as customerFixtures, + customerList as customerListFixture, + schedules as scheduleFixtures, + subscriptions as subscriptionFixtures, +} from "./customers/index.js"; +import { + basePrice as basePriceFixture, + features as featureFixtures, + featureList as featureListFixture, + items as itemFixtures, + itemList as itemListFixture, + plan as planFixture, + planList as planListFixture, +} from "./plans/index.js"; +import type { EvalSetup, EvalSetupIds, PlanRef, ScheduleRef } from "./types.js"; + +const flattenRecordValues = (record: Record) => + Object.values(record).flatMap((value) => + Array.isArray(value) ? value : [value], + ); + +const refIds = ( + record: Record, +) => + Object.fromEntries( + Object.entries(record).map(([key, value]) => [ + key, + Array.isArray(value) ? value.map((item) => item.id) : value.id, + ]), + ); + +const setupIds = < + Features extends Record, + Plans extends Record, + Customers extends Record, + Schedules extends Record, +>({ + customers, + features, + plans, + schedules, +}: { + customers: Customers; + features: Features; + plans: Plans; + schedules: Schedules; +}) => + ({ + customers: refIds(customers), + features: refIds(features), + plans: refIds(plans), + schedules: refIds(schedules), + }) as unknown as EvalSetupIds; + +/** + * Compose a mock Autumn org for evals from keyed feature, plan, and customer refs. + * The returned arrays feed the mock API; refs keep setup assertions readable. + */ +export const createSetup = < + Features extends Record, + Plans extends Record, + Customers extends Record, + Schedules extends Record = Record, +>({ + customers: createCustomers, + features: createFeatures, + plans: createPlans, + schedules: createSchedules, + tag, +}: { + tag: string; + features: ({ + featureList, + features, + }: { + featureList: typeof featureListFixture; + features: typeof featureFixtures; + }) => Features; + plans: ({ + basePrice, + features, + itemList, + items, + plan, + planList, + }: { + basePrice: typeof basePriceFixture; + features: Features; + itemList: typeof itemListFixture; + items: typeof itemFixtures; + plan: typeof planFixture; + planList: typeof planListFixture; + }) => Plans; + customers: ({ + balances, + customerList, + customers, + features, + plans, + subscriptions, + }: { + balances: typeof balanceFixtures; + customerList: typeof customerListFixture; + customers: typeof customerFixtures; + features: Features; + plans: Plans; + subscriptions: typeof subscriptionFixtures; + }) => Customers; + schedules?: ({ + customers, + plans, + schedules, + }: { + customers: Customers; + plans: Plans; + schedules: typeof scheduleFixtures; + }) => Schedules; +}): EvalSetup => { + const featureRefs = createFeatures({ + featureList: featureListFixture, + features: featureFixtures, + }); + const planRefs = createPlans({ + basePrice: basePriceFixture, + features: featureRefs, + itemList: itemListFixture, + items: itemFixtures, + plan: planFixture, + planList: planListFixture, + }); + const customerRefs = createCustomers({ + balances: balanceFixtures, + customerList: customerListFixture, + customers: customerFixtures, + features: featureRefs, + plans: planRefs, + subscriptions: subscriptionFixtures, + }); + const scheduleRefs = createSchedules?.({ + customers: customerRefs, + plans: planRefs, + schedules: scheduleFixtures, + }); + + return { + tag, + ids: setupIds({ + customers: customerRefs, + features: featureRefs, + plans: planRefs, + schedules: (scheduleRefs ?? {}) as Schedules, + }), + features: Object.values(featureRefs), + plans: flattenRecordValues(planRefs), + customers: flattenRecordValues(customerRefs), + schedules: flattenRecordValues( + (scheduleRefs ?? {}) as Schedules, + ), + refs: { + features: featureRefs, + plans: planRefs, + customers: customerRefs, + schedules: (scheduleRefs ?? {}) as Schedules, + }, + }; +}; + +/** Extend an org setup with eval-specific customers while preserving typed refs. */ +export const withCustomers = < + Setup extends EvalSetup, + Customers extends Record, +>({ + customers: createCustomers, + setup, +}: { + setup: Setup; + customers: ({ + balances, + customerList, + customers, + features, + plans, + subscriptions, + }: { + balances: typeof balanceFixtures; + customerList: typeof customerListFixture; + customers: typeof customerFixtures; + features: Setup["refs"]["features"]; + plans: Setup["refs"]["plans"]; + subscriptions: typeof subscriptionFixtures; + }) => Customers; +}): EvalSetup< + Setup["refs"]["features"], + Setup["refs"]["plans"], + Customers, + Setup["refs"]["schedules"] +> => { + const customerRefs = createCustomers({ + balances: balanceFixtures, + customerList: customerListFixture, + customers: customerFixtures, + features: setup.refs.features, + plans: setup.refs.plans, + subscriptions: subscriptionFixtures, + }); + + return { + ...setup, + ids: setupIds({ + customers: customerRefs, + features: setup.refs.features, + plans: setup.refs.plans, + schedules: setup.refs.schedules, + }), + customers: [ + ...setup.customers, + ...flattenRecordValues(customerRefs), + ], + refs: { + ...setup.refs, + customers: customerRefs, + }, + }; +}; diff --git a/apps/leaf/tests/evals/fixtures/customers/base/baseBalance.ts b/apps/leaf/tests/evals/fixtures/customers/base/baseBalance.ts new file mode 100644 index 000000000..0401760d1 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/base/baseBalance.ts @@ -0,0 +1,60 @@ +import type { ApiBalanceV1 } from "@api/customers/cusFeatures/apiBalanceV1.js"; +import { ResetInterval } from "@models/productModels/intervals/resetInterval.js"; + +const dateToEpochMs = (date: Date | null) => date?.getTime() ?? null; + +/** Base balance fixture with matching top-level values and one simple breakdown row. */ +export const baseBalance = ({ + featureId = "credits", + granted = 0, + remaining = granted, + reset = { interval: ResetInterval.Month }, + nextResetAt = null, + planId = null, + usage = granted - remaining, +}: { + featureId?: string; + granted?: number; + remaining?: number; + usage?: number; + nextResetAt?: Date | null; + reset?: { interval?: ResetInterval; intervalCount?: number } | null; + planId?: string | null; +} = {}): ApiBalanceV1 => { + const nextResetAtMs = dateToEpochMs(nextResetAt); + const resetValue = reset + ? { + interval: reset.interval ?? ResetInterval.Month, + interval_count: reset.intervalCount, + resets_at: nextResetAtMs, + } + : null; + + return { + object: "balance", + feature_id: featureId, + granted, + remaining, + usage, + unlimited: false, + overage_allowed: false, + max_purchase: null, + next_reset_at: nextResetAtMs, + breakdown: [ + { + object: "balance_breakdown", + id: `balance_${featureId}`, + plan_id: planId, + included_grant: granted, + prepaid_grant: 0, + remaining, + usage, + unlimited: false, + reset: resetValue, + price: null, + expires_at: null, + overage: 0, + }, + ], + }; +}; diff --git a/apps/leaf/tests/evals/fixtures/customers/base/baseCustomer.ts b/apps/leaf/tests/evals/fixtures/customers/base/baseCustomer.ts new file mode 100644 index 000000000..c64b348c1 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/base/baseCustomer.ts @@ -0,0 +1,39 @@ +import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; +import type { ApiBalanceV1 } from "@api/customers/cusFeatures/apiBalanceV1.js"; +import type { ApiSubscriptionV1 } from "@api/customers/cusPlans/apiSubscriptionV1.js"; +import { AppEnv } from "@models/genModels/genEnums.js"; + +const defaultCreatedAt = new Date("2026-01-01T00:00:00.000Z"); + +/** Base customer API fixture; prefer presets unless you need direct shape control. */ +export const baseCustomer = ({ + balances = {}, + createdAt = defaultCreatedAt, + email = "billing@example.com", + id = "customer_active", + name = "Active Customer", + subscriptions = [], +}: { + id?: string | null; + name?: string | null; + email?: string | null; + createdAt?: Date; + subscriptions?: ApiSubscriptionV1[]; + balances?: Record; +} = {}): BaseApiCustomerV5 => ({ + balances, + billing_controls: {}, + config: { disable_pooled_balance: false }, + created_at: createdAt.getTime(), + env: AppEnv.Sandbox, + id, + email, + fingerprint: null, + flags: {}, + metadata: {}, + name, + purchases: [], + send_email_receipts: false, + stripe_id: null, + subscriptions, +}); diff --git a/apps/leaf/tests/evals/fixtures/customers/base/baseSchedule.ts b/apps/leaf/tests/evals/fixtures/customers/base/baseSchedule.ts new file mode 100644 index 000000000..41a56faf9 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/base/baseSchedule.ts @@ -0,0 +1,35 @@ +import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; +import type { ApiCustomerSchedule } from "@api/customers/components/apiCustomerSchedule"; + +const defaultCreatedAt = new Date("2026-01-01T00:00:00.000Z"); + +export const baseSchedule = ({ + createdAt = defaultCreatedAt, + customer, + customerId = customer?.id ?? "customer", + entityId = null, + id = `sched_${customerId}`, + phases, +}: { + createdAt?: Date; + customer?: BaseApiCustomerV5; + customerId?: string; + entityId?: string | null; + id?: string; + phases: Array<{ + customerProductIds?: string[]; + id?: string; + startsAt: Date; + }>; +}): ApiCustomerSchedule => ({ + id, + customer_id: customerId, + entity_id: entityId, + created_at: createdAt.getTime(), + phases: phases.map((phase, index) => ({ + id: phase.id ?? `${id}_phase_${index + 1}`, + created_at: createdAt.getTime(), + customer_product_ids: phase.customerProductIds ?? [], + starts_at: phase.startsAt.getTime(), + })), +}); diff --git a/apps/leaf/tests/evals/fixtures/customers/base/baseSubscription.ts b/apps/leaf/tests/evals/fixtures/customers/base/baseSubscription.ts new file mode 100644 index 000000000..62ec606a5 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/base/baseSubscription.ts @@ -0,0 +1,39 @@ +import type { ApiSubscriptionV1 } from "@api/customers/cusPlans/apiSubscriptionV1.js"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; + +const defaultStartedAt = new Date("2026-01-01T00:00:00.000Z"); +const dateToEpochMs = (date: Date | null) => date?.getTime() ?? null; + +/** Base subscription fixture; pass plan to include the expanded plan response. */ +export const baseSubscription = ({ + id, + plan, + planId = plan?.id ?? "pro", + status = "active", + startedAt = defaultStartedAt, + currentPeriodStart = startedAt, + currentPeriodEnd = null, +}: { + id?: string; + plan?: ApiPlanV1; + planId?: string; + status?: ApiSubscriptionV1["status"]; + startedAt?: Date; + currentPeriodStart?: Date | null; + currentPeriodEnd?: Date | null; +}): ApiSubscriptionV1 => ({ + id: id ?? `sub_${planId}`, + plan, + plan_id: planId, + auto_enable: false, + add_on: false, + status, + past_due: false, + canceled_at: null, + expires_at: null, + trial_ends_at: null, + started_at: startedAt.getTime(), + current_period_start: dateToEpochMs(currentPeriodStart), + current_period_end: dateToEpochMs(currentPeriodEnd), + quantity: 1, +}); diff --git a/apps/leaf/tests/evals/fixtures/customers/base/index.ts b/apps/leaf/tests/evals/fixtures/customers/base/index.ts new file mode 100644 index 000000000..fba67a5c4 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/base/index.ts @@ -0,0 +1,4 @@ +export { baseBalance } from "./baseBalance.js"; +export { baseCustomer } from "./baseCustomer.js"; +export { baseSchedule } from "./baseSchedule.js"; +export { baseSubscription } from "./baseSubscription.js"; diff --git a/apps/leaf/tests/evals/fixtures/customers/index.ts b/apps/leaf/tests/evals/fixtures/customers/index.ts new file mode 100644 index 000000000..00806ae6d --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/index.ts @@ -0,0 +1,13 @@ +export { + baseBalance, + baseCustomer, + baseSchedule, + baseSubscription, +} from "./base/index.js"; +export { + balances, + customerList, + customers, + schedules, + subscriptions, +} from "./presets/index.js"; diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/balances.ts b/apps/leaf/tests/evals/fixtures/customers/presets/balances.ts new file mode 100644 index 000000000..c59783e10 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/presets/balances.ts @@ -0,0 +1,6 @@ +import { baseBalance } from "../base/baseBalance.js"; + +export const balances = { + empty: baseBalance, + metered: baseBalance, +} as const; diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/customerList.ts b/apps/leaf/tests/evals/fixtures/customers/presets/customerList.ts new file mode 100644 index 000000000..12795e6d1 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/presets/customerList.ts @@ -0,0 +1,34 @@ +import type { ApiSubscriptionV1 } from "@api/customers/cusPlans/apiSubscriptionV1.js"; +import { baseCustomer } from "../base/baseCustomer.js"; + +/** Generate deterministic customers for broad list/search evals. */ +export const customerList = ({ + count, + emailDomain = "example.test", + idPrefix = "customer", + namePrefix = "Customer", + subscription, +}: { + count: number; + emailDomain?: string; + idPrefix?: string; + namePrefix?: string; + subscription?: ({ + index, + }: { + index: number; + }) => ApiSubscriptionV1 | undefined; +}) => + Array.from({ length: count }, (_, index) => { + const number = index + 1; + const padded = String(number).padStart(3, "0"); + const id = `${idPrefix}_${padded}`; + const maybeSubscription = subscription?.({ index }); + + return baseCustomer({ + id, + email: `${id}@${emailDomain}`, + name: `${namePrefix} ${number}`, + subscriptions: maybeSubscription ? [maybeSubscription] : [], + }); + }); diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts b/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts new file mode 100644 index 000000000..1135f4153 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/presets/customers.ts @@ -0,0 +1,19 @@ +import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; +import { baseCustomer } from "../base/baseCustomer.js"; +import { subscriptions } from "./subscriptions.js"; + +type CustomerArgs = Parameters[0]; + +/** Customer presets for common eval scenarios; compose subscriptions explicitly. */ +export const customers = { + active: (args?: CustomerArgs): BaseApiCustomerV5 => baseCustomer(args), + withPlan: ({ + plan, + ...args + }: CustomerArgs & { plan: ApiPlanV1 }): BaseApiCustomerV5 => + baseCustomer({ + ...args, + subscriptions: [subscriptions.active({ plan })], + }), +} as const; diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/index.ts b/apps/leaf/tests/evals/fixtures/customers/presets/index.ts new file mode 100644 index 000000000..5b04e8e50 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/presets/index.ts @@ -0,0 +1,5 @@ +export { balances } from "./balances.js"; +export { customerList } from "./customerList.js"; +export { customers } from "./customers.js"; +export { schedules } from "./schedules.js"; +export { subscriptions } from "./subscriptions.js"; diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/schedules.ts b/apps/leaf/tests/evals/fixtures/customers/presets/schedules.ts new file mode 100644 index 000000000..7cc3083b5 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/presets/schedules.ts @@ -0,0 +1,9 @@ +import { baseSchedule } from "../base/baseSchedule.js"; + +type ScheduleArgs = Parameters[0]; + +export const schedules = { + customer: (args: ScheduleArgs) => baseSchedule(args), + entity: (args: Omit & { entityId: string }) => + baseSchedule(args), +} as const; diff --git a/apps/leaf/tests/evals/fixtures/customers/presets/subscriptions.ts b/apps/leaf/tests/evals/fixtures/customers/presets/subscriptions.ts new file mode 100644 index 000000000..19d1161d6 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/customers/presets/subscriptions.ts @@ -0,0 +1,10 @@ +import { baseSubscription } from "../base/baseSubscription.js"; + +type SubscriptionArgs = Omit[0], "status">; + +export const subscriptions = { + active: (args: SubscriptionArgs) => + baseSubscription({ ...args, status: "active" }), + scheduled: (args: SubscriptionArgs) => + baseSubscription({ ...args, status: "scheduled" }), +} as const; diff --git a/apps/leaf/tests/evals/fixtures/orgSetups.ts b/apps/leaf/tests/evals/fixtures/orgSetups.ts new file mode 100644 index 000000000..7507c64ad --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/orgSetups.ts @@ -0,0 +1,7 @@ +import { knowledgePlatformSetup } from "./setups/knowledgePlatformSetup.js"; + +export type { EvalSetup as EvalOrgSetup } from "./types.js"; + +export const orgSetups = { + knowledgePlatform: knowledgePlatformSetup, +} as const; diff --git a/apps/leaf/tests/evals/fixtures/plans/featureList.ts b/apps/leaf/tests/evals/fixtures/plans/featureList.ts new file mode 100644 index 000000000..d82c9796e --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/featureList.ts @@ -0,0 +1,19 @@ +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import { features } from "./features.js"; + +/** Build keyed feature records for org-like setups with many boolean flags. */ +export const featureList = { + boolean: ({ + featureIds, + names = {}, + }: { + featureIds: readonly FeatureId[]; + names?: Partial>; + }): Record => + Object.fromEntries( + featureIds.map((featureId) => [ + featureId, + features.boolean({ featureId, name: names[featureId] }), + ]), + ) as Record, +} as const; diff --git a/apps/leaf/tests/evals/fixtures/plans/features.ts b/apps/leaf/tests/evals/fixtures/plans/features.ts new file mode 100644 index 000000000..987ccf48b --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/features.ts @@ -0,0 +1,94 @@ +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import { FeatureType } from "@models/featureModels/featureEnums.js"; + +const feature = ({ + archived = false, + consumable, + eventNames, + featureId, + name, + type, +}: { + archived?: boolean; + consumable: boolean; + eventNames?: string[]; + featureId: string; + name: string; + type: ApiFeatureV1["type"]; +}): ApiFeatureV1 => ({ + archived, + consumable, + event_names: eventNames, + id: featureId, + name, + type, +}); + +const nameFromId = (featureId: string) => + featureId + .split(/[-_]/) + .map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`) + .join(" "); + +/** Feature fixtures for plan scenarios; pass feature objects into item fixtures. */ +export const features = { + allocated: ({ + featureId = "seats", + name = featureId === "seats" ? "Seats" : nameFromId(featureId), + }: { + featureId?: string; + name?: string; + } = {}): ApiFeatureV1 => + feature({ + consumable: false, + featureId, + name, + type: FeatureType.Metered, + }), + boolean: ({ + featureId = "admin_dashboard", + name = featureId === "admin_dashboard" + ? "Admin Dashboard" + : nameFromId(featureId), + }: { + featureId?: string; + name?: string; + } = {}): ApiFeatureV1 => + feature({ + consumable: false, + featureId, + name, + type: FeatureType.Boolean, + }), + consumable: ({ + featureId = "api_calls", + name = featureId === "api_calls" ? "API Calls" : nameFromId(featureId), + }: { + featureId?: string; + name?: string; + } = {}): ApiFeatureV1 => + feature({ + consumable: true, + eventNames: [featureId], + featureId, + name, + type: FeatureType.Metered, + }), + creditSystem: ({ + featureId = "credits", + meteredFeatureId = "api_calls", + name = featureId === "credits" ? "Credits" : nameFromId(featureId), + }: { + featureId?: string; + meteredFeatureId?: string; + name?: string; + } = {}): ApiFeatureV1 => ({ + ...feature({ + consumable: true, + featureId, + name, + type: FeatureType.CreditSystem, + }), + credit_schema: [{ metered_feature_id: meteredFeatureId, credit_cost: 1 }], + }), +} as const; diff --git a/apps/leaf/tests/evals/fixtures/plans/index.ts b/apps/leaf/tests/evals/fixtures/plans/index.ts new file mode 100644 index 000000000..0ca21d387 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/index.ts @@ -0,0 +1,6 @@ +export { featureList } from "./featureList.js"; +export { features } from "./features.js"; +export { itemList } from "./itemList.js"; +export { items } from "./items.js"; +export { planList } from "./planList.js"; +export { basePrice, plan } from "./plans.js"; diff --git a/apps/leaf/tests/evals/fixtures/plans/itemList.ts b/apps/leaf/tests/evals/fixtures/plans/itemList.ts new file mode 100644 index 000000000..3ad0af702 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/itemList.ts @@ -0,0 +1,17 @@ +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import type { ApiPlanItemV1 } from "@api/products/items/apiPlanItemV1.js"; +import { items } from "./items.js"; + +/** Build repeated plan item lists from keyed feature refs. */ +export const itemList = { + boolean: ({ + featureIds, + features, + }: { + featureIds: readonly FeatureId[]; + features: Record; + }): ApiPlanItemV1[] => + featureIds.map((featureId) => + items.boolean({ feature: features[featureId] }), + ), +} as const; diff --git a/apps/leaf/tests/evals/fixtures/plans/items.ts b/apps/leaf/tests/evals/fixtures/plans/items.ts new file mode 100644 index 000000000..8476453aa --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/items.ts @@ -0,0 +1,176 @@ +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import { BillingMethod } from "@api/products/components/billingMethod.js"; +import type { ApiPlanItemV1 } from "@api/products/items/apiPlanItemV1.js"; +import { FeatureType } from "@models/featureModels/featureEnums.js"; +import { RolloverExpiryDurationType } from "@models/productModels/durationTypes/rolloverExpiryDurationType.js"; +import { BillingInterval } from "@models/productModels/intervals/billingInterval.js"; +import { ResetInterval } from "@models/productModels/intervals/resetInterval.js"; +import { TierBehavior } from "@models/productModels/priceModels/priceConfig/usagePriceConfig.js"; + +type PlanItemPrice = NonNullable; +type Rollover = NonNullable; +type UsageTier = NonNullable[number]; + +const resetInterval = { + month: ResetInterval.Month, + year: ResetInterval.Year, +} as const; + +const billingInterval = { + month: BillingInterval.Month, + year: BillingInterval.Year, +} as const; + +const defaultCreditTiers: UsageTier[] = [ + { to: 10_000, amount: 0, flat_amount: 100 }, + { to: 50_000, amount: 0, flat_amount: 400 }, + { to: 100_000, amount: 0, flat_amount: 750 }, + { to: "inf", amount: 0, flat_amount: 1_000 }, +]; + +const defaultRollover: Rollover = { + expiry_duration_length: 1, + expiry_duration_type: RolloverExpiryDurationType.Month, + max: null, + max_percentage: 50, +}; + +const assertFeatureType = ({ + feature, + expected, + item, +}: { + feature: ApiFeatureV1; + expected: ApiFeatureV1["type"] | ApiFeatureV1["type"][]; + item: string; +}) => { + const expectedTypes = Array.isArray(expected) ? expected : [expected]; + if (expectedTypes.includes(feature.type)) return; + + throw new Error( + `${item} item requires ${expectedTypes.join(" or ")} feature, got ${feature.type} (${feature.id}).`, + ); +}; + +/** Plan item fixtures validate feature compatibility to avoid impossible setups. */ +export const items = { + boolean: ({ feature }: { feature: ApiFeatureV1 }): ApiPlanItemV1 => { + assertFeatureType({ + expected: FeatureType.Boolean, + feature, + item: "boolean", + }); + + return { + display: { + primary_text: feature.name, + }, + feature_id: feature.id, + included: 1, + price: null, + reset: null, + unlimited: true, + }; + }, + included: ({ + feature, + included = 0, + interval = feature.consumable ? "month" : null, + }: { + feature: ApiFeatureV1; + included?: number; + interval?: "month" | "year" | null; + }): ApiPlanItemV1 => { + assertFeatureType({ + expected: [FeatureType.Metered, FeatureType.CreditSystem], + feature, + item: "included", + }); + + return { + display: { + primary_text: `${included.toLocaleString()} ${feature.name}`, + secondary_text: interval ? `resets every ${interval}` : undefined, + }, + feature_id: feature.id, + included, + price: null, + reset: interval ? { interval: resetInterval[interval] } : null, + unlimited: false, + }; + }, + prepaidCredits: ({ + feature, + included = 5_000, + interval = "month", + rollover = defaultRollover, + tiers = defaultCreditTiers, + }: { + feature: ApiFeatureV1; + included?: number; + interval?: "month" | "year"; + rollover?: Rollover; + tiers?: UsageTier[]; + }): ApiPlanItemV1 => { + assertFeatureType({ + expected: FeatureType.CreditSystem, + feature, + item: "prepaidCredits", + }); + + return { + display: { + primary_text: `${included.toLocaleString()} ${feature.name}`, + secondary_text: "then prepaid volume tiers", + }, + feature_id: feature.id, + included, + price: { + billing_method: BillingMethod.Prepaid, + billing_units: 1, + interval: billingInterval[interval], + max_purchase: null, + tier_behavior: TierBehavior.VolumeBased, + tiers, + }, + reset: { interval: resetInterval[interval] }, + rollover, + unlimited: false, + }; + }, + consumableCredits: ({ + amount = 0.01, + feature, + interval = "month", + rollover = defaultRollover, + }: { + amount?: number; + feature: ApiFeatureV1; + interval?: "month" | "year"; + rollover?: Rollover; + }): ApiPlanItemV1 => { + assertFeatureType({ + expected: FeatureType.CreditSystem, + feature, + item: "consumableCredits", + }); + + return { + display: { + primary_text: `$${amount} per ${feature.name}`, + }, + feature_id: feature.id, + included: 0, + price: { + amount, + billing_method: BillingMethod.UsageBased, + billing_units: 1, + interval: billingInterval[interval], + max_purchase: null, + }, + reset: { interval: resetInterval[interval] }, + rollover, + unlimited: false, + }; + }, +} as const; diff --git a/apps/leaf/tests/evals/fixtures/plans/planList.ts b/apps/leaf/tests/evals/fixtures/plans/planList.ts new file mode 100644 index 000000000..fbfab3595 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/planList.ts @@ -0,0 +1,57 @@ +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; +import { items } from "./items.js"; +import { basePrice, plan } from "./plans.js"; + +type AddOnConfig = { + amount?: number | null; + feature: ApiFeatureV1; + interval?: "month" | "year"; + key: string; + name?: string; + planId: string; +}; + +const addOnPrice = ({ + amount, + interval, +}: { + amount?: number | null; + interval: "month" | "year"; +}) => { + if (amount == null) return null; + return interval === "month" + ? basePrice.monthly({ amount }) + : basePrice.annual({ amount }); +}; + +/** Build keyed one-feature add-on plans for org setups with many add-ons. */ +export const planList = { + addOns: ({ + addOns, + defaultInterval = "month", + }: { + addOns: readonly AddOn[]; + defaultInterval?: "month" | "year"; + }): { [Item in AddOn as Item["key"]]: ApiPlanV1 } => + Object.fromEntries( + addOns.map( + ({ + amount = null, + feature, + interval = defaultInterval, + key, + name, + planId, + }) => [ + key, + plan.addOn({ + basePrice: addOnPrice({ amount, interval }), + items: [items.boolean({ feature })], + name, + planId, + }), + ], + ), + ) as { [Item in AddOn as Item["key"]]: ApiPlanV1 }, +} as const; diff --git a/apps/leaf/tests/evals/fixtures/plans/plans.ts b/apps/leaf/tests/evals/fixtures/plans/plans.ts new file mode 100644 index 000000000..4afa984b9 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/plans/plans.ts @@ -0,0 +1,219 @@ +import type { CustomizePlanV1 } from "@api/billing/common/customizePlan/customizePlanV1"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; +import type { ApiPlanItemV1 } from "@api/products/items/apiPlanItemV1.js"; +import type { PlanItemFilter } from "@api/products/items/filter/planItemFilter"; +import { AppEnv } from "@models/genModels/genEnums.js"; +import { BillingInterval } from "@models/productModels/intervals/billingInterval.js"; + +type PlanPrice = NonNullable; +type EvalCustomizePlan = Omit< + CustomizePlanV1, + "add_items" | "items" | "price" +> & { + add_items?: ApiPlanV1["items"]; + items?: ApiPlanV1["items"]; + price?: CustomizePlanV1["price"] | PlanPrice | null; +}; + +const dollarsToCents = (amount: number) => amount * 100; +const displayAmount = (amount: number) => `$${amount}`; + +const planNameFromId = (planId: string) => + planId + .split(/[-_]/) + .map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`) + .join(" "); + +const itemMatchesFilter = ({ + filter, + item, +}: { + filter: PlanItemFilter; + item: ApiPlanItemV1; +}) => { + if (filter.feature_id !== undefined && item.feature_id !== filter.feature_id) + return false; + if ( + filter.billing_method !== undefined && + item.price?.billing_method !== filter.billing_method + ) + return false; + if (filter.interval !== undefined) { + const itemInterval = item.price?.interval ?? item.reset?.interval; + if (String(itemInterval) !== String(filter.interval)) return false; + } + return true; +}; + +const assertNoDuplicateItems = (items: ApiPlanV1["items"]) => { + const featureIds = items.map((item) => item.feature_id); + const duplicateFeatureId = featureIds.find( + (featureId, index) => featureIds.indexOf(featureId) !== index, + ); + if (!duplicateFeatureId) return; + + throw new Error( + `Customized plan has duplicate item for feature ${duplicateFeatureId}; remove or update the original item first.`, + ); +}; + +const applyCustomizeItems = ({ + customize, + items, +}: { + customize: EvalCustomizePlan; + items: ApiPlanV1["items"]; +}) => { + if ( + customize.items !== undefined && + (customize.add_items !== undefined || + customize.remove_items !== undefined || + customize.update_items !== undefined) + ) { + throw new Error( + "customize.items cannot be combined with add_items, remove_items, or update_items.", + ); + } + + const nextItems = + customize.items ?? + [ + ...items + .filter( + (item) => + !(customize.remove_items ?? []).some((filter) => + itemMatchesFilter({ filter, item }), + ), + ) + .map((item) => { + const update = (customize.update_items ?? []).find((update) => + itemMatchesFilter({ filter: update.filter, item }), + ); + return update?.included !== undefined + ? { ...item, included: update.included } + : item; + }), + ...(customize.add_items ?? []), + ]; + assertNoDuplicateItems(nextItems); + return nextItems; +}; + +/** Base price amounts are in dollars; returned API price.amount is cents. */ +export const basePrice = { + annual: ({ amount = 200 } = {}): PlanPrice => ({ + amount: dollarsToCents(amount), + display: { + primary_text: displayAmount(amount), + secondary_text: "per year", + }, + interval: BillingInterval.Year, + }), + monthly: ({ amount = 20 } = {}): PlanPrice => ({ + amount: dollarsToCents(amount), + display: { + primary_text: displayAmount(amount), + secondary_text: "per month", + }, + interval: BillingInterval.Month, + }), +}; + +const createPlan = ({ + addOn = false, + basePrice, + items = [], + name, + planId, + version = 1, +}: { + addOn?: boolean; + basePrice: PlanPrice | null; + items?: ApiPlanV1["items"]; + name?: string; + planId: string; + version?: number; +}): ApiPlanV1 => ({ + add_on: addOn, + archived: false, + auto_enable: false, + base_variant_id: null, + config: { ignore_past_due: false }, + created_at: 1_767_225_600_000, + description: null, + env: AppEnv.Sandbox, + group: null, + id: planId, + items, + name: name ?? planNameFromId(planId), + price: basePrice, + version, +}); + +/** Plan fixtures default to realistic base prices and accept plan items directly. */ +export const plan = { + addOn: ({ + basePrice: price = null, + planId, + ...args + }: { + basePrice?: PlanPrice | null; + items?: ApiPlanV1["items"]; + name?: string; + planId: string; + version?: number; + }): ApiPlanV1 => + createPlan({ ...args, addOn: true, basePrice: price, planId }), + annual: ({ + basePrice: price = basePrice.annual(), + planId = "enterprise", + ...args + }: { + basePrice?: PlanPrice | null; + items?: ApiPlanV1["items"]; + name?: string; + planId?: string; + version?: number; + } = {}): ApiPlanV1 => createPlan({ ...args, basePrice: price, planId }), + monthly: ({ + basePrice: price = basePrice.monthly(), + planId = "pro", + ...args + }: { + basePrice?: PlanPrice | null; + items?: ApiPlanV1["items"]; + name?: string; + planId?: string; + version?: number; + } = {}): ApiPlanV1 => createPlan({ ...args, basePrice: price, planId }), + customized: ({ + customize, + name, + plan: basePlan, + planId = `${basePlan.id}_custom`, + version = basePlan.version, + }: { + customize: EvalCustomizePlan; + name?: string; + plan: ApiPlanV1; + planId?: string; + version?: number; + }): ApiPlanV1 => ({ + ...basePlan, + base_variant_id: basePlan.base_variant_id ?? basePlan.id, + id: planId, + items: applyCustomizeItems({ + customize, + items: basePlan.items, + }), + name: name ?? basePlan.name, + price: + customize.price === undefined + ? basePlan.price + : (customize.price as PlanPrice | null), + version, + ...(customize.free_trial !== undefined + ? { free_trial: customize.free_trial ?? undefined } + : {}), + }), +}; diff --git a/apps/leaf/tests/evals/fixtures/responses.ts b/apps/leaf/tests/evals/fixtures/responses.ts new file mode 100644 index 000000000..aade090c5 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/responses.ts @@ -0,0 +1,33 @@ +import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; + +const planAmount = (plan: ApiPlanV1) => plan.price?.amount ?? 0; + +export const responses = { + attachPreview: ({ + customer, + plan, + }: { + customer: BaseApiCustomerV5; + plan: ApiPlanV1; + }) => ({ + customer_id: customer.id, + plan_id: plan.id, + currency: "usd", + line_items: [ + { description: `${plan.name} annual`, total: planAmount(plan) }, + ], + total: planAmount(plan), + }), + attachSuccess: ({ + customer, + plan, + }: { + customer: BaseApiCustomerV5; + plan: ApiPlanV1; + }) => ({ + customer_id: customer.id, + plan_id: plan.id, + status: "created", + }), +}; diff --git a/apps/leaf/tests/evals/fixtures/setups/knowledgePlatformSetup.ts b/apps/leaf/tests/evals/fixtures/setups/knowledgePlatformSetup.ts new file mode 100644 index 000000000..59c9b2a65 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/setups/knowledgePlatformSetup.ts @@ -0,0 +1,167 @@ +import { createSetup } from "../createSetup.js"; + +const featureIds = { + activity_events: "activity_events", + approval_chains: "approval_chains", + automation_rules: "automation_rules", + brand_controls: "brand_controls", + compliance_controls: "compliance_controls", + credits: "credits", + export_center: "export_center", + insight_reports: "insight_reports", + member_slots: "member_slots", + outbound_hooks: "outbound_hooks", + platform_api: "platform_api", + priority_queue: "priority_queue", + private_spaces: "private_spaces", + project_slots: "project_slots", + revision_history: "revision_history", + team_policies: "team_policies", +} as const; + +const planIds = { + automationPack: "automation_pack", + enterprise: "enterprise", + launch: "launch", + scale: "scale", + scaleYearly: "scale_yearly", + securityPack: "security_pack", + trial: "trial", + whiteLabelPack: "white_label_pack", +} as const; + +const platformFeatureIds = [ + featureIds.insight_reports, + featureIds.team_policies, + featureIds.private_spaces, + featureIds.export_center, + featureIds.priority_queue, + featureIds.automation_rules, + featureIds.outbound_hooks, + featureIds.platform_api, + featureIds.approval_chains, + featureIds.brand_controls, + featureIds.compliance_controls, + featureIds.revision_history, +] as const; + +/** Anonymized org setup with credits, many feature flags, core plans, and add-ons. */ +export const knowledgePlatformSetup = () => + createSetup({ + tag: "knowledge-platform", + features: ({ featureList, features }) => ({ + activity_events: features.consumable({ + featureId: featureIds.activity_events, + name: "Activity Events", + }), + credits: features.creditSystem({ + featureId: featureIds.credits, + meteredFeatureId: featureIds.activity_events, + }), + member_slots: features.allocated({ featureId: featureIds.member_slots }), + project_slots: features.allocated({ + featureId: featureIds.project_slots, + }), + ...featureList.boolean({ featureIds: platformFeatureIds }), + }), + plans: ({ basePrice, features, itemList, items, plan, planList }) => { + const creditItems = [ + items.prepaidCredits({ feature: features.credits }), + items.consumableCredits({ feature: features.credits }), + ]; + const coreFeatures = [ + featureIds.insight_reports, + featureIds.team_policies, + featureIds.private_spaces, + featureIds.export_center, + featureIds.automation_rules, + featureIds.platform_api, + ]; + const expandedFeatures = [ + ...coreFeatures, + featureIds.priority_queue, + featureIds.outbound_hooks, + featureIds.approval_chains, + featureIds.brand_controls, + featureIds.compliance_controls, + featureIds.revision_history, + ]; + + return { + launch: plan.monthly({ + basePrice: basePrice.monthly({ amount: 300 }), + items: [ + ...creditItems, + ...itemList.boolean({ featureIds: coreFeatures, features }), + ], + planId: planIds.launch, + }), + scale: plan.monthly({ + basePrice: basePrice.monthly({ amount: 500 }), + items: [ + ...creditItems, + ...itemList.boolean({ featureIds: expandedFeatures, features }), + ], + planId: planIds.scale, + }), + scaleYearly: plan.annual({ + basePrice: basePrice.annual({ amount: 5_000 }), + items: [ + ...creditItems, + ...itemList.boolean({ featureIds: expandedFeatures, features }), + ], + planId: planIds.scaleYearly, + }), + trial: plan.monthly({ + basePrice: null, + items: [ + items.included({ feature: features.credits, included: 1_000 }), + ...itemList.boolean({ + featureIds: [ + featureIds.insight_reports, + featureIds.private_spaces, + featureIds.platform_api, + ], + features, + }), + ], + planId: planIds.trial, + }), + enterprise: plan.monthly({ + basePrice: null, + items: [ + ...creditItems, + items.included({ feature: features.member_slots, included: 25 }), + items.included({ feature: features.project_slots, included: 100 }), + ...itemList.boolean({ featureIds: expandedFeatures, features }), + ], + planId: planIds.enterprise, + }), + ...planList.addOns({ + addOns: [ + { + amount: 75, + feature: features.automation_rules, + key: "automationPack", + planId: planIds.automationPack, + }, + { + amount: 2_400, + feature: features.compliance_controls, + interval: "year", + key: "securityPack", + planId: planIds.securityPack, + }, + { + amount: 3_000, + feature: features.brand_controls, + interval: "year", + key: "whiteLabelPack", + planId: planIds.whiteLabelPack, + }, + ], + }), + }; + }, + customers: () => ({}), + }); diff --git a/apps/leaf/tests/evals/fixtures/types.ts b/apps/leaf/tests/evals/fixtures/types.ts new file mode 100644 index 000000000..f1e85d545 --- /dev/null +++ b/apps/leaf/tests/evals/fixtures/types.ts @@ -0,0 +1,79 @@ +import type { BaseApiCustomerV5 } from "@api/customers/apiCustomerV5.js"; +import type { ApiCustomerSchedule } from "@api/customers/components/apiCustomerSchedule"; +import type { ApiFeatureV1 } from "@api/features/apiFeatureV1.js"; +import type { ApiPlanV1 } from "@api/products/apiPlanV1.js"; + +export type PlanRef = ApiPlanV1 | ApiPlanV1[]; +export type ScheduleRef = ApiCustomerSchedule | ApiCustomerSchedule[]; +type RefId = Value["id"]; +type RefIds = Value extends unknown[] + ? never + : RefId; + +export type EvalSetupIds< + Features extends Record = Record, + Plans extends Record = Record, + Customers extends Record< + string, + BaseApiCustomerV5 | BaseApiCustomerV5[] + > = Record, + Schedules extends Record = Record, +> = { + features: { + [Key in keyof Features]: RefIds; + }; + plans: { + [Key in keyof Plans]: Plans[Key] extends ApiPlanV1[] + ? Array> + : Plans[Key] extends ApiPlanV1 + ? RefIds + : never; + }; + customers: { + [Key in keyof Customers]: Customers[Key] extends BaseApiCustomerV5[] + ? Array> + : Customers[Key] extends BaseApiCustomerV5 + ? RefIds + : never; + }; + schedules: { + [Key in keyof Schedules]: Schedules[Key] extends ApiCustomerSchedule[] + ? Array> + : Schedules[Key] extends ApiCustomerSchedule + ? RefIds + : never; + }; +}; + +export type EvalSetupRefs< + Features extends Record = Record, + Plans extends Record = Record, + Customers extends Record< + string, + BaseApiCustomerV5 | BaseApiCustomerV5[] + > = Record, + Schedules extends Record = Record, +> = { + features: Features; + plans: Plans; + customers: Customers; + schedules: Schedules; +}; + +export type EvalSetup< + Features extends Record = Record, + Plans extends Record = Record, + Customers extends Record< + string, + BaseApiCustomerV5 | BaseApiCustomerV5[] + > = Record, + Schedules extends Record = Record, +> = { + tag: string; + ids: EvalSetupIds; + features: ApiFeatureV1[]; + plans: ApiPlanV1[]; + customers: BaseApiCustomerV5[]; + schedules: ApiCustomerSchedule[]; + refs: EvalSetupRefs; +}; diff --git a/apps/leaf/tests/evals/harness/configs/genericMcpAgentConfig.ts b/apps/leaf/tests/evals/harness/configs/genericMcpAgentConfig.ts new file mode 100644 index 000000000..309c8c901 --- /dev/null +++ b/apps/leaf/tests/evals/harness/configs/genericMcpAgentConfig.ts @@ -0,0 +1,12 @@ +export type GenericMcpAgentDriverConfig = { + maxSteps?: number; + model?: string; +}; + +export const genericMcpAgentInstructions = + "Use Autumn MCP tools. Preview destructive writes before applying them."; + +export const defaultGenericMcpAgentConfig = { + maxSteps: 6, + model: "anthropic/claude-sonnet-4-6", +} satisfies Required; diff --git a/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts b/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts new file mode 100644 index 000000000..31a4b585b --- /dev/null +++ b/apps/leaf/tests/evals/harness/context/createAutumnApiMock.ts @@ -0,0 +1,175 @@ +import { customers } from "../../fixtures/customers/index.js"; +import { responses } from "../../fixtures/responses.js"; +import type { EvalTrace } from "../tracing/types.js"; +import type { AutumnApiMock, AutumnApiMockOverrides } from "./types.js"; + +const serverURL = "http://localhost:8080"; + +const endpointToTool = { + "/v1/balances.create": "createBalance", + "/v1/billing.attach": "attach", + "/v1/billing.preview_attach": "previewAttach", + "/v1/customers.get": "getCustomer", + "/v1/customers.get_or_create": "getOrCreateCustomer", + "/v1/customers.list": "listCustomers", + "/v1/customers.update": "updateCustomer", + "/v1/features.list": "listFeatures", + "/v1/plans.get": "getPlan", + "/v1/plans.list": "listPlans", +} as const; + +const getString = (body: Record, key: string) => + typeof body[key] === "string" ? body[key] : ""; + +const defaultHandlers = { + attach: ({ body, setup }) => { + const customer = setup.customers.find( + (customer) => customer.id === getString(body, "customer_id"), + ); + const plan = setup.plans.find( + (plan) => plan.id === getString(body, "plan_id"), + ); + if (!customer || !plan) return { error: "missing customer or plan" }; + customer.subscriptions = [ + ...customer.subscriptions, + { + add_on: plan.add_on, + auto_enable: plan.auto_enable, + canceled_at: null, + current_period_end: null, + current_period_start: 1_767_225_600_000, + expires_at: null, + id: `sub_${plan.id}`, + past_due: false, + plan_id: plan.id, + quantity: 1, + started_at: 1_767_225_600_000, + status: "active", + trial_ends_at: null, + }, + ]; + return responses.attachSuccess({ customer, plan }); + }, + createBalance: () => ({ status: "created" }), + getCustomer: ({ body, setup }) => { + const customer = setup.customers.find( + (customer) => customer.id === getString(body, "customer_id"), + ); + return customer ?? { error: "customer not found" }; + }, + getOrCreateCustomer: ({ body, setup }) => { + const customerId = getString(body, "customer_id"); + const customer = setup.customers.find( + (customer) => customer.id === customerId, + ); + if (customer) return customer; + + const created = customers.active({ id: customerId }); + setup.customers.push(created); + return created; + }, + getPlan: ({ body, setup }) => { + const plan = setup.plans.find( + (plan) => plan.id === getString(body, "plan_id"), + ); + return plan ?? { error: "plan not found" }; + }, + listCustomers: ({ body, setup }) => { + const search = getString(body, "search").toLowerCase(); + const list = search + ? setup.customers.filter((customer) => + [customer.id, customer.name, customer.email].some( + (value) => + typeof value === "string" && value.toLowerCase().includes(search), + ), + ) + : setup.customers; + + return { + limit: list.length, + list, + offset: 0, + total: setup.customers.length, + total_count: setup.customers.length, + total_filtered_count: list.length, + }; + }, + listFeatures: ({ setup }) => ({ list: setup.features }), + listPlans: ({ setup }) => ({ + list: setup.plans, + }), + previewAttach: ({ body, setup }) => { + const customer = setup.customers.find( + (customer) => customer.id === getString(body, "customer_id"), + ); + const plan = setup.plans.find( + (plan) => plan.id === getString(body, "plan_id"), + ); + if (!customer || !plan) return { error: "missing customer or plan" }; + return responses.attachPreview({ customer, plan }); + }, + updateCustomer: ({ body, setup }) => { + const customer = setup.customers.find( + (customer) => customer.id === getString(body, "customer_id"), + ); + if (!customer) return { error: "customer not found" }; + if (typeof body.email === "string") customer.email = body.email; + if (typeof body.name === "string") customer.name = body.name; + return customer; + }, +} satisfies AutumnApiMockOverrides; + +export const createAutumnApiMock = ({ + overrides = {}, + setup, + trace, +}: { + overrides?: AutumnApiMockOverrides; + setup: AutumnApiMock["setup"]; + trace?: EvalTrace; +}): AutumnApiMock => { + const calls: AutumnApiMock["calls"] = []; + const originalFetch = globalThis.fetch; + const handlers = { ...defaultHandlers, ...overrides }; + + globalThis.fetch = (async (input, init) => { + const url = new URL(String(input)); + if (url.origin !== serverURL) return originalFetch(input, init); + + const endpoint = url.pathname; + const toolName = + endpointToTool[endpoint as keyof typeof endpointToTool] ?? null; + const body = JSON.parse(String(init?.body ?? "{}")); + const call = { body, endpoint, toolName }; + calls.push(call); + trace?.event({ call, type: "api_call" }); + + if (!toolName) { + return Response.json( + { error: `Unhandled endpoint: ${endpoint}` }, + { status: 500 }, + ); + } + + const handler = handlers[toolName]; + if (!handler) { + return Response.json( + { error: `No handler for ${toolName}` }, + { status: 500 }, + ); + } + + const response = handler({ body, setup }); + trace?.event({ endpoint, response, type: "api_response" }); + return Response.json(response); + }) as typeof fetch; + + return { + calls, + restore: () => { + globalThis.fetch = originalFetch; + }, + serverURL, + setup, + }; +}; diff --git a/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts b/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts new file mode 100644 index 000000000..ada3dcaa9 --- /dev/null +++ b/apps/leaf/tests/evals/harness/context/createAutumnMcpServer.ts @@ -0,0 +1,53 @@ +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { MCPServer } from "@mastra/mcp"; +import { setAnalyticsSink } from "../../../../../../packages/mcp/src/analytics/analyticsSink.js"; +import type { AutumnMcpAuth } from "../../../../../../packages/mcp/src/server/auth/auth.js"; +import { createRawAutumnOperationTools } from "../../../../../../packages/mcp/src/tools/index.js"; +import type { EvalMcpServer } from "./types.js"; + +const closeServer = (server: Server) => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + +const createEvalMcpServer = () => + new MCPServer({ + id: "autumn-mcp-eval", + name: "Autumn MCP Eval", + version: "0.0.1", + description: "Operate on Autumn customers, plans, and billing.", + instructions: + "Use preview tools before billing writes. Write tools are destructive and should only be called after explicit user confirmation.", + tools: createRawAutumnOperationTools(), + }); + +export const createAutumnMcpServer = (auth: AutumnMcpAuth) => + new Promise((resolve) => { + setAnalyticsSink(null); + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? "/mcp", `http://${req.headers.host}`); + if (url.pathname !== "/mcp") { + res.writeHead(404).end(); + return; + } + + (req as IncomingMessage & { auth?: AutumnMcpAuth }).auth = auth; + await createEvalMcpServer().startHTTP({ + httpPath: "/mcp", + options: { serverless: true }, + req, + res, + url, + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("MCP eval server did not bind to a TCP port."); + } + resolve({ + close: () => closeServer(server), + url: new URL(`http://127.0.0.1:${address.port}/mcp`), + }); + }); + }); diff --git a/apps/leaf/tests/evals/harness/context/createEvalRuntimeContext.ts b/apps/leaf/tests/evals/harness/context/createEvalRuntimeContext.ts new file mode 100644 index 000000000..7c85a545d --- /dev/null +++ b/apps/leaf/tests/evals/harness/context/createEvalRuntimeContext.ts @@ -0,0 +1,52 @@ +import type { AutumnMcpAuth } from "../../../../../../packages/mcp/src/server/auth/auth.js"; +import type { EvalSetup } from "../../fixtures/types.js"; +import type { EvalTrace } from "../tracing/types.js"; +import { createAutumnApiMock } from "./createAutumnApiMock.js"; +import { createAutumnMcpServer } from "./createAutumnMcpServer.js"; +import type { AutumnApiMockOverrides, EvalRuntimeContext } from "./types.js"; + +const defaultAuth: AutumnMcpAuth = { + apiKey: "sk_test", + env: "sandbox", + principalId: "eval-user", + resource: "http://localhost:2718/mcp", + scopes: [ + "customers:read", + "customers:write", + "plans:read", + "billing:read", + "billing:write", + "balances:write", + ], + serverURL: "http://localhost:8080", +}; + +export const createEvalRuntimeContext = async ({ + auth = {}, + autumnApiOverrides, + setup, + trace, +}: { + auth?: Partial; + autumnApiOverrides?: AutumnApiMockOverrides; + setup: EvalSetup; + trace: EvalTrace; +}): Promise => { + const resolvedAuth = { ...defaultAuth, ...auth }; + const autumnApi = createAutumnApiMock({ + overrides: autumnApiOverrides, + setup, + trace, + }); + const mcpServer = await createAutumnMcpServer(resolvedAuth); + + return { + auth: resolvedAuth, + autumnApi, + cleanup: async () => { + autumnApi.restore(); + await mcpServer.close(); + }, + mcpServer, + }; +}; diff --git a/apps/leaf/tests/evals/harness/context/types.ts b/apps/leaf/tests/evals/harness/context/types.ts new file mode 100644 index 000000000..5bded396f --- /dev/null +++ b/apps/leaf/tests/evals/harness/context/types.ts @@ -0,0 +1,51 @@ +import type { AutumnMcpAuth } from "../../../../../../packages/mcp/src/server/auth/auth.js"; +import type { EvalSetup } from "../../fixtures/types.js"; + +export type AutumnEvalToolName = + | "attach" + | "createBalance" + | "getCustomer" + | "getOrCreateCustomer" + | "getPlan" + | "listCustomers" + | "listFeatures" + | "listPlans" + | "previewAttach" + | "updateCustomer"; + +export type AutumnApiCall = { + toolName: AutumnEvalToolName | null; + endpoint: string; + body: Record; +}; + +export type AutumnApiMockHandler = ({ + body, + setup, +}: { + body: Record; + setup: EvalSetup; +}) => unknown; + +export type AutumnApiMockOverrides = Partial< + Record +>; + +export type AutumnApiMock = { + calls: AutumnApiCall[]; + restore(): void; + serverURL: string; + setup: EvalSetup; +}; + +export type EvalMcpServer = { + close(): Promise; + url: URL; +}; + +export type EvalRuntimeContext = { + auth: AutumnMcpAuth; + autumnApi: AutumnApiMock; + cleanup(): Promise; + mcpServer: EvalMcpServer; +}; diff --git a/apps/leaf/tests/evals/harness/createEvalContext.ts b/apps/leaf/tests/evals/harness/createEvalContext.ts new file mode 100644 index 000000000..b8e02d464 --- /dev/null +++ b/apps/leaf/tests/evals/harness/createEvalContext.ts @@ -0,0 +1,107 @@ +import type { AutumnMcpAuth } from "../../../../../packages/mcp/src/server/auth/auth.js"; +import type { EvalSetup } from "../fixtures/types.js"; +import { createEvalRuntimeContext } from "./context/createEvalRuntimeContext.js"; +import type { + AutumnApiMockOverrides, + EvalRuntimeContext, +} from "./context/types.js"; +import type { EvalAgentDriver } from "./drivers/types.js"; +import { createEvalTrace } from "./tracing/createEvalTrace.js"; +import type { EvalTrace, EvalTraceLevel } from "./tracing/types.js"; + +export type EvalTurn = + | { maxSteps?: number; message: string; type: "user" } + | { maxSteps?: number; optional?: boolean; type: "approve" }; + +export type EvalTurnResult = { + text?: string; + type: EvalTurn["type"]; +}; + +export type EvalRunResult = { + apiCalls: EvalRuntimeContext["autumnApi"]["calls"]; + finalText: string; + toolCalls: ReturnType< + Awaited>["getToolCalls"] + >; + turns: EvalTurnResult[]; +}; + +export const createEvalContext = async ({ + auth, + autumnApiOverrides, + driver, + name, + setup, + today, + trace: traceConfig = {}, +}: { + auth?: Partial; + autumnApiOverrides?: AutumnApiMockOverrides; + driver: EvalAgentDriver; + name?: string; + setup: EvalSetup; + today?: Date; + trace?: { level?: EvalTraceLevel }; +}) => { + const trace: EvalTrace = createEvalTrace(traceConfig); + trace.event({ name, type: "eval_started" }); + const runtimeContext = await createEvalRuntimeContext({ + auth, + autumnApiOverrides, + setup, + trace, + }); + const runningDriver = await driver.start({ + context: runtimeContext, + name, + setup, + today, + trace, + }); + + const runConversation = async (turns: EvalTurn[]): Promise => { + const turnResults: EvalTurnResult[] = []; + for (const turn of turns) { + if (turn.type === "user") { + trace.event({ message: turn.message, type: "user_turn" }); + const output = await runningDriver.send(turn.message, { + maxSteps: turn.maxSteps, + }); + turnResults.push({ text: output.text, type: turn.type }); + continue; + } + + if (!runningDriver.hasPendingApproval()) { + if (turn.optional) { + turnResults.push({ type: turn.type }); + continue; + } + throw new Error("No pending approval to approve."); + } + const output = await runningDriver.approve({ maxSteps: turn.maxSteps }); + turnResults.push({ text: output.text, type: turn.type }); + } + + trace.event({ type: "eval_finished" }); + return { + apiCalls: runtimeContext.autumnApi.calls, + finalText: turnResults + .map((turn) => turn.text) + .filter(Boolean) + .join("\n"), + toolCalls: runningDriver.getToolCalls(), + turns: turnResults, + }; + }; + + return { + cleanup: async () => { + await runningDriver.cleanup(); + await runtimeContext.cleanup(); + }, + runConversation, + runtimeContext, + trace, + }; +}; diff --git a/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts b/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts new file mode 100644 index 000000000..f725c5dbf --- /dev/null +++ b/apps/leaf/tests/evals/harness/drivers/genericMcpAgent.ts @@ -0,0 +1,179 @@ +import { Agent } from "@mastra/core/agent"; +import type { MessageListItem } from "@mastra/core/agent/message-list"; +import { Mastra } from "@mastra/core/mastra"; +import { InMemoryStore } from "@mastra/core/storage"; +import { MCPClient } from "@mastra/mcp"; +import { createRequestContext } from "../../../../../../packages/mcp/src/server/auth/auth.js"; +import { createLeafTracingOptions } from "../../../../src/internal/observability/leafTracingOptions.js"; +import { createMastraBraintrustObservability } from "../../../../src/providers/braintrust/index.js"; +import { + defaultGenericMcpAgentConfig, + type GenericMcpAgentDriverConfig, + genericMcpAgentInstructions, +} from "../configs/genericMcpAgentConfig.js"; +import type { + EvalAgentDriver, + EvalDriverStartInput, + EvalToolCall, +} from "./types.js"; + +type ToolWithApproval = { + execute?: unknown; + mcp?: { annotations?: { destructiveHint?: boolean } }; + needsApprovalFn?: unknown; + requireApproval?: unknown; +}; + +const applyToolApprovalPolicy = (tools: Record) => { + for (const tool of Object.values(tools)) { + const requiresApproval = tool.mcp?.annotations?.destructiveHint === true; + tool.requireApproval = requiresApproval; + if (!requiresApproval) tool.needsApprovalFn = undefined; + } +}; + +const toEvalToolCall = (call: { + args?: Record; + name: string; +}): EvalToolCall => ({ + args: call.args ?? {}, + name: call.name, +}); + +const instrumentToolCalls = ({ + tools, + toolCalls, + trace, +}: { + tools: Record; + toolCalls: EvalToolCall[]; + trace: EvalDriverStartInput["trace"]; +}) => { + for (const [name, tool] of Object.entries(tools)) { + if (typeof tool.execute !== "function") continue; + const execute = tool.execute.bind(tool) as ( + args: Record, + ...rest: unknown[] + ) => Promise; + tool.execute = async ( + args: Record, + ...rest: unknown[] + ) => { + const call = toEvalToolCall({ args, name }); + toolCalls.push(call); + trace.event({ call, type: "tool_call" }); + return execute(args, ...rest); + }; + } +}; + +export const createGenericMcpAgentDriver = ({ + maxSteps = defaultGenericMcpAgentConfig.maxSteps, + model = defaultGenericMcpAgentConfig.model, +}: GenericMcpAgentDriverConfig = {}): EvalAgentDriver => ({ + name: "generic-mcp-agent", + start: async ({ context, setup, today, trace }: EvalDriverStartInput) => { + const mcpClient = new MCPClient({ + id: `leaf-eval-${crypto.randomUUID()}`, + servers: { + autumn: { + requireToolApproval: ({ annotations }) => + annotations?.destructiveHint === true, + url: context.mcpServer.url, + }, + }, + }); + const { toolsets, errors } = await mcpClient.listToolsetsWithErrors(); + if (Object.keys(errors).length) { + throw new Error(`MCP tool discovery failed: ${JSON.stringify(errors)}`); + } + + const tools = toolsets.autumn ?? {}; + applyToolApprovalPolicy(tools); + const toolCalls: EvalToolCall[] = []; + instrumentToolCalls({ toolCalls, tools, trace }); + + const agent = new Agent({ + id: "leaf-mcp-eval-agent", + name: "Leaf MCP Eval Agent", + description: "A generic agent using Autumn MCP tools.", + instructions: genericMcpAgentInstructions, + model, + tools, + }); + const mastra = new Mastra({ + agents: { eval: agent }, + logger: false, + observability: createMastraBraintrustObservability(), + storage: new InMemoryStore({ id: `leaf-eval-${crypto.randomUUID()}` }), + }); + const evalAgent = mastra.getAgent("eval"); + let messages: MessageListItem[] = []; + let pendingApproval: { runId: string; toolCallId?: string } | null = null; + + const options = (stepLimit?: number) => ({ + context: today + ? [ + { + content: `Current date: ${today.toISOString()}.`, + role: "system" as const, + }, + ] + : undefined, + maxSteps: stepLimit ?? maxSteps, + requestContext: createRequestContext(context.auth), + tracingOptions: createLeafTracingOptions({ + env: context.auth.env, + orgId: context.auth.orgId, + setup: setup.tag, + source: "eval", + }), + }); + + const rememberApproval = (output: { + finishReason?: string; + runId?: string; + suspendPayload?: { toolCallId?: string }; + }) => { + pendingApproval = + output.finishReason === "suspended" && output.runId + ? { + runId: output.runId, + toolCallId: output.suspendPayload?.toolCallId, + } + : null; + if (pendingApproval) trace.event({ type: "approval_pending" }); + }; + + return { + approve: async ({ maxSteps: stepLimit } = {}) => { + if (!pendingApproval) { + throw new Error("No pending approval to approve."); + } + trace.event({ type: "approval_approved" }); + const output = await evalAgent.approveToolCallGenerate({ + ...options(stepLimit), + runId: pendingApproval.runId, + toolCallId: pendingApproval.toolCallId, + }); + messages = output.messages; + rememberApproval(output); + trace.event({ text: output.text ?? "", type: "agent_text" }); + return { text: output.text }; + }, + cleanup: async () => { + await mcpClient.disconnect(); + }, + getToolCalls: () => [...toolCalls], + hasPendingApproval: () => pendingApproval !== null, + send: async (message, { maxSteps: stepLimit } = {}) => { + messages.push({ content: message, role: "user" }); + const output = await evalAgent.generate(messages, options(stepLimit)); + messages = output.messages; + rememberApproval(output); + trace.event({ text: output.text ?? "", type: "agent_text" }); + return { text: output.text }; + }, + }; + }, +}); diff --git a/apps/leaf/tests/evals/harness/drivers/types.ts b/apps/leaf/tests/evals/harness/drivers/types.ts new file mode 100644 index 000000000..709d5a954 --- /dev/null +++ b/apps/leaf/tests/evals/harness/drivers/types.ts @@ -0,0 +1,36 @@ +import type { EvalSetup } from "../../fixtures/types.js"; +import type { EvalRuntimeContext } from "../context/types.js"; +import type { EvalTrace } from "../tracing/types.js"; + +export type EvalToolCall = { + args: Record; + name: string; +}; + +export type EvalAgentOutput = { + text?: string; +}; + +export type EvalDriverStartInput = { + context: EvalRuntimeContext; + name?: string; + setup: EvalSetup; + today?: Date; + trace: EvalTrace; +}; + +export type RunningEvalDriver = { + approve(options?: { maxSteps?: number }): Promise; + cleanup(): Promise; + getToolCalls(): EvalToolCall[]; + hasPendingApproval(): boolean; + send( + message: string, + options?: { maxSteps?: number }, + ): Promise; +}; + +export type EvalAgentDriver = { + name: string; + start(input: EvalDriverStartInput): Promise; +}; diff --git a/apps/leaf/tests/evals/harness/index.ts b/apps/leaf/tests/evals/harness/index.ts new file mode 100644 index 000000000..c59c919c9 --- /dev/null +++ b/apps/leaf/tests/evals/harness/index.ts @@ -0,0 +1,36 @@ +export { + defaultGenericMcpAgentConfig, + type GenericMcpAgentDriverConfig, + genericMcpAgentInstructions, +} from "./configs/genericMcpAgentConfig.js"; +export { createAutumnApiMock } from "./context/createAutumnApiMock.js"; +export { createAutumnMcpServer } from "./context/createAutumnMcpServer.js"; +export { createEvalRuntimeContext } from "./context/createEvalRuntimeContext.js"; +export type { + AutumnApiCall, + AutumnApiMock, + AutumnApiMockHandler, + AutumnApiMockOverrides, + AutumnEvalToolName, + EvalMcpServer, + EvalRuntimeContext, +} from "./context/types.js"; +export type { + EvalRunResult, + EvalTurn, + EvalTurnResult, +} from "./createEvalContext.js"; +export { createEvalContext } from "./createEvalContext.js"; +export { createGenericMcpAgentDriver } from "./drivers/genericMcpAgent.js"; +export type { + EvalAgentDriver, + EvalAgentOutput, + EvalToolCall, + RunningEvalDriver, +} from "./drivers/types.js"; +export { createEvalTrace } from "./tracing/createEvalTrace.js"; +export type { + EvalTrace, + EvalTraceEvent, + EvalTraceLevel, +} from "./tracing/types.js"; diff --git a/apps/leaf/tests/evals/harness/tracing/createEvalTrace.ts b/apps/leaf/tests/evals/harness/tracing/createEvalTrace.ts new file mode 100644 index 000000000..977e655f6 --- /dev/null +++ b/apps/leaf/tests/evals/harness/tracing/createEvalTrace.ts @@ -0,0 +1,26 @@ +import { formatTraceEvent } from "./formatTrace.js"; +import type { EvalTrace, EvalTraceEvent, EvalTraceLevel } from "./types.js"; + +export const createEvalTrace = ({ + level = "steps", +}: { + level?: EvalTraceLevel; +} = {}): EvalTrace => { + const events: EvalTraceEvent[] = []; + const printEvent = (event: EvalTraceEvent) => { + if (level === "off") return; + const line = formatTraceEvent(event); + if (line) console.error(line); + }; + + return { + entries: () => [...events], + event: (event) => { + events.push(event); + printEvent(event); + }, + print: () => { + for (const event of events) printEvent(event); + }, + }; +}; diff --git a/apps/leaf/tests/evals/harness/tracing/formatTrace.ts b/apps/leaf/tests/evals/harness/tracing/formatTrace.ts new file mode 100644 index 000000000..830cff26b --- /dev/null +++ b/apps/leaf/tests/evals/harness/tracing/formatTrace.ts @@ -0,0 +1,108 @@ +import type { AutumnApiCall } from "../context/types.js"; +import type { EvalToolCall } from "../drivers/types.js"; +import type { EvalTraceEvent } from "./types.js"; + +const truncate = ({ text, max = 160 }: { text: string; max?: number }) => + text.length > max ? `${text.slice(0, max - 3)}...` : text; + +const bodyOf = (value: Record) => + value.request && typeof value.request === "object" + ? (value.request as Record) + : value; + +const compactFields = (body: Record) => + [ + ["customer", body.customer_id], + ["plan", body.plan_id], + ["entity", body.entity_id], + ["feature", body.feature_id], + ["email", body.email], + ["search", body.search], + [ + "invoice_mode", + typeof body.invoice_mode === "object" && body.invoice_mode !== null + ? "true" + : undefined, + ], + ] + .flatMap(([key, value]) => + typeof value === "string" && value ? [`${key}=${value}`] : [], + ) + .join(" "); + +const formatToolCall = (call: EvalToolCall) => { + const fields = compactFields(bodyOf(call.args)); + return `[tool] ${call.name}${fields ? ` ${fields}` : ""}`; +}; + +const formatApiCall = (call: AutumnApiCall) => { + const fields = compactFields(call.body); + return `[api] POST ${call.endpoint}${fields ? ` ${fields}` : ""}`; +}; + +const summarizeRecord = (record: Record) => + [ + ["id", record.id], + ["name", record.name], + [ + "subscriptions", + Array.isArray(record.subscriptions) + ? record.subscriptions.length + : undefined, + ], + ["plan_id", record.plan_id], + ] + .flatMap(([key, value]) => + typeof value === "string" || typeof value === "number" + ? [`${key}=${value}`] + : [], + ) + .join(" "); + +const formatApiResponse = ({ + endpoint, + response, +}: { + endpoint: string; + response: unknown; +}) => { + if (!response || typeof response !== "object") { + return `[api:response] ${endpoint} ${String(response)}`; + } + + const record = response as Record; + if (Array.isArray(record.list)) { + const first = record.list[0]; + const firstSummary = + first && typeof first === "object" + ? summarizeRecord(first as Record) + : ""; + return `[api:response] ${endpoint} list=${record.list.length}${firstSummary ? ` first(${firstSummary})` : ""}`; + } + + const summary = summarizeRecord(record); + return `[api:response] ${endpoint}${summary ? ` ${summary}` : ""}`; +}; + +export const formatTraceEvent = (event: EvalTraceEvent): string | null => { + switch (event.type) { + case "agent_text": + return event.text ? `[agent] ${truncate({ text: event.text })}` : null; + case "api_call": + return formatApiCall(event.call); + case "api_response": + return formatApiResponse(event); + case "approval_approved": + return "[approval] approved pending tool call"; + case "approval_pending": + return "[approval] pending"; + case "eval_finished": + return "[eval] finished"; + case "eval_started": + return `[eval] ${event.name ?? "started"}`; + case "tool_call": + return formatToolCall(event.call); + case "user_turn": + return `[user] ${truncate({ text: event.message })}`; + } +}; diff --git a/apps/leaf/tests/evals/harness/tracing/types.ts b/apps/leaf/tests/evals/harness/tracing/types.ts new file mode 100644 index 000000000..25e86a506 --- /dev/null +++ b/apps/leaf/tests/evals/harness/tracing/types.ts @@ -0,0 +1,21 @@ +import type { AutumnApiCall } from "../context/types.js"; +import type { EvalToolCall } from "../drivers/types.js"; + +export type EvalTraceLevel = "off" | "steps"; + +export type EvalTraceEvent = + | { type: "eval_started"; name?: string } + | { type: "user_turn"; message: string } + | { type: "agent_text"; text: string } + | { type: "tool_call"; call: EvalToolCall } + | { type: "api_call"; call: AutumnApiCall } + | { type: "api_response"; endpoint: string; response: unknown } + | { type: "approval_pending" } + | { type: "approval_approved" } + | { type: "eval_finished" }; + +export type EvalTrace = { + event(event: EvalTraceEvent): void; + entries(): EvalTraceEvent[]; + print(): void; +}; diff --git a/apps/leaf/tests/evals/mcp/billing/customer-plan-summary.eval.ts b/apps/leaf/tests/evals/mcp/billing/customer-plan-summary.eval.ts new file mode 100644 index 000000000..f89ff888c --- /dev/null +++ b/apps/leaf/tests/evals/mcp/billing/customer-plan-summary.eval.ts @@ -0,0 +1,106 @@ +import { Eval } from "braintrust"; +import { withCustomers } from "../../fixtures/createSetup.js"; +import { orgSetups } from "../../fixtures/orgSetups.js"; +import { + createEvalContext, + createGenericMcpAgentDriver, +} from "../../harness/index.js"; +import { + type EvalExpected, + type EvalOutput, + expectedApiCalls, + expectedToolCalls, + finalTextIncludes, +} from "../../utils/scorers.js"; + +type EvalInput = { + confirmation: string; + prompt: string; +}; + +type EvalMetadata = { + domain: "billing"; + setup: string; +}; + +const experimentName = "customer-plan"; + +const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers, plans, subscriptions }) => ({ + joe: customers.active({ + id: "joe_customer", + name: "Joe", + subscriptions: [ + subscriptions.active({ + currentPeriodEnd: new Date("2026-02-07T00:00:00.000Z"), + currentPeriodStart: new Date("2026-01-01T00:00:00.000Z"), + id: "sub_joe_scale_custom", + plan: plans.scale, + }), + ], + }), + }), +}); +const customer = setup.refs.customers.joe; + +Eval( + "leaf", + { + experimentName, + data: [ + { + expected: { + apiCalls: [{ toolName: "listCustomers" }], + finalTextIncludes: [ + "Joe", + "Scale", + "$500", + "Credits", + "Insight Reports", + ], + toolCalls: ["listCustomers"], + }, + input: { + confirmation: `Yes, use ${customer.id}.`, + prompt: "what plan is Joe on?", + }, + metadata: { + domain: "billing", + setup: setup.tag, + }, + }, + ], + scores: [ + (args) => ({ + name: "Expected tool calls", + score: expectedToolCalls(args), + }), + (args) => ({ + name: "Expected API calls", + score: expectedApiCalls(args), + }), + (args) => ({ + name: "Final text includes", + score: finalTextIncludes(args), + }), + ], + task: async (input) => { + const context = await createEvalContext({ + driver: createGenericMcpAgentDriver(), + name: experimentName, + setup, + }); + try { + return await context.runConversation([ + { message: input.prompt, type: "user" }, + { message: input.confirmation, type: "user" }, + ]); + } finally { + await context.cleanup(); + } + }, + timeout: 45_000, + }, + { noSendLogs: !process.env.BRAINTRUST_API_KEY }, +); diff --git a/apps/leaf/tests/evals/utils/scorers.ts b/apps/leaf/tests/evals/utils/scorers.ts new file mode 100644 index 000000000..fa5906401 --- /dev/null +++ b/apps/leaf/tests/evals/utils/scorers.ts @@ -0,0 +1,91 @@ +import type { + AutumnApiCall, + AutumnEvalToolName, +} from "../harness/context/types.js"; + +export type EvalOutput = { + apiCalls: AutumnApiCall[]; + finalText: string; + toolCalls: Array<{ name: string; args: Record }>; +}; + +export type EvalExpected = { + apiCalls?: Array<{ + body?: Record; + toolName: AutumnEvalToolName; + }>; + finalTextIncludes?: string[]; + toolCalls?: AutumnEvalToolName[]; +}; + +const includesObject = ( + actual: Record, + expected: Record, +) => + Object.entries(expected).every(([key, value]) => + typeof value === "object" && value !== null + ? JSON.stringify(actual[key]) === JSON.stringify(value) + : actual[key] === value, + ); + +export const expectedApiCalls = ({ + expected, + output, +}: { + expected?: EvalExpected; + output: EvalOutput; +}) => { + const expectedCalls = expected?.apiCalls ?? []; + if (!expectedCalls.length) return 1; + return expectedCalls.every((expectedCall) => + output.apiCalls.some( + (call) => + call.toolName === expectedCall.toolName && + (!expectedCall.body || includesObject(call.body, expectedCall.body)), + ), + ) + ? 1 + : 0; +}; + +export const expectedToolCalls = ({ + expected, + output, +}: { + expected?: EvalExpected; + output: EvalOutput; +}) => { + const expectedTools = expected?.toolCalls ?? []; + if (!expectedTools.length) return 1; + return expectedTools.every((toolName) => + output.toolCalls.some((call) => call.name === toolName), + ) + ? 1 + : 0; +}; + +export const finalTextIncludes = ({ + expected, + output, +}: { + expected?: EvalExpected; + output: EvalOutput; +}) => { + const phrases = expected?.finalTextIncludes ?? []; + if (!phrases.length) return 1; + const text = output.finalText.toLowerCase(); + return phrases.every((phrase) => text.includes(phrase.toLowerCase())) ? 1 : 0; +}; + +export const noAttachBeforePreview = ({ output }: { output: EvalOutput }) => { + const attachIndex = output.apiCalls.findIndex( + (call) => call.toolName === "attach", + ); + const previewIndex = output.apiCalls.findIndex( + (call) => call.toolName === "previewAttach", + ); + return attachIndex === -1 || + (previewIndex !== -1 && previewIndex < attachIndex) + ? 1 + : 0; +}; diff --git a/apps/leaf/tests/unit/evals/mock-autumn-server.test.ts b/apps/leaf/tests/unit/evals/mock-autumn-server.test.ts new file mode 100644 index 000000000..bfeb015df --- /dev/null +++ b/apps/leaf/tests/unit/evals/mock-autumn-server.test.ts @@ -0,0 +1,381 @@ +import { describe, expect, test } from "bun:test"; +import { BillingMethod } from "@api/products/components/billingMethod.js"; +import { FeatureType } from "@models/featureModels/featureEnums.js"; +import { TierBehavior } from "@models/productModels/priceModels/priceConfig/usagePriceConfig.js"; +import { + createSetup, + withCustomers, +} from "../../evals/fixtures/createSetup.js"; +import { orgSetups } from "../../evals/fixtures/orgSetups.js"; +import { createAutumnApiMock } from "../../evals/harness/index.js"; +import { + expectedApiCalls, + expectedToolCalls, +} from "../../evals/utils/scorers.js"; + +const createCustomerPlanSetup = () => + createSetup({ + tag: "joe-customized-pro-plan", + features: ({ features }) => ({ + credits: features.creditSystem(), + dashboard: features.boolean(), + }), + plans: ({ basePrice, features, items, plan }) => ({ + pro: plan.monthly({ + basePrice: basePrice.monthly({ amount: 79 }), + items: [ + items.included({ feature: features.credits, included: 25_000 }), + items.boolean({ feature: features.dashboard }), + ], + planId: "pro", + }), + }), + customers: ({ customers, plans, subscriptions }) => ({ + joe: customers.active({ + id: "joe_customer", + name: "Joe", + subscriptions: [subscriptions.active({ plan: plans.pro })], + }), + }), + }); + +describe("eval mock Autumn server", () => { + test("composes boolean feature lists into setup refs", () => { + const setup = createSetup({ + tag: "boolean-feature-list", + features: ({ featureList }) => ({ + ...featureList.boolean({ + featureIds: ["sso", "audit_logs"], + names: { sso: "SSO" }, + }), + }), + plans: ({ features, items, plan }) => ({ + pro: plan.monthly({ + items: [items.boolean({ feature: features.sso })], + planId: "pro", + }), + }), + customers: () => ({}), + }); + + expect(setup.refs.features.sso).toMatchObject({ + id: "sso", + name: "SSO", + type: FeatureType.Boolean, + }); + expect(setup.ids.features.sso).toBe("sso"); + expect(setup.refs.features.audit_logs).toMatchObject({ + id: "audit_logs", + name: "Audit Logs", + type: FeatureType.Boolean, + }); + expect(setup.plans[0]?.items[0]?.feature_id).toBe("sso"); + }); + + test("composes anonymized knowledge platform org setup", () => { + const setup = orgSetups.knowledgePlatform(); + const enterprise = setup.refs.plans.enterprise; + const automationPack = setup.plans.find( + (plan) => plan.id === setup.ids.plans.automationPack, + ); + if (Array.isArray(enterprise) || !automationPack) { + throw new Error("Expected single plan refs."); + } + + const creditItems = enterprise.items.filter( + (item) => item.feature_id === "credits", + ); + const featureIds = setup.features.map((feature) => feature.id); + + expect(setup.refs.features.credits).toMatchObject({ + type: FeatureType.CreditSystem, + }); + expect(enterprise.price).toBeNull(); + expect(setup.ids.features.insight_reports).toBe("insight_reports"); + expect(setup.ids.plans.enterprise).toBe("enterprise"); + expect(creditItems).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + price: expect.objectContaining({ + billing_method: BillingMethod.Prepaid, + tier_behavior: TierBehavior.VolumeBased, + }), + }), + expect.objectContaining({ + price: expect.objectContaining({ + billing_method: BillingMethod.UsageBased, + }), + }), + ]), + ); + expect(featureIds.filter((id) => id !== "credits").length).toBeGreaterThan( + 10, + ); + expect(automationPack).toMatchObject({ + add_on: true, + items: [{ feature_id: "automation_rules" }], + }); + expect(featureIds).not.toContain("AI_CHAT"); + expect(featureIds).not.toContain("AI_CREDITS"); + }); + + test("extends reusable org setup with typed eval customers", () => { + const setup = withCustomers({ + setup: orgSetups.knowledgePlatform(), + customers: ({ customers, plans, subscriptions }) => ({ + joe: customers.active({ + id: "joe_customer", + subscriptions: [subscriptions.active({ plan: plans.scale })], + }), + }), + }); + + expect(setup.ids.customers.joe).toBe("joe_customer"); + expect(setup.refs.customers.joe.subscriptions[0]?.plan_id).toBe( + setup.ids.plans.scale, + ); + }); + + test("creates customized plan variants for customer subscriptions", () => { + const setup = createSetup({ + tag: "custom-plan-version", + features: ({ features }) => ({ + audit_logs: features.boolean({ featureId: "audit_logs" }), + credits: features.creditSystem(), + dashboard: features.boolean(), + }), + plans: ({ basePrice, features, items, plan }) => { + const pro = plan.monthly({ + basePrice: basePrice.monthly({ amount: 79 }), + items: [ + items.included({ feature: features.credits, included: 25_000 }), + items.boolean({ feature: features.dashboard }), + ], + planId: "pro", + }); + + return { + pro, + proCustom: plan.customized({ + customize: { + add_items: [items.boolean({ feature: features.audit_logs })], + price: basePrice.monthly({ amount: 99 }), + remove_items: [{ feature_id: features.dashboard.id }], + }, + plan: pro, + planId: "pro_custom", + }), + }; + }, + customers: ({ customers, plans, subscriptions }) => ({ + joe: customers.active({ + id: "joe_customer", + subscriptions: [subscriptions.active({ plan: plans.proCustom })], + }), + }), + }); + + const subscription = setup.refs.customers.joe.subscriptions[0]; + expect(subscription?.plan_id).toBe("pro_custom"); + expect(subscription?.plan).toMatchObject({ + base_variant_id: "pro", + id: "pro_custom", + price: { amount: 9_900 }, + }); + expect(subscription?.plan?.items.map((item) => item.feature_id)).toEqual([ + "credits", + "audit_logs", + ]); + }); + + test("requires customized plan replacements to remove original items", () => { + const setup = createSetup({ + tag: "custom-plan-duplicate-item", + features: ({ features }) => ({ + credits: features.creditSystem(), + }), + plans: ({ features, items, plan }) => { + const pro = plan.monthly({ + items: [items.included({ feature: features.credits, included: 100 })], + planId: "pro", + }); + + expect(() => + plan.customized({ + customize: { + add_items: [ + items.included({ feature: features.credits, included: 1_000 }), + ], + }, + plan: pro, + }), + ).toThrow("duplicate item"); + + return { pro }; + }, + customers: () => ({}), + }); + + expect(setup.ids.plans.pro).toBe("pro"); + }); + + test("composes customer schedule refs alongside scheduled subscriptions", () => { + const setup = createSetup({ + tag: "customer-schedule", + features: ({ features }) => ({ + credits: features.creditSystem(), + }), + plans: ({ features, items, plan }) => ({ + yearOne: plan.annual({ + items: [items.included({ feature: features.credits, included: 5_000 })], + planId: "year_one", + }), + yearTwo: plan.annual({ + items: [ + items.included({ feature: features.credits, included: 10_000 }), + ], + planId: "year_two", + }), + }), + customers: ({ customers, plans, subscriptions }) => ({ + joe: customers.active({ + id: "joe_customer", + subscriptions: [ + subscriptions.active({ plan: plans.yearOne }), + subscriptions.scheduled({ + plan: plans.yearTwo, + startedAt: new Date("2027-01-01T00:00:00.000Z"), + }), + ], + }), + }), + schedules: ({ customers, schedules }) => ({ + joeContract: schedules.customer({ + customer: customers.joe, + id: "sched_joe_contract", + phases: [ + { + customerProductIds: ["cus_prod_year_one"], + startsAt: new Date("2026-01-01T00:00:00.000Z"), + }, + { + customerProductIds: ["cus_prod_year_two"], + startsAt: new Date("2027-01-01T00:00:00.000Z"), + }, + ], + }), + }), + }); + + expect(setup.ids.schedules.joeContract).toBe("sched_joe_contract"); + expect(setup.schedules[0]?.phases.map((phase) => phase.starts_at)).toEqual([ + 1_767_225_600_000, + 1_798_761_600_000, + ]); + expect(setup.refs.customers.joe.subscriptions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ plan_id: "year_two", status: "scheduled" }), + ]), + ); + }); + + test("generates customer search and get-or-create responses from setup state", async () => { + const setup = createCustomerPlanSetup(); + const server = createAutumnApiMock({ setup }); + const customer = setup.refs.customers.joe; + if (!customer) throw new Error("Eval setup is missing customer."); + + try { + const listed = await fetch(`${server.serverURL}/v1/customers.list`, { + body: JSON.stringify({ search: "Joe" }), + method: "POST", + }).then((response) => response.json()); + const fetched = await fetch( + `${server.serverURL}/v1/customers.get_or_create`, + { + body: JSON.stringify({ customer_id: customer.id }), + method: "POST", + }, + ).then((response) => response.json()); + + expect(listed).toMatchObject({ + list: [{ id: customer.id, subscriptions: [{ plan_id: "pro" }] }], + total_filtered_count: 1, + }); + expect(fetched).toMatchObject({ + id: customer.id, + subscriptions: [{ plan: { id: "pro", name: "Pro" } }], + }); + expect(server.calls.map((call) => call.toolName)).toEqual([ + "listCustomers", + "getOrCreateCustomer", + ]); + } finally { + server.restore(); + } + }); + + test("creates a customer through get-or-create when missing", async () => { + const setup = createCustomerPlanSetup(); + const server = createAutumnApiMock({ setup }); + + try { + const created = await fetch( + `${server.serverURL}/v1/customers.get_or_create`, + { + body: JSON.stringify({ customer_id: "new_customer" }), + method: "POST", + }, + ).then((response) => response.json()); + + expect(created).toMatchObject({ id: "new_customer" }); + expect(setup.customers.map((customer) => customer.id)).toContain( + "new_customer", + ); + } finally { + server.restore(); + } + }); + + test("scores expected tool and API calls", () => { + const output = { + apiCalls: [ + { + body: { search: "Joe" }, + endpoint: "/v1/customers.list", + toolName: "listCustomers" as const, + }, + { + body: { customer_id: "joe_customer" }, + endpoint: "/v1/customers.get_or_create", + toolName: "getOrCreateCustomer" as const, + }, + ], + finalText: "Joe is on Pro for $79 per month.", + toolCalls: [ + { args: {}, name: "listCustomers" }, + { args: {}, name: "getOrCreateCustomer" }, + ], + }; + + expect( + expectedApiCalls({ + expected: { + apiCalls: [ + { + body: { customer_id: "joe_customer" }, + toolName: "getOrCreateCustomer", + }, + ], + }, + output, + }), + ).toBe(1); + expect( + expectedToolCalls({ + expected: { toolCalls: ["listCustomers", "getOrCreateCustomer"] }, + output, + }), + ).toBe(1); + }); +}); diff --git a/bun.lock b/bun.lock index 2a139b889..d7a9cf06b 100644 --- a/bun.lock +++ b/bun.lock @@ -95,9 +95,13 @@ "@chat-adapter/slack": "^4.29.0", "@chat-adapter/state-pg": "^4.29.0", "@hono/node-server": "^1.19.5", + "@mastra/braintrust": "^1.1.3", "@mastra/core": "^1.36.0", "@mastra/mcp": "^1.8.0", + "@mastra/observability": "^1.14.1", "@mendable/firecrawl-js": "^4.25.1", + "autoevals": "^0.0.132", + "braintrust": "^3.14.0", "chat": "^4.29.0", "date-fns": "^4.1.0", "drizzle-orm": "catalog:", @@ -724,7 +728,7 @@ "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.32.1", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg=="], - "@apm-js-collab/code-transformer": ["@apm-js-collab/code-transformer@0.8.2", "", {}, "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA=="], + "@apm-js-collab/code-transformer": ["@apm-js-collab/code-transformer@0.12.0", "", { "dependencies": { "@types/estree": "^1.0.8", "astring": "^1.9.0", "esquery": "^1.7.0", "meriyah": "^6.1.4", "semifies": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-5F2ob4cMYezbaUGAk+YltbDvb9BFIghN92ubct9Ho/0MFx4FkChCxYV99NkU6Kx+RAgaqBV6yxKuWreQ6K8SOw=="], "@apm-js-collab/tracing-hooks": ["@apm-js-collab/tracing-hooks@0.3.1", "", { "dependencies": { "@apm-js-collab/code-transformer": "^0.8.0", "debug": "^4.4.1", "module-details-from-path": "^1.0.4" } }, "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw=="], @@ -1384,6 +1388,10 @@ "@kubiks/otel-drizzle": ["@kubiks/otel-drizzle@2.1.0", "", { "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <2.0.0", "drizzle-orm": ">=0.28.0" } }, "sha512-9UHb0od3jwa6zTWMyEYPIZcUq5PDaziCmQLMLakSK2zeqy12SFZ3SAGWXJTgEr8valn/Wa+DKVs+Z3aqKQUpvg=="], + "@kwsites/file-exists": ["@kwsites/file-exists@1.1.1", "", { "dependencies": { "debug": "^4.1.1" } }, "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw=="], + + "@kwsites/promise-deferred": ["@kwsites/promise-deferred@1.1.1", "", {}, "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="], + "@langchain/core": ["@langchain/core@1.1.47", "", { "dependencies": { "@cfworker/json-schema": "^4.0.2", "@standard-schema/spec": "^1.1.0", "js-tiktoken": "^1.0.12", "langsmith": ">=0.5.0 <1.0.0", "mustache": "^4.2.0", "p-queue": "^6.6.2", "zod": "^3.25.76 || ^4" } }, "sha512-+fiPu6ZFnJMrZyKeM77OIVPoMPAY6OKWacnPlojHtXTbMMzb2cEOKAJV0U07cDl86NHSCIYYa0i4CyKZzXbHQQ=="], "@langchain/langgraph": ["@langchain/langgraph@1.3.2", "", { "dependencies": { "@langchain/langgraph-checkpoint": "^1.0.2", "@langchain/langgraph-sdk": "~1.9.4", "@langchain/protocol": "^0.0.15", "@standard-schema/spec": "1.1.0", "uuid": "^10.0.0" }, "peerDependencies": { "@langchain/core": "^1.1.44", "zod": "^3.25.32 || ^4.2.0", "zod-to-json-schema": "^3.x" }, "optionalPeers": ["zod-to-json-schema"] }, "sha512-SL7Ktsr681R7da+1b2MVOWEbaCoFJOXEJPTGOjg4JIG4C7quWbTYC8DzxhcCxte6D/8cGp0rYDBnbKLXEpNqlA=="], @@ -1402,10 +1410,14 @@ "@lukeed/uuid": ["@lukeed/uuid@2.0.1", "", { "dependencies": { "@lukeed/csprng": "^1.1.0" } }, "sha512-qC72D4+CDdjGqJvkFMMEAtancHUQ7/d/tAiHf64z8MopFDmcrtbcJuerDtFceuAfQJ2pDSfCKCtbqoGBNnwg0w=="], + "@mastra/braintrust": ["@mastra/braintrust@1.1.3", "", { "dependencies": { "@mastra/observability": "1.14.1", "braintrust": "^2.2.2" }, "peerDependencies": { "@mastra/core": ">=1.16.0-0 <2.0.0-0", "zod": "^3.25.34 || ^4.0.0" } }, "sha512-5NxE+7gFPXR3p+K947Dri0Ta7gDl53wsYCGj6KBfXsuEIaLlxwCu+vsWdAiuXDoyTj22bVQkr2jX92CfqitI8g=="], + "@mastra/core": ["@mastra/core@1.36.0", "", { "dependencies": { "@a2a-js/sdk": "~0.3.13", "@ai-sdk/provider-utils-v5": "npm:@ai-sdk/provider-utils@3.0.25", "@ai-sdk/provider-utils-v6": "npm:@ai-sdk/provider-utils@4.0.27", "@ai-sdk/provider-v5": "npm:@ai-sdk/provider@2.0.3", "@ai-sdk/provider-v6": "npm:@ai-sdk/provider@3.0.10", "@ai-sdk/ui-utils-v5": "npm:@ai-sdk/ui-utils@1.2.11", "@isaacs/ttlcache": "^2.1.4", "@lukeed/uuid": "^2.0.1", "@mastra/schema-compat": "1.2.10", "@modelcontextprotocol/sdk": "^1.29.0", "@sindresorhus/slugify": "^2.2.1", "@standard-schema/spec": "^1.1.0", "ajv": "^8.18.0", "chat": "^4.29.0", "croner": "^10.0.1", "dotenv": "^17.3.1", "execa": "^9.6.1", "fastq": "^1.19.1", "gray-matter": "^4.0.3", "hono": "^4.12.8", "hono-openapi": "^1.3.0", "ignore": "^7.0.5", "json-schema": "^0.4.0", "lru-cache": "^11.2.7", "p-map": "^7.0.4", "p-retry": "^7.1.1", "picomatch": "^4.0.3", "posthog-node": "^5.30.6", "tokenx": "^1.3.0", "ws": "^8.20.0", "xxhash-wasm": "^1.1.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-BEhDZPQeDcJ6jQRHtpfFLuoRiWAuv9dTCIjeWbXokzwDamI3D9jkyNzpBFJwFwy2S/a4jBTu4+d61nOaP7knTQ=="], "@mastra/mcp": ["@mastra/mcp@1.8.0", "", { "dependencies": { "@modelcontextprotocol/ext-apps": "^1.7.1", "@modelcontextprotocol/sdk": "^1.29.0", "exit-hook": "^5.1.0", "fast-deep-equal": "^3.1.3" }, "peerDependencies": { "@mastra/core": ">=1.0.0-0 <2.0.0-0", "zod": "^3.25.0 || ^4.0.0" } }, "sha512-kA1YhDa/W/ZuhZ/AZpUFuKKFhINSVvLf+hDNmbCZMsM46rYjyqqgQR0xgqNaysCwv3Anta6KqDz8fp6mJ7RyuA=="], + "@mastra/observability": ["@mastra/observability@1.14.1", "", { "peerDependencies": { "@mastra/core": ">=1.16.0-0 <2.0.0-0", "zod": "^3.25.0 || ^4.0.0" } }, "sha512-VKfn3mE1mNFOXgY9Pr9sy5L4q4xLnoLj501gFMOr0teNnRMvAiANVB+p4rnooDfM4i8nzLDG9icM80uMsgXTzQ=="], + "@mastra/schema-compat": ["@mastra/schema-compat@1.2.10", "", { "dependencies": { "json-schema-to-zod": "^2.7.0", "zod-from-json-schema": "^0.5.2", "zod-from-json-schema-v3": "npm:zod-from-json-schema@^0.0.5", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-8Fg8PeO7GsRPOrEZAzc5udZgsF9ZDxih5JSoxjgnR79d0ImjKffhcoysPW6wIYXPEZ5i6/QDNR7rCazZZSD5Tg=="], "@mdx-js/loader": ["@mdx-js/loader@3.1.1", "", { "dependencies": { "@mdx-js/mdx": "^3.0.0", "source-map": "^0.7.0" }, "peerDependencies": { "webpack": ">=5" }, "optionalPeers": ["webpack"] }, "sha512-0TTacJyZ9mDmY+VefuthVshaNIyCGZHJG2fMnGaDttCt8HmjUF7SizlHJpaCDoGnN635nK1wpzfpx/Xx5S4WnQ=="], @@ -1468,7 +1480,7 @@ "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], - "@next/env": ["@next/env@16.2.4", "", {}, "sha512-dKkkOzOSwFYe5RX6y26fZgkSpVAlIOJKQHIiydQcrWH6y/97+RceSOAdjZ14Qa3zLduVUy0TXcn+EiM6t4rPgw=="], + "@next/env": ["@next/env@14.2.35", "", {}, "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="], "@next/eslint-plugin-next": ["@next/eslint-plugin-next@16.2.1", "", { "dependencies": { "fast-glob": "3.3.1" } }, "sha512-r0epZGo24eT4g08jJlg2OEryBphXqO8aL18oajoTKLzHJ6jVr6P6FI58DLMug04MwD3j8Fj0YK0slyzneKVyzA=="], @@ -2132,6 +2144,10 @@ "@sideway/pinpoint": ["@sideway/pinpoint@2.0.0", "", {}, "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ=="], + "@simple-git/args-pathspec": ["@simple-git/args-pathspec@1.0.3", "", {}, "sha512-ngJMaHlsWDTfjyq9F3VIQ8b7NXbBLq5j9i5bJ6XLYtD6qlDXT7fdKY2KscWWUF8t18xx052Y/PUO1K1TRc9yKA=="], + + "@simple-git/argv-parser": ["@simple-git/argv-parser@1.1.1", "", { "dependencies": { "@simple-git/args-pathspec": "^1.0.3" } }, "sha512-Q9lBcfQ+VQCpQqGJFHe5yooOS5hGdLFFbJ5R+R5aDsnkPCahtn1hSkMcORX65J2Z5lxSkD0lQorMsncuBQxYUw=="], + "@simplewebauthn/browser": ["@simplewebauthn/browser@13.3.0", "", {}, "sha512-BE/UWv6FOToAdVk0EokzkqQQDOWtNydYlY6+OrmiZ5SCNmb41VehttboTetUM3T/fr6EAFYVXjz4My2wg230rQ=="], "@simplewebauthn/server": ["@simplewebauthn/server@13.3.0", "", { "dependencies": { "@hexagon/base64": "^1.1.27", "@levischuck/tiny-cbor": "^0.2.2", "@peculiar/asn1-android": "^2.6.0", "@peculiar/asn1-ecc": "^2.6.1", "@peculiar/asn1-rsa": "^2.6.1", "@peculiar/asn1-schema": "^2.6.0", "@peculiar/asn1-x509": "^2.6.1", "@peculiar/x509": "^1.14.3" } }, "sha512-MLHYFrYG8/wK2i+86XMhiecK72nMaHKKt4bo+7Q1TbuG9iGjlSdfkPWKO5ZFE/BX+ygCJ7pr8H/AJeyAj1EaTQ=="], @@ -2556,6 +2572,8 @@ "@types/normalize-package-data": ["@types/normalize-package-data@2.4.4", "", {}, "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA=="], + "@types/nunjucks": ["@types/nunjucks@3.2.6", "", {}, "sha512-pHiGtf83na1nCzliuAdq8GowYiXvH5l931xZ0YEHaLMNFgynpEqx+IPStlu7UaDkehfvl01e4x/9Tpwhy7Ue3w=="], + "@types/pako": ["@types/pako@1.0.7", "", {}, "sha512-YBtzT2ztNF6R/9+UXj2wTGFnC9NklAnASt3sC0h2m1bbH7G6FyBIkt4AN8ThZpNfxUo1b2iMVO0UawiJymEt8A=="], "@types/parse-json": ["@types/parse-json@4.0.2", "", {}, "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw=="], @@ -2714,6 +2732,8 @@ "@vercel/analytics": ["@vercel/analytics@2.0.1", "", { "peerDependencies": { "@remix-run/react": "^2", "@sveltejs/kit": "^1 || ^2", "next": ">= 13", "nuxt": ">= 3", "react": "^18 || ^19 || ^19.0.0-rc", "svelte": ">= 4", "vue": "^3", "vue-router": "^4" }, "optionalPeers": ["@remix-run/react", "@sveltejs/kit", "next", "nuxt", "react", "svelte", "vue", "vue-router"] }, "sha512-MTQG6V9qQrt1tsDeF+2Uoo5aPjqbVPys1xvnIftXSJYG2SrwXRHnqEvVoYID7BTruDz4lCd2Z7rM1BdkUehk2g=="], + "@vercel/functions": ["@vercel/functions@1.6.0", "", { "peerDependencies": { "@aws-sdk/credential-provider-web-identity": "*" }, "optionalPeers": ["@aws-sdk/credential-provider-web-identity"] }, "sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA=="], + "@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="], "@vercel/sdk": ["@vercel/sdk@1.21.5", "", { "dependencies": { "@modelcontextprotocol/sdk": "^1.26.0", "zod": "^3.25.0 || ^4.0.0" }, "bin": { "mcp": "bin/mcp-server.js" } }, "sha512-R1/j1ixylaHQ+d3y+QhG9848Ruv8XBH0g2MChzNek6F1SKNGvgHaG2hX0crndIYTVplB11Ynt3g2mKIkEKBAPQ=="], @@ -2758,6 +2778,8 @@ "@xtuc/long": ["@xtuc/long@4.2.2", "", {}, "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ=="], + "a-sync-waterfall": ["a-sync-waterfall@1.0.1", "", {}, "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA=="], + "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], "abort-controller-x": ["abort-controller-x@0.4.3", "", {}, "sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA=="], @@ -2806,6 +2828,8 @@ "ajv-keywords": ["ajv-keywords@5.1.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3" }, "peerDependencies": { "ajv": "^8.8.2" } }, "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw=="], + "ansi-align": ["ansi-align@3.0.1", "", { "dependencies": { "string-width": "^4.1.0" } }, "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w=="], + "ansi-escapes": ["ansi-escapes@7.3.0", "", { "dependencies": { "environment": "^1.0.0" } }, "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg=="], "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], @@ -2822,7 +2846,7 @@ "arg": ["arg@4.1.3", "", {}, "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="], - "argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "aria-hidden": ["aria-hidden@1.2.6", "", { "dependencies": { "tslib": "^2.0.0" } }, "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA=="], @@ -2908,6 +2932,8 @@ "auto-bind": ["auto-bind@5.0.1", "", {}, "sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg=="], + "autoevals": ["autoevals@0.0.132", "", { "dependencies": { "ajv": "^8.17.1", "compute-cosine-similarity": "^1.1.0", "js-levenshtein": "^1.1.6", "js-yaml": "^4.1.0", "linear-sum-assignment": "^1.0.7", "mustache": "^4.2.0", "openai": "^6.3.0", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" } }, "sha512-x033hXLO1Vyggbv68Y1QeoZlrdKHcNexcutPhVaDhDJ4SO6TU1rG6vME77g/zKvH4VWFVBPDWM1pRbPxAFF+sA=="], + "autumn-js": ["autumn-js@workspace:packages/autumn-js"], "ava": ["ava@5.3.1", "", { "dependencies": { "acorn": "^8.8.2", "acorn-walk": "^8.2.0", "ansi-styles": "^6.2.1", "arrgv": "^1.0.2", "arrify": "^3.0.0", "callsites": "^4.0.0", "cbor": "^8.1.0", "chalk": "^5.2.0", "chokidar": "^3.5.3", "chunkd": "^2.0.1", "ci-info": "^3.8.0", "ci-parallel-vars": "^1.0.1", "clean-yaml-object": "^0.1.0", "cli-truncate": "^3.1.0", "code-excerpt": "^4.0.0", "common-path-prefix": "^3.0.0", "concordance": "^5.0.4", "currently-unhandled": "^0.4.1", "debug": "^4.3.4", "emittery": "^1.0.1", "figures": "^5.0.0", "globby": "^13.1.4", "ignore-by-default": "^2.1.0", "indent-string": "^5.0.0", "is-error": "^2.2.2", "is-plain-object": "^5.0.0", "is-promise": "^4.0.0", "matcher": "^5.0.0", "mem": "^9.0.2", "ms": "^2.1.3", "p-event": "^5.0.1", "p-map": "^5.5.0", "picomatch": "^2.3.1", "pkg-conf": "^4.0.0", "plur": "^5.1.0", "pretty-ms": "^8.0.0", "resolve-cwd": "^3.0.0", "stack-utils": "^2.0.6", "strip-ansi": "^7.0.1", "supertap": "^3.0.1", "temp-dir": "^3.0.0", "write-file-atomic": "^5.0.1", "yargs": "^17.7.2" }, "peerDependencies": { "@ava/typescript": "*" }, "optionalPeers": ["@ava/typescript"], "bin": { "ava": "entrypoints/cli.mjs" } }, "sha512-Scv9a4gMOXB6+ni4toLuhAm9KYWEjsgBglJl+kMGI5+IVDt120CCDZyB5HNU9DjmLI2t4I0GbnxGLmmRfGTJGg=="], @@ -2966,6 +2992,8 @@ "binary-extensions": ["binary-extensions@2.3.0", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="], + "binary-search": ["binary-search@1.3.6", "", {}, "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA=="], + "bindings": ["bindings@1.5.0", "", { "dependencies": { "file-uri-to-path": "1.0.0" } }, "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ=="], "bintrees": ["bintrees@1.0.2", "", {}, "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw=="], @@ -2982,10 +3010,14 @@ "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], + "boxen": ["boxen@8.0.1", "", { "dependencies": { "ansi-align": "^3.0.1", "camelcase": "^8.0.0", "chalk": "^5.3.0", "cli-boxes": "^3.0.0", "string-width": "^7.2.0", "type-fest": "^4.21.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0" } }, "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw=="], + "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], + "braintrust": ["braintrust@3.14.0", "", { "dependencies": { "@apm-js-collab/code-transformer": "^0.12.0", "@next/env": "^14.2.3", "@vercel/functions": "^1.0.2", "ajv": "^8.20.0", "argparse": "^2.0.1", "cli-progress": "^3.12.0", "cli-table3": "^0.6.5", "cors": "^2.8.5", "dc-browser": "^1.0.4", "dotenv": "^16.4.5", "esbuild": "0.28.0", "eventsource-parser": "^1.1.2", "express": "^5.2.1", "http-errors": "^2.0.0", "minimatch": "^10.2.5", "module-details-from-path": "^1.0.4", "mustache": "^4.2.0", "pluralize": "^8.0.0", "simple-git": "^3.36.0", "source-map": "^0.7.4", "termi-link": "^1.0.1", "unplugin": "^2.3.5", "uuid": "^11.1.1", "zod-to-json-schema": "^3.25.0" }, "peerDependencies": { "zod": "^3.25.34 || ^4.0" }, "bin": { "braintrust": "dist/cli.js" } }, "sha512-B1ZvfYP4uWqCt39ACkfvglTtIg3VbiLXxs+diaqt90vTelABJ4B7tCMmv5hiSyQmt6HxIHCu+VmoBTnlsx30Xg=="], + "browser-stdout": ["browser-stdout@1.3.1", "", {}, "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw=="], "browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="], @@ -3072,6 +3104,8 @@ "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], + "cheminfo-types": ["cheminfo-types@1.15.0", "", {}, "sha512-shv45WN2u0yN9EHH1bisNrv+fy4Cw+eLM5lOoriP67mePrwbHZ1kJqg90C8GEU7K1A8gJsicEoVZHcuBbuul/w=="], + "chevrotain": ["chevrotain@10.5.0", "", { "dependencies": { "@chevrotain/cst-dts-gen": "10.5.0", "@chevrotain/gast": "10.5.0", "@chevrotain/types": "10.5.0", "@chevrotain/utils": "10.5.0", "lodash": "4.17.21", "regexp-to-ast": "0.5.0" } }, "sha512-Pkv5rBY3+CsHOYfV5g/Vs5JY9WTHHDEKOlohI2XeygaZhUeqhAlldZ8Hz9cRmxu709bvS08YzxHdTPHhffc13A=="], "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], @@ -3104,6 +3138,8 @@ "cli-cursor": ["cli-cursor@4.0.0", "", { "dependencies": { "restore-cursor": "^4.0.0" } }, "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg=="], + "cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="], + "cli-spinners": ["cli-spinners@2.9.2", "", {}, "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg=="], "cli-table3": ["cli-table3@0.6.5", "", { "dependencies": { "string-width": "^4.2.0" }, "optionalDependencies": { "@colors/colors": "1.5.0" } }, "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ=="], @@ -3164,6 +3200,12 @@ "component-emitter": ["component-emitter@1.3.1", "", {}, "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ=="], + "compute-cosine-similarity": ["compute-cosine-similarity@1.1.0", "", { "dependencies": { "compute-dot": "^1.1.0", "compute-l2norm": "^1.1.0", "validate.io-array": "^1.0.5", "validate.io-function": "^1.0.2" } }, "sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw=="], + + "compute-dot": ["compute-dot@1.1.0", "", { "dependencies": { "validate.io-array": "^1.0.3", "validate.io-function": "^1.0.2" } }, "sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg=="], + + "compute-l2norm": ["compute-l2norm@1.1.0", "", { "dependencies": { "validate.io-array": "^1.0.3", "validate.io-function": "^1.0.2" } }, "sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg=="], + "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], "concordance": ["concordance@5.0.4", "", { "dependencies": { "date-time": "^3.1.0", "esutils": "^2.0.3", "fast-diff": "^1.2.0", "js-string-escape": "^1.0.1", "lodash": "^4.17.15", "md5-hex": "^3.0.1", "semver": "^7.3.2", "well-known-symbols": "^2.0.0" } }, "sha512-OAcsnTEYu1ARJqWVGwf4zh4JDfHZEaSNlNccFmt8YjB2l/n19/PF2viLINHc57vO4FKIAFl2FWASIGZZWZ2Kxw=="], @@ -3332,6 +3374,8 @@ "dayjs": ["dayjs@1.11.20", "", {}, "sha512-YbwwqR/uYpeoP4pu043q+LTDLFBLApUP6VxRihdfNTqu4ubqMlGDLd6ErXhEgsyvY0K6nCs7nggYumAN+9uEuQ=="], + "dc-browser": ["dc-browser@1.0.4", "", {}, "sha512-7oEtnzNlcE+hr4OvO3GR6Gndgw8BhW+wKOEwMqSleyY7N29jbAxzyW5BaJl7qBCw+6OIxfMWtY0T+6dxq8RWLw=="], + "debounce": ["debounce@2.2.0", "", {}, "sha512-Xks6RUDLZFdz8LIdR6q0MTH44k7FikOmnh5xkSjMig6ch45afc8sjTjRQf3P6ax8dMgcQrYO/AR2RGWURrruqw=="], "debounce-fn": ["debounce-fn@6.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-rBMW+F2TXryBwB54Q0d8drNEI+TfoS9JpNTAoVpukbWEhjXQq4rySFYLaqXMFXwdv61Zb2OHtj5bviSoimqxRQ=="], @@ -3648,7 +3692,7 @@ "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], - "eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "eventsource-parser": ["eventsource-parser@1.1.2", "", {}, "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA=="], "evt": ["evt@2.5.9", "", { "dependencies": { "minimal-polyfills": "^2.2.3", "run-exclusive": "^2.2.19", "tsafe": "^1.8.5" } }, "sha512-GpjX476FSlttEGWHT8BdVMoI8wGXQGbEOtKcP4E+kggg+yJzXBZN2n4x7TS/zPBJ1DZqWI+rguZZApjjzQ0HpA=="], @@ -3726,6 +3770,8 @@ "fflate": ["fflate@0.4.8", "", {}, "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA=="], + "fft.js": ["fft.js@4.0.4", "", {}, "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw=="], + "figures": ["figures@5.0.0", "", { "dependencies": { "escape-string-regexp": "^5.0.0", "is-unicode-supported": "^1.2.0" } }, "sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg=="], "file-entry-cache": ["file-entry-cache@8.0.0", "", { "dependencies": { "flat-cache": "^4.0.0" } }, "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ=="], @@ -4094,6 +4140,8 @@ "is-alphanumerical": ["is-alphanumerical@2.0.1", "", { "dependencies": { "is-alphabetical": "^2.0.0", "is-decimal": "^2.0.0" } }, "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw=="], + "is-any-array": ["is-any-array@3.0.0", "", {}, "sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww=="], + "is-array-buffer": ["is-array-buffer@3.0.5", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "get-intrinsic": "^1.2.6" } }, "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A=="], "is-arrayish": ["is-arrayish@0.2.1", "", {}, "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg=="], @@ -4248,6 +4296,8 @@ "joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="], + "js-levenshtein": ["js-levenshtein@1.1.6", "", {}, "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g=="], + "js-string-escape": ["js-string-escape@1.0.1", "", {}, "sha512-Smw4xcfIQ5LVjAOuJCvN/zIodzA/BBSsluuoSykP+lUvScIi4U6RJLfwHet5cxFnCswUjISV8oAXaqaJDY3chg=="], "js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="], @@ -4256,7 +4306,7 @@ "js-types": ["js-types@1.0.0", "", {}, "sha512-bfwqBW9cC/Lp7xcRpug7YrXm0IVw+T9e3g4mCYnv0Pjr3zIzU9PCQElYU9oSGAWzXlbdl9X5SAMPejO9sxkeUw=="], - "js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], + "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], "jsep": ["jsep@1.4.0", "", {}, "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw=="], @@ -4372,6 +4422,8 @@ "line-column-path": ["line-column-path@3.0.0", "", { "dependencies": { "type-fest": "^2.0.0" } }, "sha512-Atocnm7Wr9nuvAn97yEPQa3pcQI5eLQGBz+m6iTb+CVw+IOzYB9MrYK7jI7BfC9ISnT4Fu0eiwhAScV//rp4Hw=="], + "linear-sum-assignment": ["linear-sum-assignment@1.0.9", "", { "dependencies": { "cheminfo-types": "^1.8.1", "ml-matrix": "^6.12.1", "ml-spectra-processing": "^14.18.0" } }, "sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ=="], + "lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], "listr2": ["listr2@9.0.5", "", { "dependencies": { "cli-truncate": "^5.0.0", "colorette": "^2.0.20", "eventemitter3": "^5.0.1", "log-update": "^6.1.0", "rfdc": "^1.4.1", "wrap-ansi": "^9.0.0" } }, "sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g=="], @@ -4526,6 +4578,8 @@ "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + "meriyah": ["meriyah@6.1.4", "", {}, "sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ=="], + "mermaid": ["mermaid@11.15.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.1.1", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.1", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.19", "dompurify": "^3.3.1", "es-toolkit": "^1.45.1", "katex": "^0.16.25", "khroma": "^2.1.0", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" } }, "sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw=="], "methods": ["methods@1.1.2", "", {}, "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="], @@ -4652,6 +4706,18 @@ "mkdirp-classic": ["mkdirp-classic@0.5.3", "", {}, "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="], + "ml-array-max": ["ml-array-max@2.0.0", "", { "dependencies": { "is-any-array": "^3.0.0" } }, "sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A=="], + + "ml-array-min": ["ml-array-min@2.0.0", "", { "dependencies": { "is-any-array": "^3.0.0" } }, "sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg=="], + + "ml-array-rescale": ["ml-array-rescale@2.0.0", "", { "dependencies": { "is-any-array": "^3.0.0", "ml-array-max": "^2.0.0", "ml-array-min": "^2.0.0" } }, "sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg=="], + + "ml-matrix": ["ml-matrix@6.12.2", "", { "dependencies": { "is-any-array": "^3.0.0", "ml-array-rescale": "^2.0.0" } }, "sha512-GC+BnW+pBh8Auap8goAxY0senAmF0IEoc3HNVSfnfbvGw0buuDIYb9kAKMS1l+GiwJ1rfK2bzJ8IHhwjzATSFA=="], + + "ml-spectra-processing": ["ml-spectra-processing@14.29.0", "", { "dependencies": { "binary-search": "^1.3.6", "cheminfo-types": "^1.15.0", "fft.js": "^4.0.4", "is-any-array": "^3.0.0", "ml-matrix": "^6.12.2", "ml-xsadd": "^3.0.1" } }, "sha512-825CS864krbjMv7OB0mbjgAmyOL5ymj1OGa0gAzz1h1Dcd3Eeol2DaOimSiPYmRhW+iYhpeQnb7cSU0mlSK6+g=="], + + "ml-xsadd": ["ml-xsadd@3.0.1", "", {}, "sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg=="], + "mlly": ["mlly@1.8.2", "", { "dependencies": { "acorn": "^8.16.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.3" } }, "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA=="], "mocha": ["mocha@11.7.5", "", { "dependencies": { "browser-stdout": "^1.3.1", "chokidar": "^4.0.1", "debug": "^4.3.5", "diff": "^7.0.0", "escape-string-regexp": "^4.0.0", "find-up": "^5.0.0", "glob": "^10.4.5", "he": "^1.2.0", "is-path-inside": "^3.0.3", "js-yaml": "^4.1.0", "log-symbols": "^4.1.0", "minimatch": "^9.0.5", "ms": "^2.1.3", "picocolors": "^1.1.1", "serialize-javascript": "^6.0.2", "strip-json-comments": "^3.1.1", "supports-color": "^8.1.1", "workerpool": "^9.2.0", "yargs": "^17.7.2", "yargs-parser": "^21.1.1", "yargs-unparser": "^2.0.0" }, "bin": { "mocha": "bin/mocha.js", "_mocha": "bin/_mocha" } }, "sha512-mTT6RgopEYABzXWFx+GcJ+ZQ32kp4fMf0xvpZIIfSq9Z8lC/++MtcCnQ9t5FP2veYEP95FIYSvW+U9fV4xrlig=="], @@ -4754,6 +4820,8 @@ "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "nunjucks": ["nunjucks@3.2.4", "", { "dependencies": { "a-sync-waterfall": "^1.0.0", "asap": "^2.0.3", "commander": "^5.1.0" }, "peerDependencies": { "chokidar": "^3.3.0" }, "optionalPeers": ["chokidar"], "bin": { "nunjucks-precompile": "bin/precompile" } }, "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ=="], + "nuqs": ["nuqs@2.8.9", "", { "dependencies": { "@standard-schema/spec": "1.0.0" }, "peerDependencies": { "@remix-run/react": ">=2", "@tanstack/react-router": "^1", "next": ">=14.2.0", "react": ">=18.2.0 || ^19.0.0-0", "react-router": "^5 || ^6 || ^7", "react-router-dom": "^5 || ^6 || ^7" }, "optionalPeers": ["@remix-run/react", "@tanstack/react-router", "next", "react-router", "react-router-dom"] }, "sha512-8ou6AEwsxMWSYo2qkfZtYFVzngwbKmg4c00HVxC1fF6CEJv3Fwm6eoZmfVPALB+vw8Udo7KL5uy96PFcYe1BIQ=="], "nypm": ["nypm@0.5.4", "", { "dependencies": { "citty": "^0.1.6", "consola": "^3.4.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "tinyexec": "^0.3.2", "ufo": "^1.5.4" }, "bin": { "nypm": "dist/cli.mjs" } }, "sha512-X0SNNrZiGU8/e/zAB7sCTtdxWTMSIO73q+xuKgglm2Yvzwlo8UoC5FNySQFCvl84uPaeADkqHUZUkWy4aH4xOA=="], @@ -5344,6 +5412,8 @@ "selderee": ["selderee@0.11.0", "", { "dependencies": { "parseley": "^0.12.0" } }, "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA=="], + "semifies": ["semifies@1.0.0", "", {}, "sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw=="], + "semver": ["semver@7.8.0", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA=="], "send": ["send@0.19.2", "", { "dependencies": { "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "~0.5.2", "http-errors": "~2.0.1", "mime": "1.6.0", "ms": "2.1.3", "on-finished": "~2.4.1", "range-parser": "~1.2.1", "statuses": "~2.0.2" } }, "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg=="], @@ -5392,6 +5462,8 @@ "simple-get": ["simple-get@4.0.1", "", { "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } }, "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA=="], + "simple-git": ["simple-git@3.36.0", "", { "dependencies": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", "@simple-git/args-pathspec": "^1.0.3", "@simple-git/argv-parser": "^1.1.0", "debug": "^4.4.0" } }, "sha512-cGQjLjK8bxJw4QuYT7gxHw3/IouVESbhahSsHrX97MzCL1gu2u7oy38W6L2ZIGECEfIBG4BabsWDPjBxJENv9Q=="], + "simple-swizzle": ["simple-swizzle@0.2.4", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw=="], "simple-update-notifier": ["simple-update-notifier@2.0.0", "", { "dependencies": { "semver": "^7.5.3" } }, "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w=="], @@ -5582,6 +5654,8 @@ "tempy": ["tempy@3.1.0", "", { "dependencies": { "is-stream": "^3.0.0", "temp-dir": "^3.0.0", "type-fest": "^2.12.2", "unique-string": "^3.0.0" } }, "sha512-7jDLIdD2Zp0bDe5r3D2qtkd1QOCacylBuL7oa4udvN6v2pqr4+LcCr67C8DR1zkpaZ8XosF5m1yQSabKAW6f2g=="], + "termi-link": ["termi-link@1.1.0", "", {}, "sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA=="], + "terminal-size": ["terminal-size@4.0.1", "", {}, "sha512-avMLDQpUI9I5XFrklECw1ZEUPJhqzcwSWsyyI8blhRLT+8N1jLJWLWWYQpB2q2xthq8xDvjZPISVh53T/+CLYQ=="], "terser": ["terser@5.47.1", "", { "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, "bin": { "terser": "bin/terser" } }, "sha512-tPbLXTI6ohPASb/1YViL428oEHu6/qv1OxqYnfaonVCFHqx4+wCd95pHrQWsL5X4pl90CTyW9piSAsS2L0VoMw=="], @@ -5822,7 +5896,7 @@ "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], - "unplugin": ["unplugin@1.0.1", "", { "dependencies": { "acorn": "^8.8.1", "chokidar": "^3.5.3", "webpack-sources": "^3.2.3", "webpack-virtual-modules": "^0.5.0" } }, "sha512-aqrHaVBWW1JVKBHmGo33T5TxeL0qWzfvjWokObHA9bYmN7eNDkwOxmLjhioHl9878qDFMAaT51XNroRyuz7WxA=="], + "unplugin": ["unplugin@2.3.11", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "acorn": "^8.15.0", "picomatch": "^4.0.3", "webpack-virtual-modules": "^0.6.2" } }, "sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww=="], "unrs-resolver": ["unrs-resolver@1.12.2", "", { "dependencies": { "napi-postinstall": "^0.3.4" }, "optionalDependencies": { "@unrs/resolver-binding-android-arm-eabi": "1.12.2", "@unrs/resolver-binding-android-arm64": "1.12.2", "@unrs/resolver-binding-darwin-arm64": "1.12.2", "@unrs/resolver-binding-darwin-x64": "1.12.2", "@unrs/resolver-binding-freebsd-x64": "1.12.2", "@unrs/resolver-binding-linux-arm-gnueabihf": "1.12.2", "@unrs/resolver-binding-linux-arm-musleabihf": "1.12.2", "@unrs/resolver-binding-linux-arm64-gnu": "1.12.2", "@unrs/resolver-binding-linux-arm64-musl": "1.12.2", "@unrs/resolver-binding-linux-loong64-gnu": "1.12.2", "@unrs/resolver-binding-linux-loong64-musl": "1.12.2", "@unrs/resolver-binding-linux-ppc64-gnu": "1.12.2", "@unrs/resolver-binding-linux-riscv64-gnu": "1.12.2", "@unrs/resolver-binding-linux-riscv64-musl": "1.12.2", "@unrs/resolver-binding-linux-s390x-gnu": "1.12.2", "@unrs/resolver-binding-linux-x64-gnu": "1.12.2", "@unrs/resolver-binding-linux-x64-musl": "1.12.2", "@unrs/resolver-binding-openharmony-arm64": "1.12.2", "@unrs/resolver-binding-wasm32-wasi": "1.12.2", "@unrs/resolver-binding-win32-arm64-msvc": "1.12.2", "@unrs/resolver-binding-win32-ia32-msvc": "1.12.2", "@unrs/resolver-binding-win32-x64-msvc": "1.12.2" } }, "sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ=="], @@ -5862,6 +5936,10 @@ "validate-npm-package-name": ["validate-npm-package-name@7.0.2", "", {}, "sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A=="], + "validate.io-array": ["validate.io-array@1.0.6", "", {}, "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg=="], + + "validate.io-function": ["validate.io-function@1.0.2", "", {}, "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ=="], + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], "vaul": ["vaul@1.1.2", "", { "dependencies": { "@radix-ui/react-dialog": "^1.1.1" }, "peerDependencies": { "react": "^16.8 || ^17.0 || ^18.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-ZFkClGpWyI2WUQjdLJ/BaGuV6AVQiJ3uELGk3OYtP+B6yCO7Cmn9vPFXVJkRaGkOJu3m8bQMgtyzNHixULceQA=="], @@ -5910,7 +5988,7 @@ "webpack-sources": ["webpack-sources@3.4.1", "", {}, "sha512-eACpxRN02yaawnt+uUNIF7Qje6A9zArxBbcAJjK1PK3S9Ycg5jIuJ8pW4q8EMnwNZCEGltcjkRx1QzOxOkKD8A=="], - "webpack-virtual-modules": ["webpack-virtual-modules@0.5.0", "", {}, "sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw=="], + "webpack-virtual-modules": ["webpack-virtual-modules@0.6.2", "", {}, "sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ=="], "well-known-symbols": ["well-known-symbols@2.0.0", "", {}, "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q=="], @@ -6022,8 +6100,14 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@ai-sdk/provider-utils/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "@ai-sdk/provider-utils-v5/@ai-sdk/provider": ["@ai-sdk/provider@2.0.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-h88OPkavHTiN9tMn2l5awAznGB0lXzjcLhgR1/rvjB2zlLprsNxbM2tt6OJsHUxduLC3klq0/eqaSf6fX5XVww=="], + "@ai-sdk/provider-utils-v5/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + + "@ai-sdk/provider-utils-v6/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "@ai-sdk/ui-utils-v5/@ai-sdk/provider": ["@ai-sdk/provider@1.1.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg=="], "@ai-sdk/ui-utils-v5/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@2.2.8", "", { "dependencies": { "@ai-sdk/provider": "1.1.3", "nanoid": "^3.3.8", "secure-json-parse": "^2.7.0" }, "peerDependencies": { "zod": "^3.23.8" } }, "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA=="], @@ -6036,6 +6120,10 @@ "@anthropic-ai/sdk/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], + "@apm-js-collab/code-transformer/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + + "@apm-js-collab/tracing-hooks/@apm-js-collab/code-transformer": ["@apm-js-collab/code-transformer@0.8.2", "", {}, "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA=="], + "@artilleryio/int-core/chalk": ["chalk@2.4.2", "", { "dependencies": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" } }, "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="], "@artilleryio/int-core/csv-parse": ["csv-parse@4.16.3", "", {}, "sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg=="], @@ -6046,8 +6134,6 @@ "@asyncapi/parser/ajv-formats": ["ajv-formats@2.1.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA=="], - "@asyncapi/parser/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@asyncapi/parser/node-fetch": ["node-fetch@2.6.7", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ=="], "@autumn/auth/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], @@ -6260,8 +6346,6 @@ "@eslint/eslintrc/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - "@eslint/eslintrc/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@eslint/eslintrc/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], "@eslint/eslintrc/strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], @@ -6300,6 +6384,8 @@ "@langchain/langgraph-sdk/uuid": ["uuid@13.0.2", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw=="], + "@mastra/braintrust/braintrust": ["braintrust@2.2.2", "", { "dependencies": { "@ai-sdk/provider": "^1.1.3", "@next/env": "^14.2.3", "@types/nunjucks": "^3.2.6", "@vercel/functions": "^1.0.2", "ajv": "^8.17.1", "argparse": "^2.0.1", "boxen": "^8.0.1", "chalk": "^4.1.2", "cli-progress": "^3.12.0", "cli-table3": "^0.6.5", "cors": "^2.8.5", "dotenv": "^16.4.5", "esbuild": "^0.27.0", "eventsource-parser": "^1.1.2", "express": "^4.21.2", "graceful-fs": "^4.2.11", "http-errors": "^2.0.0", "minimatch": "^9.0.3", "mustache": "^4.2.0", "nunjucks": "^3.2.4", "pluralize": "^8.0.0", "simple-git": "^3.21.0", "source-map": "^0.7.4", "termi-link": "^1.0.1", "uuid": "^9.0.1", "zod-to-json-schema": "^3.25.0" }, "peerDependencies": { "zod": "^3.25.34 || ^4.0" }, "bin": { "braintrust": "dist/cli.js" } }, "sha512-g8TPnfZb7X8ziJG3w2iYRBMiIbSTV6YW79rjhDyDAeVCwa4hq52ns4JzQeQTPRWusm7vE3gXAEgIxuBc9q18uQ=="], + "@mastra/core/dotenv": ["dotenv@17.4.2", "", {}, "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw=="], "@mastra/core/hono": ["hono@4.12.22", "", {}, "sha512-7fvVPbB92zNRsQke+uiRGwtTuef0tB2Dg4hWxYfFNvkQhIltWoyi0ONReM5LWA+jJWS3nfT5lTq+qbsIpX0IQw=="], @@ -6320,8 +6406,6 @@ "@mintlify/cli/inquirer": ["inquirer@12.3.0", "", { "dependencies": { "@inquirer/core": "^10.1.2", "@inquirer/prompts": "^7.2.1", "@inquirer/type": "^3.0.2", "ansi-escapes": "^4.3.2", "mute-stream": "^2.0.0", "run-async": "^3.0.0", "rxjs": "^7.8.1" }, "peerDependencies": { "@types/node": ">=18" } }, "sha512-3NixUXq+hM8ezj2wc7wC37b32/rHq1MwNZDYdvx+d6jokOD+r+i8Q4Pkylh9tISYP114A128LCX8RKhopC5RfQ=="], - "@mintlify/cli/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@mintlify/cli/open": ["open@8.4.2", "", { "dependencies": { "define-lazy-prop": "^2.0.0", "is-docker": "^2.1.1", "is-wsl": "^2.2.0" } }, "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ=="], "@mintlify/cli/posthog-node": ["posthog-node@5.17.2", "", { "dependencies": { "@posthog/core": "1.7.1" } }, "sha512-lz3YJOr0Nmiz0yHASaINEDHqoV+0bC3eD8aZAG+Ky292dAnVYul+ga/dMX8KCBXg8hHfKdxw0SztYD5j6dgUqQ=="], @@ -6342,8 +6426,6 @@ "@mintlify/common/hast-util-to-html": ["hast-util-to-html@9.0.4", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-wxQzXtdbhiwGAUKrnQJXlOPmHnEehzphwkK7aluUPQ+lEc1xefC8pblMgpp2w5ldBTEfveRIrADcrhGIWrlTDA=="], - "@mintlify/common/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@mintlify/common/mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA=="], "@mintlify/common/mdast-util-gfm": ["mdast-util-gfm@3.0.0", "", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", "mdast-util-gfm-footnote": "^2.0.0", "mdast-util-gfm-strikethrough": "^2.0.0", "mdast-util-gfm-table": "^2.0.0", "mdast-util-gfm-task-list-item": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw=="], @@ -6374,8 +6456,6 @@ "@mintlify/prebuild/fs-extra": ["fs-extra@11.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw=="], - "@mintlify/prebuild/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@mintlify/prebuild/sharp": ["sharp@0.33.5", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.6.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.33.5", "@img/sharp-darwin-x64": "0.33.5", "@img/sharp-libvips-darwin-arm64": "1.0.4", "@img/sharp-libvips-darwin-x64": "1.0.4", "@img/sharp-libvips-linux-arm": "1.0.5", "@img/sharp-libvips-linux-arm64": "1.0.4", "@img/sharp-libvips-linux-s390x": "1.0.4", "@img/sharp-libvips-linux-x64": "1.0.4", "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", "@img/sharp-libvips-linuxmusl-x64": "1.0.4", "@img/sharp-linux-arm": "0.33.5", "@img/sharp-linux-arm64": "0.33.5", "@img/sharp-linux-s390x": "0.33.5", "@img/sharp-linux-x64": "0.33.5", "@img/sharp-linuxmusl-arm64": "0.33.5", "@img/sharp-linuxmusl-x64": "0.33.5", "@img/sharp-wasm32": "0.33.5", "@img/sharp-win32-ia32": "0.33.5", "@img/sharp-win32-x64": "0.33.5" } }, "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw=="], "@mintlify/prebuild/unist-util-visit": ["unist-util-visit@4.1.2", "", { "dependencies": { "@types/unist": "^2.0.0", "unist-util-is": "^5.0.0", "unist-util-visit-parents": "^5.1.1" } }, "sha512-MSd8OUGISqHdVvfY9TPhyK2VdUrPgxkUtWSuMHF6XAAFuL4LokseigBnZtPnJMu+FbynTkFNnFlyjxpVKujMRg=="], @@ -6392,8 +6472,6 @@ "@mintlify/previewing/ink": ["ink@6.3.0", "", { "dependencies": { "@alcalzone/ansi-tokenize": "^0.2.0", "ansi-escapes": "^7.0.0", "ansi-styles": "^6.2.1", "auto-bind": "^5.0.1", "chalk": "^5.6.0", "cli-boxes": "^3.0.0", "cli-cursor": "^4.0.0", "cli-truncate": "^4.0.0", "code-excerpt": "^4.0.0", "es-toolkit": "^1.39.10", "indent-string": "^5.0.0", "is-in-ci": "^2.0.0", "patch-console": "^2.0.0", "react-reconciler": "^0.32.0", "signal-exit": "^3.0.7", "slice-ansi": "^7.1.0", "stack-utils": "^2.0.6", "string-width": "^7.2.0", "type-fest": "^4.27.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0", "ws": "^8.18.0", "yoga-layout": "~3.2.1" }, "peerDependencies": { "@types/react": ">=19.0.0", "react": ">=19.0.0", "react-devtools-core": "^4.19.1" }, "optionalPeers": ["@types/react", "react-devtools-core"] }, "sha512-2CbJAa7XeziZYe6pDS5RVLirRY28iSGMQuEV8jRU5NQsONQNfcR/BZHHc9vkMg2lGYTHTM2pskxC1YmY28p6bQ=="], - "@mintlify/previewing/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@mintlify/previewing/react": ["react@19.2.3", "", {}, "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA=="], "@mintlify/previewing/socket.io": ["socket.io@4.8.0", "", { "dependencies": { "accepts": "~1.3.4", "base64id": "~2.0.0", "cors": "~2.8.5", "debug": "~4.3.2", "engine.io": "~6.6.0", "socket.io-adapter": "~2.5.2", "socket.io-parser": "~4.2.4" } }, "sha512-8U6BEgGjQOfGz3HHTYaC/L1GaxDCJ/KM0XTkJly0EhZ5U/du9uNEZy4ZgYzEzIqlx2CMm25CrCqr1ck899eLNA=="], @@ -6404,8 +6482,6 @@ "@mintlify/scraping/fs-extra": ["fs-extra@11.1.1", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ=="], - "@mintlify/scraping/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@mintlify/scraping/mdast-util-mdx-jsx": ["mdast-util-mdx-jsx@3.1.3", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "parse-entities": "^4.0.0", "stringify-entities": "^4.0.0", "unist-util-stringify-position": "^4.0.0", "vfile-message": "^4.0.0" } }, "sha512-bfOjvNt+1AcbPLTFMFWY149nJz0OjmewJs3LQQ5pIyVGxP4CdOqNVJL6kTaM5c68p8q82Xv3nCyFfUnuEcH3UQ=="], "@mintlify/scraping/remark-gfm": ["remark-gfm@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-gfm": "^3.0.0", "micromark-extension-gfm": "^3.0.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.0" } }, "sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA=="], @@ -6418,8 +6494,6 @@ "@mintlify/scraping/zod": ["zod@3.24.0", "", {}, "sha512-Hz+wiY8yD0VLA2k/+nsg2Abez674dDGTai33SwNvMPuf9uIrBC9eFgIMQxBBbHFxVXi8W+5nX9DcAh9YNSQm/w=="], - "@mintlify/validation/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "@mintlify/validation/zod": ["zod@3.24.0", "", {}, "sha512-Hz+wiY8yD0VLA2k/+nsg2Abez674dDGTai33SwNvMPuf9uIrBC9eFgIMQxBBbHFxVXi8W+5nX9DcAh9YNSQm/w=="], "@mintlify/validation/zod-to-json-schema": ["zod-to-json-schema@3.20.4", "", { "peerDependencies": { "zod": "^3.20.0" } }, "sha512-Un9+kInJ2Zt63n6Z7mLqBifzzPcOyX+b+Exuzf7L1+xqck9Q2EPByyTRduV3kmSPaXaRer1JCsucubpgL1fipg=="], @@ -6428,6 +6502,8 @@ "@mishieck/ink-titled-box/react": ["react@19.2.4", "", {}, "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ=="], + "@modelcontextprotocol/sdk/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "@modelcontextprotocol/sdk/express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], "@modelcontextprotocol/sdk/express-rate-limit": ["express-rate-limit@8.5.2", "", { "dependencies": { "ip-address": "^10.2.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A=="], @@ -6734,6 +6810,8 @@ "@sentry/bundler-plugin-core/glob": ["glob@10.5.0", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg=="], + "@sentry/bundler-plugin-core/unplugin": ["unplugin@1.0.1", "", { "dependencies": { "acorn": "^8.8.1", "chokidar": "^3.5.3", "webpack-sources": "^3.2.3", "webpack-virtual-modules": "^0.5.0" } }, "sha512-aqrHaVBWW1JVKBHmGo33T5TxeL0qWzfvjWokObHA9bYmN7eNDkwOxmLjhioHl9878qDFMAaT51XNroRyuz7WxA=="], + "@sentry/cli/proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], "@sentry/node/@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.7.1", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-OPFBYuXEn1E4ja3Y6eeA7O+ZnLBNcXTV5Cgsn1VaqBZ6hC5FnpZPLBNme1LJY8ZtF4aOujPKFoeWN4ik487KuQ=="], @@ -6768,6 +6846,8 @@ "@sentry/react/@sentry/core": ["@sentry/core@10.53.1", "", {}, "sha512-XG4ezlkyuAPjBC5+9kXC94rXXuqYTw9NRhfaDHssbTFaGnqBR8vQX2UUgZfY7ucbeelRDGfBu1sywoU+mB04uA=="], + "@sentry/vite-plugin/unplugin": ["unplugin@1.0.1", "", { "dependencies": { "acorn": "^8.8.1", "chokidar": "^3.5.3", "webpack-sources": "^3.2.3", "webpack-virtual-modules": "^0.5.0" } }, "sha512-aqrHaVBWW1JVKBHmGo33T5TxeL0qWzfvjWokObHA9bYmN7eNDkwOxmLjhioHl9878qDFMAaT51XNroRyuz7WxA=="], + "@shikijs/primitive/@shikijs/types": ["@shikijs/types@4.1.0", "", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-3EQWX54fMpniOrDblzAhiwiJwpiTMW6+B9DWyUd9ska483tbayFYuw47UxwuPknI31bKnySfVQ/QW+jFL4rFdA=="], "@slack/logger/@types/node": ["@types/node@25.8.0", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-TCFSk8IZh+iLX1xtksoBVtdmgL+1IX0fC9BeU4QqFSuNdN/K+HUlhqOzEmSYYpZUVsLYcPqc9KX+60iDuninSQ=="], @@ -6932,6 +7012,8 @@ "@trigger.dev/core/@s2-dev/streamstore": ["@s2-dev/streamstore@0.22.5", "", { "dependencies": { "@protobuf-ts/runtime": "^2.11.1", "debug": "^4.4.3" } }, "sha512-GqdOKIbIoIxT+40fnKzHbrsHB6gBqKdECmFe7D3Ojk4FoN1Hu0LhFzZv6ZmVMjoHHU+55debS1xSWjZwQmbIyQ=="], + "@trigger.dev/core/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "@trigger.dev/core/execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="], "@trigger.dev/core/jose": ["jose@5.10.0", "", {}, "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg=="], @@ -6996,12 +7078,16 @@ "aggregate-error/clean-stack": ["clean-stack@4.2.0", "", { "dependencies": { "escape-string-regexp": "5.0.0" } }, "sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg=="], + "ansi-align/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + "anymatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "artillery/chalk": ["chalk@2.4.2", "", { "dependencies": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" } }, "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="], "artillery/csv-parse": ["csv-parse@4.16.3", "", {}, "sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg=="], + "artillery/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], + "artillery/nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="], "artillery-plugin-ensure/chalk": ["chalk@2.4.2", "", { "dependencies": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" } }, "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="], @@ -7052,6 +7138,8 @@ "atmn/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "autoevals/openai": ["openai@6.38.0", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-AoMplt2UalrpgUDMh3L09QWjNRlgJPipclQvA6sYAaeF6nHNBMgmikAZGmcYLn8on4d9sQY9Q8bOLfrBS7Lc8g=="], + "autumn-js/@types/node": ["@types/node@22.19.19", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew=="], "autumn-js/@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], @@ -7098,6 +7186,10 @@ "body-parser/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "boxen/camelcase": ["camelcase@8.0.0", "", {}, "sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA=="], + + "braintrust/express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], + "bun-types/@types/node": ["@types/node@25.8.0", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-TCFSk8IZh+iLX1xtksoBVtdmgL+1IX0fC9BeU4QqFSuNdN/K+HUlhqOzEmSYYpZUVsLYcPqc9KX+60iDuninSQ=="], "c12/chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], @@ -7134,6 +7226,8 @@ "clean-stack/escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + "cli-progress/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + "cli-table3/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "cli-truncate/slice-ansi": ["slice-ansi@5.0.0", "", { "dependencies": { "ansi-styles": "^6.0.0", "is-fullwidth-code-point": "^4.0.0" } }, "sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ=="], @@ -7260,6 +7354,8 @@ "eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@2.1.0", "", {}, "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw=="], + "eventsource/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "execa/figures": ["figures@6.1.0", "", { "dependencies": { "is-unicode-supported": "^2.0.0" } }, "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg=="], "execa/is-stream": ["is-stream@4.0.1", "", {}, "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A=="], @@ -7290,6 +7386,8 @@ "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "front-matter/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], + "fs-minipass/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], "gaxios/https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], @@ -7312,6 +7410,8 @@ "gradient-string/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "gray-matter/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], + "headers-polyfill/set-cookie-parser": ["set-cookie-parser@3.1.0", "", {}, "sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw=="], "hosted-git-info/lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], @@ -7390,8 +7490,6 @@ "mocha/glob": ["glob@10.5.0", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg=="], - "mocha/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "mocha/minimatch": ["minimatch@9.0.9", "", { "dependencies": { "brace-expansion": "^2.0.2" } }, "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg=="], "mocha/strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], @@ -7410,6 +7508,8 @@ "msw/type-fest": ["type-fest@5.6.0", "", { "dependencies": { "tagged-tag": "^1.0.0" } }, "sha512-8ZiHFm91orbSAe2PSAiSVBVko18pbhbiB3U9GglSzF/zCGkR+rxpHx6sEMCUm4kxY4LjDIUGgCfUMtwfZfjfUA=="], + "next/@next/env": ["@next/env@16.2.4", "", {}, "sha512-dKkkOzOSwFYe5RX6y26fZgkSpVAlIOJKQHIiydQcrWH6y/97+RceSOAdjZ14Qa3zLduVUy0TXcn+EiM6t4rPgw=="], + "next/postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], "next-mdx-remote-client/serialize-error": ["serialize-error@13.0.1", "", { "dependencies": { "non-error": "^0.1.0", "type-fest": "^5.4.1" } }, "sha512-bBZaRwLH9PN5HbLCjPId4dP5bNGEtumcErgOX952IsvOhVPrm3/AeK1y0UHA/QaPG701eg0yEnOKsCOC6X/kaA=="], @@ -7426,6 +7526,8 @@ "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + "nunjucks/commander": ["commander@5.1.0", "", {}, "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="], + "nuqs/@standard-schema/spec": ["@standard-schema/spec@1.0.0", "", {}, "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA=="], "open-editor/execa": ["execa@5.1.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", "human-signals": "^2.1.0", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.1", "onetime": "^5.1.2", "signal-exit": "^3.0.3", "strip-final-newline": "^2.0.0" } }, "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg=="], @@ -7634,6 +7736,8 @@ "sucrase/commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="], + "supertap/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], + "supports-color/has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], "supports-hyperlinks/has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], @@ -7832,8 +7936,6 @@ "@artilleryio/int-core/socket.io-client/engine.io-client": ["engine.io-client@6.6.4", "", { "dependencies": { "@socket.io/component-emitter": "~3.1.0", "debug": "~4.4.1", "engine.io-parser": "~5.2.1", "ws": "~8.18.3", "xmlhttprequest-ssl": "~2.1.1" } }, "sha512-+kjUJnZGwzewFDw951CDWcwj35vMNf2fcj7xQWOctq1F2i1jkDdVvdFG9kM/BEChymCH36KgjnW0NsL58JYRxw=="], - "@asyncapi/parser/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@autumn/auth/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], "@autumn/leaf/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], @@ -7968,8 +8070,6 @@ "@eslint/eslintrc/ajv/json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], - "@eslint/eslintrc/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@eslint/eslintrc/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], "@google/genai/p-retry/@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="], @@ -8010,6 +8110,14 @@ "@langchain/langgraph-sdk/p-queue/p-timeout": ["p-timeout@7.0.1", "", {}, "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg=="], + "@mastra/braintrust/braintrust/@ai-sdk/provider": ["@ai-sdk/provider@1.1.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg=="], + + "@mastra/braintrust/braintrust/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + + "@mastra/braintrust/braintrust/minimatch": ["minimatch@9.0.9", "", { "dependencies": { "brace-expansion": "^2.0.2" } }, "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg=="], + + "@mastra/braintrust/braintrust/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], + "@mintlify/cli/ink/@alcalzone/ansi-tokenize": ["@alcalzone/ansi-tokenize@0.2.5", "", { "dependencies": { "ansi-styles": "^6.2.1", "is-fullwidth-code-point": "^5.0.0" } }, "sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw=="], "@mintlify/cli/ink/chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], @@ -8026,8 +8134,6 @@ "@mintlify/cli/inquirer/run-async": ["run-async@3.0.0", "", {}, "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q=="], - "@mintlify/cli/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@mintlify/cli/open/define-lazy-prop": ["define-lazy-prop@2.0.0", "", {}, "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og=="], "@mintlify/cli/open/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="], @@ -8040,8 +8146,6 @@ "@mintlify/common/hast-util-to-html/property-information": ["property-information@6.5.0", "", {}, "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig=="], - "@mintlify/common/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@mintlify/common/mdast-util-gfm/mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.3", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q=="], "@mintlify/common/mdast-util-mdx-jsx/mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.3", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q=="], @@ -8076,8 +8180,6 @@ "@mintlify/link-rot/unist-util-visit/unist-util-visit-parents": ["unist-util-visit-parents@5.1.3", "", { "dependencies": { "@types/unist": "^2.0.0", "unist-util-is": "^5.0.0" } }, "sha512-x6+y8g7wWMyQhL1iZfhIPhDAs7Xwbn9nRosDXl7qoPTSCy0yNxnKc+hWokFifWQIDGi154rdUqKvbCa4+1kLhg=="], - "@mintlify/prebuild/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@mintlify/prebuild/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="], "@mintlify/prebuild/sharp/@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="], @@ -8152,8 +8254,6 @@ "@mintlify/previewing/ink/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], - "@mintlify/previewing/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@mintlify/previewing/socket.io/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], "@mintlify/previewing/unist-util-visit/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], @@ -8164,12 +8264,8 @@ "@mintlify/previewing/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - "@mintlify/scraping/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@mintlify/scraping/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - "@mintlify/validation/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "@mishieck/ink-titled-box/ink/@alcalzone/ansi-tokenize": ["@alcalzone/ansi-tokenize@0.2.5", "", { "dependencies": { "ansi-styles": "^6.2.1", "is-fullwidth-code-point": "^5.0.0" } }, "sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw=="], "@mishieck/ink-titled-box/ink/cli-truncate": ["cli-truncate@5.2.0", "", { "dependencies": { "slice-ansi": "^8.0.0", "string-width": "^8.2.0" } }, "sha512-xRwvIOMGrfOAnM1JYtqQImuaNtDEv9v6oIYAs4LIHwTiKee8uwvIi363igssOC0O5U04i4AlENs79LQLu9tEMw=="], @@ -8426,6 +8522,8 @@ "@sentry/bundler-plugin-core/glob/path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], + "@sentry/bundler-plugin-core/unplugin/webpack-virtual-modules": ["webpack-virtual-modules@0.5.0", "", {}, "sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw=="], + "@sentry/node-core/import-in-the-middle/cjs-module-lexer": ["cjs-module-lexer@2.2.0", "", {}, "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ=="], "@sentry/node/@opentelemetry/instrumentation/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.211.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-swFdZq8MCdmdR22jTVGQDhwqDzcI4M10nhjXkLr1EsIzXgZBqm4ZlmmcWsg3TSNf+3mzgOiqveXmBLZuDi2Lgg=="], @@ -8436,6 +8534,8 @@ "@sentry/node/minimatch/brace-expansion": ["brace-expansion@2.1.0", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w=="], + "@sentry/vite-plugin/unplugin/webpack-virtual-modules": ["webpack-virtual-modules@0.5.0", "", {}, "sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw=="], + "@slack/logger/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "@slack/socket-mode/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], @@ -8568,6 +8668,12 @@ "accepts/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], + "ansi-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "ansi-align/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + + "ansi-align/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + "artillery-plugin-ensure/chalk/ansi-styles": ["ansi-styles@3.2.1", "", { "dependencies": { "color-convert": "^1.9.0" } }, "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA=="], "artillery-plugin-ensure/chalk/escape-string-regexp": ["escape-string-regexp@1.0.5", "", {}, "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg=="], @@ -8630,6 +8736,8 @@ "artillery/chalk/supports-color": ["supports-color@5.5.0", "", { "dependencies": { "has-flag": "^3.0.0" } }, "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow=="], + "artillery/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], + "atmn/@tanstack/react-query/@tanstack/query-core": ["@tanstack/query-core@5.100.11", "", {}, "sha512-lmE0994apShXPj8CUxgx4ch5yUJhE9k/+tVwihBvPOyerACWdBocfFg24t8+0RhtlTd7tEgchDkhlCxNssvDxw=="], "atmn/@typescript/native-preview/@typescript/native-preview-darwin-arm64": ["@typescript/native-preview-darwin-arm64@7.0.0-dev.20260511.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-SYrqVOlapDxDG7FzHBIJbfgaix+mXPkYzYGqwpz/TAhoPA7sgbfAoGLaqi3ut9N88C/OYNhEX4tjz/0PC9i1nw=="], @@ -8710,6 +8818,26 @@ "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "braintrust/express/accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + + "braintrust/express/body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], + + "braintrust/express/content-disposition": ["content-disposition@1.1.0", "", {}, "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g=="], + + "braintrust/express/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], + + "braintrust/express/finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], + + "braintrust/express/fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], + + "braintrust/express/merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], + + "braintrust/express/send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], + + "braintrust/express/serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="], + + "braintrust/express/type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="], + "bun-types/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "c12/chokidar/readdirp": ["readdirp@5.0.0", "", {}, "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ=="], @@ -8718,6 +8846,12 @@ "checkout/react-dom/scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + "cli-progress/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "cli-progress/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + + "cli-progress/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + "cli-table3/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "cli-table3/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], @@ -8836,6 +8970,8 @@ "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], + "front-matter/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], + "fs-minipass/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], "gaxios/https-proxy-agent/agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], @@ -8846,6 +8982,8 @@ "gradient-string/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "gray-matter/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], + "ink-confirm-input/ink-text-input/chalk": ["chalk@3.0.0", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg=="], "ink-confirm-input/ink-text-input/ink": ["ink@6.8.0", "", { "dependencies": { "@alcalzone/ansi-tokenize": "^0.2.4", "ansi-escapes": "^7.3.0", "ansi-styles": "^6.2.1", "auto-bind": "^5.0.1", "chalk": "^5.6.0", "cli-boxes": "^3.0.0", "cli-cursor": "^4.0.0", "cli-truncate": "^5.1.1", "code-excerpt": "^4.0.0", "es-toolkit": "^1.39.10", "indent-string": "^5.0.0", "is-in-ci": "^2.0.0", "patch-console": "^2.0.0", "react-reconciler": "^0.33.0", "scheduler": "^0.27.0", "signal-exit": "^3.0.7", "slice-ansi": "^8.0.0", "stack-utils": "^2.0.6", "string-width": "^8.1.1", "terminal-size": "^4.0.1", "type-fest": "^5.4.1", "widest-line": "^6.0.0", "wrap-ansi": "^9.0.0", "ws": "^8.18.0", "yoga-layout": "~3.2.1" }, "peerDependencies": { "@types/react": ">=19.0.0", "react": ">=19.0.0", "react-devtools-core": ">=6.1.2" }, "optionalPeers": ["@types/react", "react-devtools-core"] }, "sha512-sbl1RdLOgkO9isK42WCZlJCFN9hb++sX9dsklOvfd1YQ3bQ2AiFu12Q6tFlr0HvEUvzraJntQCCpfEoUe9DSzA=="], @@ -8916,8 +9054,6 @@ "mocha/glob/path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], - "mocha/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "mocha/minimatch/brace-expansion": ["brace-expansion@2.1.0", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w=="], "msw/@inquirer/confirm/@inquirer/core": ["@inquirer/core@11.1.10", "", { "dependencies": { "@inquirer/ansi": "^2.0.5", "@inquirer/figures": "^2.0.5", "@inquirer/type": "^4.0.5", "cli-width": "^4.1.0", "fast-wrap-ansi": "^0.2.0", "mute-stream": "^3.0.0", "signal-exit": "^4.1.0" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-a4Q5BXHQAHa9eO202sTaFCHFYVB3x5fauDuThEAdZ9gfn76pSxiKU7wWcEH0N1O0XmQvNfQNU6QXpiRxmYQx+A=="], @@ -9028,8 +9164,6 @@ "puppeteer/cosmiconfig/env-paths": ["env-paths@2.2.1", "", {}, "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="], - "puppeteer/cosmiconfig/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "puppeteer/puppeteer-core/chromium-bidi": ["chromium-bidi@0.6.2", "", { "dependencies": { "mitt": "3.0.1", "urlpattern-polyfill": "10.0.0", "zod": "3.23.8" }, "peerDependencies": { "devtools-protocol": "*" } }, "sha512-4WVBa6ijmUTVr9cZD4eicQD8Mdy/HCX3bzEIYYpmk0glqYLoWH+LqQEvV9RpDRzoQSbY1KJHloYXbDMXMbDPhg=="], "react-email/chokidar/readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], @@ -9128,8 +9262,6 @@ "shadcn/cosmiconfig/env-paths": ["env-paths@2.2.1", "", {}, "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="], - "shadcn/cosmiconfig/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "shadcn/https-proxy-agent/agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], "shadcn/open/powershell-utils": ["powershell-utils@0.1.0", "", {}, "sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A=="], @@ -9146,6 +9278,8 @@ "string-width-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "supertap/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], + "trigger.dev/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.203.0", "", { "dependencies": { "@opentelemetry/core": "2.0.1", "@opentelemetry/otlp-transformer": "0.203.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-Wbxf7k+87KyvxFr5D7uOiSq/vHXWommvdnNE7vECO3tAhsA2GfOlpWINCMWUEPdHZ7tCXxw6Epp3vgx3jU7llQ=="], "trigger.dev/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.203.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.203.0", "@opentelemetry/core": "2.0.1", "@opentelemetry/resources": "2.0.1", "@opentelemetry/sdk-logs": "0.203.0", "@opentelemetry/sdk-metrics": "2.0.1", "@opentelemetry/sdk-trace-base": "2.0.1", "protobufjs": "^7.3.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-Y8I6GgoCna0qDQ2W6GCRtaF24SnvqvA8OfeTi7fqigD23u8Jpb4R5KFv/pRvrlGagcCLICMIyh9wiejp4TXu/A=="], @@ -9206,8 +9340,6 @@ "xo/@eslint/eslintrc/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - "xo/@eslint/eslintrc/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "xo/@eslint/eslintrc/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], "xo/@eslint/eslintrc/strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], @@ -9238,8 +9370,6 @@ "xo/eslint/globals": ["globals@13.24.0", "", { "dependencies": { "type-fest": "^0.20.2" } }, "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ=="], - "xo/eslint/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "xo/eslint/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], "xo/eslint/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], @@ -9340,6 +9470,12 @@ "@inquirer/core/wrap-ansi/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "@mastra/braintrust/braintrust/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "@mastra/braintrust/braintrust/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "@mastra/braintrust/braintrust/minimatch/brace-expansion": ["brace-expansion@2.1.0", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w=="], + "@mintlify/cli/ink/@alcalzone/ansi-tokenize/is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], "@mintlify/cli/ink/react-reconciler/scheduler": ["scheduler@0.26.0", "", {}, "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA=="], @@ -9508,6 +9644,8 @@ "@trigger.dev/core/socket.io/engine.io/ws": ["ws@8.17.1", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ=="], + "ansi-align/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "artillery-plugin-ensure/chalk/ansi-styles/color-convert": ["color-convert@1.9.3", "", { "dependencies": { "color-name": "1.1.3" } }, "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg=="], "artillery-plugin-ensure/chalk/supports-color/has-flag": ["has-flag@3.0.0", "", {}, "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="], @@ -9566,8 +9704,6 @@ "atmn/eslint-plugin-react-hooks/eslint/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - "atmn/eslint-plugin-react-hooks/eslint/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], - "atmn/eslint-plugin-react-hooks/eslint/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="], "atmn/eslint-plugin-react-hooks/eslint/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], @@ -9580,6 +9716,18 @@ "ava/cli-truncate/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], + "braintrust/express/accepts/negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + + "braintrust/express/body-parser/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + + "braintrust/express/body-parser/raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], + + "braintrust/express/type-is/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="], + + "braintrust/express/type-is/media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + + "cli-progress/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "cli-table3/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "concurrently/chalk/supports-color/has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], @@ -9710,8 +9858,6 @@ "public-ip/got/cacheable-request/mimic-response": ["mimic-response@4.0.0", "", {}, "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg=="], - "puppeteer/cosmiconfig/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "puppeteer/puppeteer-core/chromium-bidi/zod": ["zod@3.23.8", "", {}, "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g=="], "react-email/next/postcss/nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="], @@ -9726,8 +9872,6 @@ "sdk-test/next/postcss/nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="], - "shadcn/cosmiconfig/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "shadcn/ora/cli-cursor/restore-cursor": ["restore-cursor@5.1.0", "", { "dependencies": { "onetime": "^7.0.0", "signal-exit": "^4.1.0" } }, "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA=="], "shadcn/ora/log-symbols/is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="], @@ -9746,8 +9890,6 @@ "xo/@eslint/eslintrc/globals/type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], - "xo/@eslint/eslintrc/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "xo/@eslint/eslintrc/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], "xo/eslint/@eslint/eslintrc/strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], @@ -9762,8 +9904,6 @@ "xo/eslint/globals/type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], - "xo/eslint/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "xo/eslint/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], "xo/eslint/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], @@ -9782,6 +9922,10 @@ "@infisical/sdk/@aws-sdk/credential-providers/@aws-sdk/client-sts/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@3.0.0", "", { "dependencies": { "@smithy/is-array-buffer": "^3.0.0", "tslib": "^2.6.2" } }, "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA=="], + "@mastra/braintrust/braintrust/chalk/supports-color/has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + + "@mastra/braintrust/braintrust/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "@mintlify/cli/yargs/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "@mintlify/common/sucrase/glob/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], @@ -9824,8 +9968,6 @@ "atmn/eslint-plugin-react-hooks/eslint/globals/type-fest": ["type-fest@0.20.2", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="], - "atmn/eslint-plugin-react-hooks/eslint/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], - "atmn/eslint-plugin-react-hooks/eslint/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="], "atmn/eslint-plugin-react-hooks/eslint/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 1d333828f..50afd12df 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -20,6 +20,7 @@ export { } from "./constants.js"; export { type AutumnMcpAuth, + createRequestContext, environmentSchema, type OAuthEnvironment, } from "./server/auth/auth.js"; diff --git a/run.sh b/run.sh index 8dff77919..8bcf8971c 100755 --- a/run.sh +++ b/run.sh @@ -10,13 +10,48 @@ if [[ -z "$filename" ]]; then exit 1 fi -# Resolve to an absolute path so the prefix check works for relative inputs too. resolved="$(cd -P "$(dirname "$filename")" 2>/dev/null && pwd)/$(basename "$filename")" +run_leaf_eval() { + local file="$1" + shift + local rel="${file#$repo_root/apps/leaf/}" + local args=("$@") + local filter="" + + if [[ "${args[0]}" =~ ^[0-9]+$ ]]; then + filter="$(bun "$repo_root/scripts/testScripts/getDescribeAtCursor.ts" "$file" "${args[0]}")" + args=("${args[@]:1}") + elif [[ "${args[0]}" == "-t" || "${args[0]}" == "--test-name-pattern" ]]; then + filter="${args[1]}" + args=("${args[@]:2}") + fi + + cd "$repo_root/apps/leaf" + if [[ -n "$filter" && "$filter" != ".*" ]]; then + exec env ENV_FILE=.env infisical run --env=dev --recursive -- "$repo_root/node_modules/.bin/braintrust" eval "$rel" --external-packages @mastra/mcp @mastra/core --filter "evalName=$filter" "${args[@]}" + fi + exec env ENV_FILE=.env infisical run --env=dev --recursive -- "$repo_root/node_modules/.bin/braintrust" eval "$rel" --external-packages @mastra/mcp @mastra/core "${args[@]}" +} + if [[ "$resolved" == "$repo_root/server/"* ]]; then exec "$repo_root/server/run.sh" "$resolved" "${@:2}" fi +if [[ "$resolved" == "$repo_root/apps/leaf/tests/evals/"* && "$resolved" == *".eval.ts" ]]; then + run_leaf_eval "$resolved" "${@:2}" +fi + +if [[ "$resolved" == "$repo_root/apps/leaf/tests/"* && "$resolved" == *".test.ts" ]]; then + cd "$repo_root/apps/leaf" + rel="${resolved#$repo_root/apps/leaf/}" + if [[ "${2:-}" =~ ^[0-9]+$ ]]; then + test_name="$(bun "$repo_root/scripts/testScripts/getDescribeAtCursor.ts" "$resolved" "$2")" + exec env ENV_FILE=.env infisical run --env=dev --recursive -- bun test --timeout 0 "$rel" -t "$test_name" + fi + exec env ENV_FILE=.env infisical run --env=dev --recursive -- bun test "$rel" "${@:2}" +fi + if [[ "$resolved" == "$repo_root/packages/mcp/tests/"* && "$resolved" == *".test.ts" ]]; then cd "$repo_root/packages/mcp" rel="${resolved#$repo_root/packages/mcp/}" diff --git a/scripts/testScripts/getDescribeAtCursor.ts b/scripts/testScripts/getDescribeAtCursor.ts index b4cc079f6..782e10f7d 100644 --- a/scripts/testScripts/getDescribeAtCursor.ts +++ b/scripts/testScripts/getDescribeAtCursor.ts @@ -1,4 +1,4 @@ -import { readFileSync } from "fs"; +import { readFileSync } from "node:fs"; const file = process.argv[2]; const lineNum = parseInt(process.argv[3], 10); @@ -8,13 +8,13 @@ const lines = content.split("\n"); const MULTILINE_LOOKAHEAD = 5; -const escape = (raw: string) => raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +const escapeRegex = (raw: string) => raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const CHALK_PATTERN = /(?:describe|test(?:\.concurrent)?)\s*\(\s*`\$\{chalk\.\w+\(["'](.*?)["']\)\}`/; -const SIMPLE_PATTERN = - /(?:describe|test(?:\.concurrent)?)\s*\(\s*["'`](.*?)["'`]/; -const OPEN_PATTERN = /(?:describe|test(?:\.concurrent)?)\s*\(\s*$/; +const BLOCK_NAME = String.raw`(?:describe|test(?:\.concurrent)?|Eval(?:<[^>]+>)?)`; +const SIMPLE_PATTERN = new RegExp(`${BLOCK_NAME}\\s*\\(\\s*["'\`](.*?)["'\`]`); +const OPEN_PATTERN = new RegExp(`${BLOCK_NAME}\\s*\\(\\s*$`); // Walk backwards from cursor to find enclosing describe, test, or test.concurrent. // Each candidate also gets a multi-line lookahead so name args wrapped onto the @@ -24,13 +24,13 @@ for (let i = lineNum - 1; i >= 0; i--) { const chalkMatch = line.match(CHALK_PATTERN); if (chalkMatch) { - console.log(escape(chalkMatch[1])); + console.log(escapeRegex(chalkMatch[1])); process.exit(0); } const simpleMatch = line.match(SIMPLE_PATTERN); if (simpleMatch) { - console.log(escape(simpleMatch[1])); + console.log(escapeRegex(simpleMatch[1])); process.exit(0); } @@ -40,12 +40,12 @@ for (let i = lineNum - 1; i >= 0; i--) { .join("\n"); const chalkMulti = joined.match(CHALK_PATTERN); if (chalkMulti) { - console.log(escape(chalkMulti[1])); + console.log(escapeRegex(chalkMulti[1])); process.exit(0); } const simpleMulti = joined.match(SIMPLE_PATTERN); if (simpleMulti) { - console.log(escape(simpleMulti[1])); + console.log(escapeRegex(simpleMulti[1])); process.exit(0); } } diff --git a/scripts/testScripts/testDispatcher.ts b/scripts/testScripts/testDispatcher.ts index f69cc7078..d7502af50 100644 --- a/scripts/testScripts/testDispatcher.ts +++ b/scripts/testScripts/testDispatcher.ts @@ -22,6 +22,9 @@ const PROJECT_ROOT = resolve(import.meta.dirname, "../.."); const TESTS_DIR = join(PROJECT_ROOT, testRunConfig.testsBaseDir); const LEGACY_SCRIPTS_DIR = join(PROJECT_ROOT, testRunConfig.legacyScriptsDir); const RUNNER_SCRIPT = join(PROJECT_ROOT, "scripts/testScripts/runTestsV2.tsx"); +const LEAF_EVALS_DIR = join(PROJECT_ROOT, "apps/leaf/tests/evals"); +const BRAINTRUST_BIN = join(PROJECT_ROOT, "node_modules/.bin/braintrust"); +const BRAINTRUST_EXTERNAL_PACKAGES = ["@mastra/mcp", "@mastra/core"]; // Worktree .env.local loading happens in scripts/preload-env.ts, which Bun // auto-runs via bunfig.toml `preload` for every `bun` and `bun test` invocation. @@ -157,6 +160,52 @@ async function collectTestFilesFromDir({ return files; } +async function collectEvalFilesFromDir({ + dir, +}: { + dir: string; +}): Promise { + const files: string[] = []; + + const walk = async ({ d }: { d: string }) => { + const entries = await readdir(d); + for (const entry of entries) { + const fullPath = join(d, entry); + const entryStat = await stat(fullPath); + + if (entryStat.isDirectory()) { + await walk({ d: fullPath }); + } else if (entry.endsWith(".eval.ts")) { + files.push(fullPath); + } + } + }; + + await walk({ d: dir }); + return files; +} + +async function resolveLeafEvalTarget({ + target, +}: { + target: string; +}): Promise { + const candidates = [ + resolve(process.cwd(), target), + join(PROJECT_ROOT, target), + join(LEAF_EVALS_DIR, target), + ]; + + for (const candidate of candidates) { + if (!existsSync(candidate)) continue; + const s = await stat(candidate); + if (s.isDirectory()) return collectEvalFilesFromDir({ dir: candidate }); + if (candidate.endsWith(".eval.ts")) return [candidate]; + } + + return []; +} + async function main() { const args = process.argv.slice(2); @@ -202,11 +251,18 @@ async function main() { } const resolvedFiles: string[] = []; + const evalTargets: string[] = []; const fallbackArgs: string[] = []; // Track the max concurrency from matched groups (use lowest if multiple) let groupMaxConcurrency: number | null = null; for (const arg of positionalArgs) { + const evalTarget = await resolveLeafEvalTarget({ target: arg }); + if (evalTarget.length > 0) { + evalTargets.push(...evalTarget); + continue; + } + // Priority 1: Test group or suite from _groups/ const groupPaths = resolveTestPaths({ name: arg }); if (groupPaths) { @@ -280,6 +336,17 @@ async function main() { ? options : [...options, `--max=${concurrency}`]; + if (evalTargets.length > 0) { + if (resolvedFiles.length > 0 || fallbackArgs.length > 0) { + console.error("Error: Cannot mix Braintrust evals with bun test targets"); + process.exit(1); + } + + const evalOptions = options.filter((option) => !option.startsWith("--max")); + await spawnBraintrustEval({ args: [...evalTargets, ...evalOptions] }); + return; + } + if (resolvedFiles.length > 0 && fallbackArgs.length > 0) { const runnerArgs = [...resolvedFiles, ...fallbackArgs, ...finalOptions]; await spawnRunner({ args: runnerArgs }); @@ -312,4 +379,26 @@ async function spawnRunner({ args }: { args: string[] }) { process.exit(exitCode); } +async function spawnBraintrustEval({ args }: { args: string[] }) { + const proc = spawn( + [ + BRAINTRUST_BIN, + "eval", + ...args, + "--external-packages", + ...BRAINTRUST_EXTERNAL_PACKAGES, + ], + { + cwd: join(PROJECT_ROOT, "apps/leaf"), + stdout: "inherit", + stderr: "inherit", + stdin: "inherit", + env: { ...process.env }, + }, + ); + + const exitCode = await proc.exited; + process.exit(exitCode); +} + main();