diff --git a/README.md b/README.md index 5df561f..ceef2e9 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,50 @@ export OPENAI_CUSTOM_HEADERS='cf-aig-metadata: {"app":"my-evals"}' Leave every override unset and both agents run vendor-direct against api.anthropic.com and api.openai.com — nothing is rendered, nothing changes. +#### Per-experiment routing (`agent.endpoint`) + +The variables above are per adapter: every codex arm in a run reads the +same `OPENAI_BASE_URL`. To route ONE experiment elsewhere — a second +gateway, or a gateway's unified OpenAI-compatible API serving a third-party +model — the experiment names the host variables that carry its key, base +URL and headers, and the host sets them next to the vendor ones: + +```ts +export default defineExperiment({ + id: "acme-gateway-mcp", + comparisonGroup: "acme-gateway", + treatment: "mcp", + agent: { + adapter: "codex", + provider: "acme", // recorded as-is in the manifest and report + model: "acme/coder-9", // whatever the endpoint expects in `model` + reasoning: "high", + endpoint: { + apiKeyEnv: "GATEWAY_API_TOKEN", // secret-named: redacted like a key + baseUrlEnv: "GATEWAY_BASE_URL", // not secret-named: identity + egress + headersEnv: "GATEWAY_HEADERS", // secret-named; `Name: Value` lines + wireApi: "chat", // codex only: chat completions + }, + }, + runtime: ..., +}); +``` + +Unset fields keep the vendor defaults. The names must be POSIX env var +names; `defineExperiment` refuses a key or headers variable that is not +secret-named, a base URL variable that is, and `wireApi` on anything but +codex. The forwarded values travel exactly as before: the key as a provider +key, the base URL into the pair environment and the egress allowlist, the +headers redacted from artifacts. Codex renders them into its custom model +provider (`env_key`, `base_url`, `http_headers`, `wire_api`); Claude Code +only reads the vendor names, so the adapter re-exports custom ones under +them inside the container before exec — a custom key as both +`ANTHROPIC_API_KEY` (`x-api-key`) and `ANTHROPIC_AUTH_TOKEN` +(`Authorization: Bearer`), since a provider-native Anthropic route reads +the former and a gateway's unified API the latter. The preflight key probe +follows the experiment's variables, so a bad gateway token still fails +before any container starts. + ## How it works Every attempt is one point in a 2-D plan: diff --git a/src/adapters/claude.ts b/src/adapters/claude.ts index b0121c5..3111476 100644 --- a/src/adapters/claude.ts +++ b/src/adapters/claude.ts @@ -13,6 +13,8 @@ * across experiments. */ +import { type ResolvedEndpoint, resolveEndpoint } from "../experiments/endpoint.ts"; +import type { AgentEndpointSpec } from "../experiments/types.ts"; import type { McpServerConfig } from "../isolation/mcp.ts"; import { endpointUrl, parseHeaderLines } from "../secrets.ts"; import { parseClaudeStream } from "./parse.ts"; @@ -21,6 +23,18 @@ import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } f /** Container path of the MCP config Claude Code auto-discovers. */ export const CLAUDE_MCP_CONFIG_PATH = "/workspace/.mcp.json"; +/** + * The vendor's env names: what every claude experiment gets unless its + * `agent.endpoint` names something else (experiments/endpoint.ts). The wire + * is fixed — Claude Code speaks the Messages API and nothing else. + */ +export const CLAUDE_DEFAULT_ENDPOINT: ResolvedEndpoint = { + apiKeyEnv: "ANTHROPIC_API_KEY", + baseUrlEnv: "ANTHROPIC_BASE_URL", + headersEnv: "ANTHROPIC_CUSTOM_HEADERS", + wireApi: "responses", +}; + /** * Project-scope `.mcp.json` rendered from the runtime's MCP servers * (streamable HTTP). The configs arrive credential-free — authed servers @@ -38,17 +52,26 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { readonly name: string; readonly identity; readonly versionCommand = ["claude", "--version"] as const; - readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const; /** - * Endpoint overrides, forwarded only when set. ANTHROPIC_BASE_URL routes + * The env names this experiment's traffic is routed by: the vendor's + * (ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL / ANTHROPIC_CUSTOM_HEADERS) + * unless `agent.endpoint` named others. Claude Code only ever reads the + * vendor names, so custom names are re-exported under them by a shell shim + * in `command` — inside the container, from env the harness already placed. + */ + readonly endpoint: ResolvedEndpoint; + /** The key variable: required on the host, forwarded into the container. */ + readonly requiredHostEnv: readonly string[]; + /** + * Endpoint overrides, forwarded only when set. The base URL variable routes * the CLI (and the key probe) through an Anthropic-compatible gateway — * e.g. Cloudflare AI Gateway's `/anthropic` endpoint, where the gateway - * token doubles as ANTHROPIC_API_KEY; ANTHROPIC_CUSTOM_HEADERS (`Name: - * Value` lines) carries whatever else that gateway wants, such as + * token doubles as the API key; the headers variable (`Name: Value` + * lines) carries whatever else that gateway wants, such as * `cf-aig-authorization` or `cf-aig-metadata`. Claude Code reads both - * natively, so no flag changes. + * natively under the vendor names, so no flag changes. */ - readonly optionalHostEnv = ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"] as const; + readonly optionalHostEnv: readonly string[]; /** * Egress-proxy allowlist (harness-default layer). Claude Code documents * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is @@ -59,10 +82,16 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { */ readonly egressHosts = ["api.anthropic.com", "statsig.anthropic.com", "*.sentry.io"] as const; - constructor(name = "claude", config: { model?: string; reasoning?: string } = {}) { + constructor( + name = "claude", + config: { model?: string; reasoning?: string; provider?: string; endpoint?: AgentEndpointSpec } = {}, + ) { this.name = name; + this.endpoint = resolveEndpoint(name, CLAUDE_DEFAULT_ENDPOINT, config.endpoint); + this.requiredHostEnv = [this.endpoint.apiKeyEnv]; + this.optionalHostEnv = [this.endpoint.baseUrlEnv, this.endpoint.headersEnv]; this.identity = { - provider: "anthropic", + provider: config.provider ?? "anthropic", model: config.model ?? "UNPINNED", reasoning: config.reasoning ?? "UNPINNED", cliVersion: "unknown", @@ -78,15 +107,43 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { keyProbe(env: Record): { url: string; headers: Record } { return { - url: endpointUrl(env["ANTHROPIC_BASE_URL"], "https://api.anthropic.com", "v1/models"), + url: endpointUrl(env[this.endpoint.baseUrlEnv], "https://api.anthropic.com", "v1/models"), headers: { - ...parseHeaderLines(env["ANTHROPIC_CUSTOM_HEADERS"]), - "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", + ...parseHeaderLines(env[this.endpoint.headersEnv]), + "x-api-key": env[this.endpoint.apiKeyEnv] ?? "", "anthropic-version": "2023-06-01", }, }; } + /** + * `export VENDOR="$CUSTOM"` lines for every endpoint variable whose name + * differs from the vendor's — empty when nothing differs, so the default + * invocation stays the bare `claude` argv. Optional variables are only + * re-exported when set: an empty ANTHROPIC_BASE_URL would be a base URL. + * + * A custom key variable is presented both ways Claude Code can send a + * credential — `x-api-key` (ANTHROPIC_API_KEY) and `Authorization: + * Bearer` (ANTHROPIC_AUTH_TOKEN) — because gateways differ on which one + * they read: a provider-native Anthropic route takes the key header, a + * unified API takes the bearer. The vendor's own route only ever sees + * the vendor variable, so this changes nothing for it. + */ + private endpointExports(): string[] { + const lines: string[] = []; + const { apiKeyEnv, baseUrlEnv, headersEnv } = this.endpoint; + if (apiKeyEnv !== CLAUDE_DEFAULT_ENDPOINT.apiKeyEnv) { + lines.push(`export ANTHROPIC_API_KEY="$${apiKeyEnv}"`, `export ANTHROPIC_AUTH_TOKEN="$${apiKeyEnv}"`); + } + if (baseUrlEnv !== CLAUDE_DEFAULT_ENDPOINT.baseUrlEnv) { + lines.push(`[ -n "\${${baseUrlEnv}:-}" ] && export ANTHROPIC_BASE_URL="$${baseUrlEnv}"`); + } + if (headersEnv !== CLAUDE_DEFAULT_ENDPOINT.headersEnv) { + lines.push(`[ -n "\${${headersEnv}:-}" ] && export ANTHROPIC_CUSTOM_HEADERS="$${headersEnv}"`); + } + return lines; + } + setupFiles(ctx: ContainerAttemptContext): Record { if (ctx.mcpServers === undefined) return {}; return { [CLAUDE_MCP_CONFIG_PATH]: renderMcpJson(ctx.mcpServers) }; @@ -114,7 +171,12 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { args.push("--mcp-config", CLAUDE_MCP_CONFIG_PATH); } if (ctx.webPolicy !== "native-web-allowed") args.push("--disallowedTools", "WebSearch,WebFetch"); - return args; + const exports = this.endpointExports(); + if (exports.length === 0) return args; + // Custom endpoint names: re-export them under the names Claude Code reads, + // then exec the same argv. Names only reach the shell — resolveEndpoint + // admits POSIX env names and nothing else; values stay in the environment. + return ["sh", "-c", `${exports.join("; ")}; exec "$@"`, "sh", ...args]; } parseStream(raw: string): AttemptMetrics { diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index 7f9dbce..9c5b2d9 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -25,6 +25,8 @@ * talks to api.openai.com — vendor-direct stays the default. */ +import { type ResolvedEndpoint, type WireApi, resolveEndpoint } from "../experiments/endpoint.ts"; +import type { AgentEndpointSpec } from "../experiments/types.ts"; import type { McpServerConfig } from "../isolation/mcp.ts"; import { endpointUrl, parseHeaderLines } from "../secrets.ts"; import { parseCodexStream } from "./parse.ts"; @@ -40,13 +42,31 @@ export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway"; const OPENAI_DEFAULT_BASE_URL = "https://api.openai.com/v1"; /** - * A custom model provider pointing at `baseUrl` (the Responses API, which - * codex requires), carrying `headers` on every request, selected as the - * default. Top-level keys must precede any table, so this block always - * renders first in the file. JSON string escapes are valid TOML basic-string - * escapes, which keeps header values such as JSON metadata intact. + * The vendor's env names and wire: what every codex experiment gets unless + * its `agent.endpoint` names something else (experiments/endpoint.ts). */ -function renderCodexProviderToml(baseUrl: string, headers: Readonly>): string { +export const CODEX_DEFAULT_ENDPOINT: ResolvedEndpoint = { + apiKeyEnv: "OPENAI_API_KEY", + baseUrlEnv: "OPENAI_BASE_URL", + headersEnv: "OPENAI_CUSTOM_HEADERS", + wireApi: "responses", +}; + +/** + * A custom model provider pointing at `baseUrl`, speaking `wireApi` (the + * Responses API codex prefers, or chat completions for endpoints that only + * offer that), reading its key from `envKey`, carrying `headers` on every + * request, selected as the default. Top-level keys must precede any table, + * so this block always renders first in the file. JSON string escapes are + * valid TOML basic-string escapes, which keeps header values such as JSON + * metadata intact. + */ +function renderCodexProviderToml( + baseUrl: string, + headers: Readonly>, + envKey: string, + wireApi: WireApi, +): string { const entries = Object.entries(headers); return [ `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`, @@ -54,8 +74,8 @@ function renderCodexProviderToml(baseUrl: string, headers: Readonly 0 ? [`http_headers = { ${entries.map(([name, value]) => `${JSON.stringify(name)} = ${JSON.stringify(value)}`).join(", ")} }`] : []), @@ -81,17 +101,25 @@ export class CodexAdapter implements ContainerAgentAdapter { readonly name: string; readonly identity; readonly versionCommand = ["codex", "--version"] as const; - readonly requiredHostEnv = ["OPENAI_API_KEY"] as const; /** - * Endpoint overrides, forwarded only when set: OPENAI_BASE_URL (vendor - * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's - * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY — - * and OPENAI_CUSTOM_HEADERS (`Name: Value` lines) for whatever else the + * The env names this experiment's traffic is routed by: the vendor's + * (OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_CUSTOM_HEADERS) unless + * `agent.endpoint` named others — the way two codex arms in one run reach + * two gateways, or one gateway's unified API serving a third-party model. + */ + readonly endpoint: ResolvedEndpoint; + /** The key variable: required on the host, forwarded into the container. */ + readonly requiredHostEnv: readonly string[]; + /** + * Endpoint overrides, forwarded only when set: the base URL variable + * (vendor default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's + * `/openai` endpoint, where the gateway token doubles as the API key — + * and the headers variable (`Name: Value` lines) for whatever else the * endpoint wants on each request, such as `cf-aig-metadata`. Both are * rendered into config.toml by setupFiles (see the module comment); the * key probe follows them. */ - readonly optionalHostEnv = ["OPENAI_BASE_URL", "OPENAI_CUSTOM_HEADERS"] as const; + readonly optionalHostEnv: readonly string[]; /** * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based; * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) — @@ -103,10 +131,16 @@ export class CodexAdapter implements ContainerAgentAdapter { */ readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const; - constructor(name = "codex", config: { model?: string; reasoning?: string } = {}) { + constructor( + name = "codex", + config: { model?: string; reasoning?: string; provider?: string; endpoint?: AgentEndpointSpec } = {}, + ) { this.name = name; + this.endpoint = resolveEndpoint(name, CODEX_DEFAULT_ENDPOINT, config.endpoint); + this.requiredHostEnv = [this.endpoint.apiKeyEnv]; + this.optionalHostEnv = [this.endpoint.baseUrlEnv, this.endpoint.headersEnv]; this.identity = { - provider: "openai", + provider: config.provider ?? "openai", model: config.model ?? "UNPINNED", reasoning: config.reasoning ?? "UNPINNED", cliVersion: "unknown", @@ -122,18 +156,30 @@ export class CodexAdapter implements ContainerAgentAdapter { keyProbe(env: Record): { url: string; headers: Record } { return { - url: endpointUrl(env["OPENAI_BASE_URL"], OPENAI_DEFAULT_BASE_URL, "models"), - headers: { ...parseHeaderLines(env["OPENAI_CUSTOM_HEADERS"]), Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` }, + url: endpointUrl(env[this.endpoint.baseUrlEnv], OPENAI_DEFAULT_BASE_URL, "models"), + headers: { + ...parseHeaderLines(env[this.endpoint.headersEnv]), + Authorization: `Bearer ${env[this.endpoint.apiKeyEnv] ?? ""}`, + }, }; } setupFiles(ctx: ContainerAttemptContext): Record { const sections: string[] = []; - const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"]; - const headers = parseHeaderLines(ctx.overrides?.["OPENAI_CUSTOM_HEADERS"]); - // Either override needs the custom provider; headers alone keep the vendor endpoint. - if (baseUrl !== undefined || Object.keys(headers).length > 0) { - sections.push(renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers)); + const baseUrl = ctx.overrides?.[this.endpoint.baseUrlEnv]; + const headers = parseHeaderLines(ctx.overrides?.[this.endpoint.headersEnv]); + // Either override needs the custom provider (headers alone keep the + // vendor endpoint); so does a non-vendor key variable or wire, which the + // built-in provider cannot express. + const custom = + baseUrl !== undefined || + Object.keys(headers).length > 0 || + this.endpoint.apiKeyEnv !== CODEX_DEFAULT_ENDPOINT.apiKeyEnv || + this.endpoint.wireApi !== CODEX_DEFAULT_ENDPOINT.wireApi; + if (custom) { + sections.push( + renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers, this.endpoint.apiKeyEnv, this.endpoint.wireApi), + ); } if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers)); if (sections.length === 0) return {}; @@ -145,9 +191,11 @@ export class CodexAdapter implements ContainerAgentAdapter { "sh", "-c", // codex 0.144+ ignores the OPENAI_API_KEY env var; mint auth.json from - // it in-container. Login chatter is silenced so stdout stays pure JSONL - // for parseStream; the key reaches login via stdin, never argv. - 'printenv OPENAI_API_KEY | codex login --with-api-key >/dev/null 2>&1 && exec "$@"', + // the key variable in-container. Login chatter is silenced so stdout + // stays pure JSONL for parseStream; the key reaches login via stdin, + // never argv. The variable NAME is interpolated — resolveEndpoint + // admits only POSIX env names, so nothing else can reach the shell. + `printenv ${this.endpoint.apiKeyEnv} | codex login --with-api-key >/dev/null 2>&1 && exec "$@"`, "sh", "codex", "exec", diff --git a/src/experiments/endpoint.ts b/src/experiments/endpoint.ts new file mode 100644 index 0000000..945e972 --- /dev/null +++ b/src/experiments/endpoint.ts @@ -0,0 +1,138 @@ +/** + * `agent.endpoint` resolution: the experiment names the HOST ENV VARS that + * carry an agent's API key, base URL and extra headers; the adapter fills in + * its vendor defaults for whatever is unset and checks the names before any + * container starts. Names, never values, live here — the values stay on the + * host and ride the same channels the vendor defaults always have. + */ + +import { isSecretEnvName } from "../secrets.ts"; +import type { AgentEndpointSpec } from "./types.ts"; + +/** POSIX env var names the shell shims interpolate verbatim; nothing else is accepted. */ +const ENV_NAME = /^[A-Z][A-Z0-9_]*$/; + +export const WIRE_APIS = ["responses", "chat"] as const; +export type WireApi = (typeof WIRE_APIS)[number]; + +/** An adapter's endpoint with every field decided. */ +export interface ResolvedEndpoint { + apiKeyEnv: string; + baseUrlEnv: string; + headersEnv: string; + wireApi: WireApi; +} + +/** Thrown for an `agent.endpoint` the harness refuses; the message names the field. */ +export class InvalidAgentEndpointError extends Error { + constructor(detail: string) { + super(`agent.endpoint: ${detail}`); + this.name = "InvalidAgentEndpointError"; + } +} + +/** + * Check an experiment's `agent.endpoint` without an adapter: every name must + * be a POSIX env var name, the key and headers variables must be + * secret-named (so they are redacted from artifacts like a key), the base + * URL variable must not be (its value is identity and joins the egress + * allowlist), and the three must differ. `wireApi` is codex's alone. + */ +export function validateAgentEndpoint(spec: AgentEndpointSpec | undefined, adapter: string): void { + if (spec === undefined) return; + if (spec === null || typeof spec !== "object" || Array.isArray(spec)) { + throw new InvalidAgentEndpointError("must be an object"); + } + const names = new Map(); + for (const field of ["apiKeyEnv", "baseUrlEnv", "headersEnv"] as const) { + const value = spec[field]; + if (value === undefined) continue; + if (typeof value !== "string" || !ENV_NAME.test(value)) { + throw new InvalidAgentEndpointError( + `${field} must be an env var name matching ${ENV_NAME} (got ${JSON.stringify(value)})`, + ); + } + const clash = names.get(value); + if (clash !== undefined) throw new InvalidAgentEndpointError(`${field} and ${clash} both name ${value}`); + names.set(value, field); + } + if (spec.apiKeyEnv !== undefined && !isSecretEnvName(spec.apiKeyEnv)) { + throw new InvalidAgentEndpointError( + `apiKeyEnv ${spec.apiKeyEnv} must be secret-named (contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL) so its value is redacted`, + ); + } + if (spec.headersEnv !== undefined && !isSecretEnvName(spec.headersEnv)) { + throw new InvalidAgentEndpointError( + `headersEnv ${spec.headersEnv} must be secret-named (a HEADERS suffix will do) so its value is redacted`, + ); + } + if (spec.baseUrlEnv !== undefined && isSecretEnvName(spec.baseUrlEnv)) { + throw new InvalidAgentEndpointError( + `baseUrlEnv ${spec.baseUrlEnv} must not be secret-named: a base URL is identity and must survive redaction`, + ); + } + if (spec.wireApi !== undefined) { + if (!(WIRE_APIS as readonly string[]).includes(spec.wireApi)) { + throw new InvalidAgentEndpointError(`wireApi must be one of ${WIRE_APIS.join(", ")} (got ${JSON.stringify(spec.wireApi)})`); + } + if (adapter !== "codex") { + throw new InvalidAgentEndpointError(`wireApi applies to the codex adapter only (this experiment uses "${adapter}")`); + } + } +} + +const NAME_FIELDS = ["apiKeyEnv", "baseUrlEnv", "headersEnv"] as const; + +/** + * Validate `spec` for `adapter` and fill the unset fields from `defaults`. + * A declared name may not be one of the adapter's vendor variables for a + * different role: `apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS"` would make the + * shim read the header bundle as the key and then overwrite it, and the + * three resolved names must stay distinct for the same reason. + */ +export function resolveEndpoint(adapter: string, defaults: ResolvedEndpoint, spec: AgentEndpointSpec = {}): ResolvedEndpoint { + validateAgentEndpoint(spec, adapter); + for (const field of NAME_FIELDS) { + const value = spec[field]; + if (value === undefined) continue; + for (const role of NAME_FIELDS) { + if (role !== field && value === defaults[role]) { + throw new InvalidAgentEndpointError( + `${field} ${value} is the ${adapter} adapter's ${role} variable; a name cannot serve two roles`, + ); + } + } + } + const resolved: ResolvedEndpoint = { + apiKeyEnv: spec.apiKeyEnv ?? defaults.apiKeyEnv, + baseUrlEnv: spec.baseUrlEnv ?? defaults.baseUrlEnv, + headersEnv: spec.headersEnv ?? defaults.headersEnv, + wireApi: spec.wireApi ?? defaults.wireApi, + }; + const seen = new Map(); + for (const field of NAME_FIELDS) { + const clash = seen.get(resolved[field]); + if (clash !== undefined) { + throw new InvalidAgentEndpointError(`${field} and ${clash} both resolve to ${resolved[field]}`); + } + seen.set(resolved[field], field); + } + return resolved; +} + +/** + * The endpoint as identity: the declared fields, verbatim, in a fixed key + * order — what the run manifest records and comparison groups diff. Two + * arms that differ only here are two configurations, never one treatment. + * `undefined` when nothing is declared, so an experiment without an + * endpoint keeps the identity it had before the field existed. + */ +export function endpointIdentity(spec: AgentEndpointSpec | undefined): Record | undefined { + if (spec === undefined) return undefined; + const identity: Record = {}; + for (const field of [...NAME_FIELDS, "wireApi"] as const) { + const value = spec[field]; + if (value !== undefined) identity[field] = value; + } + return Object.keys(identity).length === 0 ? undefined : identity; +} diff --git a/src/experiments/groups.ts b/src/experiments/groups.ts index af7c471..14ea952 100644 --- a/src/experiments/groups.ts +++ b/src/experiments/groups.ts @@ -24,7 +24,7 @@ import { pairKey } from "../execution-policy.ts"; import { diffIdentity, type IdentityMismatch } from "../identity-diff.ts"; -import type { RunManifest } from "../manifest.ts"; +import { agentEndpointComponent, type RunManifest } from "../manifest.ts"; import type { Experiment } from "./types.ts"; export interface ComparisonGroup { @@ -62,6 +62,7 @@ function comparableView(experiment: Experiment, manifest: RunManifest): Record } { + const endpoint = endpointIdentity(spec); + return endpoint === undefined ? {} : { endpoint }; +} + /** Per-experiment identity components — phase 2's comparison-group diff reads these. */ export interface ExperimentIdentityComponents { /** Hash of the experiment's defining module source. */ @@ -76,6 +88,8 @@ export interface ExperimentIdentityComponents { model: string; reasoning: string; cliVersion: string; + /** The declared `agent.endpoint` (experiments/endpoint.ts endpointIdentity); absent when none. */ + endpoint?: Record; }; /** Hash of the canonicalized runtime configuration. */ runtime: string; @@ -192,6 +206,7 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest model: experiment.agent.model ?? "unspecified", reasoning: experiment.agent.reasoning ?? "unspecified", cliVersion: options.cliVersions?.[experiment.id] ?? experiment.agent.cliVersion ?? "unspecified", + ...agentEndpointComponent(experiment.agent.endpoint), }, runtime: contentHash({ kind: experiment.runtime.kind, diff --git a/src/runner/execute.ts b/src/runner/execute.ts index ced770b..09c2364 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -172,6 +172,16 @@ async function inStage(kind: ErrorKind, work: () => Promise): Promise { } } +/** + * What makes one key probe distinct from another: the adapter and the host + * variables it reads. Same adapter, same variables, same values — one probe + * covers every experiment that shares them; a different `agent.endpoint` + * names different variables and gets its own. + */ +export function keyProbeIdentity(adapter: Pick): string { + return [adapter.name, [...adapter.requiredHostEnv].join(","), [...(adapter.optionalHostEnv ?? [])].join(",")].join("|"); +} + function defaultHostAdapters(): Record { const echo = new EchoAdapter(); const oracle = new OracleAdapter(); @@ -180,8 +190,20 @@ function defaultHostAdapters(): Record { function defaultContainerAdapters(): Record ContainerAgentAdapter> { return { - claude: (agent) => new ClaudeCodeAdapter("claude", { model: agent.model, reasoning: agent.reasoning }), - codex: (agent) => new CodexAdapter("codex", { model: agent.model, reasoning: agent.reasoning }), + claude: (agent) => + new ClaudeCodeAdapter("claude", { + model: agent.model, + reasoning: agent.reasoning, + provider: agent.provider, + endpoint: agent.endpoint, + }), + codex: (agent) => + new CodexAdapter("codex", { + model: agent.model, + reasoning: agent.reasoning, + provider: agent.provider, + endpoint: agent.endpoint, + }), oracle: () => new OracleContainerAdapter(), }; } @@ -405,19 +427,24 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise(); + // unique endpoint once — before any docker, sidecar, or run-row work — + // and say plainly whether the run starts. "Unique" is the adapter plus + // the host variables it reads (keyProbeIdentity): two codex arms routed + // by different `agent.endpoint` variables are two probes, while arms + // sharing the vendor defaults stay one. Adapters without a probe (tests, + // custom adapters) skip it. + const probesByEndpoint = new Map(); for (const adapter of containerAgents.values()) { - if (adapter.keyProbe === undefined || probesByAdapter.has(adapter.name)) continue; - probesByAdapter.set(adapter.name, { + if (adapter.keyProbe === undefined) continue; + const identity = keyProbeIdentity(adapter); + if (probesByEndpoint.has(identity)) continue; + probesByEndpoint.set(identity, { name: adapter.name, envVars: adapter.requiredHostEnv, ...adapter.keyProbe({ ...collectProviderKeys([adapter]), ...collectOptionalHostEnv([adapter]) }), }); } - await validateProviderKeys([...probesByAdapter.values()], { log }); + await validateProviderKeys([...probesByEndpoint.values()], { log }); log(`run starting: ${plan.length} attempt(s)`); // Resolve image digests and agent CLI versions BEFORE building the manifest: diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 0f00f13..ba55794 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -2,9 +2,10 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { ClaudeCodeAdapter, CLAUDE_MCP_CONFIG_PATH } from "../src/adapters/claude.ts"; -import { CodexAdapter, CODEX_CONFIG_PATH, CODEX_GATEWAY_PROVIDER } from "../src/adapters/codex.ts"; +import { CodexAdapter, CODEX_CONFIG_PATH, CODEX_DEFAULT_ENDPOINT, CODEX_GATEWAY_PROVIDER } from "../src/adapters/codex.ts"; import { parseClaudeStream, parseCodexStream, parseJsonlLines } from "../src/adapters/parse.ts"; import type { ContainerAttemptContext } from "../src/adapters/types.ts"; +import { keyProbeIdentity } from "../src/runner/execute.ts"; const STREAMS_DIR = join(import.meta.dir, "__fixtures__", "streams"); @@ -253,3 +254,127 @@ describe("endpoint overrides (LLM gateway routing)", () => { expect(adapter.setupFiles({ ...ctxFor("none"), overrides: { ANTHROPIC_BASE_URL: base } })).toEqual({}); }); }); + +describe("per-experiment endpoint routing (agent.endpoint)", () => { + const endpoint = { + apiKeyEnv: "GATEWAY_API_TOKEN", + baseUrlEnv: "GATEWAY_BASE_URL", + headersEnv: "GATEWAY_HEADERS", + } as const; + const base = "https://gateway.example/v1/acct/ai/v1"; + + test("codex: the experiment's variables replace the vendor's for the host, the probe and the login shim", () => { + const adapter = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "high", provider: "acme", endpoint: { ...endpoint, wireApi: "chat" } }); + expect(adapter.endpoint).toEqual({ ...endpoint, wireApi: "chat" }); + expect(adapter.requiredHostEnv).toEqual(["GATEWAY_API_TOKEN"]); + expect(adapter.optionalHostEnv).toEqual(["GATEWAY_BASE_URL", "GATEWAY_HEADERS"]); + expect(adapter.identity.provider).toBe("acme"); + + // The probe reads the experiment's variables; the vendor's are ignored. + const probe = adapter.keyProbe({ + OPENAI_API_KEY: "vendor", + OPENAI_BASE_URL: "https://vendor.example", + GATEWAY_API_TOKEN: "gw", + GATEWAY_BASE_URL: base, + GATEWAY_HEADERS: "cf-aig-gateway-id: evals", + }); + expect(probe.url).toBe(`${base}/models`); + expect(probe.headers).toEqual({ "cf-aig-gateway-id": "evals", Authorization: "Bearer gw" }); + + // The shim pipes the experiment's key variable into login — by name, never value. + const command = adapter.command(ctxFor("none")); + expect(command[2]).toContain("printenv GATEWAY_API_TOKEN | codex login --with-api-key"); + expect(command[2]).not.toContain("OPENAI_API_KEY"); + }); + + test("codex: the custom provider renders the experiment's key variable and wire, from the overrides it named", () => { + const adapter = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "high", endpoint: { ...endpoint, wireApi: "chat" } }); + const files = adapter.setupFiles({ + ...ctxFor("none"), + overrides: { GATEWAY_BASE_URL: base, GATEWAY_HEADERS: "cf-aig-gateway-id: evals", OPENAI_BASE_URL: "https://ignored.example" }, + }); + const toml = files[CODEX_CONFIG_PATH] as string; + expect(toml).toContain(`base_url = ${JSON.stringify(base)}`); + expect(toml).toContain('env_key = "GATEWAY_API_TOKEN"'); + expect(toml).toContain('wire_api = "chat"'); + expect(toml).toContain('http_headers = { "cf-aig-gateway-id" = "evals" }'); + expect(toml).not.toContain("ignored.example"); + + // A custom key variable or wire alone still needs the provider: the built-in one cannot express either. + const keyOnly = new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "GATEWAY_API_TOKEN" } }); + const keyToml = keyOnly.setupFiles(ctxFor("none"))[CODEX_CONFIG_PATH] as string; + expect(keyToml).toContain('env_key = "GATEWAY_API_TOKEN"'); + expect(keyToml).toContain('base_url = "https://api.openai.com/v1"'); + expect(keyToml).toContain('wire_api = "responses"'); + const chatOnly = new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { wireApi: "chat" } }); + expect(chatOnly.setupFiles(ctxFor("none"))[CODEX_CONFIG_PATH]).toContain('wire_api = "chat"'); + + // Vendor defaults with nothing set render nothing, exactly as before. + expect(new CodexAdapter("codex", { model: "m", reasoning: "r" }).setupFiles(ctxFor("none"))).toEqual({}); + }); + + test("claude: custom variables are re-exported under the vendor names by a shim; the default argv is untouched", () => { + const plain = new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r" }); + expect(plain.command(ctxFor("none"))[0]).toBe("claude"); + + const adapter = new ClaudeCodeAdapter("claude", { model: "acme/coder-9", reasoning: "high", provider: "acme", endpoint }); + expect(adapter.requiredHostEnv).toEqual(["GATEWAY_API_TOKEN"]); + expect(adapter.optionalHostEnv).toEqual(["GATEWAY_BASE_URL", "GATEWAY_HEADERS"]); + expect(adapter.identity.provider).toBe("acme"); + expect(adapter.keyProbe({ GATEWAY_API_TOKEN: "gw", GATEWAY_BASE_URL: base }).url).toBe(`${base}/v1/models`); + expect(adapter.keyProbe({ GATEWAY_API_TOKEN: "gw" }).headers["x-api-key"]).toBe("gw"); + + const ctx = ctxFor("mcp"); + const command = adapter.command(ctx); + expect(command.slice(0, 2)).toEqual(["sh", "-c"]); + const shim = command[2] as string; + // The custom key goes out as both x-api-key and a bearer: gateways differ on which they read. + expect(shim).toContain('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"; export ANTHROPIC_AUTH_TOKEN="$GATEWAY_API_TOKEN"'); + expect(shim).toContain('[ -n "${GATEWAY_BASE_URL:-}" ] && export ANTHROPIC_BASE_URL="$GATEWAY_BASE_URL"'); + expect(shim).toContain('[ -n "${GATEWAY_HEADERS:-}" ] && export ANTHROPIC_CUSTOM_HEADERS="$GATEWAY_HEADERS"'); + expect(shim.endsWith('exec "$@"')).toBeTrue(); + // The argv after the shim is the plain invocation, prompt byte-identical. + expect(command.slice(3)).toEqual(["sh", ...plain.command(ctx).map((arg) => (arg === "m" ? "acme/coder-9" : arg === "r" ? "high" : arg))]); + + // Only the names that differ are re-exported. + const keyOnly = new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "GATEWAY_API_TOKEN" } }); + const keyShim = keyOnly.command(ctxFor("none"))[2] as string; + expect(keyShim).toBe('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"; export ANTHROPIC_AUTH_TOKEN="$GATEWAY_API_TOKEN"; exec "$@"'); + }); + + test("both adapters refuse an endpoint the harness cannot route safely", () => { + expect(() => new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "lower" } })).toThrow(/env var name/); + expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { wireApi: "chat" } })).toThrow(/codex adapter only/); + expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { baseUrlEnv: "GATEWAY_SECRET_URL" } })).toThrow(/must not be secret-named/); + }); + + test("a vendor variable cannot be borrowed for another role: the shim would read it, then overwrite it", () => { + // Swapped roles pass the name-shape checks (both secret-named) but would clobber each other. + expect( + () => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS", headersEnv: "ANTHROPIC_API_KEY" } }), + ).toThrow(/apiKeyEnv ANTHROPIC_CUSTOM_HEADERS is the claude adapter's headersEnv variable/); + // One borrowed name with the other role left at its default collides on resolution. + expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS" } })).toThrow( + /is the claude adapter's headersEnv variable/, + ); + expect(() => new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { headersEnv: "OPENAI_API_KEY" } })).toThrow( + /headersEnv OPENAI_API_KEY is the codex adapter's apiKeyEnv variable/, + ); + // The vendor name for its OWN role is simply the default, spelled out. + const explicit = new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "OPENAI_API_KEY" } }); + expect(explicit.endpoint).toEqual(CODEX_DEFAULT_ENDPOINT); + }); + + test("key probes are deduplicated per endpoint, not per adapter", () => { + const vendor = new CodexAdapter("codex", { model: "m", reasoning: "r" }); + const vendorAgain = new CodexAdapter("codex", { model: "m2", reasoning: "r" }); + const gateway = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "r", endpoint }); + const gatewayChat = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "r", endpoint: { ...endpoint, wireApi: "chat" } }); + // Same variables ⇒ one probe; the wire does not change what the probe reads. + expect(keyProbeIdentity(vendor)).toBe(keyProbeIdentity(vendorAgain)); + expect(keyProbeIdentity(gateway)).toBe(keyProbeIdentity(gatewayChat)); + // Different variables ⇒ a probe each, so a rejected gateway token still stops the run. + expect(keyProbeIdentity(vendor)).not.toBe(keyProbeIdentity(gateway)); + expect(keyProbeIdentity(new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r" }))).not.toBe(keyProbeIdentity(vendor)); + }); +}); diff --git a/tests/experiments-v3.test.ts b/tests/experiments-v3.test.ts index 2b020b4..c20b2fd 100644 --- a/tests/experiments-v3.test.ts +++ b/tests/experiments-v3.test.ts @@ -97,6 +97,39 @@ describe("defineExperiment", () => { ).toThrow(/comparisonGroup/); }); + test("agent.endpoint names host variables by role: secret-named key and headers, plain base URL, wireApi for codex only", () => { + const make = (adapter: string, endpoint: Record) => + defineExperiment({ + id: "x", + agent: { adapter, model: "m", reasoning: "r", endpoint }, + runtime: { kind: "container", image: "img:1" }, + }); + const ok = make("codex", { + apiKeyEnv: "GATEWAY_API_TOKEN", + baseUrlEnv: "GATEWAY_BASE_URL", + headersEnv: "GATEWAY_HEADERS", + wireApi: "chat", + }); + expect(ok.agent.endpoint).toEqual({ + apiKeyEnv: "GATEWAY_API_TOKEN", + baseUrlEnv: "GATEWAY_BASE_URL", + headersEnv: "GATEWAY_HEADERS", + wireApi: "chat", + }); + expect(Object.isFrozen(ok.agent.endpoint)).toBeTrue(); + // Partial specs keep the vendor defaults for the rest. + expect(make("claude", { baseUrlEnv: "GATEWAY_BASE_URL" }).agent.endpoint).toEqual({ baseUrlEnv: "GATEWAY_BASE_URL" }); + + expect(() => make("codex", { apiKeyEnv: "gateway-token" })).toThrow(/apiKeyEnv must be an env var name/); + expect(() => make("codex", { apiKeyEnv: "GATEWAY_AUTH" })).toThrow(/apiKeyEnv GATEWAY_AUTH must be secret-named/); + expect(() => make("codex", { headersEnv: "GATEWAY_EXTRA" })).toThrow(/headersEnv GATEWAY_EXTRA must be secret-named/); + expect(() => make("codex", { baseUrlEnv: "GATEWAY_TOKEN_URL" })).toThrow(/baseUrlEnv GATEWAY_TOKEN_URL must not be secret-named/); + expect(() => make("codex", { apiKeyEnv: "SAME_TOKEN", headersEnv: "SAME_TOKEN" })).toThrow(/both name SAME_TOKEN/); + expect(() => make("codex", { wireApi: "grpc" })).toThrow(/wireApi must be one of responses, chat/); + expect(() => make("claude", { wireApi: "chat" })).toThrow(/wireApi applies to the codex adapter only/); + expect(() => make("codex", { apiKeyEnv: 'X"; rm -rf /; echo "' })).toThrow(/must be an env var name/); + }); + test('runtime.egress "open" is refused on a host runtime (never proxied) and accepted on a container', () => { expect(() => defineExperiment({ id: "open-host", agent: { adapter: "echo" }, runtime: { kind: "host", egress: "open" } }), @@ -134,6 +167,38 @@ describe("comparison groups (withhold-and-diff)", () => { expect(groups[0]?.details).toEqual([]); }); + test("a different agent.endpoint within a group is withheld: the manifest records it and the diff names the field", () => { + const endpoint = { apiKeyEnv: "GATEWAY_API_TOKEN", baseUrlEnv: "GATEWAY_BASE_URL", wireApi: "chat" as const }; + const a = loadedExperiment({ + id: "g-a", + comparisonGroup: "g", + treatment: "docs", + agent: { ...agent, endpoint }, + runtime: { kind: "container", image: "img:1", command: ["true"], config: { docs: {} } }, + }); + const b = loadedExperiment({ + id: "g-b", + comparisonGroup: "g", + treatment: "mcp", + agent: { ...agent, endpoint: { ...endpoint, wireApi: "responses" } }, + runtime: { kind: "container", image: "img:1", command: ["true"], config: { mcp: {} } }, + }); + const manifest = manifestFor([a, b]); + // Recorded verbatim, in a fixed key order; an experiment without one records nothing new. + expect(manifest.experiments["g-a"]?.components.agent.endpoint).toEqual(endpoint); + expect(manifest.experiments["g-b"]?.components.agent.endpoint).toEqual({ ...endpoint, wireApi: "responses" }); + const plain = loadedExperiment({ id: "g-c", agent, runtime: { kind: "container", image: "img:1", command: ["true"] } }); + expect(manifestFor([plain]).experiments["g-c"]?.components.agent).not.toHaveProperty("endpoint"); + + const [group] = compareGroups([a.experiment, b.experiment], manifest); + expect(group?.comparison).toBe("withheld"); + expect(group?.mismatches.map((mismatch) => mismatch.path)).toEqual(["identity.agent.endpoint.wireApi"]); + + // Same endpoint on both arms compares like any other agreed component. + const same = loadedExperiment({ ...b.experiment, agent: { ...agent, endpoint } }); + expect(compareGroups([a.experiment, same.experiment], manifestFor([a, same]))[0]?.comparison).toBe("allowed"); + }); + test("a mismatched model within a group is withheld and the diff names the component", () => { const a = loadedExperiment({ id: "g-a",