From 3f2008e144b915fa6e4ab1e0f39e300b3048e1bc Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Tue, 10 Nov 2026 12:38:23 -0800 Subject: [PATCH 1/3] feat(adapters): per-experiment endpoint routing An adapter's endpoint overrides are per sweep: every codex arm reads the same OPENAI_BASE_URL, so two codex arms could not reach two endpoints in one run. A benchmark that drives OpenAI's model and third-party models through a gateway's unified API needs exactly that, a different base URL and token than the provider-native route, per experiment, without the adapter growing a vendor table. `agent.endpoint` names the host variables an experiment's key, base URL and headers come from; unset fields keep the vendor defaults. Naming variables rather than values keeps every existing guarantee: the key variable must be secret-named so it is redacted, the base URL variable must not be so its value stays identity and joins the egress allowlist, and the key probe follows the experiment's variables so a bad gateway token still fails before any container. Codex renders the names into its custom provider; Claude Code only reads the vendor names, so a shell shim re-exports custom ones under them in the container, and the default argv is untouched. --- README.md | 41 ++++++++++++++ src/adapters/claude.ts | 77 ++++++++++++++++++++++----- src/adapters/codex.ts | 100 ++++++++++++++++++++++++++--------- src/experiments/endpoint.ts | 93 ++++++++++++++++++++++++++++++++ src/experiments/types.ts | 38 +++++++++++++ src/index.ts | 9 +++- src/runner/execute.ts | 16 +++++- tests/adapters.test.ts | 93 ++++++++++++++++++++++++++++++++ tests/experiments-v3.test.ts | 33 ++++++++++++ 9 files changed, 459 insertions(+), 41 deletions(-) create mode 100644 src/experiments/endpoint.ts diff --git a/README.md b/README.md index 5df561f..2afd48f 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,47 @@ export OPENAI_CUSTOM_HEADERS='cf-aig-metadata: {"app":"my-evals"}' Leave every override unset and both agents run vendor-direct against api.anthropic.com and api.openai.com — nothing is rendered, nothing changes. +#### Per-experiment routing (`agent.endpoint`) + +The variables above are per adapter: every codex arm in a run reads the +same `OPENAI_BASE_URL`. To route ONE experiment elsewhere — a second +gateway, or a gateway's unified OpenAI-compatible API serving a third-party +model — the experiment names the host variables that carry its key, base +URL and headers, and the host sets them next to the vendor ones: + +```ts +export default defineExperiment({ + id: "acme-gateway-mcp", + comparisonGroup: "acme-gateway", + treatment: "mcp", + agent: { + adapter: "codex", + provider: "acme", // recorded as-is in the manifest and report + model: "acme/coder-9", // whatever the endpoint expects in `model` + reasoning: "high", + endpoint: { + apiKeyEnv: "GATEWAY_API_TOKEN", // secret-named: redacted like a key + baseUrlEnv: "GATEWAY_BASE_URL", // not secret-named: identity + egress + headersEnv: "GATEWAY_HEADERS", // secret-named; `Name: Value` lines + wireApi: "chat", // codex only: chat completions + }, + }, + runtime: ..., +}); +``` + +Unset fields keep the vendor defaults. The names must be POSIX env var +names; `defineExperiment` refuses a key or headers variable that is not +secret-named, a base URL variable that is, and `wireApi` on anything but +codex. The forwarded values travel exactly as before: the key as a provider +key, the base URL into the pair environment and the egress allowlist, the +headers redacted from artifacts. Codex renders them into its custom model +provider (`env_key`, `base_url`, `http_headers`, `wire_api`); Claude Code +only reads the vendor names, so the adapter re-exports custom ones under +them inside the container before exec. The preflight key probe follows the +experiment's variables, so a bad gateway token still fails before any +container starts. + ## How it works Every attempt is one point in a 2-D plan: diff --git a/src/adapters/claude.ts b/src/adapters/claude.ts index b0121c5..0469ccd 100644 --- a/src/adapters/claude.ts +++ b/src/adapters/claude.ts @@ -13,6 +13,8 @@ * across experiments. */ +import { type ResolvedEndpoint, resolveEndpoint } from "../experiments/endpoint.ts"; +import type { AgentEndpointSpec } from "../experiments/types.ts"; import type { McpServerConfig } from "../isolation/mcp.ts"; import { endpointUrl, parseHeaderLines } from "../secrets.ts"; import { parseClaudeStream } from "./parse.ts"; @@ -21,6 +23,18 @@ import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } f /** Container path of the MCP config Claude Code auto-discovers. */ export const CLAUDE_MCP_CONFIG_PATH = "/workspace/.mcp.json"; +/** + * The vendor's env names: what every claude experiment gets unless its + * `agent.endpoint` names something else (experiments/endpoint.ts). The wire + * is fixed — Claude Code speaks the Messages API and nothing else. + */ +export const CLAUDE_DEFAULT_ENDPOINT: ResolvedEndpoint = { + apiKeyEnv: "ANTHROPIC_API_KEY", + baseUrlEnv: "ANTHROPIC_BASE_URL", + headersEnv: "ANTHROPIC_CUSTOM_HEADERS", + wireApi: "responses", +}; + /** * Project-scope `.mcp.json` rendered from the runtime's MCP servers * (streamable HTTP). The configs arrive credential-free — authed servers @@ -38,17 +52,26 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { readonly name: string; readonly identity; readonly versionCommand = ["claude", "--version"] as const; - readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const; /** - * Endpoint overrides, forwarded only when set. ANTHROPIC_BASE_URL routes + * The env names this experiment's traffic is routed by: the vendor's + * (ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL / ANTHROPIC_CUSTOM_HEADERS) + * unless `agent.endpoint` named others. Claude Code only ever reads the + * vendor names, so custom names are re-exported under them by a shell shim + * in `command` — inside the container, from env the harness already placed. + */ + readonly endpoint: ResolvedEndpoint; + /** The key variable: required on the host, forwarded into the container. */ + readonly requiredHostEnv: readonly string[]; + /** + * Endpoint overrides, forwarded only when set. The base URL variable routes * the CLI (and the key probe) through an Anthropic-compatible gateway — * e.g. Cloudflare AI Gateway's `/anthropic` endpoint, where the gateway - * token doubles as ANTHROPIC_API_KEY; ANTHROPIC_CUSTOM_HEADERS (`Name: - * Value` lines) carries whatever else that gateway wants, such as + * token doubles as the API key; the headers variable (`Name: Value` + * lines) carries whatever else that gateway wants, such as * `cf-aig-authorization` or `cf-aig-metadata`. Claude Code reads both - * natively, so no flag changes. + * natively under the vendor names, so no flag changes. */ - readonly optionalHostEnv = ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"] as const; + readonly optionalHostEnv: readonly string[]; /** * Egress-proxy allowlist (harness-default layer). Claude Code documents * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is @@ -59,10 +82,16 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { */ readonly egressHosts = ["api.anthropic.com", "statsig.anthropic.com", "*.sentry.io"] as const; - constructor(name = "claude", config: { model?: string; reasoning?: string } = {}) { + constructor( + name = "claude", + config: { model?: string; reasoning?: string; provider?: string; endpoint?: AgentEndpointSpec } = {}, + ) { this.name = name; + this.endpoint = resolveEndpoint(name, CLAUDE_DEFAULT_ENDPOINT, config.endpoint); + this.requiredHostEnv = [this.endpoint.apiKeyEnv]; + this.optionalHostEnv = [this.endpoint.baseUrlEnv, this.endpoint.headersEnv]; this.identity = { - provider: "anthropic", + provider: config.provider ?? "anthropic", model: config.model ?? "UNPINNED", reasoning: config.reasoning ?? "UNPINNED", cliVersion: "unknown", @@ -78,15 +107,34 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { keyProbe(env: Record): { url: string; headers: Record } { return { - url: endpointUrl(env["ANTHROPIC_BASE_URL"], "https://api.anthropic.com", "v1/models"), + url: endpointUrl(env[this.endpoint.baseUrlEnv], "https://api.anthropic.com", "v1/models"), headers: { - ...parseHeaderLines(env["ANTHROPIC_CUSTOM_HEADERS"]), - "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", + ...parseHeaderLines(env[this.endpoint.headersEnv]), + "x-api-key": env[this.endpoint.apiKeyEnv] ?? "", "anthropic-version": "2023-06-01", }, }; } + /** + * `export VENDOR="$CUSTOM"` lines for every endpoint variable whose name + * differs from the vendor's — empty when nothing differs, so the default + * invocation stays the bare `claude` argv. Optional variables are only + * re-exported when set: an empty ANTHROPIC_BASE_URL would be a base URL. + */ + private endpointExports(): string[] { + const lines: string[] = []; + const { apiKeyEnv, baseUrlEnv, headersEnv } = this.endpoint; + if (apiKeyEnv !== CLAUDE_DEFAULT_ENDPOINT.apiKeyEnv) lines.push(`export ANTHROPIC_API_KEY="$${apiKeyEnv}"`); + if (baseUrlEnv !== CLAUDE_DEFAULT_ENDPOINT.baseUrlEnv) { + lines.push(`[ -n "\${${baseUrlEnv}:-}" ] && export ANTHROPIC_BASE_URL="$${baseUrlEnv}"`); + } + if (headersEnv !== CLAUDE_DEFAULT_ENDPOINT.headersEnv) { + lines.push(`[ -n "\${${headersEnv}:-}" ] && export ANTHROPIC_CUSTOM_HEADERS="$${headersEnv}"`); + } + return lines; + } + setupFiles(ctx: ContainerAttemptContext): Record { if (ctx.mcpServers === undefined) return {}; return { [CLAUDE_MCP_CONFIG_PATH]: renderMcpJson(ctx.mcpServers) }; @@ -114,7 +162,12 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { args.push("--mcp-config", CLAUDE_MCP_CONFIG_PATH); } if (ctx.webPolicy !== "native-web-allowed") args.push("--disallowedTools", "WebSearch,WebFetch"); - return args; + const exports = this.endpointExports(); + if (exports.length === 0) return args; + // Custom endpoint names: re-export them under the names Claude Code reads, + // then exec the same argv. Names only reach the shell — resolveEndpoint + // admits POSIX env names and nothing else; values stay in the environment. + return ["sh", "-c", `${exports.join("; ")}; exec "$@"`, "sh", ...args]; } parseStream(raw: string): AttemptMetrics { diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index 7f9dbce..9c5b2d9 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -25,6 +25,8 @@ * talks to api.openai.com — vendor-direct stays the default. */ +import { type ResolvedEndpoint, type WireApi, resolveEndpoint } from "../experiments/endpoint.ts"; +import type { AgentEndpointSpec } from "../experiments/types.ts"; import type { McpServerConfig } from "../isolation/mcp.ts"; import { endpointUrl, parseHeaderLines } from "../secrets.ts"; import { parseCodexStream } from "./parse.ts"; @@ -40,13 +42,31 @@ export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway"; const OPENAI_DEFAULT_BASE_URL = "https://api.openai.com/v1"; /** - * A custom model provider pointing at `baseUrl` (the Responses API, which - * codex requires), carrying `headers` on every request, selected as the - * default. Top-level keys must precede any table, so this block always - * renders first in the file. JSON string escapes are valid TOML basic-string - * escapes, which keeps header values such as JSON metadata intact. + * The vendor's env names and wire: what every codex experiment gets unless + * its `agent.endpoint` names something else (experiments/endpoint.ts). */ -function renderCodexProviderToml(baseUrl: string, headers: Readonly>): string { +export const CODEX_DEFAULT_ENDPOINT: ResolvedEndpoint = { + apiKeyEnv: "OPENAI_API_KEY", + baseUrlEnv: "OPENAI_BASE_URL", + headersEnv: "OPENAI_CUSTOM_HEADERS", + wireApi: "responses", +}; + +/** + * A custom model provider pointing at `baseUrl`, speaking `wireApi` (the + * Responses API codex prefers, or chat completions for endpoints that only + * offer that), reading its key from `envKey`, carrying `headers` on every + * request, selected as the default. Top-level keys must precede any table, + * so this block always renders first in the file. JSON string escapes are + * valid TOML basic-string escapes, which keeps header values such as JSON + * metadata intact. + */ +function renderCodexProviderToml( + baseUrl: string, + headers: Readonly>, + envKey: string, + wireApi: WireApi, +): string { const entries = Object.entries(headers); return [ `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`, @@ -54,8 +74,8 @@ function renderCodexProviderToml(baseUrl: string, headers: Readonly 0 ? [`http_headers = { ${entries.map(([name, value]) => `${JSON.stringify(name)} = ${JSON.stringify(value)}`).join(", ")} }`] : []), @@ -81,17 +101,25 @@ export class CodexAdapter implements ContainerAgentAdapter { readonly name: string; readonly identity; readonly versionCommand = ["codex", "--version"] as const; - readonly requiredHostEnv = ["OPENAI_API_KEY"] as const; /** - * Endpoint overrides, forwarded only when set: OPENAI_BASE_URL (vendor - * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's - * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY — - * and OPENAI_CUSTOM_HEADERS (`Name: Value` lines) for whatever else the + * The env names this experiment's traffic is routed by: the vendor's + * (OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_CUSTOM_HEADERS) unless + * `agent.endpoint` named others — the way two codex arms in one run reach + * two gateways, or one gateway's unified API serving a third-party model. + */ + readonly endpoint: ResolvedEndpoint; + /** The key variable: required on the host, forwarded into the container. */ + readonly requiredHostEnv: readonly string[]; + /** + * Endpoint overrides, forwarded only when set: the base URL variable + * (vendor default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's + * `/openai` endpoint, where the gateway token doubles as the API key — + * and the headers variable (`Name: Value` lines) for whatever else the * endpoint wants on each request, such as `cf-aig-metadata`. Both are * rendered into config.toml by setupFiles (see the module comment); the * key probe follows them. */ - readonly optionalHostEnv = ["OPENAI_BASE_URL", "OPENAI_CUSTOM_HEADERS"] as const; + readonly optionalHostEnv: readonly string[]; /** * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based; * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) — @@ -103,10 +131,16 @@ export class CodexAdapter implements ContainerAgentAdapter { */ readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const; - constructor(name = "codex", config: { model?: string; reasoning?: string } = {}) { + constructor( + name = "codex", + config: { model?: string; reasoning?: string; provider?: string; endpoint?: AgentEndpointSpec } = {}, + ) { this.name = name; + this.endpoint = resolveEndpoint(name, CODEX_DEFAULT_ENDPOINT, config.endpoint); + this.requiredHostEnv = [this.endpoint.apiKeyEnv]; + this.optionalHostEnv = [this.endpoint.baseUrlEnv, this.endpoint.headersEnv]; this.identity = { - provider: "openai", + provider: config.provider ?? "openai", model: config.model ?? "UNPINNED", reasoning: config.reasoning ?? "UNPINNED", cliVersion: "unknown", @@ -122,18 +156,30 @@ export class CodexAdapter implements ContainerAgentAdapter { keyProbe(env: Record): { url: string; headers: Record } { return { - url: endpointUrl(env["OPENAI_BASE_URL"], OPENAI_DEFAULT_BASE_URL, "models"), - headers: { ...parseHeaderLines(env["OPENAI_CUSTOM_HEADERS"]), Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` }, + url: endpointUrl(env[this.endpoint.baseUrlEnv], OPENAI_DEFAULT_BASE_URL, "models"), + headers: { + ...parseHeaderLines(env[this.endpoint.headersEnv]), + Authorization: `Bearer ${env[this.endpoint.apiKeyEnv] ?? ""}`, + }, }; } setupFiles(ctx: ContainerAttemptContext): Record { const sections: string[] = []; - const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"]; - const headers = parseHeaderLines(ctx.overrides?.["OPENAI_CUSTOM_HEADERS"]); - // Either override needs the custom provider; headers alone keep the vendor endpoint. - if (baseUrl !== undefined || Object.keys(headers).length > 0) { - sections.push(renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers)); + const baseUrl = ctx.overrides?.[this.endpoint.baseUrlEnv]; + const headers = parseHeaderLines(ctx.overrides?.[this.endpoint.headersEnv]); + // Either override needs the custom provider (headers alone keep the + // vendor endpoint); so does a non-vendor key variable or wire, which the + // built-in provider cannot express. + const custom = + baseUrl !== undefined || + Object.keys(headers).length > 0 || + this.endpoint.apiKeyEnv !== CODEX_DEFAULT_ENDPOINT.apiKeyEnv || + this.endpoint.wireApi !== CODEX_DEFAULT_ENDPOINT.wireApi; + if (custom) { + sections.push( + renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers, this.endpoint.apiKeyEnv, this.endpoint.wireApi), + ); } if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers)); if (sections.length === 0) return {}; @@ -145,9 +191,11 @@ export class CodexAdapter implements ContainerAgentAdapter { "sh", "-c", // codex 0.144+ ignores the OPENAI_API_KEY env var; mint auth.json from - // it in-container. Login chatter is silenced so stdout stays pure JSONL - // for parseStream; the key reaches login via stdin, never argv. - 'printenv OPENAI_API_KEY | codex login --with-api-key >/dev/null 2>&1 && exec "$@"', + // the key variable in-container. Login chatter is silenced so stdout + // stays pure JSONL for parseStream; the key reaches login via stdin, + // never argv. The variable NAME is interpolated — resolveEndpoint + // admits only POSIX env names, so nothing else can reach the shell. + `printenv ${this.endpoint.apiKeyEnv} | codex login --with-api-key >/dev/null 2>&1 && exec "$@"`, "sh", "codex", "exec", diff --git a/src/experiments/endpoint.ts b/src/experiments/endpoint.ts new file mode 100644 index 0000000..2cebf04 --- /dev/null +++ b/src/experiments/endpoint.ts @@ -0,0 +1,93 @@ +/** + * `agent.endpoint` resolution: the experiment names the HOST ENV VARS that + * carry an agent's API key, base URL and extra headers; the adapter fills in + * its vendor defaults for whatever is unset and checks the names before any + * container starts. Names, never values, live here — the values stay on the + * host and ride the same channels the vendor defaults always have. + */ + +import { isSecretEnvName } from "../secrets.ts"; +import type { AgentEndpointSpec } from "./types.ts"; + +/** POSIX env var names the shell shims interpolate verbatim; nothing else is accepted. */ +const ENV_NAME = /^[A-Z][A-Z0-9_]*$/; + +export const WIRE_APIS = ["responses", "chat"] as const; +export type WireApi = (typeof WIRE_APIS)[number]; + +/** An adapter's endpoint with every field decided. */ +export interface ResolvedEndpoint { + apiKeyEnv: string; + baseUrlEnv: string; + headersEnv: string; + wireApi: WireApi; +} + +/** Thrown for an `agent.endpoint` the harness refuses; the message names the field. */ +export class InvalidAgentEndpointError extends Error { + constructor(detail: string) { + super(`agent.endpoint: ${detail}`); + this.name = "InvalidAgentEndpointError"; + } +} + +/** + * Check an experiment's `agent.endpoint` without an adapter: every name must + * be a POSIX env var name, the key and headers variables must be + * secret-named (so they are redacted from artifacts like a key), the base + * URL variable must not be (its value is identity and joins the egress + * allowlist), and the three must differ. `wireApi` is codex's alone. + */ +export function validateAgentEndpoint(spec: AgentEndpointSpec | undefined, adapter: string): void { + if (spec === undefined) return; + if (spec === null || typeof spec !== "object" || Array.isArray(spec)) { + throw new InvalidAgentEndpointError("must be an object"); + } + const names = new Map(); + for (const field of ["apiKeyEnv", "baseUrlEnv", "headersEnv"] as const) { + const value = spec[field]; + if (value === undefined) continue; + if (typeof value !== "string" || !ENV_NAME.test(value)) { + throw new InvalidAgentEndpointError( + `${field} must be an env var name matching ${ENV_NAME} (got ${JSON.stringify(value)})`, + ); + } + const clash = names.get(value); + if (clash !== undefined) throw new InvalidAgentEndpointError(`${field} and ${clash} both name ${value}`); + names.set(value, field); + } + if (spec.apiKeyEnv !== undefined && !isSecretEnvName(spec.apiKeyEnv)) { + throw new InvalidAgentEndpointError( + `apiKeyEnv ${spec.apiKeyEnv} must be secret-named (contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL) so its value is redacted`, + ); + } + if (spec.headersEnv !== undefined && !isSecretEnvName(spec.headersEnv)) { + throw new InvalidAgentEndpointError( + `headersEnv ${spec.headersEnv} must be secret-named (a HEADERS suffix will do) so its value is redacted`, + ); + } + if (spec.baseUrlEnv !== undefined && isSecretEnvName(spec.baseUrlEnv)) { + throw new InvalidAgentEndpointError( + `baseUrlEnv ${spec.baseUrlEnv} must not be secret-named: a base URL is identity and must survive redaction`, + ); + } + if (spec.wireApi !== undefined) { + if (!(WIRE_APIS as readonly string[]).includes(spec.wireApi)) { + throw new InvalidAgentEndpointError(`wireApi must be one of ${WIRE_APIS.join(", ")} (got ${JSON.stringify(spec.wireApi)})`); + } + if (adapter !== "codex") { + throw new InvalidAgentEndpointError(`wireApi applies to the codex adapter only (this experiment uses "${adapter}")`); + } + } +} + +/** Validate `spec` for `adapter` and fill the unset fields from `defaults`. */ +export function resolveEndpoint(adapter: string, defaults: ResolvedEndpoint, spec: AgentEndpointSpec = {}): ResolvedEndpoint { + validateAgentEndpoint(spec, adapter); + return { + apiKeyEnv: spec.apiKeyEnv ?? defaults.apiKeyEnv, + baseUrlEnv: spec.baseUrlEnv ?? defaults.baseUrlEnv, + headersEnv: spec.headersEnv ?? defaults.headersEnv, + wireApi: spec.wireApi ?? defaults.wireApi, + }; +} diff --git a/src/experiments/types.ts b/src/experiments/types.ts index 9110e04..3822319 100644 --- a/src/experiments/types.ts +++ b/src/experiments/types.ts @@ -8,6 +8,7 @@ import type { EvalMetadata } from "../evals/types.ts"; import { validateWorkspaceFilesRuntime, type WorkspaceFiles } from "../runtime/workspace-files.ts"; +import { validateAgentEndpoint } from "./endpoint.ts"; import type { SurfaceUsageConfig } from "../surface-usage.ts"; import type { RuntimeCapabilities } from "../runtime/types.ts"; @@ -17,6 +18,36 @@ export interface RuntimeCapacity { limit: number; } +/** + * Where an agent's LLM traffic goes, named by HOST ENV VARS rather than + * values: the experiment says which variables carry its API key, base URL + * and extra headers, and the host supplies them at run time. Every field is + * optional; unset ones keep the adapter's vendor defaults (`OPENAI_API_KEY`, + * `OPENAI_BASE_URL`, … for codex; `ANTHROPIC_*` for claude). + * + * This is what lets two experiments of one adapter reach different + * endpoints in the same run — one vendor-direct, one through a gateway's + * unified API serving a third-party model — without the adapter growing a + * vendor table. The values stay secrets or identity exactly as before: the + * key variable must be secret-named (`isSecretEnvName`), the base-URL + * variable must not be (its value joins the pair environment and the + * egress allowlist), and the headers variable is redacted like a key. + */ +export interface AgentEndpointSpec { + /** Host env var holding the API key the CLI authenticates with. Secret-named. */ + apiKeyEnv?: string; + /** Host env var holding the base URL override. Non-secret name; forwarded only when set. */ + baseUrlEnv?: string; + /** Host env var holding `Name: Value` header lines. Secret-named; forwarded only when set. */ + headersEnv?: string; + /** + * codex only: the custom provider's wire protocol. `responses` (default) + * is codex's native Responses API; `chat` is OpenAI chat completions, the + * shape most OpenAI-compatible gateways and third-party models accept. + */ + wireApi?: "responses" | "chat"; +} + /** Which agent executes the attempt, plus the identity fields recorded in the manifest. */ export interface AgentSpec { /** Adapter name, e.g. "echo" (the built-in no-op pipeline smoke agent). */ @@ -25,6 +56,8 @@ export interface AgentSpec { model?: string; reasoning?: string; cliVersion?: string; + /** Endpoint routing by host env var name; see AgentEndpointSpec. Hashed into identity like every other field. */ + endpoint?: AgentEndpointSpec; } /** @@ -142,5 +175,10 @@ export function defineExperiment(experiment: Experiment): Experiment { ); } validateWorkspaceFilesRuntime(experiment.runtime); + try { + validateAgentEndpoint(experiment.agent.endpoint, experiment.agent.adapter); + } catch (err) { + throw new Error(`defineExperiment: experiment "${experiment.id}" ${err instanceof Error ? err.message : String(err)}`); + } return deepFreeze(experiment); } diff --git a/src/index.ts b/src/index.ts index 1022d55..ab1e6a9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,14 @@ export { defineWorkspaceFiles, WORKSPACE_FILES_LIMITS } from "./runtime/workspac export type { WorkspaceFile, WorkspaceFiles, WorkspaceFilesIdentity } from "./runtime/workspace-files.ts"; export type { WorkspaceFilesEvidence } from "./isolation/workspace-files.ts"; export { defineExperiment } from "./experiments/types.ts"; -export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts"; +export type { + AgentEndpointSpec, + AgentSpec, + Experiment, + LoadedExperiment, + Runtime, + RuntimeCapacity, +} from "./experiments/types.ts"; export type { ProvisionedEnvironment, RuntimeCapabilities, diff --git a/src/runner/execute.ts b/src/runner/execute.ts index ced770b..8939ae1 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -180,8 +180,20 @@ function defaultHostAdapters(): Record { function defaultContainerAdapters(): Record ContainerAgentAdapter> { return { - claude: (agent) => new ClaudeCodeAdapter("claude", { model: agent.model, reasoning: agent.reasoning }), - codex: (agent) => new CodexAdapter("codex", { model: agent.model, reasoning: agent.reasoning }), + claude: (agent) => + new ClaudeCodeAdapter("claude", { + model: agent.model, + reasoning: agent.reasoning, + provider: agent.provider, + endpoint: agent.endpoint, + }), + codex: (agent) => + new CodexAdapter("codex", { + model: agent.model, + reasoning: agent.reasoning, + provider: agent.provider, + endpoint: agent.endpoint, + }), oracle: () => new OracleContainerAdapter(), }; } diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 0f00f13..bfee6a4 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -253,3 +253,96 @@ describe("endpoint overrides (LLM gateway routing)", () => { expect(adapter.setupFiles({ ...ctxFor("none"), overrides: { ANTHROPIC_BASE_URL: base } })).toEqual({}); }); }); + +describe("per-experiment endpoint routing (agent.endpoint)", () => { + const endpoint = { + apiKeyEnv: "GATEWAY_API_TOKEN", + baseUrlEnv: "GATEWAY_BASE_URL", + headersEnv: "GATEWAY_HEADERS", + } as const; + const base = "https://gateway.example/v1/acct/ai/v1"; + + test("codex: the experiment's variables replace the vendor's for the host, the probe and the login shim", () => { + const adapter = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "high", provider: "acme", endpoint: { ...endpoint, wireApi: "chat" } }); + expect(adapter.endpoint).toEqual({ ...endpoint, wireApi: "chat" }); + expect(adapter.requiredHostEnv).toEqual(["GATEWAY_API_TOKEN"]); + expect(adapter.optionalHostEnv).toEqual(["GATEWAY_BASE_URL", "GATEWAY_HEADERS"]); + expect(adapter.identity.provider).toBe("acme"); + + // The probe reads the experiment's variables; the vendor's are ignored. + const probe = adapter.keyProbe({ + OPENAI_API_KEY: "vendor", + OPENAI_BASE_URL: "https://vendor.example", + GATEWAY_API_TOKEN: "gw", + GATEWAY_BASE_URL: base, + GATEWAY_HEADERS: "cf-aig-gateway-id: evals", + }); + expect(probe.url).toBe(`${base}/models`); + expect(probe.headers).toEqual({ "cf-aig-gateway-id": "evals", Authorization: "Bearer gw" }); + + // The shim pipes the experiment's key variable into login — by name, never value. + const command = adapter.command(ctxFor("none")); + expect(command[2]).toContain("printenv GATEWAY_API_TOKEN | codex login --with-api-key"); + expect(command[2]).not.toContain("OPENAI_API_KEY"); + }); + + test("codex: the custom provider renders the experiment's key variable and wire, from the overrides it named", () => { + const adapter = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "high", endpoint: { ...endpoint, wireApi: "chat" } }); + const files = adapter.setupFiles({ + ...ctxFor("none"), + overrides: { GATEWAY_BASE_URL: base, GATEWAY_HEADERS: "cf-aig-gateway-id: evals", OPENAI_BASE_URL: "https://ignored.example" }, + }); + const toml = files[CODEX_CONFIG_PATH] as string; + expect(toml).toContain(`base_url = ${JSON.stringify(base)}`); + expect(toml).toContain('env_key = "GATEWAY_API_TOKEN"'); + expect(toml).toContain('wire_api = "chat"'); + expect(toml).toContain('http_headers = { "cf-aig-gateway-id" = "evals" }'); + expect(toml).not.toContain("ignored.example"); + + // A custom key variable or wire alone still needs the provider: the built-in one cannot express either. + const keyOnly = new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "GATEWAY_API_TOKEN" } }); + const keyToml = keyOnly.setupFiles(ctxFor("none"))[CODEX_CONFIG_PATH] as string; + expect(keyToml).toContain('env_key = "GATEWAY_API_TOKEN"'); + expect(keyToml).toContain('base_url = "https://api.openai.com/v1"'); + expect(keyToml).toContain('wire_api = "responses"'); + const chatOnly = new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { wireApi: "chat" } }); + expect(chatOnly.setupFiles(ctxFor("none"))[CODEX_CONFIG_PATH]).toContain('wire_api = "chat"'); + + // Vendor defaults with nothing set render nothing, exactly as before. + expect(new CodexAdapter("codex", { model: "m", reasoning: "r" }).setupFiles(ctxFor("none"))).toEqual({}); + }); + + test("claude: custom variables are re-exported under the vendor names by a shim; the default argv is untouched", () => { + const plain = new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r" }); + expect(plain.command(ctxFor("none"))[0]).toBe("claude"); + + const adapter = new ClaudeCodeAdapter("claude", { model: "acme/coder-9", reasoning: "high", provider: "acme", endpoint }); + expect(adapter.requiredHostEnv).toEqual(["GATEWAY_API_TOKEN"]); + expect(adapter.optionalHostEnv).toEqual(["GATEWAY_BASE_URL", "GATEWAY_HEADERS"]); + expect(adapter.identity.provider).toBe("acme"); + expect(adapter.keyProbe({ GATEWAY_API_TOKEN: "gw", GATEWAY_BASE_URL: base }).url).toBe(`${base}/v1/models`); + expect(adapter.keyProbe({ GATEWAY_API_TOKEN: "gw" }).headers["x-api-key"]).toBe("gw"); + + const ctx = ctxFor("mcp"); + const command = adapter.command(ctx); + expect(command.slice(0, 2)).toEqual(["sh", "-c"]); + const shim = command[2] as string; + expect(shim).toContain('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"'); + expect(shim).toContain('[ -n "${GATEWAY_BASE_URL:-}" ] && export ANTHROPIC_BASE_URL="$GATEWAY_BASE_URL"'); + expect(shim).toContain('[ -n "${GATEWAY_HEADERS:-}" ] && export ANTHROPIC_CUSTOM_HEADERS="$GATEWAY_HEADERS"'); + expect(shim.endsWith('exec "$@"')).toBeTrue(); + // The argv after the shim is the plain invocation, prompt byte-identical. + expect(command.slice(3)).toEqual(["sh", ...plain.command(ctx).map((arg) => (arg === "m" ? "acme/coder-9" : arg === "r" ? "high" : arg))]); + + // Only the names that differ are re-exported. + const keyOnly = new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "GATEWAY_API_TOKEN" } }); + const keyShim = keyOnly.command(ctxFor("none"))[2] as string; + expect(keyShim).toBe('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"; exec "$@"'); + }); + + test("both adapters refuse an endpoint the harness cannot route safely", () => { + expect(() => new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "lower" } })).toThrow(/env var name/); + expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { wireApi: "chat" } })).toThrow(/codex adapter only/); + expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { baseUrlEnv: "GATEWAY_SECRET_URL" } })).toThrow(/must not be secret-named/); + }); +}); diff --git a/tests/experiments-v3.test.ts b/tests/experiments-v3.test.ts index 2b020b4..bf0b913 100644 --- a/tests/experiments-v3.test.ts +++ b/tests/experiments-v3.test.ts @@ -97,6 +97,39 @@ describe("defineExperiment", () => { ).toThrow(/comparisonGroup/); }); + test("agent.endpoint names host variables by role: secret-named key and headers, plain base URL, wireApi for codex only", () => { + const make = (adapter: string, endpoint: Record) => + defineExperiment({ + id: "x", + agent: { adapter, model: "m", reasoning: "r", endpoint }, + runtime: { kind: "container", image: "img:1" }, + }); + const ok = make("codex", { + apiKeyEnv: "GATEWAY_API_TOKEN", + baseUrlEnv: "GATEWAY_BASE_URL", + headersEnv: "GATEWAY_HEADERS", + wireApi: "chat", + }); + expect(ok.agent.endpoint).toEqual({ + apiKeyEnv: "GATEWAY_API_TOKEN", + baseUrlEnv: "GATEWAY_BASE_URL", + headersEnv: "GATEWAY_HEADERS", + wireApi: "chat", + }); + expect(Object.isFrozen(ok.agent.endpoint)).toBeTrue(); + // Partial specs keep the vendor defaults for the rest. + expect(make("claude", { baseUrlEnv: "GATEWAY_BASE_URL" }).agent.endpoint).toEqual({ baseUrlEnv: "GATEWAY_BASE_URL" }); + + expect(() => make("codex", { apiKeyEnv: "gateway-token" })).toThrow(/apiKeyEnv must be an env var name/); + expect(() => make("codex", { apiKeyEnv: "GATEWAY_AUTH" })).toThrow(/apiKeyEnv GATEWAY_AUTH must be secret-named/); + expect(() => make("codex", { headersEnv: "GATEWAY_EXTRA" })).toThrow(/headersEnv GATEWAY_EXTRA must be secret-named/); + expect(() => make("codex", { baseUrlEnv: "GATEWAY_TOKEN_URL" })).toThrow(/baseUrlEnv GATEWAY_TOKEN_URL must not be secret-named/); + expect(() => make("codex", { apiKeyEnv: "SAME_TOKEN", headersEnv: "SAME_TOKEN" })).toThrow(/both name SAME_TOKEN/); + expect(() => make("codex", { wireApi: "grpc" })).toThrow(/wireApi must be one of responses, chat/); + expect(() => make("claude", { wireApi: "chat" })).toThrow(/wireApi applies to the codex adapter only/); + expect(() => make("codex", { apiKeyEnv: 'X"; rm -rf /; echo "' })).toThrow(/must be an env var name/); + }); + test('runtime.egress "open" is refused on a host runtime (never proxied) and accepted on a container', () => { expect(() => defineExperiment({ id: "open-host", agent: { adapter: "echo" }, runtime: { kind: "host", egress: "open" } }), From 268f7600e632065bd559c4b95ba27ac3082a58aa Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Tue, 10 Nov 2026 12:45:39 -0800 Subject: [PATCH 2/3] feat(claude): send a custom key as bearer too Gateways differ on which credential header they read. A provider-native Anthropic route takes `x-api-key`, which Claude Code sends from ANTHROPIC_API_KEY; a gateway's unified API takes `Authorization: Bearer`, which Claude Code only sends from ANTHROPIC_AUTH_TOKEN. The shim that re-exports a custom key variable now sets both, so one experiment works against either route without a second knob. The vendor route still sees only the vendor variable. --- README.md | 9 ++++++--- src/adapters/claude.ts | 11 ++++++++++- tests/adapters.test.ts | 5 +++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2afd48f..ceef2e9 100644 --- a/README.md +++ b/README.md @@ -186,9 +186,12 @@ key, the base URL into the pair environment and the egress allowlist, the headers redacted from artifacts. Codex renders them into its custom model provider (`env_key`, `base_url`, `http_headers`, `wire_api`); Claude Code only reads the vendor names, so the adapter re-exports custom ones under -them inside the container before exec. The preflight key probe follows the -experiment's variables, so a bad gateway token still fails before any -container starts. +them inside the container before exec — a custom key as both +`ANTHROPIC_API_KEY` (`x-api-key`) and `ANTHROPIC_AUTH_TOKEN` +(`Authorization: Bearer`), since a provider-native Anthropic route reads +the former and a gateway's unified API the latter. The preflight key probe +follows the experiment's variables, so a bad gateway token still fails +before any container starts. ## How it works diff --git a/src/adapters/claude.ts b/src/adapters/claude.ts index 0469ccd..3111476 100644 --- a/src/adapters/claude.ts +++ b/src/adapters/claude.ts @@ -121,11 +121,20 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { * differs from the vendor's — empty when nothing differs, so the default * invocation stays the bare `claude` argv. Optional variables are only * re-exported when set: an empty ANTHROPIC_BASE_URL would be a base URL. + * + * A custom key variable is presented both ways Claude Code can send a + * credential — `x-api-key` (ANTHROPIC_API_KEY) and `Authorization: + * Bearer` (ANTHROPIC_AUTH_TOKEN) — because gateways differ on which one + * they read: a provider-native Anthropic route takes the key header, a + * unified API takes the bearer. The vendor's own route only ever sees + * the vendor variable, so this changes nothing for it. */ private endpointExports(): string[] { const lines: string[] = []; const { apiKeyEnv, baseUrlEnv, headersEnv } = this.endpoint; - if (apiKeyEnv !== CLAUDE_DEFAULT_ENDPOINT.apiKeyEnv) lines.push(`export ANTHROPIC_API_KEY="$${apiKeyEnv}"`); + if (apiKeyEnv !== CLAUDE_DEFAULT_ENDPOINT.apiKeyEnv) { + lines.push(`export ANTHROPIC_API_KEY="$${apiKeyEnv}"`, `export ANTHROPIC_AUTH_TOKEN="$${apiKeyEnv}"`); + } if (baseUrlEnv !== CLAUDE_DEFAULT_ENDPOINT.baseUrlEnv) { lines.push(`[ -n "\${${baseUrlEnv}:-}" ] && export ANTHROPIC_BASE_URL="$${baseUrlEnv}"`); } diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index bfee6a4..2cdaea5 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -327,7 +327,8 @@ describe("per-experiment endpoint routing (agent.endpoint)", () => { const command = adapter.command(ctx); expect(command.slice(0, 2)).toEqual(["sh", "-c"]); const shim = command[2] as string; - expect(shim).toContain('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"'); + // The custom key goes out as both x-api-key and a bearer: gateways differ on which they read. + expect(shim).toContain('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"; export ANTHROPIC_AUTH_TOKEN="$GATEWAY_API_TOKEN"'); expect(shim).toContain('[ -n "${GATEWAY_BASE_URL:-}" ] && export ANTHROPIC_BASE_URL="$GATEWAY_BASE_URL"'); expect(shim).toContain('[ -n "${GATEWAY_HEADERS:-}" ] && export ANTHROPIC_CUSTOM_HEADERS="$GATEWAY_HEADERS"'); expect(shim.endsWith('exec "$@"')).toBeTrue(); @@ -337,7 +338,7 @@ describe("per-experiment endpoint routing (agent.endpoint)", () => { // Only the names that differ are re-exported. const keyOnly = new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "GATEWAY_API_TOKEN" } }); const keyShim = keyOnly.command(ctxFor("none"))[2] as string; - expect(keyShim).toBe('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"; exec "$@"'); + expect(keyShim).toBe('export ANTHROPIC_API_KEY="$GATEWAY_API_TOKEN"; export ANTHROPIC_AUTH_TOKEN="$GATEWAY_API_TOKEN"; exec "$@"'); }); test("both adapters refuse an endpoint the harness cannot route safely", () => { From ec9157f214781d248967bf9db53820958bbf7624 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Tue, 10 Nov 2026 12:51:34 -0800 Subject: [PATCH 3/3] fix(endpoint): identity, probes and roles follow the route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps the review found in per-experiment endpoint routing: The manifest recorded only adapter, provider, model, reasoning and CLI version, and comparison groups diff those fields, so two arms that differed only in wire or in the variables they read compared as one configuration. The declared endpoint is now an agent identity component and part of the group diff; an experiment without one records nothing new and keeps its prior identity. Key probes were deduplicated by adapter name, so of two codex arms on different gateways only the first token was checked and a rejected second token started containers. Probes now deduplicate by adapter plus the host variables it reads. A declared name could be another role's vendor variable — a key read from ANTHROPIC_CUSTOM_HEADERS and the headers from ANTHROPIC_API_KEY — which the Claude shim would read and then overwrite. Such names are refused, as are two roles resolving to one name. --- src/experiments/endpoint.ts | 49 ++++++++++++++++++++++++++++++++++-- src/experiments/groups.ts | 3 ++- src/manifest.ts | 17 ++++++++++++- src/runner/execute.ts | 29 +++++++++++++++------ tests/adapters.test.ts | 33 +++++++++++++++++++++++- tests/experiments-v3.test.ts | 32 +++++++++++++++++++++++ 6 files changed, 151 insertions(+), 12 deletions(-) diff --git a/src/experiments/endpoint.ts b/src/experiments/endpoint.ts index 2cebf04..945e972 100644 --- a/src/experiments/endpoint.ts +++ b/src/experiments/endpoint.ts @@ -81,13 +81,58 @@ export function validateAgentEndpoint(spec: AgentEndpointSpec | undefined, adapt } } -/** Validate `spec` for `adapter` and fill the unset fields from `defaults`. */ +const NAME_FIELDS = ["apiKeyEnv", "baseUrlEnv", "headersEnv"] as const; + +/** + * Validate `spec` for `adapter` and fill the unset fields from `defaults`. + * A declared name may not be one of the adapter's vendor variables for a + * different role: `apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS"` would make the + * shim read the header bundle as the key and then overwrite it, and the + * three resolved names must stay distinct for the same reason. + */ export function resolveEndpoint(adapter: string, defaults: ResolvedEndpoint, spec: AgentEndpointSpec = {}): ResolvedEndpoint { validateAgentEndpoint(spec, adapter); - return { + for (const field of NAME_FIELDS) { + const value = spec[field]; + if (value === undefined) continue; + for (const role of NAME_FIELDS) { + if (role !== field && value === defaults[role]) { + throw new InvalidAgentEndpointError( + `${field} ${value} is the ${adapter} adapter's ${role} variable; a name cannot serve two roles`, + ); + } + } + } + const resolved: ResolvedEndpoint = { apiKeyEnv: spec.apiKeyEnv ?? defaults.apiKeyEnv, baseUrlEnv: spec.baseUrlEnv ?? defaults.baseUrlEnv, headersEnv: spec.headersEnv ?? defaults.headersEnv, wireApi: spec.wireApi ?? defaults.wireApi, }; + const seen = new Map(); + for (const field of NAME_FIELDS) { + const clash = seen.get(resolved[field]); + if (clash !== undefined) { + throw new InvalidAgentEndpointError(`${field} and ${clash} both resolve to ${resolved[field]}`); + } + seen.set(resolved[field], field); + } + return resolved; +} + +/** + * The endpoint as identity: the declared fields, verbatim, in a fixed key + * order — what the run manifest records and comparison groups diff. Two + * arms that differ only here are two configurations, never one treatment. + * `undefined` when nothing is declared, so an experiment without an + * endpoint keeps the identity it had before the field existed. + */ +export function endpointIdentity(spec: AgentEndpointSpec | undefined): Record | undefined { + if (spec === undefined) return undefined; + const identity: Record = {}; + for (const field of [...NAME_FIELDS, "wireApi"] as const) { + const value = spec[field]; + if (value !== undefined) identity[field] = value; + } + return Object.keys(identity).length === 0 ? undefined : identity; } diff --git a/src/experiments/groups.ts b/src/experiments/groups.ts index af7c471..14ea952 100644 --- a/src/experiments/groups.ts +++ b/src/experiments/groups.ts @@ -24,7 +24,7 @@ import { pairKey } from "../execution-policy.ts"; import { diffIdentity, type IdentityMismatch } from "../identity-diff.ts"; -import type { RunManifest } from "../manifest.ts"; +import { agentEndpointComponent, type RunManifest } from "../manifest.ts"; import type { Experiment } from "./types.ts"; export interface ComparisonGroup { @@ -62,6 +62,7 @@ function comparableView(experiment: Experiment, manifest: RunManifest): Record } { + const endpoint = endpointIdentity(spec); + return endpoint === undefined ? {} : { endpoint }; +} + /** Per-experiment identity components — phase 2's comparison-group diff reads these. */ export interface ExperimentIdentityComponents { /** Hash of the experiment's defining module source. */ @@ -76,6 +88,8 @@ export interface ExperimentIdentityComponents { model: string; reasoning: string; cliVersion: string; + /** The declared `agent.endpoint` (experiments/endpoint.ts endpointIdentity); absent when none. */ + endpoint?: Record; }; /** Hash of the canonicalized runtime configuration. */ runtime: string; @@ -192,6 +206,7 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest model: experiment.agent.model ?? "unspecified", reasoning: experiment.agent.reasoning ?? "unspecified", cliVersion: options.cliVersions?.[experiment.id] ?? experiment.agent.cliVersion ?? "unspecified", + ...agentEndpointComponent(experiment.agent.endpoint), }, runtime: contentHash({ kind: experiment.runtime.kind, diff --git a/src/runner/execute.ts b/src/runner/execute.ts index 8939ae1..09c2364 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -172,6 +172,16 @@ async function inStage(kind: ErrorKind, work: () => Promise): Promise { } } +/** + * What makes one key probe distinct from another: the adapter and the host + * variables it reads. Same adapter, same variables, same values — one probe + * covers every experiment that shares them; a different `agent.endpoint` + * names different variables and gets its own. + */ +export function keyProbeIdentity(adapter: Pick): string { + return [adapter.name, [...adapter.requiredHostEnv].join(","), [...(adapter.optionalHostEnv ?? [])].join(",")].join("|"); +} + function defaultHostAdapters(): Record { const echo = new EchoAdapter(); const oracle = new OracleAdapter(); @@ -417,19 +427,24 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise(); + // unique endpoint once — before any docker, sidecar, or run-row work — + // and say plainly whether the run starts. "Unique" is the adapter plus + // the host variables it reads (keyProbeIdentity): two codex arms routed + // by different `agent.endpoint` variables are two probes, while arms + // sharing the vendor defaults stay one. Adapters without a probe (tests, + // custom adapters) skip it. + const probesByEndpoint = new Map(); for (const adapter of containerAgents.values()) { - if (adapter.keyProbe === undefined || probesByAdapter.has(adapter.name)) continue; - probesByAdapter.set(adapter.name, { + if (adapter.keyProbe === undefined) continue; + const identity = keyProbeIdentity(adapter); + if (probesByEndpoint.has(identity)) continue; + probesByEndpoint.set(identity, { name: adapter.name, envVars: adapter.requiredHostEnv, ...adapter.keyProbe({ ...collectProviderKeys([adapter]), ...collectOptionalHostEnv([adapter]) }), }); } - await validateProviderKeys([...probesByAdapter.values()], { log }); + await validateProviderKeys([...probesByEndpoint.values()], { log }); log(`run starting: ${plan.length} attempt(s)`); // Resolve image digests and agent CLI versions BEFORE building the manifest: diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 2cdaea5..ba55794 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -2,9 +2,10 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { ClaudeCodeAdapter, CLAUDE_MCP_CONFIG_PATH } from "../src/adapters/claude.ts"; -import { CodexAdapter, CODEX_CONFIG_PATH, CODEX_GATEWAY_PROVIDER } from "../src/adapters/codex.ts"; +import { CodexAdapter, CODEX_CONFIG_PATH, CODEX_DEFAULT_ENDPOINT, CODEX_GATEWAY_PROVIDER } from "../src/adapters/codex.ts"; import { parseClaudeStream, parseCodexStream, parseJsonlLines } from "../src/adapters/parse.ts"; import type { ContainerAttemptContext } from "../src/adapters/types.ts"; +import { keyProbeIdentity } from "../src/runner/execute.ts"; const STREAMS_DIR = join(import.meta.dir, "__fixtures__", "streams"); @@ -346,4 +347,34 @@ describe("per-experiment endpoint routing (agent.endpoint)", () => { expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { wireApi: "chat" } })).toThrow(/codex adapter only/); expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { baseUrlEnv: "GATEWAY_SECRET_URL" } })).toThrow(/must not be secret-named/); }); + + test("a vendor variable cannot be borrowed for another role: the shim would read it, then overwrite it", () => { + // Swapped roles pass the name-shape checks (both secret-named) but would clobber each other. + expect( + () => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS", headersEnv: "ANTHROPIC_API_KEY" } }), + ).toThrow(/apiKeyEnv ANTHROPIC_CUSTOM_HEADERS is the claude adapter's headersEnv variable/); + // One borrowed name with the other role left at its default collides on resolution. + expect(() => new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS" } })).toThrow( + /is the claude adapter's headersEnv variable/, + ); + expect(() => new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { headersEnv: "OPENAI_API_KEY" } })).toThrow( + /headersEnv OPENAI_API_KEY is the codex adapter's apiKeyEnv variable/, + ); + // The vendor name for its OWN role is simply the default, spelled out. + const explicit = new CodexAdapter("codex", { model: "m", reasoning: "r", endpoint: { apiKeyEnv: "OPENAI_API_KEY" } }); + expect(explicit.endpoint).toEqual(CODEX_DEFAULT_ENDPOINT); + }); + + test("key probes are deduplicated per endpoint, not per adapter", () => { + const vendor = new CodexAdapter("codex", { model: "m", reasoning: "r" }); + const vendorAgain = new CodexAdapter("codex", { model: "m2", reasoning: "r" }); + const gateway = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "r", endpoint }); + const gatewayChat = new CodexAdapter("codex", { model: "acme/coder-9", reasoning: "r", endpoint: { ...endpoint, wireApi: "chat" } }); + // Same variables ⇒ one probe; the wire does not change what the probe reads. + expect(keyProbeIdentity(vendor)).toBe(keyProbeIdentity(vendorAgain)); + expect(keyProbeIdentity(gateway)).toBe(keyProbeIdentity(gatewayChat)); + // Different variables ⇒ a probe each, so a rejected gateway token still stops the run. + expect(keyProbeIdentity(vendor)).not.toBe(keyProbeIdentity(gateway)); + expect(keyProbeIdentity(new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r" }))).not.toBe(keyProbeIdentity(vendor)); + }); }); diff --git a/tests/experiments-v3.test.ts b/tests/experiments-v3.test.ts index bf0b913..c20b2fd 100644 --- a/tests/experiments-v3.test.ts +++ b/tests/experiments-v3.test.ts @@ -167,6 +167,38 @@ describe("comparison groups (withhold-and-diff)", () => { expect(groups[0]?.details).toEqual([]); }); + test("a different agent.endpoint within a group is withheld: the manifest records it and the diff names the field", () => { + const endpoint = { apiKeyEnv: "GATEWAY_API_TOKEN", baseUrlEnv: "GATEWAY_BASE_URL", wireApi: "chat" as const }; + const a = loadedExperiment({ + id: "g-a", + comparisonGroup: "g", + treatment: "docs", + agent: { ...agent, endpoint }, + runtime: { kind: "container", image: "img:1", command: ["true"], config: { docs: {} } }, + }); + const b = loadedExperiment({ + id: "g-b", + comparisonGroup: "g", + treatment: "mcp", + agent: { ...agent, endpoint: { ...endpoint, wireApi: "responses" } }, + runtime: { kind: "container", image: "img:1", command: ["true"], config: { mcp: {} } }, + }); + const manifest = manifestFor([a, b]); + // Recorded verbatim, in a fixed key order; an experiment without one records nothing new. + expect(manifest.experiments["g-a"]?.components.agent.endpoint).toEqual(endpoint); + expect(manifest.experiments["g-b"]?.components.agent.endpoint).toEqual({ ...endpoint, wireApi: "responses" }); + const plain = loadedExperiment({ id: "g-c", agent, runtime: { kind: "container", image: "img:1", command: ["true"] } }); + expect(manifestFor([plain]).experiments["g-c"]?.components.agent).not.toHaveProperty("endpoint"); + + const [group] = compareGroups([a.experiment, b.experiment], manifest); + expect(group?.comparison).toBe("withheld"); + expect(group?.mismatches.map((mismatch) => mismatch.path)).toEqual(["identity.agent.endpoint.wireApi"]); + + // Same endpoint on both arms compares like any other agreed component. + const same = loadedExperiment({ ...b.experiment, agent: { ...agent, endpoint } }); + expect(compareGroups([a.experiment, same.experiment], manifestFor([a, same]))[0]?.comparison).toBe("allowed"); + }); + test("a mismatched model within a group is withheld and the diff names the component", () => { const a = loadedExperiment({ id: "g-a",