diff --git a/AGENTS.md b/AGENTS.md index 0434102..3aa76e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,7 +70,7 @@ and its resources. The source layout stays under `app/`. (the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities//` (the keyword-memory `capability`); `workspace` (and `workspace_cli`, its shell command), `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), - `paths` (user config, data and cache locations), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine + `paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request, never by a run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine says is done, and where each step is done), `gitrefs` (a repository's branches and tags, by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository), `sandbox` (Settings' docker: its Docker Hub login and the images runs keep) and `redact` at the top. - `app/templates/`, `app/static/` — the interface, grouped by the same capability @@ -110,8 +110,8 @@ remain available; the executable is `nullius` and storage paths keep that name. Harbor a recipe runs is its pin's, run through `uvx` (or the target's own, for a recipe without a pin), and the viewer never imports it. -`nullius paths` lists persistent locations. On Linux, settings and benchmark -licences live under `~/.config/nullius/`, user catalogue entries, workspaces, +`nullius paths` lists persistent locations. On Linux, settings, the model price +list and benchmark licences live under `~/.config/nullius/`, user catalogue entries, workspaces, runs and standard installs under `~/.local/share/nullius/`, and caches under `~/.cache/nullius/`. XDG variables move those roots; `NULLIUS_HOME` selects a separate profile with `config/`, `data/` and `cache/`. Existing storage overrides diff --git a/app/.env.example b/app/.env.example index fccb609..ddfd9fc 100644 --- a/app/.env.example +++ b/app/.env.example @@ -6,6 +6,7 @@ # Start-up options, not on the Settings page: # NULLIUS_HOME: a separate profile with config/, data/ and cache/ directories # NULLIUS_CATALOGUE: the user catalogue directory (default catalogue/ in the user data directory) +# NULLIUS_PRICES: the model price list file (default prices.json in the user configuration directory); set for a run on a host # APP_HOST: the address the server binds (default 127.0.0.1, or --host) # APP_PORT: the port the server listens on (default 5057, or --port) # JOB_ROOTS: job roots to open at start (or the command-line argument) diff --git a/app/nullius/cli.py b/app/nullius/cli.py index e569c4f..36a1d9a 100644 --- a/app/nullius/cli.py +++ b/app/nullius/cli.py @@ -20,6 +20,7 @@ "standard": "inspect and run standard definitions", "workspace": "manage workspaces", "paths": "show persistent storage locations", + "prices": "show and update model prices", "migrate": "copy legacy storage into the user profile", } MODULES = { @@ -29,6 +30,7 @@ "standard": "nullius.standard.cli", "workspace": "nullius.workspace_cli", "paths": "nullius.paths", + "prices": "nullius.prices", "migrate": "nullius.migrate", } diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 90b08ad..8fad11e 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -120,7 +120,7 @@ from datetime import datetime, timezone from typing import Callable, Dict, List, Optional -from nullius import hosts, paths, progress, redact, secrets, settings +from nullius import hosts, paths, prices as price_list, progress, redact, secrets, settings from nullius.harbor import source as harbor_source from nullius.standard import payload as standard_payload @@ -1450,6 +1450,15 @@ def _launch(self, part: dict) -> Optional[bool]: if rc != 0: self._fail("run", f"the script did not land on {where}: " + out.strip()[-400:]) return False + prices_on_target = "" + if self.plan.runner == STANDARD and self.on_host: + # the price list is this machine's user's, so the run takes it along (nullius.prices) - an empty one + # too: a host's own list, if it has one, never prices a run started here + prices_on_target = run_dir + "/prices.json" + rc, out = self._sh(f"cat > {self._q(prices_on_target)}", 60, stdin=json.dumps(price_list.load()) + "\n") + if rc != 0: + self._fail("run", f"the price list did not land on {where}: " + out.strip()[-400:]) + return False exports = " ".join(f"{k}={self._q(v) if v.startswith('~') else shlex.quote(v)}" for k, v in [("NULLIUS_HARBOR", self.harbor), ("NULLIUS_TASKS", self.tasks_on_target), ("NULLIUS_JOBS", run_dir + "/jobs"), ("NULLIUS_PROGRESS", "1")] @@ -1457,6 +1466,7 @@ def _launch(self, part: dict) -> Optional[bool]: if self.plan.runner == STANDARD else []) + ([("NULLIUS_CATALOGUE", self.catalogue_on_target)] if self.plan.runner == STANDARD and self.catalogue_on_target else []) + + ([("NULLIUS_PRICES", prices_on_target)] if prices_on_target else []) + ([("NULLIUS_PYTHON", sys.executable)] if self.plan.runner == STANDARD and not self.on_host else []) + ([("NULLIUS_LICENCE_DIR", self._licence_dir())] if self._licence_dir() else []) + ([("API_MODEL_NAME", self.plan.model)] if self.plan.model else [])) diff --git a/app/nullius/paths.py b/app/nullius/paths.py index 84c601c..4377a55 100644 --- a/app/nullius/paths.py +++ b/app/nullius/paths.py @@ -46,6 +46,11 @@ def settings_file(env=None) -> str: return os.path.join(config_dir(env), "settings.json") +def prices_file(env=None) -> str: + env = os.environ if env is None else env + return env.get("NULLIUS_PRICES") or os.path.join(config_dir(env), "prices.json") + + def catalogue_dir(env=None) -> str: env = os.environ if env is None else env return env.get("NULLIUS_CATALOGUE") or os.path.join(data_dir(env), "catalogue") @@ -70,7 +75,7 @@ def main(argv=None) -> int: parser = argparse.ArgumentParser(description="Show Nullius's persistent user directories (no secret values).") parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) - values = {"settings": settings_file(), "secrets": secrets_dir(), "catalogue": catalogue_dir(), + values = {"settings": settings_file(), "secrets": secrets_dir(), "prices": prices_file(), "catalogue": catalogue_dir(), "workspaces": workspaces_dir(), "runs": runs_dir(), "cache": cache_dir()} if args.json: print(json.dumps(values, indent=2)) diff --git a/app/nullius/prices.py b/app/nullius/prices.py new file mode 100644 index 0000000..cd4c5de --- /dev/null +++ b/app/nullius/prices.py @@ -0,0 +1,279 @@ +"""Model prices, the user's own: what a call's cost is estimated from when +the provider's reply does not carry one. + +Nullius ships no price. The list lives in one file of the user's +configuration, ``prices.json`` (``NULLIUS_PRICES`` names another: the file a +host is sent), in two parts: + +``models`` a public list, fetched on request (``update``) and kept whole + with where it came from and when: LiteLLM's + ``model_prices_and_context_window.json``, keyed by the model + names the providers themselves use +``own`` the user's rows, which win over the list: a price it lacks, a + negotiated one, a model served under a name of one's own + +A row is USD per million tokens: ``input`` and ``output``, optionally +``cached`` (input a cache served) and ``cache_write``, and ``max_output``, +the model's output limit in tokens. A model is priced by exactly its name or +not at all: a name that only starts like another's is another model. + +Nothing here runs during a run: a run reads the file, never the network. + + nullius prices # where the list came from, how many models, the user's own rows + nullius prices list MODEL ... # those models' rows + nullius prices update [--from URL-or-file] + nullius prices set MODEL INPUT OUTPUT [--cached X] [--cache-write X] [--max-output N] + nullius prices remove MODEL +""" +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.request +from typing import Callable, Dict, Iterable, Optional, Tuple + +from nullius import paths + +SOURCE = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +RATES = ("input", "output", "cached", "cache_write") # USD per million tokens +_LISTED = {"input": "input_cost_per_token", "output": "output_cost_per_token", + "cached": "cache_read_input_token_cost", "cache_write": "cache_creation_input_token_cost"} +_MOST = 64 * 1024 * 1024 # a list larger than this is not a price list + + +def path(env=None) -> str: + return paths.prices_file(env) + + +def load(env=None) -> dict: + """The file as ``{from, on, models, own}``; missing or unreadable, empty: + no price is better than a run that cannot start for its price list.""" + doc = {"from": "", "on": "", "models": {}, "own": {}} + try: + with open(path(env), encoding="utf-8") as f: + got = json.load(f) + except (OSError, ValueError): + return doc + if isinstance(got, dict): + for k in ("from", "on"): + if isinstance(got.get(k), str): + doc[k] = got[k] + for k in ("models", "own"): + if isinstance(got.get(k), dict): + doc[k] = {n: r for n, r in got[k].items() if _usable(r)} + return doc + + +def _usable(r) -> bool: + return isinstance(r, dict) and all(_number(r.get(k)) for k in ("input", "output")) + + +def _number(v) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) and v >= 0 + + +def save(doc: dict, env=None) -> None: + """Written whole beside the file, then renamed: a reader never finds half a list.""" + out = path(env) + os.makedirs(os.path.dirname(out) or ".", exist_ok=True) + part = f"{out}.{os.getpid()}.part" + with open(part, "w", encoding="utf-8") as f: + json.dump(doc, f, indent=1, sort_keys=True) + f.write("\n") + os.replace(part, out) + + +def table(env=None) -> Dict[str, dict]: + """Every priced model: the list, then the user's own rows over it.""" + doc = load(env) + return {**doc["models"], **doc["own"]} + + +def given(names: Iterable[Optional[str]], prices: Optional[Dict[str, dict]]) -> Dict[str, dict]: + """The rows of ``prices`` for exactly ``names``: what a relay started as + a process of its own is handed, which cannot take the whole list.""" + prices = prices or {} + return {n: prices[n] for n in dict.fromkeys(n for n in names if n) if n in prices} + + +def row(input, output, cached=None, cache_write=None, max_output=None) -> dict: + """One row, checked: rates are numbers not below zero, the limit a whole number above it.""" + out = {} + for name, v in (("input", input), ("output", output), ("cached", cached), ("cache_write", cache_write)): + if v is None: + continue + try: + v = float(v) + except (TypeError, ValueError): + raise ValueError(f"{name}: a price in USD per million tokens, not {v!r}") + if not v >= 0 or v == float("inf"): + raise ValueError(f"{name}: a price in USD per million tokens, not {v!r}") + out[name] = v + if max_output is not None: + try: + n = int(max_output) + except (TypeError, ValueError): + n = 0 + if n <= 0: + raise ValueError(f"max_output: the model's output limit in tokens, not {max_output!r}") + out["max_output"] = n + return out + + +def parse(text: str) -> Tuple[str, dict]: + """``MODEL=INPUT,OUTPUT[,CACHED]``, a run's ``--price``.""" + name, eq, rest = str(text).partition("=") + bits = [b.strip() for b in rest.split(",")] if eq else [] + if not name.strip() or not 2 <= len(bits) <= 3 or not all(bits): + raise ValueError(f"--price {text!r}: MODEL=INPUT,OUTPUT[,CACHED], in USD per million tokens") + try: + return name.strip(), row(*bits) + except ValueError as e: + raise ValueError(f"--price {text!r}: {e}") + + +def convert(listing) -> Dict[str, dict]: + """LiteLLM's list as rows: its costs are per token, a row's per million. + An entry with no input price is not a priced model (the list's own + sample, an image model priced per picture) and is left out.""" + if not isinstance(listing, dict): + raise ValueError("not a price list: a JSON object of models was expected") + out = {} + for name, e in listing.items(): + if not isinstance(e, dict) or not _number(e.get(_LISTED["input"])): + continue + r = {"input": round(e[_LISTED["input"]] * 1e6, 6), + "output": round(e[_LISTED["output"]] * 1e6, 6) if _number(e.get(_LISTED["output"])) else 0.0} + for k in ("cached", "cache_write"): + if _number(e.get(_LISTED[k])): + r[k] = round(e[_LISTED[k]] * 1e6, 6) + limit = e.get("max_output_tokens") + if isinstance(limit, int) and not isinstance(limit, bool) and limit > 0: + r["max_output"] = limit + out[str(name)] = r + if not out: + raise ValueError("not a price list: no model in it has a price") + return out + + +def _fetch(source: str) -> bytes: + if source.startswith("https://"): + req = urllib.request.Request(source, headers={"User-Agent": "nullius"}) # nothing of the user or the machine + with urllib.request.urlopen(req, timeout=60) as r: + return r.read(_MOST + 1) + if "://" in source: + raise ValueError("only an https address or a file is read") + with open(source, "rb") as f: + return f.read(_MOST + 1) + + +def update(source: Optional[str] = None, env=None, fetch: Optional[Callable[[str], bytes]] = None, + today: Optional[str] = None) -> dict: + """The public list fetched and kept in place of the last one; the user's + own rows are not touched. Nothing is written when the fetch or the list + is bad: the list held stays. Returns ``{n, from, on}``.""" + source = source or SOURCE + try: + raw = (fetch or _fetch)(source) + except (OSError, ValueError) as e: + why = e.strerror if isinstance(e, OSError) and e.filename else e # a file's error names the file again + raise ValueError(f"could not fetch the price list from {source}: {why}") + if len(raw) > _MOST: + raise ValueError(f"{source}: too large to be a price list") + try: + models = convert(json.loads(raw)) + except ValueError as e: + raise ValueError(f"{source}: {e}") + doc = load(env) + doc.update({"from": source, "on": today or time.strftime("%Y-%m-%d"), "models": models}) + save(doc, env) + return {"n": len(models), "from": doc["from"], "on": doc["on"]} + + +def set_own(name: str, r: dict, env=None) -> None: + name = str(name).strip() + if not name or not _usable(r): + raise ValueError("a model's name, and its input and output prices") + doc = load(env) + doc["own"][name] = r + save(doc, env) + + +def remove_own(name: str, env=None) -> bool: + doc = load(env) + if doc["own"].pop(str(name).strip(), None) is None: + return False + save(doc, env) + return True + + +def state(env=None) -> dict: + """What Settings shows: how many models the list prices, where it came + from and when, and the user's own rows.""" + doc = load(env) + return {"n": len(doc["models"]), "from": doc["from"], "on": doc["on"], "own": doc["own"]} + + +def said(name: str, r: dict) -> str: + bits = [f"input {r['input']:g}", f"output {r['output']:g}"] + bits += [f"{label} {r[k]:g}" for k, label in (("cached", "cached"), ("cache_write", "cache write")) if k in r] + if "max_output" in r: + bits.append(f"output limit {r['max_output']}") + return f"{name}: " + ", ".join(bits) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(prog="nullius prices", + description="The user's model prices, in USD per million tokens: what a run's cost is estimated from.") + sub = ap.add_subparsers(dest="cmd") + ls = sub.add_parser("list", help="the list's source and size, the user's own rows, and the rows of the models named") + ls.add_argument("models", nargs="*") + ls.add_argument("--json", action="store_true") + up = sub.add_parser("update", help="fetch the public list and keep it; the user's own rows stay") + up.add_argument("--from", dest="source", metavar="URL-or-file", help=f"another list in the same format (default: {SOURCE})") + st = sub.add_parser("set", help="a price of the user's own, which wins over the list") + st.add_argument("model") + st.add_argument("input") + st.add_argument("output") + st.add_argument("--cached") + st.add_argument("--cache-write") + st.add_argument("--max-output") + rm = sub.add_parser("remove", help="remove one of the user's own prices") + rm.add_argument("model") + a = ap.parse_args(argv) + try: + if a.cmd == "update": + got = update(a.source) + print(f"{got['n']} models priced, from {got['from']}") + return 0 + if a.cmd == "set": + r = row(a.input, a.output, a.cached, a.cache_write, a.max_output) + set_own(a.model, r) + print(said(a.model.strip(), r)) + return 0 + if a.cmd == "remove": + if not remove_own(a.model): + ap.exit(1, f"{a.model}: not one of your own prices\n") + return 0 + except ValueError as e: + ap.exit(1, f"{e}\n") + doc, names = load(), list(getattr(a, "models", None) or []) + both = {**doc["models"], **doc["own"]} + if getattr(a, "json", False): + print(json.dumps({"file": path(), "from": doc["from"], "on": doc["on"], "n": len(doc["models"]), "own": doc["own"], + "models": {n: both.get(n) for n in names}}, indent=2)) + return 0 + print(f"file: {path()}") + print(f"list: {len(doc['models'])} models, from {doc['from']} on {doc['on']}" if doc["models"] + else "list: none yet - nullius prices update") + for n, r in sorted(doc["own"].items()): + print("own " + said(n, r)) + for n in names: + print(said(n, both[n]) if n in both else f"{n}: no price") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/nullius/settings.py b/app/nullius/settings.py index 4796f35..7750c7d 100644 --- a/app/nullius/settings.py +++ b/app/nullius/settings.py @@ -150,6 +150,7 @@ def validate(self, value: str) -> Optional[str]: STARTUP: Dict[str, str] = { "NULLIUS_HOME": "a separate profile with config/, data/ and cache/ directories", "NULLIUS_CATALOGUE": "the user catalogue directory (default catalogue/ in the user data directory)", + "NULLIUS_PRICES": "the model price list file (default prices.json in the user configuration directory); set for a run on a host", "APP_HOST": "the address the server binds (default 127.0.0.1, or --host)", "APP_PORT": "the port the server listens on (default 5057, or --port)", "JOB_ROOTS": "job roots to open at start (or the command-line argument)", diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 143070e..efc0fb0 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,7 +6,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend docker] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--unsandboxed | --backend docker] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED] ...] [--unsandboxed | --backend docker] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -63,7 +63,7 @@ import sys import tempfile -from nullius import paths, progress, secrets, settings +from nullius import paths, prices as price_list, progress, secrets, settings # The legacy module command needs saved path overrides before install.HOME # and other runtime defaults are computed, just like the unified CLI does. if __name__ == "__main__" and "--from-run" not in sys.argv[1:]: @@ -416,6 +416,18 @@ def options(pairs) -> dict: return out +def prices(given) -> dict: + """``--price MODEL=INPUT,OUTPUT[,CACHED]``, repeatable: this run's own prices, over the user's list.""" + out = {} + for p in given or []: + try: + name, row = price_list.parse(p) + except ValueError as e: + sys.exit(str(e)) + out[name] = row + return out + + def cmd_conform(a): o = _other(a.other) try: @@ -457,7 +469,7 @@ def cmd_run(a): concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, - **({"env": run_env(a.provider)} if a.from_run else {})) + prices=prices(a.price), **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) return 2 @@ -610,6 +622,8 @@ def main(argv=None): p.add_argument("--launch", help="the launch id every shard records (default: the day, the benchmark, the agent, the model)") p.add_argument("--bench-option", action="append", metavar="NAME=VALUE", help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults") + p.add_argument("--price", action="append", metavar="MODEL=INPUT,OUTPUT[,CACHED]", + help="this run's price for a model in USD per million tokens, over your price list (repeatable; nullius prices)") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") v.add_argument("benchmark") v.add_argument("--dataset", help="one dataset (default: each)") diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index 9cdd8d2..bc60de2 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -29,6 +29,7 @@ import time from typing import Callable, Dict, List, Optional +from nullius import prices as price_list from nullius.standard import definitions, install, jsonl, relay, run SIDES = ("acp", "cli") @@ -171,7 +172,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa units = [str(u) for u in r.driver().request("units", dataset=r.ds["id"], timeout=300)["units"]] # One relay for the whole comparison, on this machine: every route's # records side by side, with the fields fidelity reads. - r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=run.PRICES, model=model, + r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 15e0198..95aaa0a 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -459,7 +459,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, - simulator: str = ""): + simulator: str = "", prices: Optional[dict] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -467,7 +467,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str several machines (``shard``), which ``merge`` reads back. ``repeat``: how many times each unit was run, said only when more than once. ``judge`` and ``simulator``: the models that graded and that played the user, when the - benchmark has them.""" + benchmark has them. ``prices``: the rows its models' costs are + estimated from and where the user's list came from, when it has any.""" with open(os.path.join(job_dir, "config.json"), "w") as f: json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius", "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, @@ -476,7 +477,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"image": image} if image else {}), **({"judge": judge} if judge else {}), **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other), **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), - **({"repeat": repeat} if repeat > 1 else {})}, + **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, "agents": [{"name": other["key"], "model_name": model}], diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index a525a99..f90dca3 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -11,8 +11,8 @@ the tokens, and an abridged request and reply. Three API shapes are read, Chat Completions, Responses and Anthropic's Messages, streamed or not. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an -estimate from ``prices`` when one is given for the model, marked as an -estimate. +estimate from ``prices`` when one is given for exactly that model (the +user's price list, ``nullius.prices``), marked as an estimate. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -380,7 +380,7 @@ def _cost(self, model: str, usage: Optional[dict]): return None, None if "cost" in usage: return usage["cost"], "provider" - p = self.prices.get(model) or next((v for k, v in self.prices.items() if model and model.startswith(k)), None) + p = self.prices.get(model) # exactly this model: one whose name only starts the same is another if not p: return None, None fresh = max(0, usage["input"] - usage["cached"]) diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 101e05d..b84d328 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -62,7 +62,7 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from typing import Callable, Dict, List, Optional -from nullius import paths, progress, providers +from nullius import paths, prices as price_list, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, match, mcp_machine, relay, shard as shards @@ -98,24 +98,6 @@ def driver_path() -> str: "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"}, "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, } -# List prices in USD per million tokens, for an estimate of what a call cost -# when the provider does not say (OpenAI does not; OpenRouter does). A model -# not here has no cost estimate rather than a wrong one. -PRICES = { - "gpt-5-mini": {"input": 0.25, "cached": 0.025, "output": 2.0}, - "gpt-5-nano": {"input": 0.05, "cached": 0.005, "output": 0.4}, - "gpt-4.1-mini": {"input": 0.4, "cached": 0.1, "output": 1.6}, - "gpt-4.1": {"input": 2.0, "cached": 0.5, "output": 8.0}, - "gpt-4o-mini": {"input": 0.15, "cached": 0.075, "output": 0.6}, - "gpt-4o-mini-2024-07-18": {"input": 0.15, "cached": 0.075, "output": 0.6}, - "gpt-4o": {"input": 2.5, "cached": 1.25, "output": 10.0}, - "gpt-4o-2024-08-06": {"input": 2.5, "cached": 1.25, "output": 10.0}, - "gpt-4-turbo": {"input": 10.0, "cached": 10.0, "output": 30.0}, # a paper's reader (a setup's original) - "gpt-3.5-turbo": {"input": 0.5, "cached": 0.5, "output": 1.5}, - "text-embedding-3-large": {"input": 0.13, "cached": 0.13, "output": 0.0}, # a search option's embeddings (tau3-bench) - "text-embedding-3-small": {"input": 0.02, "cached": 0.02, "output": 0.0}, # a capability's own embedder (Mem0) - "text-embedding-ada-002": {"input": 0.1, "cached": 0.1, "output": 0.0}, -} TURN_SECONDS = 900 DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image) @@ -326,7 +308,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None, launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None, licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, - remove_images: bool = False): + remove_images: bool = False, prices: Optional[dict] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: the caller's explicit yes to running an agent on the @@ -344,6 +326,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas ``u0042r2``; once by default, tagged ``u0042`` as always. ``simulator_model``: a benchmark whose user a model plays (``simulator``): this model instead of its official one. + ``prices``: this run's own prices, by model, over the user's list + (``nullius.prices``): what a call's cost is estimated from. ``licences``: what the run holds of the benchmark's licences (:class:`Licences`); by default its store on this machine. ``bench_options``: the benchmark's own options for this run @@ -414,6 +398,10 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas raise Refused("; ".join(bad)) needs = definitions.option_needs(bench, self.bench_options) self.embed_model = needs["embeddings"][0] if needs["embeddings"] else None + # What a call's cost is estimated from when the provider does not say it: the user's price list + # (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price. + self.run_prices = dict(prices or {}) + self.prices = {**price_list.table(), **self.run_prices} lacking = hostreq.missing(bench, needs["host"], which) if lacking: raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine " @@ -511,7 +499,7 @@ def start(self) -> str: granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend, image=self.image, options=self.options, launch=self.launch or "", bench_options=self.bench_options, shard=shards.record(*self.shard, len(everything), units, self.limit) if self.shard else None, - repeat=self.repeat, + repeat=self.repeat, prices=self.prices_record(), simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -638,7 +626,7 @@ def ready(self) -> None: # own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's. if self.embed_model and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), - prices=PRICES, model=self.embed_model, + prices=self.prices, model=self.embed_model, key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -896,6 +884,28 @@ def agent_servers(self, env_def: Optional[dict]) -> List[str]: out.append(h["value"]) return out + def prices_record(self) -> Optional[dict]: + """What the job's config keeps of its prices: the rows of the models + it calls, each with its ``source`` (``list`` the public list, ``own`` + the user's own row, ``run`` this run's ``--price``), and where and + when the list was fetched when a row is its. None when it has none.""" + rows = price_list.given(self.models_called(), self.prices) + if not rows: + return None + doc, mine = price_list.load(), getattr(self, "run_prices", {}) + rows = {n: {**r, "source": "run" if n in mine else "own" if n in doc["own"] else "list"} for n, r in rows.items()} + listed = doc["from"] and any(r["source"] == "list" for r in rows.values()) + return {**({"from": doc["from"], "on": doc["on"]} if listed else {}), "models": rows} + + def models_called(self) -> List[str]: + """Every model this run's relays may send a call to, as sent: the + run's, the judge's, the simulator's, an option's embeddings and the + agent's side models. Any other name a caller asks for is replaced by + the run's (``relay``), so these are all a run can be priced for.""" + side = [(b or a) for a, _, b in (str(m).partition("=") for m in (self.other.get("model") or {}).get("side_models") or ())] + mine = [getattr(self, n, None) for n in ("model", "judge_model", "simulator_model", "embed_model")] + return [m for m in dict.fromkeys([*mine, *side]) if m] + def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] = None, own: bool = False): """The unit's own relay, started: in this process on the local backend, a sidecar container on docker. Either answers ``url`` (the @@ -909,7 +919,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] side = tuple((self.other.get("model") or {}).get("side_models") or ()) key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": - rl = relay.Relay(self.base, self.key, record, prices=PRICES, model=self.model, side_models=side, + rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header).start() rl.unit_url = rl.url(tag) return rl @@ -928,7 +938,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}") door = getattr(self, "loopback", None) rl = container.Unit(self.name, tag, record, image=self.image, app=APP, base=door.base if door else self.base, key=self.key, - model=self.model, routes=routes, prices=PRICES, side_models=side, key_header=key_header, + model=self.model, routes=routes, side_models=side, key_header=key_header, + prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None) rl.mcp_listeners = listeners try: diff --git a/app/packaging/verify.py b/app/packaging/verify.py index b11edd9..ae740b3 100644 --- a/app/packaging/verify.py +++ b/app/packaging/verify.py @@ -182,7 +182,7 @@ def run(*command, expected=0): assert 'serve' in run(*prefix, '--help') assert set(json.loads(run(*prefix, 'import', '--list', '--json'))) == {'atif', 'otel'} assert run(*prefix, '--version').strip() == 'nullius ' + backend['version'] - for subcommand in ('runs', 'compare', 'standard', 'workspace', 'paths', 'migrate'): + for subcommand in ('runs', 'compare', 'standard', 'workspace', 'paths', 'prices', 'migrate'): assert 'usage:' in run(*prefix, subcommand, '--help') run(*prefix, 'import', root / 'missing', expected=3) locations = json.loads(run(command, 'paths', '--json')) diff --git a/app/site/mkdocs.yml b/app/site/mkdocs.yml index b702a9b..5034a18 100644 --- a/app/site/mkdocs.yml +++ b/app/site/mkdocs.yml @@ -111,6 +111,7 @@ nav: - System One provider: settings/system-one.md - LLM judge provider: settings/llm-judge.md - Agent keys: settings/agent-keys.md + - Model prices: settings/model-prices.md - Run hosts: settings/run-hosts.md - Sandbox: settings/sandbox.md - Other: settings/other.md diff --git a/app/static/settings/prices.js b/app/static/settings/prices.js new file mode 100644 index 0000000..060e6d8 --- /dev/null +++ b/app/static/settings/prices.js @@ -0,0 +1,64 @@ +// The Model prices section: the price list's state, its Update, and the user's own prices. +// The server sends the state with the page and again after each change; this draws it. +(() => { + const section = document.getElementById('prices'); + if (!section) return; + const $ = sel => section.querySelector(sel); + const csrf = section.dataset.csrf; + const tell = (message, error = false) => window.nulliusNotify ? window.nulliusNotify(message, {error}) : null; + const post = async (url, fields) => { + try { + const r = await fetch(url, {method: 'POST', headers: {'X-CSRF-Token': csrf}, body: new URLSearchParams(fields)}); + const j = await r.json(); + return j.ok ? j : {ok: false, error: j.error || 'HTTP ' + r.status}; + } catch (err) { return {ok: false, error: String(err)}; } + }; + const rate = v => String(Number(v)); + + const show = state => { + $('[data-prices-state]').textContent = state.n + ? `${state.n.toLocaleString('en-US')} models · ${state.on}` : 'No price list yet'; + const own = Object.entries(state.own || {}).sort(([a], [b]) => a.localeCompare(b)); + $('[data-own-rows]').replaceChildren(...own.map(([model, r]) => { + const row = document.createElement('div'); row.className = 'st-row'; + const name = document.createElement('span'); name.className = 'st-row-label mono'; name.textContent = model; + const value = document.createElement('span'); value.className = 'st-row-value st-value'; + value.append(`input ${rate(r.input)} · output ${rate(r.output)}` + (r.cached === undefined ? '' : ` · cached ${rate(r.cached)}`)); + if (csrf) { + const button = document.createElement('button'); button.type = 'button'; button.className = 'btn sm ghost'; button.textContent = 'Remove'; + button.setAttribute('aria-label', `Remove the price of ${model}`); + button.addEventListener('click', async () => { + button.disabled = true; + const got = await post(section.dataset.remove, {model}); + if (got.ok) show(got); else { button.disabled = false; tell(got.error, true); } + }); + value.append(' ', button); + } + row.append(name, value); + return row; + })); + $('[data-own-none]').hidden = own.length > 0; + }; + + const update = $('[data-prices-update]'); + if (update) update.addEventListener('click', async () => { + const label = update.querySelector('span'); + update.disabled = true; update.classList.add('busy'); label.textContent = 'Updating…'; + const got = await post(section.dataset.update, {}); + update.disabled = false; update.classList.remove('busy'); label.textContent = 'Update'; + tell(got.ok ? `${got.n.toLocaleString('en-US')} models priced` : got.error, !got.ok); + if (got.ok) show(got); + }); + + const form = $('[data-own-add]'); + if (form) form.addEventListener('submit', async e => { + e.preventDefault(); + const f = new FormData(form), button = form.querySelector('[type=submit]'); + button.disabled = true; + const got = await post(section.dataset.set, {model: f.get('model'), input: f.get('input'), output: f.get('output'), cached: f.get('cached')}); + button.disabled = false; + if (got.ok) { form.reset(); show(got); } else tell(got.error, true); + }); + + try { show(JSON.parse(section.dataset.state)); } catch (err) { show({n: 0, own: {}}); } +})(); diff --git a/app/static/settings/settings.css b/app/static/settings/settings.css index 65ab761..a33a8e4 100644 --- a/app/static/settings/settings.css +++ b/app/static/settings/settings.css @@ -225,5 +225,10 @@ .st-sandbox-rows { display: grid; gap: 14px; } .st-sandbox-rows > .st-row + .st-row { padding-top: 14px; border-top: 1px solid var(--border-soft); } .settings-page .st-sandbox-rows .st-input { height: 34px; min-height: 34px; padding-block: 0; } -.settings-page #sandbox .st-row-value.st-value { flex-direction: row; justify-content: flex-start; align-items: center; } +.settings-page #sandbox .st-row-value.st-value, .settings-page #prices .st-row-value.st-value { flex-direction: row; justify-content: flex-start; align-items: center; } +#prices [data-prices-update] .ic { width: 15px; height: 15px; } +#prices [data-prices-update].busy .ic { animation: st-prices-spin .9s linear infinite; } +@keyframes st-prices-spin { to { transform: rotate(360deg); } } +@media (prefers-reduced-motion: reduce) { #prices [data-prices-update].busy .ic { animation: none; } } +.st-prices-add { margin-top: 14px; padding-top: 14px; border-top: 1px solid var(--border-soft); } .st-input[list]::-webkit-calendar-picker-indicator { opacity: 1; display: block; cursor: pointer; } diff --git a/app/templates/base.html b/app/templates/base.html index 98e6f39..e61f298 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -44,6 +44,7 @@ + diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index c368d0d..d43f158 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -74,6 +74,7 @@

Settings

{{ section_link('providers', 'Model providers', 'capability', 'openrouter openai anthropic local endpoint api key model_api_url') }} {{ section_link('agent-keys', 'Agent keys', 'agent', 'credentials cursor copilot token agent_keys') }} {{ section_link('judge', 'Judges', 'compare', 'typesafe system one llm model judgement') }} + {{ section_link('prices', 'Model prices', 'dollar', 'cost price estimate usd tokens list update') }}
Execution
@@ -239,6 +240,32 @@

Run hosts

+ {# the user's price list (nullius.prices), outside the form: prices.js draws the state and answers the buttons #} +
+

Model prices

What a run's cost is estimated from, in USD per million tokens.

+
+
Price list{% if writable %} {% endif %}
+
+
+ Your prices +
+ + {% if writable %} +
+
+
+
+
+
+
+
+
+
+
+ {% endif %} +
+
+ {# kept in this browser, not in the file: onboarding.js fills the state and answers the buttons #}
@@ -276,6 +303,7 @@

General

+ {% if writable %} {# a host: one of the ssh config's, or by hand — who and where, and how ssh gets in without a prompt. Test before Add; no password is ever stored, one is used once to put this machine's key on the host. #} diff --git a/app/tests/parallel.py b/app/tests/parallel.py index f35f1dd..b89faa6 100644 --- a/app/tests/parallel.py +++ b/app/tests/parallel.py @@ -80,7 +80,7 @@ def main(argv) -> int: if not docker: os.environ.pop("NULLIUS_TEST_DOCKER", None) for name in ("NULLIUS_CATALOGUE", "WORKSPACES_DIR", "RUNS_DIR", "SECRETS_DIR", - "STANDARD_DIR", "STANDARD_DATA_DIR", "HARBOR_PINS"): + "STANDARD_DIR", "STANDARD_DATA_DIR", "HARBOR_PINS", "NULLIUS_PRICES"): os.environ.pop(name, None) return _suite(argv) diff --git a/app/tests/test_execution.py b/app/tests/test_execution.py index e636a34..72e06e0 100644 --- a/app/tests/test_execution.py +++ b/app/tests/test_execution.py @@ -12,7 +12,7 @@ from unittest import mock from unittest.mock import patch -from nullius import execution, hosts, settings +from nullius import execution, hosts, prices as price_list, settings from nullius.harbor import source from nullius import providers @@ -725,6 +725,22 @@ def test_app_is_sent_at_its_content_address_and_the_part_runs_from_it(self): self.assertEqual(json.loads((self.tmp / "runs" / "std-shard-2of2" / "run.json").read_text())["catalogue_on_target"], "~/.nullius/runs/std-shard-2of2/catalogue") + def test_the_price_list_goes_with_the_run_even_empty_so_the_host_s_own_is_never_read(self): + ex = self.run_it() + sent = next(s for c, s in self.calls if c[0] == "ssh" and "cat > " in c[-1] and "/prices.json" in c[-1]) + self.assertEqual((json.loads(sent)["models"], json.loads(sent)["own"]), ({}, {})) # no list here: an empty one + launch = next(c for c, _ in self.calls if c[0] == "ssh" and "setsid nohup" in c[-1]) + self.assertIn("NULLIUS_PRICES=", launch[-1]) + self.calls.clear(); self.polls = 0 + price_list.set_own("gpt-5-mini", price_list.row(1, 2)) + self.plan.name = "std-priced" + ex = self.run_it() + self.assertEqual((ex.state, ex.exit_code), ("done", 0), ex.lines) + sent = next(s for c, s in self.calls if c[0] == "ssh" and "cat > " in c[-1] and "/prices.json" in c[-1]) + self.assertEqual(json.loads(sent)["own"], {"gpt-5-mini": {"input": 1.0, "output": 2.0}}) + launch = next(c for c, _ in self.calls if c[0] == "ssh" and "setsid nohup" in c[-1]) + self.assertIn('NULLIUS_PRICES="$HOME/.nullius/runs/std-priced/prices.json"', launch[-1]) + def test_what_it_does_is_said_in_stages_its_own_and_the_cli_s(self): """Getting the host ready is said stage by stage in the run's log, and the cli's own stages come through the part's log: the page folds them.""" diff --git a/app/tests/test_prices.py b/app/tests/test_prices.py new file mode 100644 index 0000000..5b6994c --- /dev/null +++ b/app/tests/test_prices.py @@ -0,0 +1,207 @@ +"""The user's price list: nothing shipped, a public list fetched on request, +the user's own rows over it, a model priced by exactly its name.""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import shutil +import tempfile +import unittest +from unittest import mock + +from nullius import prices +from nullius.standard import run + +# LiteLLM's shape: costs per token, a sample entry and an image model without an input price. +LISTING = { + "sample_spec": {"input_cost_per_token": "a number", "max_output_tokens": "a number"}, + "model-a": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 1.25e-06, + "max_output_tokens": 16384, "litellm_provider": "one"}, + "model-b": {"input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, "max_output_tokens": 128000}, + "embedder": {"input_cost_per_token": 2e-08, "output_cost_per_token": 0.0}, + "picture-model": {"output_cost_per_image": 0.04}, +} + + +class Profile(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="nullius-prices-") + self.addCleanup(shutil.rmtree, self.tmp, True) + env = mock.patch.dict(os.environ, {"NULLIUS_HOME": self.tmp}) + env.start() + self.addCleanup(env.stop) + os.environ.pop("NULLIUS_PRICES", None) + + def read(self): + with open(prices.path()) as f: + return f.read() + + def fetch(self, listing=LISTING): + return lambda source: json.dumps(listing).encode() + + +class Nothing(Profile): + def test_no_price_is_written_in_the_code(self): + self.assertFalse(hasattr(run, "PRICES")) + self.assertEqual(prices.table(), {}) + self.assertEqual(prices.state(), {"n": 0, "from": "", "on": "", "own": {}}) + + def test_an_unreadable_file_is_no_prices_rather_than_a_run_that_cannot_start(self): + os.makedirs(os.path.dirname(prices.path())) + with open(prices.path(), "w") as f: + f.write("{not json") + self.assertEqual(prices.table(), {}) + + def test_reading_the_list_never_fetches(self): + prices.update(fetch=self.fetch()) + with mock.patch("urllib.request.urlopen", side_effect=AssertionError("the network was asked")): + self.assertEqual(len(prices.table()), 3) + prices.state() + prices.given(["model-a"], prices.table()) + + +class Update(Profile): + def test_the_list_is_kept_per_million_tokens_with_its_source_and_day(self): + got = prices.update(fetch=self.fetch(), today="2026-11-10") + self.assertEqual(got, {"n": 3, "from": prices.SOURCE, "on": "2026-11-10"}) + self.assertEqual(prices.table(), { + "model-a": {"input": 2.5, "output": 10.0, "cached": 1.25, "max_output": 16384}, + "model-b": {"input": 3.0, "output": 15.0, "cached": 0.3, "cache_write": 3.75, "max_output": 128000}, + "embedder": {"input": 0.02, "output": 0.0}}) + self.assertEqual(prices.path(), os.path.join(self.tmp, "config", "prices.json")) + + def test_a_list_that_is_not_one_changes_nothing(self): + prices.update(fetch=self.fetch(), today="2026-11-10") + before = self.read() + for bad in (b"", b"[]", json.dumps({"x": {"mode": "chat"}}).encode()): + with self.assertRaises(ValueError): + prices.update(fetch=lambda source, bad=bad: bad) + with self.assertRaises(ValueError): + prices.update(fetch=mock.Mock(side_effect=OSError("no route"))) + self.assertEqual(self.read(), before) + + def test_a_file_is_a_source_and_only_https_is_fetched(self): + listing = os.path.join(self.tmp, "listing.json") + with open(listing, "w") as f: + json.dump(LISTING, f) + self.assertEqual(prices.update(listing)["n"], 3) + with mock.patch("urllib.request.urlopen", side_effect=AssertionError("fetched")): + for source in ("http://example.invalid/prices.json", "ftp://example.invalid/prices.json"): + with self.assertRaises(ValueError) as e: + prices.update(source) + self.assertEqual(str(e.exception).count(source), 1) # the address once, not twice + with self.assertRaises(ValueError) as e: + prices.update(os.path.join(self.tmp, "none.json")) + self.assertEqual(str(e.exception).count("none.json"), 1) + + def test_the_fetch_says_nothing_of_the_user(self): + seen = [] + + class Reply(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def urlopen(req, timeout=None): + seen.append(req) + return Reply(json.dumps(LISTING).encode()) + with mock.patch("urllib.request.urlopen", urlopen): + prices.update() + self.assertEqual(seen[0].full_url, prices.SOURCE) + self.assertEqual(dict(seen[0].header_items()), {"User-agent": "nullius"}) + self.assertIsNone(seen[0].data) + + +class Own(Profile): + def test_the_users_own_price_wins_and_survives_an_update(self): + prices.update(fetch=self.fetch()) + prices.set_own("model-a", prices.row(1, 2)) + prices.set_own("my-gateway/model", prices.row("0.5", "1.5", cached="0.05", max_output="4096")) + prices.update(fetch=self.fetch()) + t = prices.table() + self.assertEqual(t["model-a"], {"input": 1.0, "output": 2.0}) + self.assertEqual(t["my-gateway/model"], {"input": 0.5, "output": 1.5, "cached": 0.05, "max_output": 4096}) + self.assertTrue(prices.remove_own("model-a")) + self.assertFalse(prices.remove_own("model-a")) + self.assertEqual(prices.table()["model-a"]["input"], 2.5) # the list's again + + def test_a_row_is_checked(self): + for bad in ((-1, 2), ("x", 2), (1, None), (float("inf"), 1), (float("nan"), 1)): + with self.assertRaises((ValueError, TypeError), msg=bad): + prices.set_own("m", prices.row(*bad)) + with self.assertRaises(ValueError): + prices.row(1, 2, max_output=0) + with self.assertRaises(ValueError): + prices.set_own(" ", prices.row(1, 2)) + + def test_a_runs_price_is_model_input_output_and_optionally_cached(self): + self.assertEqual(prices.parse("m=1,2"), ("m", {"input": 1.0, "output": 2.0})) + self.assertEqual(prices.parse("org/m:free = 0.5, 1.5, 0.05"), ("org/m:free", {"input": 0.5, "output": 1.5, "cached": 0.05})) + for bad in ("m", "m=1", "=1,2", "m=1,2,3,4", "m=1,x", "m=1,", "m=-1,2"): + with self.assertRaises(ValueError, msg=bad): + prices.parse(bad) + + def test_another_file_is_named_by_the_environment(self): + other = os.path.join(self.tmp, "sent", "prices.json") + with mock.patch.dict(os.environ, {"NULLIUS_PRICES": other}): + prices.set_own("m", prices.row(1, 2)) + self.assertEqual(prices.table(), {"m": {"input": 1.0, "output": 2.0}}) + self.assertTrue(os.path.isfile(other)) + self.assertEqual(prices.table(), {}) + + +class Exactly(Profile): + def test_only_the_models_named_are_given_and_only_by_their_exact_name(self): + t = {"model-a": {"input": 1.0, "output": 2.0}, "model-a-mini": {"input": 0.1, "output": 0.2}} + self.assertEqual(prices.given(["model-a", None, "", "model-a", "model-a-2026"], t), {"model-a": t["model-a"]}) + self.assertEqual(prices.given(["model-a"], None), {}) + + def test_a_run_is_priced_for_every_model_its_relays_send(self): + r = run.Run.__new__(run.Run) + r.model, r.judge_model, r.simulator_model, r.embed_model = "m", "judge", None, "embedder" + r.other = {"model": {"side_models": ["side", "asked=sent", "m"]}} + self.assertEqual(r.models_called(), ["m", "judge", "embedder", "side", "sent"]) + + +class Cli(Profile): + def say(self, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = prices.main(list(argv)) + return code, out.getvalue() + + def test_set_list_and_remove(self): + self.assertEqual(self.say("set", "m", "1", "2", "--cached", "0.1")[0], 0) + code, out = self.say("list", "m", "unknown") + self.assertEqual(code, 0) + self.assertIn("list: none yet - nullius prices update", out) + self.assertIn("own m: input 1, output 2, cached 0.1", out) + self.assertIn("unknown: no price", out) + got = json.loads(self.say("list", "m", "--json")[1]) + self.assertEqual((got["n"], got["models"]), (0, {"m": {"input": 1.0, "output": 2.0, "cached": 0.1}})) + self.assertEqual(self.say("remove", "m")[0], 0) + self.assertEqual(prices.table(), {}) + + def test_an_update_from_a_file_says_how_many(self): + listing = os.path.join(self.tmp, "listing.json") + with open(listing, "w") as f: + json.dump(LISTING, f) + code, out = self.say("update", "--from", listing) + self.assertEqual((code, out.strip()), (0, f"3 models priced, from {listing}")) + self.assertIn(f"list: 3 models, from {listing} on ", self.say()[1]) + + def test_a_bad_price_is_refused_with_its_reason(self): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit) as e: + prices.main(["set", "m", "free", "2"]) + self.assertEqual(e.exception.code, 1) + self.assertIn("input: a price in USD per million tokens", err.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index e275d6a..da11793 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -98,7 +98,6 @@ def test_an_embeddings_reply_is_recorded_by_its_usage_never_its_vectors(self): got = relay.read_reply("/embeddings", body, False) self.assertEqual((got["embeddings"], got["usage"]["input"]), (3, 1000)) self.assertNotIn("0.1", json.dumps(got)) - self.assertIn("text-embedding-3-large", run.PRICES) class Refused(unittest.TestCase): diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index a01ca91..3d79e5a 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -93,6 +93,15 @@ def test_plain_reply_passes_through_and_the_real_key_goes_out(self): self.assertAlmostEqual(rec["cost_usd"], (6 * 1.0 + 4 * 0.1 + 2 * 10.0) / 1e6) self.assertEqual(rec["cost_from"], "estimate") + def test_a_model_is_priced_by_exactly_its_name(self): + """``m`` has a price; a model whose name only starts with it is another model, and has no cost.""" + status, _ = self.post("u1x", "/chat/completions", {"model": "m-large", "messages": [{"role": "user", "content": "hi"}]}) + self.assertEqual(status, 200) + rec = self.relay.calls("u1x")[-1] + self.assertEqual(rec["model"], "m-large") + self.assertNotIn("cost_usd", rec) + self.assertNotIn("cost_from", rec) + def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): status, body = self.post("u2", "/chat/completions", {"model": "m", "stream": True, "messages": []}) self.assertEqual(body, CHAT_SSE) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 479ec4b..c9856e8 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -254,6 +254,52 @@ def test_an_agent_run_reads_back_with_its_outcomes(self): self.assert_no_key(job_dir) self.assert_unit_relays(job_dir) + def relay_calls(self, job_dir): + out = [] + for root, _, files in os.walk(job_dir): + for n in files: + if n.endswith(".jsonl") and os.path.basename(root) == "relay": + with open(os.path.join(root, n)) as f: + out += [json.loads(line) for line in f if line.strip()] + return out + + def test_a_run_given_a_price_estimates_its_calls_and_records_the_price(self): + price = {"input": 1.0, "output": 10.0} + job_dir = self.go("fakebench", "fake", prices={"m-1": price, "another": {"input": 9.0, "output": 9.0}}) + calls = self.relay_calls(job_dir) + self.assertTrue(calls) + for c in calls: + self.assertAlmostEqual(c["cost_usd"], (11 * 1.0 + 3 * 10.0) / 1e6) + self.assertEqual(c["cost_from"], "estimate") + with open(os.path.join(job_dir, "config.json")) as f: + # the models it calls, no other; the row is this run's, so no list is credited with it + self.assertEqual(json.load(f)["standard"]["prices"], {"models": {"m-1": {**price, "source": "run"}}}) + + def test_the_record_says_whose_each_price_is_and_names_the_list_only_for_its_own(self): + listed, own = {"input": 1.0, "output": 10.0}, {"input": 2.0, "output": 4.0} + held = os.path.join(self.tmp, "prices.json") + + def recorded(doc): + with open(held, "w") as f: + json.dump({"from": "https://example.invalid/list.json", "on": "2026-11-10", **doc}, f) + with mock.patch.dict(os.environ, {"NULLIUS_PRICES": held}): + job_dir = self.go("fakebench", "fake") + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"]["prices"] + + self.assertEqual(recorded({"models": {"m-1": listed}}), + {"from": "https://example.invalid/list.json", "on": "2026-11-10", "models": {"m-1": {**listed, "source": "list"}}}) + # the user's own row over the list's: the list did not say this price, so it is not named + self.assertEqual(recorded({"models": {"m-1": listed}, "own": {"m-1": own}}), {"models": {"m-1": {**own, "source": "own"}}}) + + def test_a_run_with_no_price_has_no_cost_and_says_nothing_of_prices(self): + job_dir = self.go("fakebench", "fake") + calls = self.relay_calls(job_dir) + self.assertTrue(calls) + self.assertFalse(any("cost_usd" in c for c in calls)) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertNotIn("prices", json.load(f)["standard"]) + def test_the_trajectory_carries_the_prompt_the_tool_call_and_the_answer(self): job_dir = self.go("fakebench", "fake", limit=1) trial = [d for d in os.listdir(job_dir) if d.startswith("good__")][0] diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index 3f6ddd6..d328c65 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -5,6 +5,8 @@ that refuses before anything is written.""" from __future__ import annotations +import html as html_text +import json import os import re import shutil @@ -14,7 +16,7 @@ from pathlib import Path from unittest.mock import patch -from nullius import hosts, sandbox, settings +from nullius import hosts, prices, sandbox, settings from .test_routes import web @@ -434,6 +436,39 @@ def test_images_are_removed_per_benchmark_only_from_this_machines_browser(self): environ_base={"REMOTE_ADDR": "10.0.0.7"}).status_code, 403) + def test_the_price_list_is_updated_and_priced_only_from_this_machines_browser(self): + self.stack.enter_context(patch.dict(os.environ, {"NULLIUS_PRICES": str(self.tmp / "prices.json")})) + page = self.page() + section = page[page.index('id="prices"'):page.index('id="general"')] + self.assertNotIn('name="value:', section) # outside Save: nothing of it lands in the file + state = json.loads(html_text.unescape(re.search(r"data-state='([^']*)'", section).group(1))) + self.assertEqual(state, {"n": 0, "from": "", "on": "", "own": {}}) # nothing ships with Nullius + headers = {"Origin": "http://localhost", "X-CSRF-Token": self.token()} + listing = json.dumps({"model-a": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1e-05}}).encode() + with patch.object(prices, "_fetch", lambda source: listing): + r = self.client.post("/settings/prices/update", headers=headers); self.addCleanup(r.close) + self.assertEqual((r.status_code, r.json["ok"], r.json["n"], r.json["from"]), (200, True, 1, prices.SOURCE)) + self.assertEqual(prices.table(), {"model-a": {"input": 2.5, "output": 10.0}}) + with patch.object(prices, "_fetch", lambda source: b"not a list"): + r = self.client.post("/settings/prices/update", headers=headers); self.addCleanup(r.close) + self.assertEqual((r.status_code, r.json["ok"]), (502, False)) + self.assertEqual(prices.state()["n"], 1) # the list held stays + r = self.client.post("/settings/prices/set", data={"model": "model-a", "input": "1", "output": "2", "cached": ""}, headers=headers); self.addCleanup(r.close) + self.assertEqual(r.json["own"], {"model-a": {"input": 1.0, "output": 2.0}}) + self.assertEqual(prices.table()["model-a"]["input"], 1.0) # the user's own, over the list + for bad in ({"model": "m", "input": "free", "output": "2"}, {"model": "m", "input": "1"}, {"model": "", "input": "1", "output": "2"}): + r = self.client.post("/settings/prices/set", data=bad, headers=headers); self.addCleanup(r.close) + self.assertEqual((r.status_code, r.json["ok"]), (400, False), bad) + r = self.client.post("/settings/prices/remove", data={"model": "model-a"}, headers=headers); self.addCleanup(r.close) + self.assertEqual(r.json["own"], {}) + self.assertEqual(self.client.post("/settings/prices/remove", data={"model": "model-a"}, headers=headers).status_code, 404) + # the same guards as Save: token, origin, loopback + for route in ("update", "set", "remove"): + for kw in ({"headers": {"Origin": "http://localhost"}}, {"headers": headers, "environ_base": {"REMOTE_ADDR": "10.0.0.7"}}): + self.assertEqual(self.client.post(f"/settings/prices/{route}", data={"model": "x", "input": "1", "output": "2"}, **kw).status_code, 403) + self.assertIsNone(self.file()) + + class StartUp(unittest.TestCase): """app.py's command line: the address and port come from APP_HOST and APP_PORT (app/.env is loaded before it is read), the flags win, and a diff --git a/app/web/settings.py b/app/web/settings.py index 1f3d4d5..c00ca8b 100644 --- a/app/web/settings.py +++ b/app/web/settings.py @@ -3,7 +3,7 @@ import os from flask import abort, flash, jsonify, redirect, render_template, request, url_for -from nullius import execution, hosts, paths, sandbox, settings +from nullius import execution, hosts, paths, prices, sandbox, settings from nullius.standard import container from nullius.judge import client @@ -30,9 +30,11 @@ def settings_page(): env_file=shared.ENV_FILE, storage_paths=[("Settings", shared.ENV_FILE), ("User catalogue", paths.catalogue_dir()), ("Workspaces", paths.workspaces_dir()), ("Runs", paths.runs_dir()), - ("Benchmark licences", paths.secrets_dir()), ("Cache", paths.cache_dir())], + ("Benchmark licences", paths.secrets_dir()), ("Model prices", paths.prices_file()), + ("Cache", paths.cache_dir())], unread=[dict(u, inherited=u["name"] not in withheld, startup=u["instead"] in settings.STARTUP) for u in settings.unread(shared.ENV_FILE)], + prices=prices.state(), ssh_hosts=hosts.scan_ssh_config(), judges=(client.system_one(os.environ), client.llm_judge(os.environ)), llm_providers=client.LLM_PROVIDERS, s1_providers=client.SYSTEM_ONE_PROVIDERS, agent_keys=settings.agent_key_names()) @@ -150,6 +152,38 @@ def settings_sandbox_remove(): return jsonify(got), (200 if got["ok"] else 409) +@app.route("/settings/prices/update", methods=["POST"]) +def settings_prices_update(): + """The public price list fetched and kept (``nullius.prices``): the one + fetch, asked for here and never by a run. Same guards as Save.""" + _sandbox_guard("Prices are updated") + try: + prices.update() + except ValueError as e: + return jsonify(ok=False, error=str(e)), 502 + return jsonify(ok=True, **prices.state()) + + +@app.route("/settings/prices/set", methods=["POST"]) +def settings_prices_set(): + """A price of the user's own for one model, over the list.""" + _sandbox_guard("Prices are set") + f = request.form + try: + prices.set_own(f.get("model", ""), prices.row(f.get("input") or None, f.get("output") or None, f.get("cached") or None)) + except (ValueError, TypeError) as e: + return jsonify(ok=False, error=str(e)), 400 + return jsonify(ok=True, **prices.state()) + + +@app.route("/settings/prices/remove", methods=["POST"]) +def settings_prices_remove(): + _sandbox_guard("Prices are removed") + if not prices.remove_own(request.form.get("model", "")): + return jsonify(ok=False, error="not one of your own prices"), 404 + return jsonify(ok=True, **prices.state()) + + def _settings_return(): return url_for("settings_page", embedded="1") if request.form.get("embedded") == "1" else url_for("settings_page") diff --git a/docs/cli/README.md b/docs/cli/README.md index 95f073c..d8b045e 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -24,6 +24,7 @@ also works. | [Fidelity](fidelity.md) | `nullius standard fidelity` | | [Shard and merge](shard-and-merge.md) | `nullius standard run --shard`, `merge` | | [Storage](../settings/README.md#storage) | `nullius paths [--json]` | +| [Model prices](../settings/model-prices.md) | `nullius prices [list\|update\|set\|remove]` | | [Migration](../getting-started.md#migrate-an-existing-checkout) | `nullius migrate --from CHECKOUT [--secrets PATH] [--apply]` | | [Pages from a shell](pages.md) | | | [Exit codes and output](exit-codes.md) | | diff --git a/docs/cli/run.md b/docs/cli/run.md index 71146ad..7b8c06b 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -5,6 +5,7 @@ One benchmark and one agent, capability, control or setup, into a job. ```bash title="Run" nullius standard run [@version] [--model M] [--dataset D] [--limit N] [-j N] \ [--unsandboxed | --backend docker] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ + [--price MODEL=INPUT,OUTPUT[,CACHED]]... \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` @@ -26,6 +27,7 @@ nullius standard run locomo model --model gpt-4o-mini | `--backend` | `local` (default) or `docker`, each unit in a container with its relay | | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | +| `--price MODEL=INPUT,OUTPUT[,CACHED]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | | `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | | `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--repeat`); by default they stay for the next run, listed and removed in Settings, [Sandbox](../settings/sandbox.md) | | `--judge-model` | Instead of the official judge | diff --git a/docs/field-guide.md b/docs/field-guide.md index c2fc57b..c0fede5 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -163,7 +163,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, the unit's setup, the benchmark's driver and its scorer. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | The cached part of the prompt tokens: a subset, never added to them. | measured | any | -| `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from a price table. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | +| `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | | `n_steps / n_tool_calls` | Trajectory steps and tool calls; empty placeholder calls are not counted. | measured | any | | `exception_type` | What ended the trial, if anything. It decides the infra / agent split. | measured | any | | `n_rate_limited_retries` | 429s counted in `agent/gateway.log`: on Harbor, the ones the agent's SDK retried and survived; on a standard run, every model call the relay saw refused with 429. No log reads *not recorded*, never 0. | inferred | any | @@ -237,7 +237,7 @@ Eleven classes. The three marked **yes** under Infrastructure leave the adjusted ## Caveats {#caveats} -- **Cost is an estimate, and only where one was recorded.** A Harbor agent prices its own usage; a standard run's relay keeps the provider's figure when the reply carries one and estimates otherwise. No bill is read. A trial without a cost shows —, and a run's total covers only the trials that have one. +- **Cost is an estimate, and only where one was recorded.** A Harbor agent prices its own usage; a standard run's relay keeps the provider's figure when the reply carries one and otherwise estimates from the user's price list, when it prices that exact model. No bill is read. A trial without a cost shows —, and a run's total covers only the trials that have one. - **The infra/agent split is an explicit list.** Which exceptions leave the adjusted denominator and which count against the agent is pinned in `classify.py` and tested against Harbor's own classes; anything else is *Unknown*. diff --git a/docs/settings/README.md b/docs/settings/README.md index 773558e..af155a7 100644 --- a/docs/settings/README.md +++ b/docs/settings/README.md @@ -9,6 +9,7 @@ results open without credentials. Changes save automatically. | [System One provider](system-one.md) | The System One judgement | | [LLM judge provider](llm-judge.md) | The second judge | | [Agent keys](agent-keys.md) | A Harbor agent's vendor credentials | +| [Model prices](model-prices.md) | What a run's cost is estimated from | | [Run hosts](run-hosts.md) | SSH machines | | [Sandbox](sandbox.md) | Docker: its Docker Hub login and the images runs keep | | [General](other.md) | Tour and checklist; start-up options remain in the environment | @@ -46,6 +47,7 @@ and job roots also have command-line options. The app does not load `app/.env`. | `RUNS_DIR`, `HARBOR_PINS` | `runs/` in the user data directory; `harbor-pins/` beside the runs | | `STANDARD_DIR`, `STANDARD_DATA_DIR`, `SECRETS_DIR` | `standard/` in the data directory; `datasets/` inside it; `secrets/` in the configuration directory | | `NULLIUS_CATALOGUE` | `catalogue/` in the user data directory; whole-entry overrides of the bundled catalogue | +| `NULLIUS_PRICES` | `prices.json` in the configuration directory; the [model price list](model-prices.md) | | `DEBUG` | Development debugger and reloader; off by default, enabled by `1` | ## Save @@ -71,6 +73,7 @@ directory and installation: | --- | --- | | Saved settings | `~/.config/nullius/settings.json` | | Benchmark licences | `~/.config/nullius/secrets//` | +| Model prices | `~/.config/nullius/prices.json` | | User catalogue | `~/.local/share/nullius/catalogue/` | | Workspaces | `~/.local/share/nullius/workspaces/` | | Runs started from Review | `~/.local/share/nullius/runs/` | diff --git a/docs/settings/model-prices.md b/docs/settings/model-prices.md new file mode 100644 index 0000000..5dec9ed --- /dev/null +++ b/docs/settings/model-prices.md @@ -0,0 +1,37 @@ +# Model prices + +What a run's cost is estimated from, in USD per million tokens. Under +**Evaluation**. Nullius ships no price: a new profile shows no estimate until +its list is filled once. + +| Card | Does | +| --- | --- | +| Price list | How many models are priced and the day the list was fetched; **Update** fetches it again | +| Your prices | A model's price of your own: input, output and optionally cached input; **Remove** takes one out | + +| Rule | | +| --- | --- | +| The list | LiteLLM's public `model_prices_and_context_window.json`, keyed by the model names the providers themselves use, kept whole with its address and day | +| Fetched | Only when **Update** or `nullius prices update` asks; a run reads the saved list and never the network | +| Your prices | Win over the list: a model it lacks, a negotiated price, a model served under your own name. An update leaves them as they are | +| One run | `--price MODEL=INPUT,OUTPUT[,CACHED]` on [`nullius standard run`](../cli/run.md) prices that run only, over both | +| A call's cost | The provider's own figure when its reply carries one (OpenRouter's does); else the price of exactly that model name; else none, shown as — | +| Exact names | `gpt-4.1-nano` is not priced as `gpt-4.1`: a model whose name only starts like another's is another model | +| Recorded | A job's `config.json` keeps the prices of the models it calls, each with its source (`list`, `own`, or `run` for `--price`), and where and when the list was fetched when a price is the list's | +| Docker and hosts | A unit's relay in a container gets its models' prices from the run; a run on a host takes this machine's list with it | +| An estimate | List prices, not a bill: [Metrics](../field-guide.md) | +| Code | `app/nullius/prices.py` | + +## From a shell + +```bash +nullius prices # the list's source and size, and your own prices +nullius prices list gpt-4o --json # those models' prices +nullius prices update # fetch the public list +nullius prices update --from FILE # another list in the same format: a mirror, a saved copy +nullius prices set my-model 1.5 6 --cached 0.15 +nullius prices remove my-model +``` + +The list is `prices.json` in the configuration directory +([Storage](README.md#storage)); `NULLIUS_PRICES` names another file.