fix(standard): give the relay's container its key as a file

.claude/rules/running.md

@@ -16,8 +16,9 @@ through the execution backend (`docs/builder.md`).
   are included; configuration and credential stores are excluded. Several
   machines run one shard each.
 - **Keys**: a run gets only its provider's key and endpoint, as environment
-  (over stdin to a host), never written; logs are redacted. The standard's
-  agents and upstream scripts get only the relay's word, never a key.
+  (over stdin to a host, and the key to a docker relay as a file); logs are
+  redacted. The standard's agents and upstream scripts get only the relay's
+  word, never a key.
 - What a recipe needs of each machine is asked before Run and again as it
   starts (`nullius/prerequisites.py`). `runs/<name>/run.json` lets a
   restarted Nullius pick a run up again.

app/nullius/execution.py

@@ -78,9 +78,9 @@
 run without that endpoint fails before it starts.
 They are read on this machine and handed to the run as environment
 (API_TOKEN, OPENAI_API_KEY, LLM_API_KEY; API_URL, OPENAI_BASE_URL,
-LLM_BASE_URL) - over stdin to the launcher, never in argv, never written
-anywhere; a stdin cut short starts nothing and fails the run at that part
-(``_launch``), and one that holds a line break is never sent
+LLM_BASE_URL) - over stdin to the launcher, never in argv (the key to a
+docker relay as a file); a stdin cut short starts nothing and fails the
+run at that part (``_launch``), and one that holds a line break is never sent
 (``_line_break``) - and they are the only credentials it has: its commands get this
 process's environment less every secret in Settings, whichever provider it
 belongs to (``withheld``); a host gets nothing of it but stdin.

app/nullius/standard/container.py

@@ -32,12 +32,13 @@
 bridge *and* on that network as ``relay``; the agent's container on the
 internal network alone. Its model endpoint is ``http://relay:8080/<tag>/v1``
 and nothing else answers it. The real key reaches the relay's container
-only, as ``-e NULLIUS_RELAY_UPSTREAM_KEY`` with the value in the docker
-program's own environment - never on an argument list, which ``ps`` and
-``docker inspect`` show. The relay publishes its port on 127.0.0.1 so the
-driver, which stays on this machine (it owns the answers), reaches the
-judge's route and a capability benchmark's reader. Its record folder is mounted
-read-write, so the host reads each call as the local relay's are read.
+only, as an owner-only file under :func:`key_root`, mounted read-only and
+removed from this machine once the relay listens: never on an argument list
+(``ps``) or in the container's environment (``docker inspect``). The relay
+publishes its port on 127.0.0.1 so the driver, which stays on this machine
+(it owns the answers), reaches the judge's route and a capability
+benchmark's reader. Its record folder is mounted read-write, so the host
+reads each call as the local relay's are read.
 
 **A licence server is one more door.** A unit whose environment needs a
 licence of kind ``server`` (``host:port``) gets a forwarder beside its relay
@@ -102,6 +103,7 @@
 import urllib.request
 from typing import Dict, Iterable, List, Optional, Tuple
 
+from nullius import paths
 from nullius.standard import definitions, install
 
 DOCKER = "docker"
@@ -114,6 +116,8 @@
 EGRESS_IN = "/opt/nullius/egress.py"       # the check's proxy's one file, in its container
 EGRESS_PORT = 3128
 EGRESS_ALIAS = "egress"
+RELAY_KEY_IN = "/opt/nullius/relay-key"    # the key's one file, in the relay's container
+KEY_STALE = 600                            # older than any start can take: a killed run left it
 IMAGE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
 NODE_VERSION = "22.23.3"
@@ -465,20 +469,23 @@ def install_argv(image: str, where: str, cmd: List[str]) -> List[str]:
             "-e", f"HOME={where}/.npm-home", "-e", f"npm_config_cache={where}/.npm-cache", image] + list(cmd)
 
 
-def relay_argv(image: str, *, name: str, job: str, app: str, record: str, base: str, model: Optional[str],
-               routes: Dict[str, str], prices: Optional[dict] = None, side_models=(),
+def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str,
+               model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(),
                key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None) -> List[str]:
     """``docker run`` for a unit's relay: on the default bridge (its way to the
-    provider), its port published on 127.0.0.1 only. The key is named, never
-    given: ``-e NULLIUS_RELAY_UPSTREAM_KEY``. ``hosts``: names it resolves
+    provider), its port published on 127.0.0.1 only. The key is never given:
+    ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with
+    ``--mount``, which fails when the engine cannot see it (``-v`` would mount
+    an empty folder). ``hosts``: names it resolves
     (``--add-host``), the machine's own for an endpoint on its loopback."""
     a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", "--user", _user(),
          "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--init",
          *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")],
-         "-p", f"127.0.0.1::{RELAY_PORT}", "-v", f"{app}:{app}:ro", "-v", f"{record}:{record}:rw", "-w", app,
-         "-e", "PYTHONPATH=" + app, "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "NULLIUS_RELAY_UPSTREAM_KEY",
+         "-p", f"127.0.0.1::{RELAY_PORT}", "-v", f"{app}:{app}:ro", "-v", f"{record}:{record}:rw",
+         "--mount", f"type=bind,source={key_file},target={RELAY_KEY_IN},readonly", "-w", app,
+         "-e", "PYTHONPATH=" + app, "-e", "PYTHONDONTWRITEBYTECODE=1",
          image, "python3", "-m", "nullius.standard.relay", "serve", "--listen", f"0.0.0.0:{RELAY_PORT}",
-         "--base", base, "--record", record, "--public", f"http://{RELAY_ALIAS}:{RELAY_PORT}"]
+         "--base", base, "--record", record, "--key-file", RELAY_KEY_IN, "--public", f"http://{RELAY_ALIAS}:{RELAY_PORT}"]
     if model:
         a += ["--model", model]
     for tag, m in sorted(routes.items()):
@@ -657,19 +664,22 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str,
         self.host: Optional[str] = None
         self.started: List[str] = []           # containers to remove
         self.network = False
+        self.key_dir: Optional[str] = None     # the key file's folder, until the relay listens
         os.makedirs(record_dir, exist_ok=True)
 
     def start(self) -> "Unit":
         try:
             _docker(["network", "create", "--internal", "--label", f"{LABEL}={self.job}", self.names["network"]])
             self.network = True
             argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir,
-                              base=self.base, model=self.model, routes=self.routes, prices=self.prices,
-                              side_models=self.side_models, key_header=self.key_header, hosts=self.hosts)
+                              key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes,
+                              prices=self.prices, side_models=self.side_models, key_header=self.key_header,
+                              hosts=self.hosts)
             self.started.append(self.names["relay"])
-            _docker(argv[1:], env=client_env({"NULLIUS_RELAY_UPSTREAM_KEY": self.key}))
+            _docker(argv[1:])
             _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]])
             self._wait(self.names["relay"], "the relay's container")
+            self._forget_key()
             port = _docker(["port", self.names["relay"], f"{RELAY_PORT}/tcp"]).stdout.strip().splitlines()[0]
             self.host = "http://" + port.replace("0.0.0.0", "127.0.0.1")
             self.reach = {}
@@ -682,6 +692,24 @@ def start(self) -> "Unit":
             raise
         return self
 
+    def _key_file(self) -> str:
+        """Write the key to an owner-only file in a new folder under
+        :func:`key_root`, and return its path. Not the record folder: it is kept."""
+        root = key_root()
+        os.makedirs(root, exist_ok=True)
+        os.chmod(root, 0o700)
+        forget_stale_keys()
+        self.key_dir = tempfile.mkdtemp(prefix=key_prefix(self.job), dir=root)
+        path = os.path.join(self.key_dir, "key")
+        with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as f:
+            f.write(self.key)
+        return path
+
+    def _forget_key(self):
+        if self.key_dir:
+            shutil.rmtree(self.key_dir, ignore_errors=True)
+            self.key_dir = None
+
     def _forward(self, i: int, address: str) -> str:
         """A licence server's forwarder, started and on the unit's network
         under the server's name: the address the agent reaches it by. It joins
@@ -746,8 +774,9 @@ def wrap(self, cmd: List[str], env: dict, workdir: str, mounted: List[Tuple[str,
         return argv, client_env({k: v for k, v in inside.items() if k != "PATH"})
 
     def stop(self):
-        """Remove the unit's containers and network; the relay's log is kept
-        beside its record first."""
+        """Remove the unit's containers and network, and the key's file if the
+        relay has not read it; the relay's log is kept beside its record first."""
+        self._forget_key()
         if self.names["relay"] in self.started:
             try:
                 r = _docker(["logs", self.names["relay"]], check=False, timeout=30)
@@ -1369,8 +1398,34 @@ def stop(self):
             self.own_net = False
 
 
+def key_root() -> str:
+    """The relays' key folders, in the user's cache: Colima and Lima share the
+    home with their engine, not the temporary folder."""
+    return os.path.join(paths.cache_dir(), "relay-keys")
+
+
+def forget_stale_keys():
+    """Remove every job's key folders older than ``KEY_STALE``."""
+    old = time.time() - KEY_STALE
+    for d in glob.glob(os.path.join(glob.escape(key_root()), "*")):
+        try:
+            if os.lstat(d).st_mtime < old:
+                shutil.rmtree(d, ignore_errors=True)
+        except OSError:
+            pass
+
+
+def key_prefix(job: str) -> str:
+    """A job's key folder prefix: a hash, so no job's prefix starts another's."""
+    return f"nullius-relay-key-{hashlib.sha256(job.encode()).hexdigest()[:12]}-"
+
+
 def sweep(job: str):
-    """Remove every container and network a job's units left, by its label."""
+    """Remove every container and network a job's units left, by its label,
+    its relays' key folders, and stale key folders."""
+    for d in glob.glob(os.path.join(glob.escape(key_root()), glob.escape(key_prefix(job)) + "*")):
+        shutil.rmtree(d, ignore_errors=True)
+    forget_stale_keys()
     try:
         ids = _docker(["ps", "-aq", "--filter", f"label={LABEL}={job}"], check=False).stdout.split()
         if ids:

app/nullius/standard/relay.py

@@ -570,36 +570,39 @@ def failure(calls: List[dict]) -> Optional[tuple]:
 
 
 # -- the relay as a process of its own: a container backend's sidecar ----------
-KEY_VAR = "NULLIUS_RELAY_UPSTREAM_KEY"
-
-
 def serve(argv=None) -> int:
     """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL
-    --record DIR [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME]
+    --record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME]
     [--public URL] [--prices JSON]``.
 
     ``--base`` is the provider's endpoint, ``--route`` a route sent as another
     model (a benchmark's judge), ``--public`` the address ``url`` hands out.
-    The real key is read from ``NULLIUS_RELAY_UPSTREAM_KEY`` and nowhere else:
-    an argument is visible to anyone who lists processes or inspects a
-    container. One JSON line, ``{"listening": "http://host:port"}``, says it
+    The real key is read from ``--key-file`` and nowhere else: an argument or
+    an environment variable is visible to anyone who lists processes or
+    inspects a container. One JSON line, ``{"listening": "http://host:port"}``, says it
     is ready; SIGTERM (or SIGINT) stops it."""
     import argparse
     import signal
     ap = argparse.ArgumentParser(prog="nullius.standard.relay serve")
     ap.add_argument("--listen", default="127.0.0.1:0", help="HOST:PORT")
     ap.add_argument("--base", required=True, help="the provider's endpoint")
     ap.add_argument("--record", required=True, help="the folder each route's calls are recorded in")
+    ap.add_argument("--key-file", required=True, help="the file that holds the provider's key")
     ap.add_argument("--model", help="every call sent as this model")
     ap.add_argument("--route", action="append", default=[], help="TAG=MODEL")
     ap.add_argument("--side-model", action="append", default=[], help="a model the agent may call as asked")
     ap.add_argument("--key-header", default="authorization", choices=("authorization", "x-api-key"))
     ap.add_argument("--public", help="the address url() hands out, when not the listening one")
     ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate")
     a = ap.parse_args(argv)
-    key = os.environ.get(KEY_VAR)
+    try:
+        with open(a.key_file) as f:
+            key = f.read().strip()
+    except OSError as e:
+        print(json.dumps({"error": f"cannot read --key-file: {e}"}), flush=True)
+        return 2
     if not key:
-        print(json.dumps({"error": f"{KEY_VAR} is not set"}), flush=True)
+        print(json.dumps({"error": f"--key-file {a.key_file} contains no key"}), flush=True)
         return 2
     host, _, port = a.listen.rpartition(":")
     r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model,
@@ -625,5 +628,5 @@ def serve(argv=None) -> int:
 if __name__ == "__main__":
     import sys
     if sys.argv[1:2] != ["serve"]:
-        sys.exit("python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR [--model M]")
+        sys.exit("python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M]")
     sys.exit(serve(sys.argv[2:]))