From d2331f7bef1448ecd67b312932249d5c2524ecb8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 09:30:59 -0800 Subject: [PATCH 1/4] fix(computers): the session's supervisord logs to the console again, so a disk form's boot check sees its desktop Since #649 root's supervisord runs the session's supervisord as a program, and wrote its output to a root-only file. A disk form's console then showed root's programs alone (clock, egress, session), so the release's boot check, which waits on the console for every program supervisord.conf declares, timed out after 900 s on every tag since v0.9.1522, though nothing failed to start. - root.conf sends the session's supervisord's output to root's stdout, the console, as when it was the computer's first process. - The boot check waits for root's programs as well as the session's. - A unit test pins the session program's console logging to what the check reads. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 10 +++--- .../artifacts/ubuntu-24.04/root.conf | 6 ++-- tst/unit/test_infra_build_script.py | 31 ++++++++++++++----- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 0937e4343..36ae8ef82 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -72,7 +72,7 @@ _LOG_TAIL_LINES = 30 # of a container that never turned healthy, in the error # How long a disk form may take to boot to its desktop: seconds with KVM, minutes under TCG. _BOOTED_WITHIN_SECONDS = 900 -# What supervisord prints on a disk form's console as each program starts. +# What supervisord prints on a disk form's console as each program starts, root's and the session's. _RUNNING = re.compile(r"success: (\S+) entered RUNNING") _PROGRAM = re.compile(r"^\[program:([^\]]+)\]", re.MULTILINE) _PRINTED_CHARS = 200 # of what docker printed in place of a manifest, in the error @@ -415,9 +415,11 @@ def _smoke(name: str, image: str, launcher: str | None) -> int: def _boots(name: str, image: str, launcher: str) -> int: """Boot ``image``, a build of the disk form ``name``, in ``launcher``, with KVM where it can, and wait for every - program its base computer's supervisord.conf declares to run, which its console shows.""" - programs = set(_PROGRAM.findall((REPO / Path(IMAGES[DISK_ARGS[name]["ROOTFS"]]).with_name("supervisord.conf")) - .read_text())) + program its base computer declares to run, which its console shows: root's supervisord's (root.conf) and the + session's it starts (supervisord.conf).""" + base = REPO / Path(IMAGES[DISK_ARGS[name]["ROOTFS"]]) + programs = {program for conf in ("root.conf", "supervisord.conf") + for program in _PROGRAM.findall(base.with_name(conf).read_text())} arch = subprocess.run(["docker", "image", "inspect", "--format", "{{.Architecture}}", image], capture_output=True, text=True, check=True).stdout.strip() machine = f"{image}-machine" diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf b/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf index 96faf63e5..f3b6a243f 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf @@ -1,7 +1,8 @@ ; Root's supervisord, which the base's init starts as root once it has made what the session must not touch: the clock ; follower and the egress proxy, each as its own user, on real time, and the session's supervisord (supervisord.conf) ; as the session user, which runs every session program. Its socket and logs are root's alone, so the session user can -; neither stop nor restart any of them. The follower follows nothing until core writes its config, then picks it up +; neither stop nor restart any of them. The session's supervisord logs to the console, as it did when it was the +; computer's first process, so a disk form's console shows each session program start. The follower follows nothing until core writes its config, then picks it up ; within a quarter second. The egress proxy logs a JSON line per connection. [supervisord] nodaemon=true @@ -52,5 +53,6 @@ priority=10 autorestart=true stopwaitsecs=30 redirect_stderr=true -stdout_logfile=/var/log/agentenv-root/session.log +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 environment=HOME="/home/user",USER="user",LOGNAME="user" diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index 5fa6b5020..4ded1f9b3 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -5,6 +5,7 @@ import ast import base64 +import configparser import json import subprocess import sys @@ -323,8 +324,9 @@ def test_the_service_db_smoke_waits_for_its_health_check_and_removes_the_contain (True, "x86_64", "under TCG")], ids=["kvm", "no-kvm", "another-arch"]) def test_a_disk_forms_smoke_boots_it_in_the_launcher_until_its_desktop_runs(script, monkeypatch, capsys, kvm, machine, how): - calls, logs = [], iter(["agentenv-init: starting", "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\n" - "success: desktop entered RUNNING"]) + console = "".join(f"success: {program} entered RUNNING\n" + for program in ("clock", "egress", "session", "xvfb", "dbus", "desktop")) + calls, logs = [], iter(["agentenv-init: starting", console]) def run(command, **kw): calls.append((command, kw.get("input"))) @@ -347,15 +349,19 @@ def run(command, **kw): assert f"booted to its desktop {how}" in capsys.readouterr().out -def test_a_disk_forms_smoke_waits_for_every_program_its_bases_supervisord_conf_declares(script, monkeypatch, capsys, - tmp_path): - conf = tmp_path / Path(script.IMAGES["agentenv-ubuntu-24.04"]).with_name("supervisord.conf") - conf.parent.mkdir(parents=True) - conf.write_text("[supervisord]\nnodaemon=true\n\n[program:xvfb]\ncommand=Xvfb\n\n[program:recorder]\ncommand=rec\n") +@pytest.mark.parametrize("missing", ["recorder", "session"]) +def test_a_disk_forms_smoke_waits_for_every_program_its_bases_root_and_session_supervisords_declare( + script, monkeypatch, capsys, tmp_path, missing): + base = tmp_path / Path(script.IMAGES["agentenv-ubuntu-24.04"]) + base.parent.mkdir(parents=True) + base.with_name("root.conf").write_text("[supervisord]\nnodaemon=true\n\n[program:session]\ncommand=supervisord\n") + base.with_name("supervisord.conf").write_text("[supervisord]\nnodaemon=true\n\n[program:xvfb]\ncommand=Xvfb\n\n" + "[program:recorder]\ncommand=rec\n") monkeypatch.setattr(script, "REPO", tmp_path) monkeypatch.setattr(script, "_BOOTED_WITHIN_SECONDS", 0.01) monkeypatch.setattr(script.time, "sleep", lambda seconds: None) - console = "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\nsuccess: desktop entered RUNNING\n" + console = "".join(f"success: {program} entered RUNNING\n" for program in ("session", "xvfb", "recorder") + if program != missing) monkeypatch.setattr(script.subprocess, "run", lambda command, **kw: subprocess.CompletedProcess( command, 0, {"image": "amd64\n", "run": "c1\n", "logs": console}.get(command[1], ""), "")) @@ -364,6 +370,15 @@ def test_a_disk_forms_smoke_waits_for_every_program_its_bases_supervisord_conf_d assert "::error::disk:1 didn't boot to its desktop within 0.01s" in capsys.readouterr().out +def test_the_sessions_supervisord_logs_to_the_console_where_a_disk_forms_smoke_reads_its_programs_start(script): + root = configparser.ConfigParser(interpolation=None) + root.read_string((script.REPO / Path(script.IMAGES["agentenv-ubuntu-24.04"]).with_name("root.conf")).read_text()) + + assert (root["program:session"]["stdout_logfile"], root["program:session"]["stdout_logfile_maxbytes"]) == ( + "/dev/stdout", "0") + assert root["supervisord"]["nodaemon"] == "true" + + def test_a_disk_forms_smoke_needs_the_launcher_to_boot_it_in(script, monkeypatch, capsys): with pytest.raises(SystemExit) as exited: _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1") From 05d4f083f19c1d4650f81b345d8d06f41c6509b8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 09:49:19 -0800 Subject: [PATCH 2/4] fix(computers): a disk form's guest has /dev/stdout, so the session's output reaches its console The disk's init mounted devtmpfs, which has no /dev/fd, /dev/stdin, /dev/stdout or /dev/stderr: a container runtime or udev makes those links. Root's supervisord then wrote the session's output to a plain file named /dev/stdout instead of the console. The init makes the four links before it starts the computer. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/disk/linux/boot/init | 6 ++++++ tst/unit/test_infra_build_script.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init index 44480b86e..d6cd873f2 100755 --- a/src/agent_env/computers/disk/linux/boot/init +++ b/src/agent_env/computers/disk/linux/boot/init @@ -64,6 +64,12 @@ mount -t devpts -o gid=5,mode=620,ptmxmode=666 devpts /newroot/dev/pts case "$AGENTENV_SHM_MB" in ''|0|*[!0-9]*) shm_mb=512 ;; *) shm_mb=$AGENTENV_SHM_MB ;; esac unset AGENTENV_SHM_MB mount -t tmpfs -o mode=1777,size="${shm_mb}m" shm /newroot/dev/shm +# The links udev or a container runtime makes, which programs name their standard streams by: the computer's root +# supervisord sends the session's output to the console through /dev/stdout, and a shell's <(...) reads /dev/fd. +ln -sf /proc/self/fd /newroot/dev/fd +ln -sf /proc/self/fd/0 /newroot/dev/stdin +ln -sf /proc/self/fd/1 /newroot/dev/stdout +ln -sf /proc/self/fd/2 /newroot/dev/stderr log "starting $* as $user ($uid:$gid) in $workdir" # Last, as busybox runs its applets through /proc/self/exe. mount --move /proc /newroot/proc diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index 4ded1f9b3..359084bb8 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -377,6 +377,8 @@ def test_the_sessions_supervisord_logs_to_the_console_where_a_disk_forms_smoke_r assert (root["program:session"]["stdout_logfile"], root["program:session"]["stdout_logfile_maxbytes"]) == ( "/dev/stdout", "0") assert root["supervisord"]["nodaemon"] == "true" + init = (script.REPO / Path(script.IMAGES["agentenv-ubuntu-24.04-disk"]).with_name("init")).read_text() + assert "ln -sf /proc/self/fd/1 /newroot/dev/stdout\n" in init def test_a_disk_forms_smoke_needs_the_launcher_to_boot_it_in(script, monkeypatch, capsys): From 9f14ba476431ce609f9662f52492212360e84c10 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 10:22:42 -0800 Subject: [PATCH 3/4] fix(computers): two composes of one Linux disk on a machine take turns, so a boot's pinned disk stays put Two composes of the same new disk key on one machine (the slow tier's two pytest workers, each composing the shared desktop) both found nothing stored and both built it. Each build tags localhost:5000/agentenv-computers: and its bytes differ, so the second moved the tag off the image the first had pinned by digest. The first's probe then mounted name:tag@digest with --mount type=image, which Docker doesn't pull for, and failed with "No such image", failing launcher_test's three tests with KeyError: 'image' on every run that built the disk rather than copied it from the mirror (eac7660e and d1fcdf82 on main, and PRs that change the computer images). The Linux disk is now looked up, made and pushed holding a lock of its key, as Windows and Omarchy disks are, so the second compose waits and reuses the first's. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/composer.py | 62 ++++++++++--------- tst/unit/env/envs/test_computer_disk.py | 43 ++++++++++++- 2 files changed, 75 insertions(+), 30 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 4ab6da95d..f3863976e 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -677,7 +677,9 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: """The status of the disk form of ``arch`` once built from ``container``, its container's result, and pushed, or reused, or the stage that failed and why. Its key is of the container's key, the base disk and the disk recipe; the base disk is the base's own, the one published for it, or else one made on the builder from the base's - container and the boot recipe. Its build runs no program of ``arch``'s but that recipe's.""" + container and the boot recipe. Its build runs no program of ``arch``'s but that recipe's. Two composes of one disk + on this machine take turns, so the second reuses the first's: built again, its tag would move off the image the + first pinned, which a boot mounts by that name and Docker doesn't pull for a mount.""" pair, platform = f"disk/{arch}", f"linux/{arch}" with (_attempting(env, "disk", arch) as attempt, tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened): @@ -687,34 +689,36 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: base_disk, base_identity, recipe = *_base_disk(base, arch), _digest(context) key = attempt.key = _key(container=container["key"], base_disk=base_identity, recipe=recipe, form="disk", arch=arch, format=FORMAT) - store, ref, pinned = _stored(env, key) - if pinned is not None: - progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") - return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": now()} - link = builds.link(platform, base.image()) - progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") - on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) - on.login(store.auth(container["image"])) - if base_disk is not None: - on.login(get_config().get_image_store().auth(base_disk)) - if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: - tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" - on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, - context=f"{builder.CONTEXT}/tools") - dockerfile = (context / "Dockerfile").read_text() - if base_disk is None: - progress.at(pair, "making the base's disk") - base_disk = _made_base_disk(on, base, arch, f"{base_identity} {recipe}", tools, dockerfile) - progress.at(pair, "making the disk") - local = f"agentenv-computer:{key[7:23]}-disk-{arch}" - on.build(dockerfile, local, platform, {KEY_LABEL: key}, build_args={ - "ROOTFS": container["image"], "ROOTFS_CONFIG": _encoded(_config(on, container["image"], arch)), - "ESP": base_disk, "TOOLS": tools}) - attempt.stage = "push" - progress.at(pair, "pushing the disk") - on.push(local, ref, store) - return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, - "builder": on.name, "built_at": now()} + with holding_locks([f"linux-disk:{key}"], lambda: progress.at( + pair, "waiting", "waiting for the compose of the same disk under way on this machine")): + store, ref, pinned = _stored(env, key) + if pinned is not None: + progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") + return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": now()} + link = builds.link(platform, base.image()) + progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") + on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) + on.login(store.auth(container["image"])) + if base_disk is not None: + on.login(get_config().get_image_store().auth(base_disk)) + if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: + tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" + on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, + context=f"{builder.CONTEXT}/tools") + dockerfile = (context / "Dockerfile").read_text() + if base_disk is None: + progress.at(pair, "making the base's disk") + base_disk = _made_base_disk(on, base, arch, f"{base_identity} {recipe}", tools, dockerfile) + progress.at(pair, "making the disk") + local = f"agentenv-computer:{key[7:23]}-disk-{arch}" + on.build(dockerfile, local, platform, {KEY_LABEL: key}, build_args={ + "ROOTFS": container["image"], "ROOTFS_CONFIG": _encoded(_config(on, container["image"], arch)), + "ESP": base_disk, "TOOLS": tools}) + attempt.stage = "push" + progress.at(pair, "pushing the disk") + on.push(local, ref, store) + return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, + "builder": on.name, "built_at": now()} return attempt.result diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py index daff46e01..119de0b0a 100644 --- a/tst/unit/env/envs/test_computer_disk.py +++ b/tst/unit/env/envs/test_computer_disk.py @@ -7,6 +7,7 @@ import asyncio import base64 +import contextlib import json import threading from datetime import datetime @@ -22,7 +23,7 @@ from agent_env.env import Env from agent_env.config.runtime import Config from agent_env.env.envs.computer.env import ComputerEnv -from agent_env.env.envs.computer.utils import boot, composer, probing +from agent_env.env.envs.computer.utils import boot, builder, composer, probing from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.published_images import DISK from agent_env import providers @@ -89,6 +90,46 @@ def test_a_disk_is_reused_by_its_key_of_its_containers_key_the_base_disk_and_the assert len(docker.disk_builds) > made +def test_a_disk_is_looked_up_made_and_pushed_holding_a_lock_of_its_key_so_composes_of_it_take_turns( + disk_base, driver, docker, monkeypatch): + held, events = set(), [] + real_locks, real_stored = composer.holding_locks, composer._stored + real_build, real_push = docker.build_image, docker.push + + @contextlib.contextmanager + def holding(ids, on_wait=None): + with real_locks(ids, on_wait): + held.update(ids) + try: + yield + finally: + held.difference_update(ids) + + def stored(env, key): + events.append(("looked up", key, set(held))) + return real_stored(env, key) + + def build(dockerfile, context, tag, **kw): + events.append(("made", tag, set(held))) + return real_build(dockerfile, context, tag, **kw) + + def push(tag, ref, store): + events.append(("pushed", tag, set(held))) + return real_push(tag, ref, store) + + monkeypatch.setattr(composer, "holding_locks", holding) + monkeypatch.setattr(composer, "_stored", stored) + monkeypatch.setattr(builder, "build_image", build) + monkeypatch.setattr(builder, "push_local_image", push) + + disks = Env.get(_put(disk_base, driver).id).metadata["composed"]["disk"] + + for arch, result in disks.items(): + names = (result["key"], f"agentenv-computer:{result['key'][7:23]}-disk-{arch}") + assert [(what, f"linux-disk:{result['key']}" in locks) for what, name, locks in events if name in names] == [ + ("looked up", True), ("made", True), ("pushed", True)] + + def test_a_bases_disk_size_is_in_the_key_and_the_build_of_the_disk_made_for_it(disk_base, driver, docker): bigger = _base(disk_base.image(), id="@local/~/kit/bigger", forms={ **disk_base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, "min_disk_gb": 32}}) From 3a8f4b48e1d3f5f7490a61c2d1630b70e39b3ec3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 10:48:15 -0800 Subject: [PATCH 4/4] test: the integration tier's workers share the locks that guard this machine's Docker and registry agent-env keeps its locks in the state root, and each integration test has a state root of its own, so the disk lock alone didn't stop two pytest workers composing one computer at once: on this PR, both still built the desktop's disk under the same tag and the first's boot failed as before. A session fixture keeps the locks whose ids name this machine's image work (composes, disks, disk exports, image builds, pulls and saves) in one folder for the run, as one user's processes share theirs, so such work takes turns across workers and the second reuses what the first pushed. Co-Authored-By: Claude Opus 5.5 --- tst/integration/conftest.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tst/integration/conftest.py diff --git a/tst/integration/conftest.py b/tst/integration/conftest.py new file mode 100644 index 000000000..6f79b6565 --- /dev/null +++ b/tst/integration/conftest.py @@ -0,0 +1,32 @@ +"""The integration tier's setup shared by every test.""" + +import os +import tempfile +from pathlib import Path + +import pytest + +from agent_env.store import local_state + +#: The locks that guard this machine's Docker and image registry, by the start of their ids: a run's tests keep stores +#: and state roots of their own, but all of them, on every pytest worker, use the one Docker and registry. +MACHINE_LOCKS = ("computer-compose:", "linux-disk:", "windows-disk:", "omarchy-disk:", "disk-export:", "image-build:", + "image-pull:", "image-save:") + + +@pytest.fixture(scope="session", autouse=True) +def machine_locks_shared_by_the_runs_workers(): + """Each test's state root, where agent-env keeps its locks, is its own, so two pytest workers composing one + computer took no lock in common and both built it under the same tags: the second moved them off the image the + first had pinned, or their pushes collided. The locks guarding this machine's Docker and registry are kept in one + folder for the run, as one user's processes keep theirs, so such work takes turns and the second reuses the first's. + """ + run = os.environ.get("PYTEST_XDIST_TESTRUNUID") or str(os.getpid()) + folder = Path(tempfile.gettempdir()) / f"agent-env-machine-locks-{run}" + folder.mkdir(mode=0o700, exist_ok=True) + per_test = local_state._lock_path + patch = pytest.MonkeyPatch() + patch.setattr(local_state, "_lock_path", + lambda id: folder / per_test(id).name if id.startswith(MACHINE_LOCKS) else per_test(id)) + yield + patch.undo()