diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5fd12fcab..3217a0a0f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -70,6 +70,16 @@ updates: groups: infra-base-images: patterns: ["*"] + # The base computer's egress proxy, a Go program its Dockerfile builds: its modules' fixes, uTLS's and + # golang.org/x/crypto's among them. A bump changes the base's sources, so the next release rebuilds and publishes it. + - package-ecosystem: "gomod" + directory: "/src/agent_env/computers/artifacts/ubuntu-24.04/egress" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + groups: + egress-go-modules: + patterns: ["*"] - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 3b08d5c52..f27123a54 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -25,6 +25,21 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: + # The base computer's egress proxy (src/agent_env/computers/artifacts/ubuntu-24.04/egress/): gofmt, go vet and its + # tests under the race detector, in the Go image its Dockerfile builds with, so a failing Go test fails here rather + # than as a broken base image in the slow tier. + egress: + name: egress + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + # ECR Public's anonymous pulls fail here for hours at a time. + - name: Pull the Go image from mirror.gcr.io + run: python3 .github/scripts/mirror_base_images.py src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile + - name: gofmt, go vet and go test -race + run: make egress-test + # The unit tier on two runners, AGENTENV_TEST_SHARD picking each one's share by the times in # tst/integration/slow_durations.json (tst/util/shards.py); unit passes when both shards do. The # lockfile and lint checks run on the first shard only. diff --git a/AGENTS.md b/AGENTS.md index c80ddc3a8..c27c5f7df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ or one of them only where it is used. | `make int-test-slow` | the `int_test_slow` tests, serially (they build the same Docker tags from module fixtures and race under xdist) | | `make installer-test` | `tst/installer`: `plugin add` / `remove` through the real pip, uv and pipx, offline against wheels built from the checkout, and the container user journey (`python:3.12-slim`, `--network none`) with the PEP 668 refusal; needs uv, pipx and Docker | | `make clock-test` | `tst/clock`: agent-env's libfaketime (`src/agent_env/computers/artifacts/ubuntu-24.04/clock`) and stock 0.9.12 built in an Ubuntu 24.04 image for this machine's Docker, and its gates run there: every wait API in C, Python, Java and Node at several rates, jumps, cancelled waits, timerfd duplicates, condvar and semaphore stress, the waits again with `futex_waitv` refused (as before Linux 5.16 and under gVisor), a stalled writer, a dead one, torn reads, the marker, the environment race, forks while another thread closes a timerfd, the TLS shim's certificate checks on real time (curl, Python and `openssl s_client` at 182 days back: live sites verify, expired and untrusted certificates don't; and two copies of OpenSSL in one process, the system's 3.0 and a 1.1, each checking with its own), and Chromium launched 40 times with it and 10 with stock, side by side, lateness held to `tst/clock/bounds.py`; needs Docker, and fetches Chrome's archive, upstream's tarball and OpenSSL 1.1.1w's once into `AGENTENV_TEST_CLOCK_CACHE` | +| `make egress-test` | the base computer's egress proxy (`src/agent_env/computers/artifacts/ubuntu-24.04/egress`, Go): `gofmt`, `go vet` and `go test -race` in the Go image its Dockerfile builds with; needs Docker, not Go | | `make trl-test` | `tst/trl`: one `GRPOTrainer` step on CPU through `agent_env.training.trl`, with a tiny model from the Hub; needs trl and CPU torch, which the `dev` extra leaves out | | `make nemo-gym-test` | `tst/nemo_gym`: rollouts through `agent_env.training.nemo_gym`'s resources server in-process; needs the `nemo-gym` extra, on Python 3.13.14 or later, which the `dev` extra leaves out (`uv sync --extra dev --extra nemo-gym`) | | `make verl-test` | `tst/verl`: rows played through `agent_env.training.verl`'s agent loop on verl itself, with Qwen3's tokenizer from the Hub and a stand-in rollout server; needs verl and CPU torch, which the `dev` extra leaves out | @@ -138,7 +139,7 @@ and deserialized through a registry. | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | -| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, settings as `AGENTENV_SETTING_`; it starts as root, its container form's `start_as`, whose supervisord, `root.conf`, runs the clock follower as `agentenv-clock` and the session's supervisord as the session user, which runs every session program), and `omarchy-4.5`, a disk alone built on the composing machine from Omarchy's own VM image (its `[forms.disk]` `source`) by `disk/omarchy/`'s recipe, its services systemd user units of the graphical session that start through `with-seed`, booted by the launcher as a `linux-desktop` guest (`env/envs/computer/utils/omarchy_disk.py`, on `thin_disk.py`, which Windows' disks use too), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. `artifacts/ubuntu-24.04/clock/` is the computer's clock, in the base's folder since an image builds only from its own: agent-env's build of libfaketime 0.9.12 (`libfaketime.patch` and the two `agentenv_clock_*.c` files it includes, GPL-2.0-only as libfaketime is, built by `build.sh` from upstream's sha256-pinned tarball, in a stage of the base's Dockerfile of its own) and the TLS shim, which `/etc/ld.so.preload` loads into every process and which fake only the session user's: clock reads and waits for a set clock time follow the page the follower (`clock-follower`) writes (`clockpage.py`), monotonic and relative waits staying real, with its default settings (`clock.conf`); `tst/clock` gates it. The follower long-polls the gateway's `/clock/watch` under the config core writes through the root channel when a multi env deploys the computer and after a reset (`env/envs/computer/utils/clock.py`), each time waiting on the computer's `sync_time` until it follows, a warning naming why when it doesn't (arming the clock refuses such a computer), and the driver serves `clock/v1` `sync_time` where the follower is (`agentenv_protocol.computer.clock`), not on a disk guest, whose follower core can't reach. | +| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, settings as `AGENTENV_SETTING_`; it starts as root, its container form's `start_as`, whose supervisord, `root.conf`, runs the clock follower as `agentenv-clock`, the egress proxy as `agentenv-egress` (`egress/`, a Go program its Dockerfile builds and tests, on real time, through which every session program goes out, so a live site loads at any universe date: it passes a site's TLS through when the universe is inside the site's certificate, else re-signs it with the computer's own CA, minted at boot and valid from 1950 to 9999, handshaking with the site as the client did, and refuses any destination that isn't global) and the session's supervisord as the session user, which runs every session program), and `omarchy-4.5`, a disk alone built on the composing machine from Omarchy's own VM image (its `[forms.disk]` `source`) by `disk/omarchy/`'s recipe, its services systemd user units of the graphical session that start through `with-seed`, booted by the launcher as a `linux-desktop` guest (`env/envs/computer/utils/omarchy_disk.py`, on `thin_disk.py`, which Windows' disks use too), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. `artifacts/ubuntu-24.04/clock/` is the computer's clock, in the base's folder since an image builds only from its own: agent-env's build of libfaketime 0.9.12 (`libfaketime.patch` and the two `agentenv_clock_*.c` files it includes, GPL-2.0-only as libfaketime is, built by `build.sh` from upstream's sha256-pinned tarball, in a stage of the base's Dockerfile of its own) and the TLS shim, which `/etc/ld.so.preload` loads into every process and which fake only the session user's: clock reads and waits for a set clock time follow the page the follower (`clock-follower`) writes (`clockpage.py`), monotonic and relative waits staying real, with its default settings (`clock.conf`); `tst/clock` gates it. The follower long-polls the gateway's `/clock/watch` under the config core writes through the root channel when a multi env deploys the computer and after a reset (`env/envs/computer/utils/clock.py`), each time waiting on the computer's `sync_time` until it follows, a warning naming why when it doesn't (arming the clock refuses such a computer), and the driver serves `clock/v1` `sync_time` where the follower is (`agentenv_protocol.computer.clock`), not on a disk guest, whose follower core can't reach. | | `browsers/` | The `browsers` bundle: the website browsers agent-env ships, today `chromium`, a `computer_app` env (Chrome for Testing at the build Playwright pins, for amd64 and arm64, set up for unattended use by managed policies in `/etc/opt/chrome_for_testing/policies/managed`, opened at boot on its `start_urls`, its profile its data). All of it, its pin, install step, policies and command, is in `envs/chromium/chromium.py`; `tst/integration/computer/chromium_app_test.py` runs it. Like `drivers`, it has no tasks. | | `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, the page driver aliased `page`, a browser computer's, whose screen is the browser's active tab and which serves Playwright MCP's `browser_*` tools beside `computer_*` from a Playwright MCP it runs attached to the browser over a CDP address the relay issues it (`browser_evaluate` and `browser_run_code_unsafe` hidden by default; each call an entry in the browser's log, begun and ended with `browser/v1` `calls/begin` and `calls/end`, that its CDP commands name as `for`; `scripts/upgrade_playwright_mcp.py` pins its version), and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | @@ -419,8 +420,8 @@ A merge into `main` needs an approving review from a code owner (`CODEOWNERS`), resolved, and the checks the `main` ruleset requires: `unit`, `integration-local` and `integration-local-slow`. The other checks below run on every pull request too and are expected to pass, but a failing one does not block the merge. -`.github/workflows/local-backends.yml` runs `unit`, `ui`, `integration-local`, `integration-local-slow`, -`integration-local-gateway` (the slow tier's `gateway_test.py`, run beside the rest of it) and +`.github/workflows/local-backends.yml` runs `unit`, `ui`, `egress` (`make egress-test`), `integration-local`, +`integration-local-slow`, `integration-local-gateway` (the slow tier's `gateway_test.py`, run beside the rest of it) and `installer`. The slow tier runs on twelve runners (`integration-local-slow (1/12)` to `(12/12)`), the gateway tests on five, and the unit and installer tiers on two each; `unit`, `integration-local-slow`, `integration-local-gateway` and `installer` each pass when all of their shards do. The Python jobs install from `uv.lock` (public PyPI only), with the uv that @@ -489,9 +490,10 @@ image's digest. Run by hand on `main`, the workflow publishes the images alone. Its jobs that run agent-env (`.github/scripts/infra_images.py`) hold no token that writes; the jobs that push images or publish to PyPI install nothing and check what they're handed (`.github/scripts/infra_build.py`, standard library and the docker CLI only). -Actions are pinned to commit SHAs; Dependabot updates them, the pinned uv, `uv.lock`, the UI's npm packages and -the infra images' base images, each pinned by digest so that a bump rebuilds and republishes the image; Dependabot -proposes only a base's digest and patch updates, and a major or minor upgrade is made by hand. +Actions are pinned to commit SHAs; Dependabot updates them, the pinned uv, `uv.lock`, the UI's npm packages, the +base computer's egress proxy's Go modules (`go.sum`), and the infra images' base images, each pinned by digest so +that a bump rebuilds and republishes the image; Dependabot proposes only a base's digest and patch updates, and a +major or minor upgrade is made by hand. ## Conventions diff --git a/Makefile b/Makefile index 167bb1dd9..56fa11de7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install lint unit-test int-test int-test-fast int-test-slow installer-test clock-test trl-test nemo-gym-test verl-test skyrl-test miles-test slime-test clean-install-test test clean +.PHONY: help install lint unit-test int-test int-test-fast int-test-slow installer-test clock-test egress-test trl-test nemo-gym-test verl-test skyrl-test miles-test slime-test clean-install-test test clean VENV := .venv VENV_BIN := $(abspath $(VENV))/bin @@ -72,6 +72,14 @@ installer-test: ## Run plugin add/remove through the real pip, uv and pipx, and clock-test: ## Build agent-env's libfaketime and run its gates in Docker (tst/clock) $(PYTHON) -m pytest tst/clock --timeout=2400 $(PYTEST_ARGS) +# The base computer's egress proxy, in the Go image its Dockerfile builds it with, so it needs Docker, not Go. +EGRESS := src/agent_env/computers/artifacts/ubuntu-24.04/egress +EGRESS_GO_IMAGE = $(shell awk '/^FROM .*golang/ {print $$2; exit}' $(EGRESS)/../Dockerfile) + +egress-test: ## gofmt, go vet and go test -race for the base computer's egress proxy, in Docker + docker run --rm -v "$(CURDIR)/$(EGRESS):/src" -w /src $(EGRESS_GO_IMAGE) \ + sh -c 'test -z "$$(gofmt -l .)" || { gofmt -l .; exit 1; }; go vet ./... && go test -race -count=1 ./...' + trl-test: ## Run one GRPOTrainer step on CPU through the TRL helper (needs trl and CPU torch, and the Hub) $(PYTHON) -m pytest tst/trl --timeout=900 $(PYTEST_ARGS) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 79edc47a8..fe86e7dd8 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -2556,6 +2556,22 @@ The folder's other files, like the rest of the repository, are licensed under th - Source: - License: GPL-2.0-only +## The egress proxy's Go modules + +`src/agent_env/computers/artifacts/ubuntu-24.04/egress/` is agent-env's egress proxy, a Go program the base +computer's Dockerfile builds, statically linked with the Go standard library and the modules below. Their source is +not in the repository: the folder's `go.sum` pins each, the build downloads them, and the program runs inside +computers. + +| Module | Version | License | +| --- | --- | --- | +| Go standard library | 1.26.11 | BSD-3-Clause | +| github.com/refraction-networking/utls | v1.8.2 | BSD-3-Clause | +| golang.org/x/crypto | v0.36.0 | BSD-3-Clause | +| golang.org/x/sys | v0.31.0 | BSD-3-Clause | +| github.com/andybalholm/brotli | v1.0.6 | MIT | +| github.com/klauspost/compress | v1.17.4 | BSD-3-Clause (a few packages the proxy doesn't link are Apache-2.0 or MIT) | + ## License texts License and notice files as shipped by each package, with copyright lines removed: they are listed with each package above. Packages whose files differ only in those lines share a text. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py index 1c259b2b1..be25c769a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py @@ -15,7 +15,8 @@ from pydantic import TypeAdapter, ValidationError from ..agent_env_environment import add_data, get_data -from ..transfers import NamespaceUploader, ReadObject, RelativePath, TransferError, WriteNamespaceGrant, download +from ..transfers import (NamespaceUploader, ReadObject, RelativePath, TransferError, WriteNamespaceGrant, client_options, + download) from ..types import DATA_OBJECTS_EXTENSION_URI, FilePart, FileWithBytes, FileWithUri, uploaded_file_part, uploaded_object_path if TYPE_CHECKING: @@ -115,7 +116,7 @@ async def _get_url(file: FileWithUri) -> bytes: if not file.uri.startswith("https://"): raise ValueError("a file part's uri is an https URL") data = bytearray() - async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: + async with httpx.AsyncClient(follow_redirects=True, timeout=60, **client_options()) as client: async with client.stream("GET", file.uri) as response: response.raise_for_status() async for chunk in response.aiter_bytes(): diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py index 81df184c8..0029e574a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py @@ -1,11 +1,13 @@ """Serve a computer driver from its file, as a composed computer's program starts it: ``python -m agentenv_protocol.computer.serve :``, run in the driver's folder, serves the class on -``MCP_PORT``. The file's folder is importable, so it may import its own modules.""" +``MCP_PORT``. The file's folder is importable, so it may import its own modules. The driver runs on real time, so its +own transfers reach the object store directly, not through the computer's egress proxy (``transfers.direct``).""" from __future__ import annotations import sys from pathlib import Path +from .. import transfers from ._load import load_class from .driver import ComputerDriverEnvironment @@ -16,6 +18,7 @@ def main(argv: list[str]) -> None: if len(argv) != 1 or ":" not in argv[0]: raise SystemExit(USAGE) file, _, name = argv[0].partition(":") + transfers.direct() driver_class(Path(file), name)().serve() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py index cb605edcd..fb58ec48e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py @@ -5,7 +5,9 @@ (``CONTROL_TOKEN_ENV``): the driver checks it in front of the app. Before it serves, it keeps a copy of the app's data paths as installed, which a reset puts back; once it serves, it adds the app's launcher and launches a boot app (``AppRuntime.start``). A server that can't start, or whose start fails once it serves, writes why to -``.agentenv/server.failure``, which its computer's driver reports, and exits. Needs the ``app`` extra.""" +``.agentenv/server.failure``, which its computer's driver reports, and exits. It runs on real time, so its own +transfers reach the object store directly, not through the computer's egress proxy (``transfers.direct``). Needs the +``app`` extra.""" from __future__ import annotations import asyncio @@ -17,6 +19,7 @@ import uvicorn +from .. import transfers from ..types import CONTROL_TOKEN_ENV from ._load import load_class from .app import FAILURE, ComputerAppEnvironment @@ -27,6 +30,7 @@ def main(argv: list[str]) -> None: FAILURE.unlink(missing_ok=True) + transfers.direct() try: _serve(argv) except (Exception, SystemExit) as e: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/transfers.py b/packages/agentenv-protocol/src/agentenv_protocol/transfers.py index 86334da3d..ed9445c05 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/transfers.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/transfers.py @@ -46,6 +46,28 @@ write=TRANSFER_IDLE_TIMEOUT_SECONDS, pool=_CONNECT_TIMEOUT_SECONDS, ) +#: Whether this process's transfers take a proxy from the environment's settings. A computer's own servers turn it +#: off (``direct``): they run on real time and reach the object store directly, while the programs they start keep +#: the settings that send them through the computer's egress proxy. +_proxy_from_environment = True + + +def direct() -> None: + """Have this process's transfers reach their objects directly, whatever proxy the environment names; the + environment, and so every process this one starts, keeps it.""" + global _proxy_from_environment + _proxy_from_environment = False + + +def client_options() -> dict: + """The options a transfer's HTTP client takes: none, or for a ``direct`` process, no proxy from the environment + but still its certificates (``SSL_CERT_FILE``, ``SSL_CERT_DIR``), since httpx takes both from it or neither and a + local store's CA is trusted through them.""" + if _proxy_from_environment: + return {} + return {"trust_env": False, "verify": httpx.create_ssl_context(trust_env=True)} + + # The longest the helpers take to give up on a transfer whose every connection stalls. TRANSFER_STALL_BUDGET_SECONDS = TRANSFER_ATTEMPTS * ( _CONNECT_TIMEOUT_SECONDS + TRANSFER_IDLE_TIMEOUT_SECONDS @@ -483,7 +505,7 @@ async def upload(target: WriteObject, source: Path | bytes) -> Uploaded: async def put() -> Uploaded: with _transfer_request(target.write.expires_at): async with httpx.AsyncClient( - follow_redirects=False, timeout=_TRANSFER_TIMEOUT + follow_redirects=False, timeout=_TRANSFER_TIMEOUT, **client_options() ) as client: async def to(url: str) -> Uploaded: @@ -519,7 +541,7 @@ async def get() -> tuple[int, str]: digest = hashlib.sha256() with _transfer_request(source.read.expires_at): async with httpx.AsyncClient( - follow_redirects=False, timeout=_TRANSFER_TIMEOUT + follow_redirects=False, timeout=_TRANSFER_TIMEOUT, **client_options() ) as client: response = await _send_to_first_listening( urls, @@ -571,7 +593,7 @@ def _post_to_namespace( fields.setdefault("Content-Type", "application/octet-stream") with ( _transfer_request(target.expires_at), - httpx.Client(follow_redirects=False, timeout=_TRANSFER_TIMEOUT) as client, + httpx.Client(follow_redirects=False, timeout=_TRANSFER_TIMEOUT, **client_options()) as client, ): def to(url: str) -> Uploaded: diff --git a/packages/agentenv-protocol/tests/test_a2a_object_transfers.py b/packages/agentenv-protocol/tests/test_a2a_object_transfers.py index 20b53c4fa..41d544461 100644 --- a/packages/agentenv-protocol/tests/test_a2a_object_transfers.py +++ b/packages/agentenv-protocol/tests/test_a2a_object_transfers.py @@ -1143,3 +1143,60 @@ def handle(request: httpx.Request) -> httpx.Response: assert sent == [f"http://127.0.0.1:8456{_STAGED}"] assert transfers.loopback_url(write.write.url, write.write.headers) == f"http://127.0.0.1:9999{_STAGED}" + + +@pytest.mark.asyncio +async def test_a_process_made_direct_takes_no_proxy_from_the_environment_but_still_its_certificates( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "local grant server CA")]) + ca = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key()) + .serial_number(1).not_valid_before(datetime(2020, 1, 1, tzinfo=_UTC)) + .not_valid_after(datetime(2040, 1, 1, tzinfo=_UTC)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256())) + (tmp_path / "ca.pem").write_bytes(ca.public_bytes(serialization.Encoding.PEM)) + monkeypatch.setenv("SSL_CERT_FILE", str(tmp_path / "ca.pem")) + made = [] + + def recording(real): + def make(**kwargs): + made.append(kwargs) + return real(transport=httpx.MockTransport(lambda request: httpx.Response(200)), **kwargs) + + return make + + monkeypatch.setattr(transfers.httpx, "AsyncClient", recording(httpx.AsyncClient)) + monkeypatch.setattr(transfers, "_proxy_from_environment", True) + await upload(_write_object(), _write(tmp_path / "before.json", b"{}")) + transfers.direct() + await upload(_write_object(), _write(tmp_path / "after.json", b"{}")) + + assert "trust_env" not in made[0] and made[1]["trust_env"] is False + trusted = {dict(item[0] for item in cert["subject"])["commonName"] for cert in made[1]["verify"].get_ca_certs()} + assert "local grant server CA" in trusted + + +def test_a_computers_own_servers_make_their_transfers_direct(monkeypatch: pytest.MonkeyPatch) -> None: + from agentenv_protocol.computer import serve, serve_app + + seen = [] + + class Driver: + def serve(self) -> None: + seen.append(transfers.client_options().get("trust_env", True)) + + monkeypatch.setattr(transfers, "_proxy_from_environment", True) + monkeypatch.setattr(serve, "driver_class", lambda file, name: Driver) + serve.main(["driver.py:Driver"]) + monkeypatch.setattr(transfers, "_proxy_from_environment", True) + monkeypatch.setattr(serve_app, "_serve", lambda argv: seen.append(transfers.client_options().get("trust_env", True))) + serve_app.main(["app.py:App"]) + + assert seen == [False, False] diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 07171af04..7ff759198 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -2,8 +2,17 @@ Base computers that come with agent-env, each a `computer` artifact a computer e `agent-env run --set envs/desktop.computer=ubuntu-24.04-browser` composes a bundle's `desktop` env on another of them for one run, the bundle left as written. -`artifacts/ubuntu-24.04/` is Ubuntu 24.04 (`amd64` and `arm64`) with XFCE on Xvfb, D-Bus and AT-SPI. tini starts -supervisord, which runs every program as the session user, `user` (uid 1000); nothing in it runs as root. Its home +`artifacts/ubuntu-24.04/` is Ubuntu 24.04 (`amd64` and `arm64`) with XFCE on Xvfb, D-Bus and AT-SPI. It starts as +root: its init sets the time zone, makes the clock page and boots the egress proxy, then root's supervisord runs the +clock follower as `agentenv-clock`, the egress proxy as `agentenv-egress` and the session's supervisord as the +session user, `user` (uid 1000), which runs every session program. Every session program goes out through the +egress proxy, on `127.0.0.1:3128` and on real time (`egress/`, a Go program the Dockerfile builds and tests): it +passes a site's TLS through untouched when the universe's date is inside the site's certificate, else verifies the +site against real time and hands the program a certificate from the computer's own CA, minted at boot, its key the +proxy's alone, and valid from 1950 to 9999, so a live site loads however far the clock is from now. Its handshake +with the site is built from the program's own, so the site sees the program's TLS fingerprint either way, and it +refuses any destination that isn't global. Java and Node's `fetch` reach it by their own settings, which the +session sets too. Its home has a desktop's standard folders (`Desktop`, `Documents`, `Downloads`, `Pictures` and the rest, from `xdg-user-dirs`). Its container form's image is built from the folder's `Dockerfile`, and a release publishes it as `ghcr.io/scaleapi/agentenv-ubuntu-24.04:src-`, tagged by its sources, so a run pulls it by digest instead of diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile index 922a3697f..8124c124b 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile @@ -1,13 +1,21 @@ # The ubuntu-24.04 base computer: Ubuntu 24.04 with XFCE on Xvfb, D-Bus and AT-SPI, whose session user's home has a # desktop's standard folders (Documents, Downloads, ...). It starts as root: root's supervisord (root.conf) runs the clock -# follower as its own user and the session's supervisord (supervisord.conf) as the session user, which runs every -# session program. Composing a driver or an app onto it adds its services as programs in /etc/supervisor/conf.d. The -# trust files agentenv adds are in /etc/agentenv from boot: ca.pem, empty until a deploy adds a CA, and ca-bundle.pem, -# the public roots and those CAs. +# follower and the egress proxy, each as its own user, and the session's supervisord (supervisord.conf) as the session +# user, which runs every session program. Composing a driver or an app onto it adds its services as programs in +# /etc/supervisor/conf.d. The trust files agentenv adds are in /etc/agentenv from boot: ca.pem, the CAs agentenv adds +# (the egress proxy's from boot, a deploy's for its sites), and ca-bundle.pem, the public roots and those CAs. # # The universe clock (clock/): agent-env's libfaketime and its TLS shim, built in a stage of their own so no compiler is # left in the image, are preloaded into every process and fake only the session user's, from the clock page the # follower writes, which the session user can read but neither write nor replace. +# +# The egress proxy (egress/), on real time, which every session program goes out through, so a live site loads at any +# universe date: built and tested in a stage of its own, so no Go toolchain is left in the image. +FROM public.ecr.aws/docker/library/golang:1.26.11-bookworm@sha256:dc40a13b61304647ed7e5d581d246b86413dfc29d63c6d018310feb93cd98fc0 AS egress +COPY egress/ /egress/ +RUN cd /egress && go vet ./... && go test -count=1 ./... \ + && CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags=-s -o /out/egress . + FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c AS clock RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential patch curl ca-certificates @@ -22,19 +30,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ dbus dbus-x11 at-spi2-core xdg-user-dirs \ supervisor tini \ python3 python3-venv python3-tk \ - fonts-dejavu-core locales tzdata ca-certificates procps \ + fonts-dejavu-core locales tzdata ca-certificates libnss3-tools procps \ && rm -rf /var/lib/apt/lists/* \ && sed -i 's/^# *en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen \ && userdel --remove ubuntu \ && useradd --create-home --uid 1000 --shell /bin/bash user \ && runuser -u user -- xdg-user-dirs-update \ && useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin agentenv-clock \ + && useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin agentenv-egress \ && mkdir -p /etc/supervisor/conf.d /var/log/agentenv /run/agentenv /var/log/agentenv-root /run/agentenv-root \ - /usr/local/lib/agentenv-clock /etc/agentenv/clock \ + /usr/local/lib/agentenv-clock /etc/agentenv/clock /usr/local/lib/agentenv-egress \ && chmod 0700 /var/log/agentenv-root /run/agentenv-root \ && chown root:agentenv-clock /etc/agentenv/clock && chmod 0750 /etc/agentenv/clock \ && touch /etc/agentenv/ca.pem \ && cp /etc/ssl/certs/ca-certificates.crt /etc/agentenv/ca-bundle.pem \ + && cp /etc/ssl/certs/ca-certificates.crt /usr/local/lib/agentenv-egress/roots.pem \ && chown user:user /var/log/agentenv /run/agentenv COPY --from=clock /out/libfaketimeMT.so.1 /out/libtls_realtime.so /usr/local/lib/agentenv-clock/ COPY clock/clockpage.py clock/clock-follower /usr/local/lib/agentenv-clock/ @@ -44,6 +54,8 @@ RUN sed -i -e "s/^AGENTENV_SESSION_UID=.*/AGENTENV_SESSION_UID=$(id -u user)/" \ && chmod 0755 /usr/local/lib/agentenv-clock/clock-follower \ && printf '%s\n' /usr/local/lib/agentenv-clock/libfaketimeMT.so.1 /usr/local/lib/agentenv-clock/libtls_realtime.so \ > /etc/ld.so.preload +COPY --from=egress /out/egress /usr/local/lib/agentenv-egress/egress +COPY egress/agentenv-egress.sh /etc/profile.d/agentenv-egress.sh COPY root.conf /etc/supervisor/root.conf COPY supervisord.conf /etc/supervisor/supervisord.conf COPY xfwm4.xml /etc/xdg/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/clock/clockpage.py b/src/agent_env/computers/artifacts/ubuntu-24.04/clock/clockpage.py index 53e57db1d..4002bb6c2 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/clock/clockpage.py +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/clock/clockpage.py @@ -47,11 +47,19 @@ def check(seq, universe_ns, anchor_ns, rate, armed, generation, mono_universe_ns ^ (mono_universe_ns & MASK) ^ (mono_anchor_ns & MASK) ^ 0x9E3779B97F4A7C15) & MASK +def encode(seq, universe_ns, anchor_ns, rate, armed, generation, mono_universe_ns, mono_anchor_ns) -> bytes: + """The page's bytes for these fields, its check among them: what every write writes and every reader checks, + agent-env's libfaketime and the egress proxy (its egress/testdata pins them).""" + return LAYOUT.pack(MAGIC, VERSION, seq, universe_ns, anchor_ns, rate, armed, 0, generation, + check(seq, universe_ns, anchor_ns, rate, armed, generation, mono_universe_ns, mono_anchor_ns), + mono_universe_ns, mono_anchor_ns) + + def open_page() -> mmap.mmap: try: fd = os.open(PAGE, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o644) os.ftruncate(fd, LAYOUT.size) - os.write(fd, LAYOUT.pack(MAGIC, VERSION, 0, 0, 0, 1.0, 0, 0, 0, check(0, 0, 0, 1.0, 0, 0, 0, 0), 0, 0)) + os.write(fd, encode(0, 0, 0, 1.0, 0, 0, 0, 0)) except FileExistsError: fd = os.open(PAGE, os.O_RDWR) if os.fstat(fd).st_uid != os.getuid(): @@ -102,9 +110,7 @@ def write(armed: int, universe: float, rate: float, anchor: float) -> dict: continued = armed and generation > 1 and not dead and 0 < old_now - new_now < CONTINUITY_NS if continued: universe_ns, anchor_ns = old_now, now_rt - body = LAYOUT.pack(MAGIC, VERSION, seq + 2, universe_ns, anchor_ns, rate, armed, 0, generation, - check(seq + 2, universe_ns, anchor_ns, rate, armed, generation, now_mono, now_mono), - now_mono, now_mono) + body = encode(seq + 2, universe_ns, anchor_ns, rate, armed, generation, now_mono, now_mono) m[16:] = body[16:] m[8:16] = struct.pack(" None: fd = os.open(PAGE, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) try: os.fchown(fd, user.pw_uid, user.pw_gid) - os.write(fd, LAYOUT.pack(MAGIC, VERSION, 0, 0, 0, 1.0, 0, 0, 0, check(0, 0, 0, 1.0, 0, 0, 0, 0), 0, 0)) + os.write(fd, encode(0, 0, 0, 1.0, 0, 0, 0, 0)) os.fchmod(fd, 0o644) finally: os.close(fd) diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/README.md b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/README.md new file mode 100644 index 000000000..f744ec21d --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/README.md @@ -0,0 +1,82 @@ +# The egress proxy + +`egress` is the `ubuntu-24.04` base computer's explicit proxy, the way every session program goes out. The session's +programs read the universe's time (the clock library in `../clock/`), so a client that checks a site's certificate with +its own TLS, Chromium, Java or an official Node build, refuses a live site once the clock is months back or ahead. The +proxy runs on real time and hands such a client a certificate that holds at the universe's date when it must. + +It is one static Go program. The base's Dockerfile builds it in a stage of its own, after `go vet` and its tests, so no +Go toolchain is left in the image. `go.sum` pins its modules, and Dependabot's `gomod` entry keeps them current. + +## What runs where + +- `egress boot` runs once, as root, in the base's init (`../start-desktop`), before root's supervisord starts: + - it mints the computer's CA once, valid from 1950 to 9999 as routed sites' certificates are, its key in + `/etc/agentenv/egress/` and readable by `agentenv-egress` alone; + - it trusts the CA in the system store, agent-env's trust files (`/etc/agentenv/ca.pem`, `ca-bundle.pem`) and the + session user's NSS database, and fails the boot naming any of these it can't; + - it trusts the CA in every certifi bundle and every JDK's store an app brings, naming any that can't take it among + its warnings, and the computer boots; + - its report is `/var/log/agentenv-root/egress-boot.log`. +- `egress serve` runs as `agentenv-egress` under root's supervisord (`../root.conf`), on `127.0.0.1:3128`. The clock + library leaves that user on real time. +- The session's supervisord (`../supervisord.conf`), and a login shell through `agentenv-egress.sh`, point + `HTTP(S)_PROXY`, `ALL_PROXY`, Java's proxy properties (`JAVA_TOOL_OPTIONS`) and Node's `fetch` + (`NODE_USE_ENV_PROXY`) at it. +- agent-env's own servers in the computer, the driver and its apps' servers, run on real time and send their own + transfers to the object store directly (`agentenv_protocol.transfers.direct`); the programs they start keep the + proxy. + +## What it does with a connection + +- **Plain HTTP** (an absolute URL): the client's own head goes on with its request line and the proxy-only headers + rewritten, then bytes are relayed both ways. An upgrade keeps its `Connection` and `Upgrade` headers; any other + request closes once answered. +- **`CONNECT`**: the destination's name is resolved, and the connection is refused unless every address is global + (`notGlobal` lists the blocks that aren't). A routed site's name, in the hosts file's `# agentenv sites` block, goes + to the site proxy there. Bytes that aren't TLS, and a routed site's TLS, are relayed untouched. +- **TLS, with the clock unarmed**: relayed untouched. +- **TLS, with the clock armed**: each of the name's addresses is decided by its own probe, a handshake the proxy makes + with the client's own ClientHello, cached per address and SNI. An address already found working is tried first; one + that doesn't answer is passed over. + - The probe's site verifies, and every universe a client may check against is more than `Margin` (8 days) inside the + dates its verified chain shares: **passed through** untouched. + - The probe's handshake doesn't finish (a site asking for the client's certificate, say): **passed through**, for + the client to answer. + - Otherwise: **intercepted**. The site is verified against real time and the image's public roots, and the client is + handed a leaf for its names from the computer's CA, or from a CA trusted nowhere when the site doesn't verify. The + proxy's own handshake with the site uses uTLS, built from the client's ClientHello, so the site sees the client's + fingerprint and ALPN offer, and h2 runs end to end. + +The universes a client may check against are now, the universe at the clock's anchor, and, for `Hold` (30 s) after a +change, the universe the clock left. The universe is read from the clock page the follower writes (`clock.go`); +`testdata/` holds pages the follower's writer wrote, which both sides' tests read. + +## Its log + +One JSON line per connection, on stdout, which root's supervisord keeps in `/var/log/agentenv-root/egress.log`, out of +the session user's reach. Its fields, each there when it applies: + +| Field | What it is | +|---|---| +| `t` | When the line was written, in seconds | +| `client`, `method`, `host`, `port` | Who asked, how, and for what | +| `allowed`, `dst` | `global` or `routed`, and the address dialled | +| `mode` | `pass`, `splice`, `intercept`, `http`, `upgrade` or `refused` | +| `decision` | `real`, `inside`, `outside`, `unverified`, `probe_failed`, `routed` or `not TLS` | +| `sni`, `alpn_offered`, `alpn` | The client's server name and protocols, and the one the site picked | +| `universes`, `upstream_valid` | The universes checked, and the verified chain's shared dates, in seconds | +| `probed`, `probe_error` | Whether this connection probed, and why a probe failed | +| `verified`, `upstream_verify`, `upstream_tls`, `minted` | An interception's upstream: verified or why not, its version and cipher, a new leaf | +| `upstream_ms`, `handshake_ms` | Milliseconds to the upstream, and to the client's handshake | +| `error` | Why the connection ended early | + +## Testing it + +```bash +make egress-test # gofmt, go vet and go test -race, in the Go image the Dockerfile builds with +``` + +It needs Docker, not Go. CI runs it as the `egress` job. `tst/integration/computer/egress_test.py` runs the proxy in +a computer: the session user's reach, the refused destinations, Java and Node 182 days back, and handshakes through week +jumps. diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/agentenv-egress.sh b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/agentenv-egress.sh new file mode 100644 index 000000000..2840d9eb4 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/agentenv-egress.sh @@ -0,0 +1,7 @@ +# The egress proxy for a login shell, as the session's supervisord gives every session program (supervisord.conf). +export HTTP_PROXY=http://127.0.0.1:3128 HTTPS_PROXY=http://127.0.0.1:3128 ALL_PROXY=http://127.0.0.1:3128 +export http_proxy=http://127.0.0.1:3128 https_proxy=http://127.0.0.1:3128 all_proxy=http://127.0.0.1:3128 +export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal no_proxy=localhost,127.0.0.1,::1,host.docker.internal +export SSL_CERT_FILE=/etc/agentenv/ca-bundle.pem REQUESTS_CA_BUNDLE=/etc/agentenv/ca-bundle.pem +export NODE_EXTRA_CA_CERTS=/etc/agentenv/ca.pem NODE_USE_ENV_PROXY=1 +export JAVA_TOOL_OPTIONS="-Dhttp.proxyHost=127.0.0.1 -Dhttp.proxyPort=3128 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|host.docker.internal" diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/boot.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/boot.go new file mode 100644 index 000000000..e1ea67894 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/boot.go @@ -0,0 +1,285 @@ +package main + +import ( + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "os/user" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +// bootPaths are where boot reads and writes; tests point them into a folder of their own. +type bootPaths struct { + Folder, PublicCA, Anchor string + // Bundles are the PEM bundles the CA is appended to when they exist: agent-env's trust files, which + // SSL_CERT_FILE, REQUESTS_CA_BUNDLE and NODE_EXTRA_CA_CERTS name. + Bundles []string + // UpdateTrust rebuilds the system store from its anchors, and its hooks Java's. + UpdateTrust []string + // CertifiRoots and JavaRoots are searched for certifi's cacert.pem and JDKs' lib/security/cacerts. + CertifiRoots, JavaRoots []string + EgressUser, SessionUser string + // NSS adds the CA to the session user's NSS database, which Chromium and Firefox read; nil leaves it. + NSS func(paths bootPaths) error +} + +func defaultBootPaths() bootPaths { + return bootPaths{ + Folder: Folder, PublicCA: PublicCA, Anchor: "/usr/local/share/ca-certificates/agentenv-egress.crt", + Bundles: []string{"/etc/agentenv/ca.pem", "/etc/agentenv/ca-bundle.pem"}, + UpdateTrust: []string{"update-ca-certificates"}, + CertifiRoots: []string{"/usr/lib/python3", "/usr/local/lib", "/opt"}, + JavaRoots: []string{"/usr/lib/jvm", "/opt"}, + EgressUser: EgressUser, SessionUser: SessionUser, + NSS: sessionNSS, + } +} + +// boot, as root before the session starts: the CA, minted once per computer, its key the egress user's alone, and +// trusted in the system store, agent-env's trust files, every certifi bundle, every JDK's store and the session +// user's NSS database. A step that fails fails the boot, naming it, but for a certifi bundle or a JDK's store, which +// an app brings: those are named among the report's warnings. +func boot(p bootPaths) (map[string]any, error) { + start := time.Now() + report := map[string]any{} + egress, err := user.Lookup(p.EgressUser) + if err != nil { + return report, err + } + uid, _ := strconv.Atoi(egress.Uid) + gid, _ := strconv.Atoi(egress.Gid) + if err := os.MkdirAll(p.Folder, 0o700); err != nil { + return report, err + } + caPath, keyPath := filepath.Join(p.Folder, "ca.pem"), filepath.Join(p.Folder, "ca.key") + if _, err := os.Stat(caPath); errors.Is(err, fs.ErrNotExist) { + if err := writeCA(caPath, keyPath); err != nil { + return report, fmt.Errorf("minting the CA: %w", err) + } + report["minted"] = true + } + for _, path := range []string{p.Folder, caPath, keyPath} { + if err := os.Chown(path, uid, gid); err != nil { + return report, err + } + } + if err := os.Chmod(p.Folder, 0o700); err != nil { + return report, err + } + caPEM, err := os.ReadFile(caPath) + if err != nil { + return report, err + } + for _, path := range []string{p.PublicCA, p.Anchor} { + if err := writeRootFile(path, caPEM); err != nil { + return report, err + } + } + if len(p.UpdateTrust) > 0 { + if out, err := exec.Command(p.UpdateTrust[0], p.UpdateTrust[1:]...).CombinedOutput(); err != nil { + return report, fmt.Errorf("%s: %w: %s", strings.Join(p.UpdateTrust, " "), err, truncate(string(out), ErrorLimit)) + } + } + appended := map[string]bool{} + for _, path := range p.Bundles { + added, err := appendOnce(path, caPEM) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return report, fmt.Errorf("trusting the CA in %s: %w", path, err) + } + appended[path] = added + } + // An app's own stores, certifi's and JDKs': one that can't take the CA is named, and the computer boots, its + // app then refusing intercepted sites as it would on a store of its own. + var warnings []string + for _, path := range certifiBundles(p.CertifiRoots) { + added, err := appendOnce(path, caPEM) + if err != nil { + warnings = append(warnings, fmt.Sprintf("trusting the CA in %s: %v", path, err)) + continue + } + appended[path] = added + } + report["bundles"] = appended + java := map[string]string{} + for _, store := range javaStores(p.JavaRoots) { + if err := trustJava(store, p.PublicCA); err != nil { + warnings = append(warnings, fmt.Sprintf("trusting the CA in %s: %v", store, err)) + continue + } + java[store] = "trusted" + } + report["java"] = java + if len(warnings) > 0 { + report["warnings"] = warnings + } + if p.NSS != nil { + if err := p.NSS(p); err != nil { + return report, fmt.Errorf("the session's NSS database: %w", err) + } + report["nss"] = "trusted" + } + report["seconds"] = time.Since(start).Seconds() + return report, nil +} + +func writeCA(caPath, keyPath string) error { + ca, err := mintCA(CAName) + if err != nil { + return err + } + key, err := x509.MarshalPKCS8PrivateKey(ca.key) + if err != nil { + return err + } + if err := writeExclusive(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}), 0o600); err != nil { + return err + } + return writeExclusive(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.cert.Raw}), 0o644) +} + +func writeExclusive(path string, data []byte, mode os.FileMode) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + return f.Close() +} + +// writeRootFile writes path whole, 0644, replacing it in one rename. +func writeRootFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + part := path + ".part" + if err := os.WriteFile(part, data, 0o644); err != nil { + return err + } + if err := os.Chmod(part, 0o644); err != nil { + return err + } + return os.Rename(part, path) +} + +// appendOnce appends pem to the bundle at path (through a symlink) unless it's there; whether it did. +func appendOnce(path string, pemData []byte) (bool, error) { + real, err := filepath.EvalSymlinks(path) + if err != nil { + return false, err + } + text, err := os.ReadFile(real) + if err != nil { + return false, err + } + block := strings.TrimSpace(string(pemData)) + if strings.Contains(string(text), block) { + return false, nil + } + f, err := os.OpenFile(real, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return false, err + } + defer f.Close() + sep := "" + if len(text) > 0 && !strings.HasSuffix(string(text), "\n") { + sep = "\n" + } + _, err = f.WriteString(sep + block + "\n") + return err == nil, err +} + +func certifiBundles(roots []string) []string { + seen := map[string]bool{} + var out []string + for _, root := range roots { + filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || d.Name() != "cacert.pem" || filepath.Base(filepath.Dir(path)) != "certifi" { + return nil + } + real, err := filepath.EvalSymlinks(path) + if err == nil && !seen[real] { + seen[real] = true + out = append(out, path) + } + return nil + }) + } + sort.Strings(out) + return out +} + +// javaStores are the JDKs' stores, lib/security/cacerts, under roots; the system store's hook builds Debian's. +func javaStores(roots []string) []string { + var out []string + for _, root := range roots { + filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || d.Name() != "cacerts" { + return nil + } + if filepath.Base(filepath.Dir(path)) == "security" && filepath.Base(filepath.Dir(filepath.Dir(path))) == "lib" { + if info, err := os.Lstat(path); err == nil && info.Mode().IsRegular() { + out = append(out, path) + } + } + return nil + }) + } + sort.Strings(out) + return out +} + +// trustJava adds the CA to a JDK's store with that JDK's own keytool, else any on the PATH. +func trustJava(store, ca string) error { + keytool := filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(store))), "bin", "keytool") + if _, err := os.Stat(keytool); err != nil { + if keytool, err = exec.LookPath("keytool"); err != nil { + return errors.New("there's no keytool to add the CA with") + } + } + opened := []string{"-keystore", store, "-storepass", "changeit"} + exec.Command(keytool, append([]string{"-delete", "-alias", "agentenv-egress"}, opened...)...).Run() + out, err := exec.Command(keytool, append([]string{"-importcert", "-noprompt", "-trustcacerts", "-alias", + "agentenv-egress", "-file", ca}, opened...)...).CombinedOutput() + if err != nil { + return fmt.Errorf("keytool: %w: %s", err, truncate(string(out), ErrorLimit)) + } + return nil +} + +// sessionNSS adds the CA to the session user's NSS database, as that user, making the database if there's none. +func sessionNSS(p bootPaths) error { + session, err := user.Lookup(p.SessionUser) + if err != nil { + return err + } + db := filepath.Join(session.HomeDir, ".pki", "nssdb") + as := []string{"setpriv", "--reuid=" + p.SessionUser, "--regid=" + p.SessionUser, "--init-groups", "env", + "HOME=" + session.HomeDir} + steps := [][]string{{"mkdir", "-p", db}} + if _, err := os.Stat(filepath.Join(db, "cert9.db")); err != nil { + steps = append(steps, []string{"certutil", "-N", "-d", "sql:" + db, "--empty-password"}) + } + steps = append(steps, []string{"certutil", "-A", "-d", "sql:" + db, "-n", NSSNickname, "-t", "C,,", "-i", + p.PublicCA}) + for _, step := range steps { + argv := append(append([]string{}, as...), step...) + if out, err := exec.Command(argv[0], argv[1:]...).CombinedOutput(); err != nil { + return fmt.Errorf("%s: %w: %s", strings.Join(step[:2], " "), err, truncate(string(out), ErrorLimit)) + } + } + return nil +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/boot_test.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/boot_test.go new file mode 100644 index 000000000..239d8db0a --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/boot_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "os" + "os/user" + "path/filepath" + "strings" + "testing" +) + +func TestBootMintsTheCAOnceItsKeyPrivateAndTrustsItInEveryBundleOnce(t *testing.T) { + dir := t.TempDir() + me, _ := user.Current() + bundle := filepath.Join(dir, "ca-bundle.pem") + os.WriteFile(bundle, []byte("-----BEGIN CERTIFICATE-----\nroot\n-----END CERTIFICATE-----"), 0o644) + certifi := filepath.Join(dir, "opt", "venv", "lib", "certifi", "cacert.pem") + os.MkdirAll(filepath.Dir(certifi), 0o755) + os.WriteFile(certifi, []byte(""), 0o644) + p := bootPaths{ + Folder: filepath.Join(dir, "egress"), PublicCA: filepath.Join(dir, "egress-ca.pem"), + Anchor: filepath.Join(dir, "anchors", "agentenv-egress.crt"), + Bundles: []string{bundle, filepath.Join(dir, "missing.pem")}, + CertifiRoots: []string{filepath.Join(dir, "opt")}, + EgressUser: me.Username, + } + report, err := boot(p) + if err != nil { + t.Fatal(err, report) + } + if report["minted"] != true { + t.Fatalf("%v", report) + } + info, _ := os.Stat(filepath.Join(p.Folder, "ca.key")) + if info.Mode().Perm() != 0o600 { + t.Fatalf("the CA key is %v", info.Mode().Perm()) + } + if info, _ := os.Stat(p.Folder); info.Mode().Perm() != 0o700 { + t.Fatalf("the folder is %v", info.Mode().Perm()) + } + ca, _ := os.ReadFile(filepath.Join(p.Folder, "ca.pem")) + public, _ := os.ReadFile(p.PublicCA) + if string(ca) != string(public) { + t.Fatal("the public CA isn't the CA") + } + if _, err := boot(p); err != nil { + t.Fatal(err) + } + again, _ := os.ReadFile(filepath.Join(p.Folder, "ca.pem")) + if string(again) != string(ca) { + t.Fatal("a second boot minted another CA") + } + for _, path := range []string{bundle, certifi} { + text, _ := os.ReadFile(path) + if n := strings.Count(string(text), strings.TrimSpace(string(ca))); n != 1 { + t.Fatalf("%s has the CA %d times", path, n) + } + } + leaves, err := LoadLeaves(p.Folder) + if err != nil || leaves.ca.cert.NotBefore != CAFrom || leaves.ca.cert.NotAfter != CAUntil { + t.Fatalf("%v", err) + } +} + +func TestAnAppsStoreThatCantTakeTheCAIsNamedAndTheBootGoesOn(t *testing.T) { + dir := t.TempDir() + me, _ := user.Current() + broken := filepath.Join(dir, "opt", "venv", "lib", "certifi") + os.MkdirAll(filepath.Join(broken, "elsewhere"), 0o755) + os.Symlink(filepath.Join(broken, "elsewhere"), filepath.Join(broken, "cacert.pem")) // a folder, not a bundle + jdk := filepath.Join(dir, "jdk", "lib", "security") + os.MkdirAll(jdk, 0o755) + os.WriteFile(filepath.Join(jdk, "cacerts"), []byte("not a keystore"), 0o644) + t.Setenv("PATH", dir) // and no keytool to add the CA with + p := bootPaths{ + Folder: filepath.Join(dir, "egress"), PublicCA: filepath.Join(dir, "egress-ca.pem"), + Anchor: filepath.Join(dir, "anchors", "agentenv-egress.crt"), + CertifiRoots: []string{filepath.Join(dir, "opt")}, JavaRoots: []string{filepath.Join(dir, "jdk")}, + EgressUser: me.Username, + } + report, err := boot(p) + if err != nil { + t.Fatalf("the boot failed: %v", err) + } + warnings, _ := report["warnings"].([]string) + if len(warnings) != 2 || !strings.Contains(strings.Join(warnings, " "), "keytool") { + t.Fatalf("%v", report) + } +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/clock.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/clock.go new file mode 100644 index 000000000..30cb880c1 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/clock.go @@ -0,0 +1,151 @@ +package main + +import ( + "encoding/binary" + "math" + "os" + "sync" + "syscall" + "time" +) + +// The clock page the follower writes (clockpage.py, version 2): magic version seq universe_ns anchor_ns rate armed +// reserved generation check mono_universe_ns mono_anchor_ns, little-endian, 80 bytes, seq odd while it's written. +const ( + pageSize = 80 + pageMagic = 0x4B434541 + checkSalt = 0x9E3779B97F4A7C15 +) + +// state is the universe as the session reads it: armed, the universe at the anchor (seconds), the anchor (real +// seconds) and the rate. +type state struct { + Armed bool + V float64 + Anchor float64 + Rate float64 +} + +var unarmed = state{Rate: 1} + +func (s state) at(t float64) float64 { + if !s.Armed { + return t + } + if s.Rate == 0 || t <= s.Anchor { + return s.V + } + return s.V + (t-s.Anchor)*s.Rate +} + +func pageCheck(seq uint64, u, a int64, rate float64, armed uint32, generation uint64, mu, ma int64) uint64 { + return seq ^ uint64(u) ^ uint64(a) ^ math.Float64bits(rate) ^ uint64(armed) ^ generation ^ uint64(mu) ^ + uint64(ma) ^ checkSalt +} + +// readPage reads a consistent state from the page's bytes, or false when it's being written, torn or not a page. +func readPage(b []byte) (state, bool) { + if len(b) < pageSize || binary.LittleEndian.Uint32(b[0:]) != pageMagic { + return state{}, false + } + seq := binary.LittleEndian.Uint64(b[8:]) + if seq&1 == 1 { + return state{}, false + } + u := int64(binary.LittleEndian.Uint64(b[16:])) + a := int64(binary.LittleEndian.Uint64(b[24:])) + rate := math.Float64frombits(binary.LittleEndian.Uint64(b[32:])) + armed := binary.LittleEndian.Uint32(b[40:]) + generation := binary.LittleEndian.Uint64(b[48:]) + check := binary.LittleEndian.Uint64(b[56:]) + mu := int64(binary.LittleEndian.Uint64(b[64:])) + ma := int64(binary.LittleEndian.Uint64(b[72:])) + if check != pageCheck(seq, u, a, rate, armed, generation, mu, ma) || binary.LittleEndian.Uint64(b[8:]) != seq { + return state{}, false + } + if armed == 0 { + return unarmed, true + } + return state{Armed: true, V: float64(u) / 1e9, Anchor: float64(a) / 1e9, Rate: rate}, true +} + +// Universe follows the clock page, read every PagePoll, so a change is seen and the universe it left is held for Hold. +type Universe struct { + path string + mu sync.Mutex + page []byte + current state + known bool + previous state + leftAt time.Time +} + +func NewUniverse(path string) *Universe { + u := &Universe{path: path, current: unarmed} + u.refresh(time.Now()) + return u +} + +func (u *Universe) Poll() { + for { + time.Sleep(PagePoll) + u.refresh(time.Now()) + } +} + +func (u *Universe) read() state { + if u.page == nil { + f, err := os.Open(u.path) + if err != nil { + return unarmed + } + mapped, err := syscall.Mmap(int(f.Fd()), 0, pageSize, syscall.PROT_READ, syscall.MAP_SHARED) + f.Close() + if err != nil { + return unarmed + } + u.page = mapped + } + for i := 0; i < PageReads; i++ { + if s, ok := readPage(u.page); ok { + return s + } + } + return u.current +} + +func (u *Universe) refresh(now time.Time) state { + u.mu.Lock() + defer u.mu.Unlock() + s := u.read() + if u.known && s != u.current { + u.previous, u.leftAt = u.current, now + } + u.current, u.known = s, true + return s +} + +// View is false when the session is on real time and was armed nowhere within Hold; else true, with the universes a +// client may check against: now, the universe at the anchor, and the one left within Hold when it was armed. +func (u *Universe) View(now time.Time) (bool, []float64) { + u.refresh(now) + u.mu.Lock() + defer u.mu.Unlock() + return view(u.current, u.previous, u.leftAt, now) +} + +func view(current, previous state, leftAt time.Time, now time.Time) (bool, []float64) { + held := !leftAt.IsZero() && now.Sub(leftAt) <= Hold && previous.Armed + if !current.Armed && !held { + return false, nil + } + t := float64(now.UnixNano()) / 1e9 + out := []float64{current.at(t)} + if current.Armed { + out = append(out, current.V) + } + if held { + out = append(out, previous.at(t)) + } + return true, out +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/clock_test.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/clock_test.go new file mode 100644 index 000000000..22bb41503 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/clock_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "encoding/binary" + "encoding/hex" + "math" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func page(seq uint64, universe, anchor time.Time, rate float64, armed bool) []byte { + b := make([]byte, pageSize) + u, a := universe.UnixNano(), anchor.UnixNano() + var on uint32 + if armed { + on = 1 + } + binary.LittleEndian.PutUint32(b[0:], pageMagic) + binary.LittleEndian.PutUint32(b[4:], 2) + binary.LittleEndian.PutUint64(b[8:], seq) + binary.LittleEndian.PutUint64(b[16:], uint64(u)) + binary.LittleEndian.PutUint64(b[24:], uint64(a)) + binary.LittleEndian.PutUint64(b[32:], math.Float64bits(rate)) + binary.LittleEndian.PutUint32(b[40:], on) + binary.LittleEndian.PutUint64(b[48:], 7) + binary.LittleEndian.PutUint64(b[56:], pageCheck(seq, u, a, rate, on, 7, a, a)) + binary.LittleEndian.PutUint64(b[64:], uint64(a)) + binary.LittleEndian.PutUint64(b[72:], uint64(a)) + return b +} + +func TestAPageReadsAsTheUniverseItHoldsAndATornOneNot(t *testing.T) { + universe, anchor := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), time.Unix(time.Now().Unix(), 0) + s, ok := readPage(page(4, universe, anchor, 2, true)) + if !ok || !s.Armed || s.Rate != 2 || int64(s.V) != universe.Unix() { + t.Fatalf("read %+v %v", s, ok) + } + if got := s.at(float64(anchor.Unix()) + 10); int64(got) != universe.Unix()+20 { + t.Fatalf("10 s on at rate 2: %v", got) + } + if _, ok := readPage(page(5, universe, anchor, 2, true)); ok { + t.Fatal("a page being written (odd seq) read as whole") + } + torn := page(4, universe, anchor, 2, true) + torn[20] ^= 1 + if _, ok := readPage(torn); ok { + t.Fatal("a torn page read as whole") + } + if s, ok := readPage(page(2, time.Time{}, time.Time{}, 1, false)); !ok || s.Armed { + t.Fatalf("an unarmed page: %+v %v", s, ok) + } +} + +func TestTheUniverseFollowsThePageFileAndHoldsTheOneItLeft(t *testing.T) { + path := filepath.Join(t.TempDir(), "page") + if err := os.WriteFile(path, page(2, time.Time{}, time.Time{}, 1, false), 0o644); err != nil { + t.Fatal(err) + } + u := NewUniverse(path) + if armed, _ := u.View(time.Now()); armed { + t.Fatal("unarmed page viewed as armed") + } + past := time.Now().Add(-180 * 24 * time.Hour) + f, _ := os.OpenFile(path, os.O_WRONLY, 0) + f.WriteAt(page(4, past, time.Now(), 1, true), 0) + f.Close() + armed, universes := u.View(time.Now()) + if !armed || len(universes) != 2 || int64(universes[1]) != past.Unix() { + t.Fatalf("armed page: %v %v", armed, universes) + } +} + +func TestAClientMayCheckAgainstTheUniverseItLeftForHoldAfterAChange(t *testing.T) { + now := time.Now() + past := state{Armed: true, V: float64(now.Add(-100 * 24 * time.Hour).Unix()), Anchor: float64(now.Unix()), Rate: 1} + if armed, _ := view(unarmed, state{}, time.Time{}, now); armed { + t.Fatal("real time viewed as armed") + } + armed, universes := view(unarmed, past, now.Add(-Hold/2), now) + if !armed || len(universes) != 2 { + t.Fatalf("within Hold of leaving an armed universe: %v %v", armed, universes) + } + if armed, _ := view(unarmed, past, now.Add(-2*Hold), now); armed { + t.Fatal("past Hold, the universe left still counts") + } +} + +// golden is a page the follower's writer wrote (clockpage.encode), as testdata keeps it. +func golden(t *testing.T, name string) []byte { + text, err := os.ReadFile(filepath.Join("testdata", "page-"+name+".hex")) + if err != nil { + t.Fatal(err) + } + b, err := hex.DecodeString(strings.TrimSpace(string(text))) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestThePagesTheFollowersWriterWritesReadAsTheUniverseTheyHold(t *testing.T) { + s, ok := readPage(golden(t, "armed")) + if !ok || !s.Armed || s.V != 1577836800 || s.Rate != 2 || math.Abs(s.Anchor-1794000000.123456789) > 1e-6 { + t.Fatalf("armed: %+v %v", s, ok) + } + if s, ok := readPage(golden(t, "unarmed")); !ok || s != unarmed { + t.Fatalf("unarmed: %+v %v", s, ok) + } +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/go.mod b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/go.mod new file mode 100644 index 000000000..7b1b97e80 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/go.mod @@ -0,0 +1,12 @@ +module agentenv/egress + +go 1.26.11 + +require github.com/refraction-networking/utls v1.8.2 + +require ( + github.com/andybalholm/brotli v1.0.6 // indirect + github.com/klauspost/compress v1.17.4 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/sys v0.31.0 // indirect +) diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/go.sum b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/go.sum new file mode 100644 index 000000000..3e8a6d1ef --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/go.sum @@ -0,0 +1,10 @@ +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/hello.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/hello.go new file mode 100644 index 000000000..381ea2993 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/hello.go @@ -0,0 +1,164 @@ +package main + +import ( + "bufio" + "encoding/binary" + "errors" + "strings" + "time" +) + +// A TLS record's content type for handshake messages, a ClientHello's handshake type, and the most plaintext one +// record carries. +const ( + recordHandshake = 22 + clientHelloType = 1 + maxRecord = 16384 +) + +// clientHello is the ClientHello handshake message the client has sent, peeked from r so the handshake stays +// untouched for whoever reads it next; it may span several records. +func clientHello(r *bufio.Reader, deadline time.Time) ([]byte, error) { + need := 5 + for { + if time.Now().After(deadline) { + return nil, errors.New("ClientHello incomplete") + } + data, err := r.Peek(need) + if err != nil { + if errors.Is(err, bufio.ErrBufferFull) { + return nil, errors.New("ClientHello larger than the read buffer") + } + return nil, err + } + if data[0] != recordHandshake { + return nil, errors.New("not TLS") + } + var body []byte + for p := 0; ; { + if p+5 > len(data) { + need = p + 5 + break + } + if data[p] != recordHandshake { + return nil, errors.New("ClientHello interrupted by another record") + } + size := int(binary.BigEndian.Uint16(data[p+3:])) + if p+5+size > len(data) { + need = p + 5 + size + break + } + body = append(body, data[p+5:p+5+size]...) + p += 5 + size + if len(body) >= 4 { + if whole := 4 + (int(body[1])<<16 | int(body[2])<<8 | int(body[3])); len(body) >= whole { + return body[:whole], nil + } + } + } + } +} + +// parseHello is the server name and the ALPN protocols a ClientHello message offers. +func parseHello(hello []byte) (string, []string, error) { + p, err := helloExtensions(hello) + if err != nil { + return "", nil, err + } + if len(hello) < p+2 { + return "", nil, nil + } + end := p + 2 + int(binary.BigEndian.Uint16(hello[p:])) + if end > len(hello) { + return "", nil, errors.New("extensions overrun the ClientHello") + } + var sni string + var alpn []string + for q := p + 2; q+4 <= end; { + kind := binary.BigEndian.Uint16(hello[q:]) + size := int(binary.BigEndian.Uint16(hello[q+2:])) + body := hello[q+4 : min(q+4+size, end)] + q += 4 + size + switch { + case kind == 0 && len(body) > 5 && body[2] == 0: + n := int(binary.BigEndian.Uint16(body[3:])) + if 5+n <= len(body) { + sni = strings.ToLower(string(body[5 : 5+n])) + } + case kind == 16 && len(body) > 2: + for r := 2; r < len(body); { + n := int(body[r]) + if r+1+n > len(body) { + break + } + alpn = append(alpn, string(body[r+1:r+1+n])) + r += 1 + n + } + } + } + return sni, alpn, nil +} + +// resumption: what only the proxy's own session with the client could mean, never the site's. +var dropped = map[uint16]bool{41: true, 42: true} + +// helloRecord is the ClientHello as one handshake record, without the extensions only a resumption with the proxy put +// there, and the extensions it left out. +func helloRecord(hello []byte) ([]byte, []uint16, error) { + p, err := helloExtensions(hello) + if err != nil { + return nil, nil, err + } + body := append([]byte{}, hello[4:p]...) + var gone []uint16 + if len(hello) >= p+2 { + end := p + 2 + int(binary.BigEndian.Uint16(hello[p:])) + if end > len(hello) { + return nil, nil, errors.New("extensions overrun the ClientHello") + } + var exts []byte + for q := p + 2; q+4 <= end; { + kind := binary.BigEndian.Uint16(hello[q:]) + size := int(binary.BigEndian.Uint16(hello[q+2:])) + if q+4+size > end { + return nil, nil, errors.New("an extension overruns the ClientHello") + } + if dropped[kind] { + gone = append(gone, kind) + } else { + exts = append(exts, hello[q:q+4+size]...) + } + q += 4 + size + } + body = binary.BigEndian.AppendUint16(body, uint16(len(exts))) + body = append(body, exts...) + } + if len(body)+4 > maxRecord { + return nil, nil, errors.New("ClientHello longer than one record") + } + msg := append([]byte{clientHelloType, byte(len(body) >> 16), byte(len(body) >> 8), byte(len(body))}, body...) + return append([]byte{recordHandshake, 3, 1, byte(len(msg) >> 8), byte(len(msg))}, msg...), gone, nil +} + +// helloExtensions is where a ClientHello message's extensions start, past its version, random, session id, cipher +// suites and compression methods, each length checked against the message. +func helloExtensions(hello []byte) (int, error) { + if len(hello) < 4 || hello[0] != clientHelloType { + return 0, errors.New("not a ClientHello") + } + p := 4 + 2 + 32 + for _, width := range []int{1, 2, 1} { // session id, cipher suites, compression methods + if len(hello) < p+width { + return 0, errors.New("short ClientHello") + } + n := int(hello[p]) + if width == 2 { + n = int(binary.BigEndian.Uint16(hello[p:])) + } + p += width + n + if p > len(hello) { + return 0, errors.New("a ClientHello field overruns the message") + } + } + return p, nil +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/hello_test.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/hello_test.go new file mode 100644 index 000000000..a3b30b2f1 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/hello_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/binary" + "net" + "reflect" + "testing" + "time" +) + +// goHello is the ClientHello message Go's TLS client sends for name, offering alpn. +func goHello(t *testing.T, name string, alpn []string) []byte { + t.Helper() + client, server := net.Pipe() + go tls.Client(client, &tls.Config{ServerName: name, NextProtos: alpn, InsecureSkipVerify: true}).Handshake() + defer client.Close() + defer server.Close() + hello, err := clientHello(bufio.NewReader(server), time.Now().Add(5*time.Second)) + if err != nil { + t.Fatal(err) + } + return hello +} + +func records(message []byte, size int) []byte { + var out []byte + for len(message) > 0 { + n := min(size, len(message)) + out = append(out, 22, 3, 1, byte(n>>8), byte(n)) + out = append(out, message[:n]...) + message = message[n:] + } + return out +} + +func TestAClientHelloIsReadWholeAcrossRecordsAndNamesItsSiteAndProtocols(t *testing.T) { + hello := goHello(t, "Example.COM", []string{"h2", "http/1.1"}) + sni, alpn, err := parseHello(hello) + if err != nil || sni != "example.com" || !reflect.DeepEqual(alpn, []string{"h2", "http/1.1"}) { + t.Fatalf("parsed %q %v %v", sni, alpn, err) + } + split := records(hello, 100) + again, err := clientHello(bufio.NewReader(bytes.NewReader(split)), time.Now().Add(time.Second)) + if err != nil || !bytes.Equal(again, hello) { + t.Fatalf("across records: %v", err) + } + if _, err := clientHello(bufio.NewReader(bytes.NewReader([]byte("GET / HTTP/1.1\r\n\r\n"))), + time.Now().Add(time.Second)); err == nil { + t.Fatal("plain HTTP read as TLS") + } +} + +func withExtension(hello []byte, kind uint16, body []byte) []byte { + p := 4 + 2 + 32 + p += 1 + int(hello[p]) + p += 2 + int(binary.BigEndian.Uint16(hello[p:])) + p += 1 + int(hello[p]) + exts := append(append([]byte{}, hello[p+2:]...), byte(kind>>8), byte(kind), byte(len(body)>>8), byte(len(body))) + exts = append(exts, body...) + out := append(append([]byte{}, hello[:p]...), byte(len(exts)>>8), byte(len(exts))) + out = append(out, exts...) + n := len(out) - 4 + out[1], out[2], out[3] = byte(n>>16), byte(n>>8), byte(n) + return out +} + +func TestTheUpstreamRecordLeavesOutWhatOnlyAResumptionWithTheProxyMeans(t *testing.T) { + hello := withExtension(goHello(t, "example.com", nil), 41, []byte{0, 1, 2}) + record, gone, err := helloRecord(hello) + if err != nil || !reflect.DeepEqual(gone, []uint16{41}) { + t.Fatalf("dropped %v %v", gone, err) + } + if record[0] != 22 || int(binary.BigEndian.Uint16(record[3:])) != len(record)-5 { + t.Fatal("not one handshake record") + } + if _, again, _ := helloRecord(record[5:]); len(again) != 0 { + t.Fatalf("the record still has %v", again) + } +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/leaves.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/leaves.go new file mode 100644 index 000000000..027ecac92 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/leaves.go @@ -0,0 +1,188 @@ +package main + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "math/big" + "net" + "os" + "path/filepath" + "sort" + "strings" + "sync" +) + +// The longest common name X.509 allows (ub-common-name). +const commonNameLimit = 64 + +// Leaves signs the leaves an intercepted client is handed: by this computer's CA when the site verified, else by a CA +// trusted nowhere, so the client refuses it as it would the site. Each is valid over all of its CA's window, so it +// holds wherever the universe goes, and is minted once per name and its alternative names. +type Leaves struct { + ca, untrusted signer + key *ecdsa.PrivateKey + mu sync.Mutex + cache map[string]*tls.Certificate + // order is the cache's keys, oldest first, so it drops the oldest past cacheLimit; a connection keeps its own. + order []string +} + +type signer struct { + cert *x509.Certificate + key crypto.Signer +} + +func LoadLeaves(folder string) (*Leaves, error) { + certPEM, err := os.ReadFile(filepath.Join(folder, "ca.pem")) + if err != nil { + return nil, err + } + keyPEM, err := os.ReadFile(filepath.Join(folder, "ca.key")) + if err != nil { + return nil, err + } + ca, err := parseSigner(certPEM, keyPEM) + if err != nil { + return nil, err + } + return NewLeaves(ca) +} + +func NewLeaves(ca signer) (*Leaves, error) { + untrusted, err := mintCA("Untrusted upstream") + if err != nil { + return nil, err + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + return &Leaves{ca: ca, untrusted: untrusted, key: key, cache: map[string]*tls.Certificate{}}, nil +} + +// For is the leaf for host and the names sans ("DNS:..." or "IP:..."), by this computer's CA when trusted, and +// whether it was minted now. +func (l *Leaves) For(host string, sans []string, trusted bool) (*tls.Certificate, bool, error) { + sorted := append([]string{}, sans...) + sort.Strings(sorted) + key := host + "|" + strings.Join(sorted, ",") + by := l.ca + if !trusted { + key, by = "untrusted|"+key, l.untrusted + } + l.mu.Lock() + defer l.mu.Unlock() + if cert, ok := l.cache[key]; ok { + return cert, false, nil + } + template := &x509.Certificate{ + SerialNumber: serial(), + Subject: pkix.Name{CommonName: truncate(host, commonNameLimit)}, + NotBefore: by.cert.NotBefore.Add(LeafInset), + NotAfter: by.cert.NotAfter.Add(-LeafInset), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + for _, name := range append([]string{host}, sans...) { + name = strings.TrimPrefix(strings.TrimPrefix(name, "DNS:"), "IP:") + if ip := net.ParseIP(strings.Trim(name, "[]")); ip != nil { + template.IPAddresses = appendIP(template.IPAddresses, ip) + } else if name != "" && !contains(template.DNSNames, name) { + template.DNSNames = append(template.DNSNames, name) + } + } + der, err := x509.CreateCertificate(rand.Reader, template, by.cert, &l.key.PublicKey, by.key) + if err != nil { + return nil, false, err + } + cert := &tls.Certificate{Certificate: [][]byte{der, by.cert.Raw}, PrivateKey: l.key} + l.cache[key] = cert + l.order = append(l.order, key) + for len(l.order) > cacheLimit { + delete(l.cache, l.order[0]) + l.order = l.order[1:] + } + return cert, true, nil +} + +// mintCA is a self-signed CA over the window, its key in memory. +func mintCA(name string) (signer, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return signer{}, err + } + template := &x509.Certificate{ + SerialNumber: serial(), + Subject: pkix.Name{CommonName: name}, + NotBefore: CAFrom, + NotAfter: CAUntil, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return signer{}, err + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return signer{}, err + } + return signer{cert: cert, key: key}, nil +} + +func parseSigner(certPEM, keyPEM []byte) (signer, error) { + block, _ := pem.Decode(certPEM) + if block == nil { + return signer{}, errors.New("no certificate in the CA's PEM") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return signer{}, err + } + block, _ = pem.Decode(keyPEM) + if block == nil { + return signer{}, errors.New("no key in the CA key's PEM") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return signer{}, err + } + s, ok := key.(crypto.Signer) + if !ok { + return signer{}, errors.New("the CA key can't sign") + } + return signer{cert: cert, key: s}, nil +} + +func serial() *big.Int { + n, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 127)) + return n +} + +func appendIP(ips []net.IP, ip net.IP) []net.IP { + for _, have := range ips { + if have.Equal(ip) { + return ips + } + } + return append(ips, ip) +} + +func contains(list []string, s string) bool { + for _, have := range list { + if strings.EqualFold(have, s) { + return true + } + } + return false +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/main.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/main.go new file mode 100644 index 000000000..0bd5ae5c5 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/main.go @@ -0,0 +1,92 @@ +// egress: the computer's explicit egress proxy, on real time, for the session's programs, which run on the universe's. +// +// It runs as its own user, agentenv-egress, a uid the clock library leaves on real time, under root's supervisord, and +// serves CONNECT and absolute-form HTTP on 127.0.0.1:3128. Session programs reach it through HTTP(S)_PROXY and +// ALL_PROXY; nothing redirects or filters, so it needs no NET_ADMIN and works on any provider. +// +// A TLS connection is spliced (the client's bytes relayed untouched) when the clock is unarmed, or when every universe +// a client may check against lies inside the site's real certificate by more than Margin either side. Otherwise it is +// intercepted: the site is verified against real time and the public roots, and the client is handed a leaf from this +// computer's CA, valid over every date the clock can show. The upstream handshake is uTLS's, built from the client's +// own ClientHello, so the site sees the client's fingerprint and ALPN offer either way. +// +// egress boot as root at the computer's start: the CA, minted once, its key this user's alone, trusted everywhere +// egress serve as agentenv-egress: serve, one JSON line per connection on stdout +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" +) + +const ( + Listen = "127.0.0.1:3128" + EgressUser = "agentenv-egress" + SessionUser = "user" + Folder = "/etc/agentenv/egress" + PublicCA = "/etc/agentenv/egress-ca.pem" + PublicRoots = "/usr/local/lib/agentenv-egress/roots.pem" + ClockPage = "/dev/shm/agentenv-clock" + Hosts = "/etc/hosts" + HostsBegin = "# agentenv sites" + HostsEnd = "# end agentenv sites" + CAName = "agentenv egress CA" + NSSNickname = "agentenv egress" + Hold = 30 * time.Second + Margin = 8 * 24 * time.Hour + ProbeTTL = time.Hour + ProbeFailedTTL = time.Minute + FirstBytes = 300 * time.Millisecond + HelloTimeout = 10 * time.Second + DialTimeout = 10 * time.Second + + // How long a client has to send its request's head, and how long either handshake of an interception, the + // client's with the proxy and the proxy's with the site, has to finish. + HeadTimeout = 15 * time.Second + HandshakeTimeout = 15 * time.Second + + // The longest request head a client may send, and the most of an error a log row keeps, in bytes. + HeadLimit = 64 << 10 + ErrorLimit = 300 + + // How far inside the CA's dates each leaf it signs is, so no leaf outlasts its CA. + LeafInset = 2 * time.Hour + + // How often the clock page is read, and how many reads may find it mid-write before the last universe stands. + PagePoll = 20 * time.Millisecond + PageReads = 1000 +) + +// The dates the CA and every leaf it signs are valid between: the widest a certificate carries, as routed sites' +// (sites.UNIVERSE_FROM and UNIVERSE_UNTIL in agent-env's site routing), so they hold at any date the clock can show. +var ( + CAFrom = time.Date(1950, 1, 1, 0, 0, 0, 0, time.UTC) + CAUntil = time.Date(9999, 12, 31, 23, 59, 59, 0, time.UTC) +) + +func main() { + command := "" + if len(os.Args) > 1 { + command = os.Args[1] + } + switch command { + case "boot": + report, err := boot(defaultBootPaths()) + out, _ := json.Marshal(report) + fmt.Println(string(out)) + if err != nil { + fmt.Fprintln(os.Stderr, "egress boot:", err) + os.Exit(1) + } + case "serve": + if err := serveForever(); err != nil { + fmt.Fprintln(os.Stderr, "egress serve:", err) + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "usage: egress boot|serve, not %q\n", command) + os.Exit(2) + } +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/proxy.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/proxy.go new file mode 100644 index 000000000..c575fe5e6 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/proxy.go @@ -0,0 +1,598 @@ +package main + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" +) + +type Proxy struct { + Leaves *Leaves + Universe interface { + View(time.Time) (bool, []float64) + } + // Roots are the public roots a site is verified against, as the image had them before any agentenv CA joined. + Roots *x509.CertPool + // Resolve, Allowed, Routed and Dial default to the system's resolver, a global address, the hosts file's agentenv + // sites block and a TCP dial; tests replace them. + Resolve func(ctx context.Context, host string) ([]netip.Addr, error) + Allowed func(netip.Addr) bool + Routed func() map[string]string + Dial func(dst string) (net.Conn, error) + Log func(row map[string]any) + probes probes +} + +func serveForever() error { + leaves, err := LoadLeaves(Folder) + if err != nil { + return fmt.Errorf("the CA (has boot run?): %w", err) + } + roots, err := os.ReadFile(PublicRoots) + if err != nil { + return err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(roots) { + return fmt.Errorf("no certificates in %s", PublicRoots) + } + universe := NewUniverse(ClockPage) + go universe.Poll() + ln, err := net.Listen("tcp", Listen) + if err != nil { + return err + } + p := &Proxy{Leaves: leaves, Universe: universe, Roots: pool, Log: jsonLines(os.Stdout)} + p.Log(map[string]any{"event": "serving", "listen": Listen, "uid": os.Getuid(), "pid": os.Getpid()}) + return p.Serve(ln) +} + +func jsonLines(w io.Writer) func(map[string]any) { + var mu sync.Mutex + encoder := json.NewEncoder(w) + return func(row map[string]any) { + row["t"] = float64(time.Now().UnixMilli()) / 1e3 + mu.Lock() + defer mu.Unlock() + encoder.Encode(row) + } +} + +func (p *Proxy) Serve(ln net.Listener) error { + for { + conn, err := ln.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return nil + } + return err + } + go p.handle(conn) + } +} + +func (p *Proxy) log(row map[string]any) { + if p.Log != nil { + p.Log(row) + } +} + +func (p *Proxy) handle(conn net.Conn) { + start := time.Now() + row := map[string]any{"client": conn.RemoteAddr().String()} + defer func() { + if r := recover(); r != nil { + row["error"] = fmt.Sprint("panic: ", r) + p.log(row) + } + conn.Close() + }() + br := bufio.NewReaderSize(conn, HeadLimit) + conn.SetReadDeadline(start.Add(HeadTimeout)) + method, target, version, headers, err := readHead(br) + if err != nil { + row["error"] = truncate("request: "+err.Error(), ErrorLimit) + p.log(row) + return + } + row["method"] = method + if method == http.MethodConnect { + host, port := splitAuthority(target, 443) + row["host"], row["port"] = host, port + dsts, why, err := p.destination(host, port) + if err != nil { + row["mode"], row["error"] = "refused", err.Error() + reply(conn, "403 Forbidden", "egress: "+err.Error()+"\n") + p.log(row) + return + } + row["allowed"] = why + if _, err := io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + p.tunnel(conn, br, host, port, dsts, why, row, start) + return + } + u, err := url.Parse(target) + if err != nil || u.Scheme != "http" || u.Host == "" { + row["mode"], row["error"] = "refused", "not an absolute http URL" + reply(conn, "400 Bad Request", "egress: CONNECT, or an absolute http:// URL\n") + p.log(row) + return + } + host, port := splitAuthority(u.Host, 80) + row["host"], row["port"], row["mode"] = host, port, "http" + dsts, why, err := p.destination(host, port) + if err != nil { + row["mode"], row["error"] = "refused", err.Error() + reply(conn, "403 Forbidden", "egress: "+err.Error()+"\n") + p.log(row) + return + } + row["allowed"] = why + up, dst, err := p.dialAny(dsts) + if err != nil { + row["error"] = truncate(err.Error(), ErrorLimit) + reply(conn, "502 Bad Gateway", "egress: "+err.Error()+"\n") + p.log(row) + return + } + defer up.Close() + row["dst"] = dst + upgrade := false + for _, h := range headers { + if headerName(h) == "upgrade" { + upgrade = true + } + } + // The client's own head, but for its request line, which names the path alone, and the headers meant for this + // proxy; a request that isn't an upgrade closes once answered, so its connection can't carry one for another site. + var head strings.Builder + head.WriteString(method + " " + u.RequestURI() + " " + version + "\r\n") + for _, h := range headers { + switch headerName(h) { + case "proxy-connection", "proxy-authorization": + continue + case "connection", "keep-alive": + if !upgrade { + continue + } + } + head.WriteString(h + "\r\n") + } + if upgrade { + row["mode"] = "upgrade" + } else { + head.WriteString("Connection: close\r\n") + } + head.WriteString("\r\n") + conn.SetReadDeadline(time.Time{}) + if _, err := io.WriteString(up, head.String()); err != nil { + row["error"] = truncate("upstream: "+err.Error(), ErrorLimit) + p.log(row) + return + } + p.log(row) + // Both ways at once: a body the client sends once told to continue, or an upgraded connection's frames. + go func() { + io.Copy(up, br) + closeWrite(up) + }() + io.Copy(conn, up) +} + +// readHead is a request's head, read through its blank line: its request line's parts and its header lines. +func readHead(r *bufio.Reader) (string, string, string, []string, error) { + var lines []string + size := 0 + for { + line, err := r.ReadString('\n') + size += len(line) + if size > HeadLimit { + return "", "", "", nil, errors.New("request head too long") + } + if err != nil { + return "", "", "", nil, err + } + line = strings.TrimRight(line, "\r\n") + if line == "" { + if len(lines) == 0 { + continue + } + break + } + lines = append(lines, line) + } + parts := strings.SplitN(lines[0], " ", 3) + if len(parts) != 3 { + return "", "", "", nil, fmt.Errorf("malformed request line %q", truncate(lines[0], 100)) + } + return parts[0], parts[1], parts[2], lines[1:], nil +} + +func headerName(line string) string { + name, _, _ := strings.Cut(line, ":") + return strings.ToLower(strings.TrimSpace(name)) +} + +// destination is the addresses to dial for host:port, in the order to try them, and why they're allowed: "routed" for +// a name the hosts file's agentenv sites block maps (the site proxy, as a client that bypasses this proxy reaches it), +// "global" for a name whose every address is global, IPv4 first. Anything else is refused. +func (p *Proxy) destination(host string, port int) ([]string, string, error) { + host = strings.TrimSuffix(strings.ToLower(strings.Trim(host, "[]")), ".") + routed := p.Routed + if routed == nil { + routed = routedSites + } + if address, ok := routed()[host]; ok { + return []string{net.JoinHostPort(address, strconv.Itoa(port))}, "routed", nil + } + resolve := p.Resolve + if resolve == nil { + resolve = func(ctx context.Context, host string) ([]netip.Addr, error) { + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) + } + } + allowed := p.Allowed + if allowed == nil { + allowed = isGlobal + } + ctx, cancel := context.WithTimeout(context.Background(), DialTimeout) + defer cancel() + addresses, err := resolve(ctx, host) + if err != nil { + return nil, "", fmt.Errorf("%s doesn't resolve: %w", host, err) + } + if len(addresses) == 0 { + return nil, "", fmt.Errorf("%s has no address", host) + } + var private []string + var v4, v6 []string + for _, a := range addresses { + a = a.Unmap() + dst := netip.AddrPortFrom(a, uint16(port)).String() + switch { + case !allowed(a): + private = append(private, a.String()) + case a.Is4(): + v4 = append(v4, dst) + default: + v6 = append(v6, dst) + } + } + if len(private) > 0 { + return nil, "", fmt.Errorf("%s:%d isn't public (%s)", host, port, strings.Join(private, ", ")) + } + return append(v4, v6...), "global", nil +} + +// notGlobal are the special-purpose blocks (IANA's registries) no global address is in. +var notGlobal = func() []netip.Prefix { + var out []netip.Prefix + for _, s := range []string{ + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", + "192.0.0.0/24", "192.0.2.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", + "224.0.0.0/4", "240.0.0.0/4", + "::/128", "::1/128", "64:ff9b:1::/48", "100::/64", "2001::/23", "2001:db8::/32", "fc00::/7", "fe80::/10", + "ff00::/8", + } { + out = append(out, netip.MustParsePrefix(s)) + } + return out +}() + +func isGlobal(a netip.Addr) bool { + a = a.Unmap() + if !a.IsValid() { + return false + } + for _, block := range notGlobal { + if block.Contains(a) { + return false + } + } + return true +} + +// routedSites is the hosts file's agentenv sites block, name to address: root's, so the session user can't add to it. +func routedSites() map[string]string { + out := map[string]string{} + data, err := os.ReadFile(Hosts) + if err != nil { + return out + } + inside := false + for _, line := range strings.Split(string(data), "\n") { + text := strings.TrimSpace(line) + switch { + case text == HostsBegin: + inside = true + case text == HostsEnd: + inside = false + case inside && text != "" && !strings.HasPrefix(text, "#"): + fields := strings.Fields(text) + for _, name := range fields[1:] { + out[strings.ToLower(name)] = fields[0] + } + } + } + return out +} + +func (p *Proxy) tunnel(conn net.Conn, br *bufio.Reader, host string, port int, dsts []string, why string, + row map[string]any, start time.Time) { + wait := FirstBytes + if port == 443 { + wait = HelloTimeout + } + conn.SetReadDeadline(time.Now().Add(wait)) + first, err := br.Peek(1) + if err != nil && !isTimeout(err) { + row["error"] = "closed before sending" + p.log(row) + return + } + if len(first) == 0 || first[0] != recordHandshake || why == "routed" { + row["mode"], row["decision"] = "pass", "not TLS" + if len(first) > 0 && first[0] == recordHandshake { + row["mode"], row["decision"] = "splice", "routed" + } + if !p.splice(conn, br, dsts, row, start) { + p.log(row) + } + return + } + conn.SetReadDeadline(time.Now().Add(HelloTimeout)) + hello, err := clientHello(br, time.Now().Add(HelloTimeout)) + if err != nil { + row["error"] = truncate("ClientHello: "+err.Error(), ErrorLimit) + p.log(row) + return + } + sni, offered, err := parseHello(hello) + if err != nil { + row["error"] = truncate("ClientHello: "+err.Error(), ErrorLimit) + p.log(row) + return + } + row["sni"], row["alpn_offered"] = sni, offered + verifyName := sni + if verifyName == "" { + verifyName = strings.Trim(host, "[]") + } + armed, universes := p.Universe.View(time.Now()) + if !armed { + row["decision"], row["mode"] = "real", "splice" + if !p.splice(conn, br, dsts, row, start) { + p.log(row) + } + return + } + row["universes"] = rounded(universes) + // Each address in turn, by its own probe: one that doesn't answer is passed over, so the connection goes to a + // server whose own certificate decided it. + var unreachable []string + for _, dst := range p.preferChecked(dsts, sni) { + delete(row, "probe_error") + delete(row, "upstream_valid") + seen, probed := p.probeAt(dst, sni, verifyName, hello) + if seen.unreachable { + unreachable = append(unreachable, seen.problem) + continue + } + row["probed"] = probed + decision := "outside" + switch { + case seen.failed: + // a handshake only the client can finish, as one asking for its certificate: the client decides + decision, row["probe_error"] = "probe_failed", seen.problem + case !seen.verified: + decision, row["probe_error"] = "unverified", seen.problem + case inside(seen.from, seen.until, universes): + decision = "inside" + } + if seen.verified { + row["upstream_valid"] = []int64{seen.from.Unix(), seen.until.Unix()} + } + row["decision"] = decision + var dialled bool + if decision == "inside" || decision == "probe_failed" { + row["mode"] = "splice" + dialled = p.splice(conn, br, []string{dst}, row, start) + } else { + row["mode"] = "intercept" + dialled = p.intercept(conn, br, dst, sni, verifyName, hello, row, start) + } + if dialled { + return + } + // it stopped answering since its probe: forget it, and go on to the next + p.forget(dst, sni) + unreachable = append(unreachable, fmt.Sprint(row["error"])) + delete(row, "error") + } + row["error"] = truncate("upstream: no address answered: "+strings.Join(unreachable, "; "), ErrorLimit) + p.log(row) +} + +// preferChecked is dsts with those whose fresh probe answered first, each group in its own order, so an address +// already found working isn't kept waiting behind one whose probe has to be made again. +func (p *Proxy) preferChecked(dsts []string, sni string) []string { + p.probes.mu.Lock() + defer p.probes.mu.Unlock() + var checked, rest []string + for _, dst := range dsts { + if hit, ok := p.probes.cache[dst+"|"+sni]; ok && hit.fresh() && !hit.unreachable { + checked = append(checked, dst) + } else { + rest = append(rest, dst) + } + } + return append(checked, rest...) +} + +// splice relays the connection to the first of dsts that answers, untouched; false, with the error in row, when none +// does. +func (p *Proxy) splice(conn net.Conn, br *bufio.Reader, dsts []string, row map[string]any, start time.Time) bool { + up, dst, err := p.dialAny(dsts) + if err != nil { + row["error"] = truncate("upstream: "+err.Error(), ErrorLimit) + return false + } + defer up.Close() + row["dst"] = dst + conn.SetReadDeadline(time.Time{}) + row["upstream_ms"] = time.Since(start).Milliseconds() + p.log(row) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + io.Copy(up, br) + closeWrite(up) + }() + io.Copy(conn, up) + closeWrite(conn) + wg.Wait() + return true +} + +// intercept hands the client a leaf of this computer's CA and relays the plaintext to dst over a handshake of the +// proxy's own; false, with the error in row, when dst doesn't answer. +func (p *Proxy) intercept(conn net.Conn, br *bufio.Reader, dst, sni, verifyName string, hello []byte, + row map[string]any, start time.Time) bool { + uc, found, err := p.handshake([]string{dst}, sni, verifyName, hello) + var unanswered dialError + if errors.As(err, &unanswered) { + row["error"] = truncate("upstream: "+err.Error(), ErrorLimit) + return false + } + if err != nil { + row["error"] = truncate("upstream: "+err.Error(), ErrorLimit) + p.log(row) + return true + } + defer uc.NetConn().Close() + row["dst"] = dst + row["alpn"], row["verified"], row["upstream_tls"] = found.ALPN, found.Verified, []string{found.Version, found.Cipher} + row["upstream_ms"] = time.Since(start).Milliseconds() + if len(found.Dropped) > 0 { + row["dropped_extensions"] = found.Dropped + } + var sans []string + if found.Verified { + // the site's own names, as its leaf lists them, so the client pools connections as it would with the site's + // certificate + for _, name := range found.SANs { + if sni == "" || strings.HasPrefix(name, "DNS:") { + sans = append(sans, name) + } + } + if sni == "" { + sans = append(sans, "IP:"+strings.Trim(hostOf(dst), "[]")) + } + } else { + row["upstream_verify"] = truncate(found.VerifyErr, 200) + } + cert, minted, err := p.Leaves.For(verifyName, sans, found.Verified) + if err != nil { + row["error"] = truncate("leaf: "+err.Error(), ErrorLimit) + p.log(row) + return true + } + row["minted"] = minted + config := &tls.Config{Certificates: []tls.Certificate{*cert}, SessionTicketsDisabled: true} + if found.ALPN != "" { + config.NextProtos = []string{found.ALPN} + } + client := tls.Server(&peeked{Conn: conn, r: br}, config) + conn.SetDeadline(time.Now().Add(HandshakeTimeout)) + if err := client.Handshake(); err != nil { + row["error"] = truncate("client handshake: "+err.Error(), ErrorLimit) + p.log(row) + return true + } + conn.SetDeadline(time.Time{}) + row["handshake_ms"] = time.Since(start).Milliseconds() + p.log(row) + done := make(chan struct{}, 2) + go func() { + io.Copy(uc, client) + done <- struct{}{} + }() + go func() { + io.Copy(client, uc) + done <- struct{}{} + }() + <-done + client.Close() + uc.NetConn().Close() + <-done + return true +} + +// peeked is a connection whose reads come through the reader that peeked its first bytes. +type peeked struct { + net.Conn + r *bufio.Reader +} + +func (c *peeked) Read(b []byte) (int, error) { return c.r.Read(b) } + +func closeWrite(c net.Conn) { + if cw, ok := c.(interface{ CloseWrite() error }); ok { + cw.CloseWrite() + } +} + +func isTimeout(err error) bool { + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() +} + +func reply(conn net.Conn, status, body string) { + fmt.Fprintf(conn, "HTTP/1.1 %s\r\nContent-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", + status, len(body), body) +} + +func splitAuthority(authority string, port int) (string, int) { + host, p, err := net.SplitHostPort(authority) + if err != nil { + return strings.Trim(authority, "[]"), port + } + n, err := strconv.Atoi(p) + if err != nil { + return host, port + } + return host, n +} + +func hostOf(hostport string) string { + host, _, err := net.SplitHostPort(hostport) + if err != nil { + return hostport + } + return host +} + +func rounded(values []float64) []int64 { + out := make([]int64, len(values)) + for i, v := range values { + out[i] = int64(v) + } + return out +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/proxy_test.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/proxy_test.go new file mode 100644 index 000000000..c8e57b3e7 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/proxy_test.go @@ -0,0 +1,564 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "sync" + "testing" + "time" +) + +type fixedUniverse struct { + armed bool + universes []float64 +} + +func (f fixedUniverse) View(time.Time) (bool, []float64) { return f.armed, f.universes } + +func TestOnlyDestinationsWhoseEveryAddressIsGlobalAreDialled(t *testing.T) { + refused := []string{"10.0.0.1", "10.255.255.254", "172.16.0.1", "172.31.255.1", "192.168.1.1", "127.0.0.1", + "127.1.2.3", "0.0.0.0", "169.254.169.254", "169.254.1.1", "100.64.0.1", "100.127.255.254", "192.0.2.1", + "198.18.0.1", "224.0.0.1", "255.255.255.255", "::1", "::", "fd00::1", "fe80::1", "::ffff:10.0.0.1", + "2001:db8::1"} + if len(refused) != 22 { + t.Fatal(len(refused)) + } + p := &Proxy{Routed: func() map[string]string { return nil }} + for _, address := range refused { + p.Resolve = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr(address)}, nil + } + if _, _, err := p.destination("somewhere.example", 443); err == nil { + t.Errorf("%s dialled", address) + } + } + p.Resolve = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("2606:4700::1111"), netip.MustParseAddr("1.1.1.1")}, nil + } + dst, why, err := p.destination("one.one.one.one", 443) + if err != nil || strings.Join(dst, " ") != "1.1.1.1:443 [2606:4700::1111]:443" || why != "global" { + t.Fatalf("a public name, IPv4 first: %q %q %v", dst, why, err) + } + p.Resolve = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("1.1.1.1"), netip.MustParseAddr("10.0.0.1")}, nil + } + if _, _, err := p.destination("split.example", 443); err == nil || !strings.Contains(err.Error(), "10.0.0.1") { + t.Fatalf("a name with one private address: %v", err) + } + p.Routed = func() map[string]string { return map[string]string{"notes.test": "127.0.0.1"} } + if dst, why, err := p.destination("Notes.Test.", 443); err != nil || strings.Join(dst, " ") != "127.0.0.1:443" || + why != "routed" { + t.Fatalf("a routed site: %q %q %v", dst, why, err) + } +} + +// harness is a proxy in front of the site server, which it reaches as example.com, and a client whose clock is at +// clientTime. +type harness struct { + t *testing.T + site *httptest.Server + proxy *Proxy + url *url.URL + ca signer + clientCert *tls.Certificate + rows []map[string]any + rowsMu sync.Mutex +} + +func newHarness(t *testing.T, site *httptest.Server, universe fixedUniverse, trustSite bool) *harness { + t.Helper() + ca, err := mintCA(CAName) + if err != nil { + t.Fatal(err) + } + leaves, err := NewLeaves(ca) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if trustSite && site.Certificate() != nil { + roots.AddCert(site.Certificate()) + } + h := &harness{t: t, site: site, ca: ca} + h.proxy = &Proxy{ + Leaves: leaves, Universe: universe, Roots: roots, + Resolve: func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil + }, + Allowed: func(netip.Addr) bool { return true }, + Routed: func() map[string]string { return nil }, + Log: func(row map[string]any) { + h.rowsMu.Lock() + h.rows = append(h.rows, row) + h.rowsMu.Unlock() + }, + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go h.proxy.Serve(ln) + h.url, _ = url.Parse("http://" + ln.Addr().String()) + return h +} + +func (h *harness) get(scheme string, clientTime time.Time, http2 bool) (*http.Response, string, error) { + roots := x509.NewCertPool() + if h.site.Certificate() != nil { + roots.AddCert(h.site.Certificate()) + } + roots.AddCert(h.ca.cert) + config := &tls.Config{RootCAs: roots, Time: func() time.Time { return clientTime }} + if h.clientCert != nil { + config.Certificates = []tls.Certificate{*h.clientCert} + } + transport := &http.Transport{Proxy: http.ProxyURL(h.url), ForceAttemptHTTP2: http2, TLSClientConfig: config} + defer transport.CloseIdleConnections() + _, port, _ := net.SplitHostPort(h.site.Listener.Addr().String()) + resp, err := (&http.Client{Transport: transport, Timeout: 10 * time.Second}).Get( + fmt.Sprintf("%s://example.com:%s/hello", scheme, port)) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return resp, string(body), err +} + +func (h *harness) lastRow() map[string]any { + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + h.rowsMu.Lock() + n := len(h.rows) + h.rowsMu.Unlock() + if n > 0 { + break + } + time.Sleep(10 * time.Millisecond) + } + h.rowsMu.Lock() + defer h.rowsMu.Unlock() + if len(h.rows) == 0 { + h.t.Fatal("no row logged") + } + return h.rows[len(h.rows)-1] +} + +func site(t *testing.T, http2 bool) *httptest.Server { + s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello over %s", r.Proto) + })) + s.EnableHTTP2 = http2 + s.StartTLS() + t.Cleanup(s.Close) + return s +} + +func TestOnRealTimeTheClientsBytesReachTheSiteUntouched(t *testing.T) { + s := site(t, false) + h := newHarness(t, s, fixedUniverse{}, true) + resp, body, err := h.get("https", time.Now(), false) + if err != nil || body != "hello over HTTP/1.1" { + t.Fatalf("%q %v", body, err) + } + if !resp.TLS.PeerCertificates[0].Equal(s.Certificate()) { + t.Fatal("the client didn't see the site's own certificate") + } + if row := h.lastRow(); row["mode"] != "splice" || row["decision"] != "real" { + t.Fatalf("%v", row) + } +} + +func TestAUniverseInsideTheSitesCertificateIsSplicedAndOneOutsideIntercepted(t *testing.T) { + s := site(t, true) + now := float64(time.Now().Unix()) + h := newHarness(t, s, fixedUniverse{armed: true, universes: []float64{now, now}}, true) + resp, _, err := h.get("https", time.Now(), true) + if err != nil || !resp.TLS.PeerCertificates[0].Equal(s.Certificate()) { + t.Fatalf("inside: %v", err) + } + if row := h.lastRow(); row["mode"] != "splice" || row["decision"] != "inside" { + t.Fatalf("%v", row) + } + + // httptest's certificate ends in 2084: a universe in 2090 is outside it. + later := time.Date(2090, 6, 1, 0, 0, 0, 0, time.UTC) + u := float64(later.Unix()) + h = newHarness(t, s, fixedUniverse{armed: true, universes: []float64{u, u}}, true) + resp, body, err := h.get("https", later, true) + if err != nil || body != "hello over HTTP/2.0" { + t.Fatalf("outside: %q %v", body, err) + } + leaf := resp.TLS.PeerCertificates[0] + if leaf.CheckSignatureFrom(h.ca.cert) != nil || !leaf.NotAfter.After(later) || leaf.NotBefore.Year() != 1950 { + t.Fatalf("the client's leaf: issuer %s, %s to %s", leaf.Issuer, leaf.NotBefore, leaf.NotAfter) + } + row := h.lastRow() + if row["mode"] != "intercept" || row["decision"] != "outside" || row["alpn"] != "h2" || row["verified"] != true { + t.Fatalf("%v", row) + } +} + +func TestASiteThatDoesntVerifyIsHandedOnUnderACATrustedNowhere(t *testing.T) { + s := site(t, false) + u := float64(time.Date(2090, 6, 1, 0, 0, 0, 0, time.UTC).Unix()) + h := newHarness(t, s, fixedUniverse{armed: true, universes: []float64{u}}, false) + _, _, err := h.get("https", time.Now(), false) + if err == nil || !strings.Contains(err.Error(), "certificate") { + t.Fatalf("the client took it: %v", err) + } + if row := h.lastRow(); row["decision"] != "unverified" { + t.Fatalf("%v", row) + } +} + +func TestPlainHTTPInAbsoluteFormIsForwarded(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "plain %s %s ua=%q", r.Method, r.URL.Path, r.UserAgent()) + })) + t.Cleanup(s.Close) + h := newHarness(t, s, fixedUniverse{}, false) + _, body, err := h.get("http", time.Now(), false) + if err != nil || !strings.HasPrefix(body, "plain GET /hello") { + t.Fatalf("%q %v", body, err) + } +} + +func TestARefusedDestinationIsAnswered403(t *testing.T) { + s := site(t, false) + h := newHarness(t, s, fixedUniverse{}, true) + h.proxy.Allowed = isGlobal + _, _, err := h.get("https", time.Now(), false) + if err == nil || !strings.Contains(err.Error(), "Forbidden") { + t.Fatalf("%v", err) + } + if row := h.lastRow(); row["mode"] != "refused" { + t.Fatalf("%v", row) + } +} + +func TestEveryCheckedAddressIsTriedInTurn(t *testing.T) { + s := site(t, false) + h := newHarness(t, s, fixedUniverse{}, true) + h.proxy.Resolve = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("127.0.0.2"), netip.MustParseAddr("127.0.0.1")}, nil + } + _, body, err := h.get("https", time.Now(), false) + if err != nil || body != "hello over HTTP/1.1" { + t.Fatalf("%q %v", body, err) + } + if row := h.lastRow(); !strings.HasPrefix(fmt.Sprint(row["dst"]), "127.0.0.1:") { + t.Fatalf("%v", row) + } +} + +func TestAnUpgradeOverPlainHTTPRelaysBothWays(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Upgrade") != "echo" || !strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade") { + w.WriteHeader(http.StatusBadRequest) + return + } + conn, rw, _ := w.(http.Hijacker).Hijack() + defer conn.Close() + rw.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: echo\r\n\r\n") + rw.Flush() + line, _ := rw.ReadString('\n') + rw.WriteString(line) + rw.Flush() + })) + t.Cleanup(s.Close) + h := newHarness(t, s, fixedUniverse{}, false) + _, port, _ := net.SplitHostPort(s.Listener.Addr().String()) + conn, err := net.Dial("tcp", h.url.Host) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(5 * time.Second)) + fmt.Fprintf(conn, "GET http://example.com:%s/ws HTTP/1.1\r\nHost: example.com:%s\r\nConnection: Upgrade\r\n"+ + "Upgrade: echo\r\n\r\n", port, port) + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, nil) + if err != nil || resp.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("%v %v", resp, err) + } + fmt.Fprint(conn, "ping\n") + if line, err := br.ReadString('\n'); err != nil || line != "ping\n" { + t.Fatalf("%q %v", line, err) + } +} + +func TestAnUploadThatWaitsFor100ContinueGoesThroughAtOnce(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + fmt.Fprint(w, len(body)) + })) + t.Cleanup(s.Close) + h := newHarness(t, s, fixedUniverse{}, false) + _, port, _ := net.SplitHostPort(s.Listener.Addr().String()) + req, _ := http.NewRequest("POST", "http://example.com:"+port+"/up", bytes.NewReader(make([]byte, 100000))) + req.Header.Set("Expect", "100-continue") + transport := &http.Transport{Proxy: http.ProxyURL(h.url), ExpectContinueTimeout: 10 * time.Second} + defer transport.CloseIdleConnections() + started := time.Now() + resp, err := (&http.Client{Transport: transport, Timeout: 20 * time.Second}).Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if string(body) != "100000" || time.Since(started) > 5*time.Second { + t.Fatalf("%q after %s", body, time.Since(started)) + } +} + +func TestEveryCertificateOfTheVerifiedChainBoundsPassingThrough(t *testing.T) { + year := func(y int) time.Time { return time.Date(y, 1, 1, 0, 0, 0, 0, time.UTC) } + leaf := &x509.Certificate{NotBefore: year(2020), NotAfter: year(2030)} + intermediate := &x509.Certificate{NotBefore: year(2019), NotAfter: year(2025)} + root := &x509.Certificate{NotBefore: year(2000), NotAfter: year(2040)} + from, until := chainWindow([][]*x509.Certificate{{leaf, intermediate, root}}) + if !from.Equal(year(2020)) || !until.Equal(year(2025)) { + t.Fatalf("%s to %s", from, until) + } + if inside(from, until, []float64{float64(year(2027).Unix())}) { + t.Fatal("a universe past the intermediate's end passed through") + } +} + +func TestASiteAskingForTheClientsCertificateIsPassedThroughForTheClientToAnswer(t *testing.T) { + s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello %d certificate", len(r.TLS.PeerCertificates)) + })) + s.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert, MaxVersion: tls.VersionTLS12} + s.StartTLS() + t.Cleanup(s.Close) + u := float64(time.Date(2090, 6, 1, 0, 0, 0, 0, time.UTC).Unix()) + h := newHarness(t, s, fixedUniverse{armed: true, universes: []float64{u}}, true) + client, err := mintCA("client") + if err != nil { + t.Fatal(err) + } + h.clientCert = &tls.Certificate{Certificate: [][]byte{client.cert.Raw}, PrivateKey: client.key} + _, body, err := h.get("https", time.Now(), false) + if err != nil || body != "hello 1 certificate" { + t.Fatalf("%q %v", body, err) + } + if row := h.lastRow(); row["mode"] != "splice" || row["decision"] != "probe_failed" { + t.Fatalf("%v", row) + } +} + +func TestTheProbeAndLeafCachesStayWithinTheirLimit(t *testing.T) { + limit := cacheLimit + cacheLimit = 3 + defer func() { cacheLimit = limit }() + ca, _ := mintCA(CAName) + leaves, _ := NewLeaves(ca) + for i := 0; i < 10; i++ { + if _, _, err := leaves.For(fmt.Sprintf("site%d.test", i), nil, true); err != nil { + t.Fatal(err) + } + } + if len(leaves.cache) != 3 || leaves.cache["site9.test|"] == nil { + t.Fatalf("%d leaves kept", len(leaves.cache)) + } + p := &Proxy{Dial: func(string) (net.Conn, error) { return nil, fmt.Errorf("unreachable") }} + for i := 0; i < 10; i++ { + p.probeAt(fmt.Sprintf("192.0.2.%d:443", i), "", "x", goHello(t, "x", nil)) + } + if len(p.probes.cache) != 3 { + t.Fatalf("%d probes kept", len(p.probes.cache)) + } +} + +func TestAConnectionTheProbeLetsThroughGoesToTheServerItProbed(t *testing.T) { + s := site(t, false) + now := float64(time.Now().Unix()) + h := newHarness(t, s, fixedUniverse{armed: true, universes: []float64{now}}, true) + h.proxy.Resolve = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("127.0.0.2"), netip.MustParseAddr("127.0.0.1")}, nil + } + var mu sync.Mutex + var dialled []string + h.proxy.Dial = func(dst string) (net.Conn, error) { + mu.Lock() + dialled = append(dialled, dst) + mu.Unlock() + return net.DialTimeout("tcp", dst, DialTimeout) + } + if _, _, err := h.get("https", time.Now(), false); err != nil { + t.Fatal(err) + } + row := h.lastRow() + mu.Lock() + defer mu.Unlock() + // the probe tried 127.0.0.2, then reached 127.0.0.1; the connection went straight to 127.0.0.1 + if row["decision"] != "inside" || len(dialled) != 3 || !strings.HasPrefix(dialled[2], "127.0.0.1:") { + t.Fatalf("%v %v", row, dialled) + } + if hit := h.proxy.probes.cache[dialled[0]+"|example.com"]; !hit.unreachable || hit.verified { + t.Fatalf("the address that didn't answer is cached as %+v", hit) + } +} + +func TestASlowProbeHoldsUpNoOtherSite(t *testing.T) { + release := make(chan struct{}) + p := &Proxy{Dial: func(dst string) (net.Conn, error) { + if strings.HasPrefix(dst, "192.0.2.1:") { + <-release + } + return nil, fmt.Errorf("unreachable") + }} + hello := goHello(t, "x", nil) + p.probeAt("192.0.2.2:443", "", "x", hello) + go p.probeAt("192.0.2.1:443", "", "x", hello) + time.Sleep(50 * time.Millisecond) + started := time.Now() + if _, probed := p.probeAt("192.0.2.2:443", "", "x", hello); probed || time.Since(started) > 100*time.Millisecond { + t.Fatalf("a cached site waited %s behind a slow one (probed again: %v)", time.Since(started), probed) + } + close(release) +} + +// standIns resolves example.com to its addresses and dials each to the site unless it's down; both may change +// between requests. +type standIns struct { + mu sync.Mutex + addresses []string + down map[string]bool +} + +func (s *standIns) set(down []string, addresses ...string) { + s.mu.Lock() + defer s.mu.Unlock() + s.addresses, s.down = addresses, map[string]bool{} + for _, a := range down { + s.down[a] = true + } +} + +func newStandIns(h *harness, addresses ...string) *standIns { + s := &standIns{addresses: addresses, down: map[string]bool{}} + h.proxy.Resolve = func(context.Context, string) ([]netip.Addr, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []netip.Addr + for _, a := range s.addresses { + out = append(out, netip.MustParseAddr(a)) + } + return out, nil + } + h.proxy.Dial = func(dst string) (net.Conn, error) { + host, _, _ := net.SplitHostPort(dst) + s.mu.Lock() + down := s.down[host] + s.mu.Unlock() + if down { + return nil, fmt.Errorf("%s is down", host) + } + return net.DialTimeout("tcp", h.site.Listener.Addr().String(), DialTimeout) + } + return s +} + +func TestAnAddressThatStopsAnsweringAfterItsProbeGivesWayToTheNext(t *testing.T) { + s := site(t, false) + now := float64(time.Now().Unix()) + h := newHarness(t, s, fixedUniverse{armed: true, universes: []float64{now}}, true) + addresses := newStandIns(h, "198.51.100.1", "198.51.100.2") + if _, _, err := h.get("https", time.Now(), false); err != nil { + t.Fatal(err) + } + if row := h.lastRow(); !strings.HasPrefix(fmt.Sprint(row["dst"]), "198.51.100.1:") { + t.Fatalf("%v", row) + } + addresses.set([]string{"198.51.100.1"}, "198.51.100.1", "198.51.100.2") + if _, _, err := h.get("https", time.Now(), false); err != nil { + t.Fatal(err) + } + if row := h.lastRow(); !strings.HasPrefix(fmt.Sprint(row["dst"]), "198.51.100.2:") || row["decision"] != "inside" { + t.Fatalf("%v", row) + } +} + +func TestAnAddressThatDidntAnswerLeavesANewAddressToItsOwnProbe(t *testing.T) { + s := site(t, false) + u := float64(time.Date(2090, 6, 1, 0, 0, 0, 0, time.UTC).Unix()) + h := newHarness(t, s, fixedUniverse{armed: true, universes: []float64{u}}, true) + addresses := newStandIns(h) + addresses.set([]string{"198.51.100.1"}, "198.51.100.1") + if _, _, err := h.get("https", time.Date(2090, 6, 1, 0, 0, 0, 0, time.UTC), false); err == nil { + t.Fatal("reached a site none of whose addresses answer") + } + addresses.set([]string{"198.51.100.1"}, "198.51.100.1", "198.51.100.2") + if _, _, err := h.get("https", time.Date(2090, 6, 1, 0, 0, 0, 0, time.UTC), false); err != nil { + t.Fatal(err) + } + row := h.lastRow() + if !strings.HasPrefix(fmt.Sprint(row["dst"]), "198.51.100.2:") || row["mode"] != "intercept" || row["decision"] != "outside" { + t.Fatalf("the new address wasn't judged by its own certificate: %v", row) + } +} + +func TestAClientHelloWhoseFieldsOverrunItIsRefusedNotPanickedOn(t *testing.T) { + hello := goHello(t, "example.com", nil) + p := 4 + 2 + 32 + p += 1 + int(hello[p]) + p += 2 + int(hello[p])<<8 | int(hello[p+1]) + bad := append([]byte{}, hello...) + bad[p] = 0xff // compression methods' length, past the message's end + bad = bad[:p+3] + if _, _, err := parseHello(bad); err == nil { + t.Fatal("parseHello took it") + } + if _, _, err := helloRecord(bad); err == nil { + t.Fatal("helloRecord took it") + } +} + +func TestAProbeThatPanicsLeavesNoOneWaitingOnIt(t *testing.T) { + p := &Proxy{Dial: func(string) (net.Conn, error) { panic("dial") }} + hello := goHello(t, "x", nil) + probe := func() (panicked bool) { + defer func() { panicked = recover() != nil }() + p.probeAt("192.0.2.9:443", "", "x", hello) + return false + } + if !probe() { + t.Fatal("the probe didn't panic") + } + done := make(chan bool) + go func() { done <- probe() }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("a second probe waited on the one that panicked") + } +} + +func TestAnAddressAlreadyFoundWorkingIsTriedBeforeOnesNotYetProbed(t *testing.T) { + p := &Proxy{} + p.probes.cache = map[string]probe{ + "[2001:db8::1]:443|x": {verified: true, at: time.Now()}, + "192.0.2.1:443|x": {unreachable: true, at: time.Now().Add(-2 * ProbeFailedTTL)}, + "192.0.2.3:443|x": {unreachable: true, at: time.Now()}, + } + got := p.preferChecked([]string{"192.0.2.1:443", "192.0.2.2:443", "192.0.2.3:443", "[2001:db8::1]:443"}, "x") + want := []string{"[2001:db8::1]:443", "192.0.2.1:443", "192.0.2.2:443", "192.0.2.3:443"} + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Fatalf("%v", got) + } +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/testdata/page-armed.hex b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/testdata/page-armed.hex new file mode 100644 index 000000000..2aecf27e8 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/testdata/page-armed.hex @@ -0,0 +1 @@ +4145434b02000000040000000000000000008ab9359ae51515cd68be7e91e51800000000000000400100000000000000070000000000000002b1a878f27237d300f2052a0100000000f2052a01000000 diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/testdata/page-unarmed.hex b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/testdata/page-unarmed.hex new file mode 100644 index 000000000..7f6a95f54 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/testdata/page-unarmed.hex @@ -0,0 +1 @@ +4145434b02000000020000000000000000000000000000000000000000000000000000000000f03f00000000000000000300000000000000147c4a7fb979c7a100000000000000000000000000000000 diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/upstream.go b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/upstream.go new file mode 100644 index 000000000..ccf32704f --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/upstream.go @@ -0,0 +1,268 @@ +package main + +import ( + "crypto/x509" + "errors" + "fmt" + "net" + "strings" + "sync" + "time" + + utls "github.com/refraction-networking/utls" +) + +// upstream is what a handshake with a site found: the protocol it picked, its leaf's validity and names, and whether +// that leaf verifies against real time and the public roots. +type upstream struct { + // Dst is the address the handshake reached. + Dst string + ALPN string + Verified bool + VerifyErr string + NotBefore time.Time + NotAfter time.Time + SANs []string + Version string + Cipher string + Dropped []uint16 +} + +// handshake is uTLS's handshake with the site at dst, built from the client's own ClientHello, so the site sees the +// client's fingerprint and ALPN offer. Records go full-size from the start and the connection closes without a +// close_notify, as Chromium's BoringSSL does. +func (p *Proxy) handshake(dsts []string, sni, verifyName string, hello []byte) (*utls.UConn, upstream, error) { + var found upstream + record, gone, err := helloRecord(hello) + if err != nil { + return nil, found, err + } + found.Dropped = gone + spec, err := (&utls.Fingerprinter{AllowBluntMimicry: true}).FingerprintClientHello(record) + if err != nil { + return nil, found, fmt.Errorf("fingerprint: %w", err) + } + raw, dst, err := p.dialAny(dsts) + if err != nil { + return nil, found, dialError{err} + } + found.Dst = dst + uc := utls.UClient(raw, &utls.Config{ServerName: sni, InsecureSkipVerify: true, OmitEmptyPsk: true, + DynamicRecordSizingDisabled: true}, utls.HelloCustom) + if err := uc.ApplyPreset(spec); err != nil { + raw.Close() + return nil, found, fmt.Errorf("apply: %w", err) + } + raw.SetDeadline(time.Now().Add(HandshakeTimeout)) + if err := uc.Handshake(); err != nil { + raw.Close() + return nil, found, fmt.Errorf("handshake: %w", err) + } + raw.SetDeadline(time.Time{}) + st := uc.ConnectionState() + found.ALPN, found.Version, found.Cipher = st.NegotiatedProtocol, fmt.Sprintf("%#04x", st.Version), + utls.CipherSuiteName(st.CipherSuite) + if len(st.PeerCertificates) == 0 { + found.VerifyErr = "no certificate" + return uc, found, nil + } + leaf := st.PeerCertificates[0] + found.NotBefore, found.NotAfter = leaf.NotBefore, leaf.NotAfter + for _, name := range leaf.DNSNames { + found.SANs = append(found.SANs, "DNS:"+name) + } + for _, ip := range leaf.IPAddresses { + found.SANs = append(found.SANs, "IP:"+ip.String()) + } + intermediates := x509.NewCertPool() + for _, c := range st.PeerCertificates[1:] { + intermediates.AddCert(c) + } + chains, err := leaf.Verify(x509.VerifyOptions{Roots: p.Roots, Intermediates: intermediates, DNSName: verifyName, + CurrentTime: time.Now()}) + if err != nil { + found.VerifyErr = err.Error() + } else { + found.Verified = true + found.NotBefore, found.NotAfter = chainWindow(chains) + } + return uc, found, nil +} + +// chainWindow is the dates every certificate of every verified chain is valid between: a client building any of +// those chains takes the site only then, so passing it through is safe only inside them. +func chainWindow(chains [][]*x509.Certificate) (time.Time, time.Time) { + var from, until time.Time + for _, chain := range chains { + for _, cert := range chain { + if from.IsZero() || cert.NotBefore.After(from) { + from = cert.NotBefore + } + if until.IsZero() || cert.NotAfter.Before(until) { + until = cert.NotAfter + } + } + } + return from, until +} + +// probe is what a handshake of the proxy's own found of one address of a site: the dates its verified chain shares, +// or that it didn't answer, or that its handshake didn't finish. It's cached per (address, SNI): a finished one for +// ProbeTTL, the others for ProbeFailedTTL, since they may be a moment's trouble. +type probe struct { + from, until time.Time + verified bool + // failed is a handshake that didn't finish; unreachable, an address that didn't answer. + failed, unreachable bool + at time.Time + problem string +} + +func (hit probe) fresh() bool { + if hit.failed || hit.unreachable { + return time.Since(hit.at) < ProbeFailedTTL + } + return time.Since(hit.at) < ProbeTTL +} + +// probes caches each address's probe, up to cacheLimit entries, the stale ones dropped first once it's full. An +// address's concurrent probes wait on the one under way; no lock is held while one is. +type probes struct { + mu sync.Mutex + cache map[string]probe + underway map[string]*underway +} + +type underway struct { + done chan struct{} + hit probe + ok bool +} + +// cacheLimit bounds the probe and leaf caches, so a computer that visits many sites keeps a bounded memory. +var cacheLimit = 4096 + +// dialError is a handshake's that never reached the site. +type dialError struct{ error } + +// probeAt is the probe of dst for sni, cached or new, and whether it probed now. +func (p *Proxy) probeAt(dst, sni, verifyName string, hello []byte) (probe, bool) { + key := dst + "|" + sni + p.probes.mu.Lock() + if p.probes.cache == nil { + p.probes.cache, p.probes.underway = map[string]probe{}, map[string]*underway{} + } + if hit, ok := p.probes.cache[key]; ok && hit.fresh() { + p.probes.mu.Unlock() + return hit, false + } + if running, ok := p.probes.underway[key]; ok { + p.probes.mu.Unlock() + <-running.done + if !running.ok { + return probe{at: time.Now(), failed: true, problem: "the probe it waited on failed"}, false + } + return running.hit, false + } + running := &underway{done: make(chan struct{})} + p.probes.underway[key] = running + p.probes.mu.Unlock() + defer func() { + p.probes.mu.Lock() + if running.ok { + p.store(key, running.hit) + } + delete(p.probes.underway, key) + p.probes.mu.Unlock() + close(running.done) + }() + + hit := probe{at: time.Now()} + uc, found, err := p.handshake([]string{dst}, sni, verifyName, hello) + var unanswered dialError + switch { + case errors.As(err, &unanswered): + hit.unreachable, hit.problem = true, truncate(err.Error(), 200) + case err != nil: + hit.failed, hit.problem = true, truncate(err.Error(), 200) + case !found.Verified: + hit.problem = truncate(found.VerifyErr, 200) + default: + hit.from, hit.until, hit.verified = found.NotBefore, found.NotAfter, true + } + if uc != nil { + uc.NetConn().Close() + } + running.hit, running.ok = hit, true + return hit, true +} + +// store keeps hit for key, dropping the stale entries, then any others, once the cache is past cacheLimit. +func (p *Proxy) store(key string, hit probe) { + p.probes.cache[key] = hit + if len(p.probes.cache) <= cacheLimit { + return + } + for k, v := range p.probes.cache { + if !v.fresh() { + delete(p.probes.cache, k) + } + } + for k := range p.probes.cache { + if len(p.probes.cache) <= cacheLimit { + break + } + if k != key { + delete(p.probes.cache, k) + } + } +} + +// forget drops dst's probe for sni, for an address that stopped answering since. +func (p *Proxy) forget(dst, sni string) { + p.probes.mu.Lock() + delete(p.probes.cache, dst+"|"+sni) + p.probes.mu.Unlock() +} + +// inside is whether every universe lies inside [from + Margin, until - Margin]. +func inside(from, until time.Time, universes []float64) bool { + lo := float64(from.Add(Margin).UnixNano()) / 1e9 + hi := float64(until.Add(-Margin).UnixNano()) / 1e9 + for _, u := range universes { + if u < lo || u > hi { + return false + } + } + return true +} + +// dialAny dials each address in turn until one answers: which, or every one's error. +func (p *Proxy) dialAny(dsts []string) (net.Conn, string, error) { + var problems []string + for _, dst := range dsts { + conn, err := p.dial(dst) + if err == nil { + return conn, dst, nil + } + problems = append(problems, err.Error()) + } + if len(problems) == 0 { + return nil, "", errors.New("no address to dial") + } + return nil, "", errors.New(strings.Join(problems, "; ")) +} + +func (p *Proxy) dial(dst string) (net.Conn, error) { + if p.Dial != nil { + return p.Dial(dst) + } + return net.DialTimeout("tcp", dst, DialTimeout) +} + +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + } + return s +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf b/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf index 49d4d57c6..96faf63e5 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/root.conf @@ -1,7 +1,8 @@ ; Root's supervisord, which the base's init starts as root once it has made what the session must not touch: the clock -; follower as its own user, on real time, and the session's supervisord (supervisord.conf) as the session user, which -; runs every session program. Its socket and logs are root's alone, so the session user can neither stop nor restart -; either. The follower follows nothing until core writes its config, then picks it up within a quarter second. +; follower and the egress proxy, each as its own user, on real time, and the session's supervisord (supervisord.conf) +; as the session user, which runs every session program. Its socket and logs are root's alone, so the session user can +; neither stop nor restart any of them. The follower follows nothing until core writes its config, then picks it up +; within a quarter second. The egress proxy logs a JSON line per connection. [supervisord] nodaemon=true user=root @@ -31,6 +32,18 @@ redirect_stderr=true stdout_logfile=/var/log/agentenv-root/clock.log environment=HOME="/nonexistent",USER="agentenv-clock",LOGNAME="agentenv-clock" +[program:egress] +command=/usr/local/lib/agentenv-egress/egress serve +user=agentenv-egress +directory=/ +priority=3 +autorestart=true +startsecs=1 +startretries=1000 +redirect_stderr=true +stdout_logfile=/var/log/agentenv-root/egress.log +environment=HOME="/nonexistent",USER="agentenv-egress",LOGNAME="agentenv-egress" + [program:session] command=/usr/bin/supervisord -c /etc/supervisor/supervisord.conf user=user diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop b/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop index 7405a3d18..423f143ac 100755 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop @@ -1,9 +1,9 @@ #!/bin/sh # The base's start, with the settings a deploy passes as AGENTENV_SETTING_, their defaults artifact.toml's. init, # as root at the computer's start, sets the time zone for every program (docker exec's included), makes what the -# session must not touch (the clock page and file, the clock user's from their first byte), then starts root's -# supervisord (root.conf), which runs the clock follower as its own user and the session's supervisord as the session -# user. That one runs xvfb at the resolution, and with-x runs a program, the XFCE session first, once X answers. +# session must not touch (the clock page and file, the clock user's from their first byte; the egress proxy's CA, its +# key the egress user's alone, trusted everywhere), then starts root's supervisord (root.conf), which runs the clock +# follower and the egress proxy, each as its own user, and the session's supervisord as the session user. That one runs xvfb at the resolution, and with-x runs a program, the XFCE session first, once X answers. set -eu resolution="${AGENTENV_SETTING_RESOLUTION:-1280x800}" width="${resolution%%x*}" height="${resolution#*x}" @@ -18,6 +18,8 @@ case "${1:-}" in ln -sfn "/usr/share/zoneinfo/$timezone" /etc/localtime && echo "$timezone" > /etc/timezone install -d -o agentenv-clock -g agentenv-clock -m 0755 /run/agentenv-clock python3 -I /usr/local/lib/agentenv-clock/clockpage.py create agentenv-clock + /usr/local/lib/agentenv-egress/egress boot > /var/log/agentenv-root/egress-boot.log 2>&1 \ + || { cat /var/log/agentenv-root/egress-boot.log >&2; echo "the egress proxy's boot failed" >&2; exit 2; } exec /usr/bin/supervisord -c /etc/supervisor/root.conf ;; xvfb) diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04/supervisord.conf index 6260b4c06..d47dc697d 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/supervisord.conf +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/supervisord.conf @@ -1,12 +1,18 @@ ; The session's programs, all run as the session user by this supervisord, which root's (root.conf) runs as that user; ; every one is on the universe's clock, and only agent-env's own servers carry the clock's real-time marker, on their -; own programs (the composer's). A composed driver or app adds its own in conf.d. +; own programs (the composer's). A composed driver or app adds its own in conf.d. Every program goes out through the +; egress proxy, on real time, by the variables below (Java and Node's fetch by their own, which ignore the others), +; and trusts its CA through the trust files agentenv adds, which root owns. [supervisord] nodaemon=true logfile=/var/log/agentenv/supervisord.log pidfile=/run/agentenv/supervisord.pid childlogdir=/var/log/agentenv -environment=DBUS_SESSION_BUS_ADDRESS="unix:path=/run/agentenv/bus" +environment=DBUS_SESSION_BUS_ADDRESS="unix:path=/run/agentenv/bus", + HTTP_PROXY="http://127.0.0.1:3128",HTTPS_PROXY="http://127.0.0.1:3128",ALL_PROXY="http://127.0.0.1:3128",NO_PROXY="localhost,127.0.0.1,::1,host.docker.internal", + http_proxy="http://127.0.0.1:3128",https_proxy="http://127.0.0.1:3128",all_proxy="http://127.0.0.1:3128",no_proxy="localhost,127.0.0.1,::1,host.docker.internal", + SSL_CERT_FILE="/etc/agentenv/ca-bundle.pem",REQUESTS_CA_BUNDLE="/etc/agentenv/ca-bundle.pem",NODE_EXTRA_CA_CERTS="/etc/agentenv/ca.pem", + NODE_USE_ENV_PROXY="1",JAVA_TOOL_OPTIONS="-Dhttp.proxyHost=127.0.0.1 -Dhttp.proxyPort=3128 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|host.docker.internal" [unix_http_server] file=/run/agentenv/supervisor.sock diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index d71c7504c..f133ca1b1 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -262,8 +262,9 @@ def _platform_check(base: DockerImageArtifact) -> str: @functools.cache def _shipped_base_from() -> str: - """The image the shipped Linux base computer's Dockerfile builds ``FROM``, pinned by its digest there.""" - return re.search(r"^FROM\s+(\S+)", COMPUTER_IMAGES[0].dockerfile.read_text(), re.MULTILINE).group(1) + """The image the shipped Linux base computer's Dockerfile builds ``FROM``, pinned by its digest there: its last + stage's, since the stages before it only build what it copies in.""" + return re.findall(r"^FROM\s+(\S+)", COMPUTER_IMAGES[0].dockerfile.read_text(), re.MULTILINE)[-1] def _resolver(source: str, arch: str, context: str, out: str) -> list[str]: diff --git a/tst/integration/computer/base_computer_test.py b/tst/integration/computer/base_computer_test.py index 2e17bbe94..4e8bd4cbb 100644 --- a/tst/integration/computer/base_computer_test.py +++ b/tst/integration/computer/base_computer_test.py @@ -22,7 +22,7 @@ from agent_env.artifact.store import reset_artifact_store from agent_env.config import configure, reset_config from tst.integration.computer.conftest import (BASE, FOLLOWER_BACK_S, PAGE_WRITER, docker, kill_follower, - only_session_and_clock, process_owners) + only_session_clock_and_egress, process_owners) pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -62,7 +62,7 @@ def test_the_desktop_runs_as_the_session_user_with_the_settings_a_deploy_passes( subprocess.run(["docker", "rm", "-f", name], capture_output=True) assert {line.split()[0] for line in running.splitlines()} == {"dbus", "desktop", "xvfb"} assert all(" RUNNING " in line for line in running.splitlines()), running - assert uid == "1000" and only_session_and_clock(owners) + assert uid == "1000" and only_session_clock_and_egress(owners) assert zone == "/usr/share/zoneinfo/Europe/Paris" assert "dimensions: 1600x900 pixels" in screen and "window id #" in wm assert "TZ=Europe/Paris" in environment.splitlines() and "DISPLAY=:99" in environment.splitlines() diff --git a/tst/integration/computer/clock_desktop_test.py b/tst/integration/computer/clock_desktop_test.py index 0a604b9f8..2507b0813 100644 --- a/tst/integration/computer/clock_desktop_test.py +++ b/tst/integration/computer/clock_desktop_test.py @@ -8,17 +8,15 @@ is drawn. Routing a site, a fresh Chrome loads it over HTTPS with the clock six years back, six years on and past 2099, its certificate checked against the universe's time. Needs Docker and the local registry on localhost:5000.""" -import ast import asyncio import platform import random import re -import subprocess import time from pathlib import PurePosixPath import pytest -from agentenv_protocol.computer.clock import FOLLOWER, FOLLOWER_USER +from agentenv_protocol.computer.clock import FOLLOWER from mcp import ClientSession from mcp.client.streamable_http import streamablehttp_client @@ -28,7 +26,7 @@ from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext from tst.clock.bounds import LATE_S -from tst.integration.computer.conftest import PAGE_WRITER +from tst.integration.computer.conftest import UniverseClock, session_run from tst.integration.computer.start_tabs import host_pages from tst.integration.computer.website_browser_test import _window_center @@ -75,11 +73,11 @@ async def test_the_desktop_follows_the_universe_through_jumps_a_rate_change_and_ env_params={"app_settings": {"chromium": {"start_urls": [PAGE]}}}) context = TaskStepContext(deployed_envs=[deployed]) try: - clock = _Clock(deployed) + clock = UniverseClock(deployed) opened = await _chrome_shows(deployed, {"universe_ns": 0}, seconds=90) assert opened is not None, "Chrome's page never opened" pids = await _pids(deployed) - loaded = await _run(deployed, f"grep -c {LIBRARY} /proc/{pids[2]}/maps") + loaded = await session_run(deployed, f"grep -c {LIBRARY} /proc/{pids[2]}/maps") clock.set(time.time() - 182 * DAY, 1) await asyncio.sleep(2) # each program maps the page the next time it reads the clock, within a second reached = {client: await inside.run(deployed, ["sh", "-c", command], timeout_s=60) @@ -136,7 +134,7 @@ async def test_a_fresh_chrome_loads_a_routed_site_six_years_either_side_of_now_a deployed = await Env.get(clock_desktop.id).deploy(env_params=params) context = TaskStepContext(deployed_envs=[deployed]) try: - clock, loaded = _Clock(deployed), {} + clock, loaded = UniverseClock(deployed), {} for years in (-6, 6, 80): clock.set(time.time() + years * 365 * DAY, 1) await asyncio.sleep(2) # each program maps the page the next time it reads the clock, within a second @@ -152,68 +150,13 @@ async def test_a_fresh_chrome_loads_a_routed_site_six_years_either_side_of_now_a assert not report.failed -class _Clock: - """The computer's clock page, written as its follower's user, as its follower writes it, and what the session - reads of it.""" - - def __init__(self, deployed): - self.container = _container(deployed) - self.page = None - - def set(self, universe: float, rate: float) -> dict: - """The page written to ``universe`` at ``rate``: the write's own answer, its rate, and when it started.""" - started = time.time() - out = subprocess.run(["docker", "exec", "-u", FOLLOWER_USER, self.container, "python3", PAGE_WRITER, "set", - repr(universe), repr(rate)], capture_output=True, text=True, check=True).stdout - self.page = ast.literal_eval(out.strip()) - self.page.update(rate=rate, started=started) - return self.page - - def advance(self, seconds: float) -> dict: - return self.set(self.now() + seconds, self.page["rate"]) - - def rate(self, rate: float) -> dict: - return self.set(self.now(), rate) - - def now(self, real: float | None = None) -> float: - """The universe at real time ``real`` (now by default), as the page has it.""" - real_ns = (time.time() if real is None else real) * 1e9 - return (self.page["universe_ns"] + max(0.0, real_ns - self.page["anchor_ns"]) * self.page["rate"]) / 1e9 - - async def read(self, deployed) -> float | bool: - """The session's ``date``, when it's the universe between the moments before and after it ran (a few ms either - side for the two machines' reads of the clock); False otherwise.""" - start = time.time() - value = float(await _run(deployed, "date +%s.%N")) - end = time.time() - rate = self.page["rate"] - return value if self.now(start) - 0.01 * max(rate, 1) <= value <= self.now(end) + 0.01 * max(rate, 1) else False - - -def _container(deployed) -> str: - """The deployed computer's container, the one of its sandbox's (each named for the sandbox) holding the page's - writer.""" - listed = subprocess.run(["docker", "ps", "-q", "--filter", f"name={deployed.sandbox_id}"], - capture_output=True, text=True, check=True).stdout.split() - for container in listed: - if subprocess.run(["docker", "exec", container, "test", "-e", PAGE_WRITER]).returncode == 0: - return container - raise AssertionError(f"no container of sandbox {deployed.sandbox_id} has {PAGE_WRITER}: {listed}") - - -async def _run(deployed, command: str) -> str: - ran = await inside.run(deployed, ["sh", "-c", command], timeout_s=30) - assert ran["exit_code"] == 0, ran - return ran["stdout"].strip() - - async def _pids(deployed) -> list[str]: - return (await _run(deployed, PIDS)).split() + return (await session_run(deployed, PIDS)).split() async def _title(deployed) -> tuple[int, str] | None: """Chrome's page's time (ms) and what was typed, from its window's title.""" - match = TITLE.search(await _run(deployed, WINDOWS)) + match = TITLE.search(await session_run(deployed, WINDOWS)) return (int(match.group(1)), match.group(2)) if match else None diff --git a/tst/integration/computer/computer_clock_test.py b/tst/integration/computer/computer_clock_test.py index 9f2c9c31d..e6fe5c1ee 100644 --- a/tst/integration/computer/computer_clock_test.py +++ b/tst/integration/computer/computer_clock_test.py @@ -6,13 +6,11 @@ current version. Needs Docker and the local registry on localhost:5000.""" import asyncio -import subprocess import time from datetime import datetime, timedelta, timezone from urllib.parse import urlsplit import pytest -from agentenv_protocol.computer.clock import FOLLOWER, FOLLOWER_USER from agent_env.env import Env from agent_env.env.env import DeployedEnv @@ -24,7 +22,7 @@ from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep -from tst.integration.computer.conftest import CONTAINER, FOLLOWER_BACK_S, kill_follower, multi_env +from tst.integration.computer.conftest import CONTAINER, FOLLOWER_BACK_S, computer_container, kill_follower, multi_env pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -47,7 +45,7 @@ async def test_a_multi_envs_computer_follows_its_gateways_clock_through_arming_a [deployed] = (await step.execute(context)).deployed_envs own = DeployedEnv.from_dict(deployed.child_envs["desktop"]) kept = own.metadata["computer"]["clock"] - container = _container(own.sandbox_id) + container = computer_container(own.sandbox_id) universe = datetime.now(timezone.utc) - timedelta(days=182) await SyncEnvClockTaskStep(id="clock", version=None, env_id=office.id, @@ -105,18 +103,3 @@ async def _session_offset(computer) -> float: ran = await inside.run(computer, ["date", "+%s.%N"], timeout_s=30) assert ran["exit_code"] == 0, ran return float(ran["stdout"]) - time.time() - - -def _follower(container: str) -> str: - return subprocess.run(["docker", "exec", container, "pgrep", "-u", FOLLOWER_USER, "-f", FOLLOWER], - capture_output=True, text=True).stdout.strip() - - -def _container(sandbox_id: str) -> str: - """The computer's container, the one of its sandbox's (each named for the sandbox) that runs the follower.""" - listed = subprocess.run(["docker", "ps", "-q", "--filter", f"name={sandbox_id}"], capture_output=True, text=True, - check=True).stdout.split() - for container in listed: - if _follower(container): - return container - raise AssertionError(f"no container of sandbox {sandbox_id} runs the follower: {listed}") diff --git a/tst/integration/computer/computer_env_test.py b/tst/integration/computer/computer_env_test.py index 2d5660bc3..8b1e58cc7 100644 --- a/tst/integration/computer/computer_env_test.py +++ b/tst/integration/computer/computer_env_test.py @@ -49,7 +49,7 @@ from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from agent_env.task_step.task_steps.run_code import RunCodeTaskStep from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep -from tst.integration.computer.conftest import docker, only_session_and_clock, process_owners +from tst.integration.computer.conftest import docker, only_session_clock_and_egress, process_owners pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -94,7 +94,7 @@ async def test_the_image_of_this_machines_arch_boots_as_the_session_user_and_pas subprocess.run(["docker", "rm", "-f", name], capture_output=True) for report in reports: assert report.passed, [f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed"] - assert only_session_and_clock(owners) + assert only_session_clock_and_egress(owners) def test_an_env_that_changes_only_its_display_and_name_reuses_the_image(desktop): @@ -112,6 +112,17 @@ def test_an_env_that_changes_only_its_display_and_name_reuses_the_image(desktop) assert took < 60, took +#: A request to an address on the internet, around the egress proxy, and through it, printing the status and the body +#: it answers. +DIRECT = "import urllib.request as u; u.build_opener(u.ProxyHandler({})).open('http://1.1.1.1', timeout=5)" +PROXIED = """import urllib.error, urllib.request +try: + urllib.request.urlopen('http://1.1.1.1', timeout=5) +except urllib.error.HTTPError as e: + print(e.code, e.read().decode()) +""" + + @pytest.mark.asyncio async def test_with_no_network_it_is_reached_reaches_nothing_comes_back_fresh_and_leaves_nothing(desktop): with namespace_routing(): @@ -129,7 +140,8 @@ async def test_with_no_network_it_is_reached_reaches_nothing_comes_back_fresh_an context = await RunCodeTaskStep(id="read", version=None, env_id=closed.id, script_artifact_id=script.id).execute(context) driver = await Driver.connect(deployed.environment_url, 30, deployed.control_token) - egress = await driver.run("python3", "-c", "import urllib.request; urllib.request.urlopen('http://1.1.1.1', timeout=5)") + direct = await driver.run("python3", "-c", DIRECT) + proxied = await driver.run("python3", "-c", PROXIED) await driver.run("sh", "-c", "echo x > Desktop/left.txt") context = await SnapshotComputerEnvTaskStep(id="snapshot", version=None, env_id=closed.id, snapshot_id=f"{closed.id}-snapshot", @@ -146,7 +158,9 @@ async def test_with_no_network_it_is_reached_reaches_nothing_comes_back_fresh_an assert context.metadata["script_results"]["read"] == {"note": "inside\n", "card": "closed"} assert snapshot == {"Desktop/left.txt": b"x\n", "Downloads/note.txt": b"inside\n"} - assert egress["exit_code"] != 0 and "Network is unreachable" in egress["stderr"], egress + # Neither the computer itself nor its egress proxy reaches anything: the proxy answers why. + assert direct["exit_code"] != 0 and "Network is unreachable" in direct["stderr"], direct + assert proxied["stdout"].startswith("502") and "network is unreachable" in proxied["stdout"], proxied assert left["exit_code"] != 0 and (restored["exit_code"], restored["stdout"]) == (0, "x\n"), restored assert [s.sandbox_id for s in report.terminated] == [sandbox] and not report.failed for listing in (["ps", "-aq", "--filter", f"label=agentenv.sandbox={sandbox}"], diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index 720388a17..fe39943b0 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -7,6 +7,7 @@ ``cdp_clients``, Playwright and Puppeteer installed to connect to a CDP address. A compose's build of an image waits for another test worker's build of the same tag.""" +import ast import fcntl import hashlib import logging @@ -14,6 +15,7 @@ import shutil import subprocess import tempfile +import time import uuid from pathlib import Path, PurePosixPath @@ -32,7 +34,7 @@ from agent_env.env import Env from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv -from agent_env.env.envs.computer.utils import builder +from agent_env.env.envs.computer.utils import builder, inside from agent_env.providers.sandbox_providers.host_images import host_platform from agent_env.store.object_store.local.grant_server import grant_server from agent_env.store.routing import namespace_routing @@ -48,6 +50,7 @@ WEBSITE_BROWSER = Path(__file__).parent / "website-browser" BROWSER_COMPUTER = Path(__file__).parent / "browser-computer" TLS = Path(__file__).parent / "tls" +EGRESS_CLIENTS = Path(__file__).parent / "egress" CLOCK_DESKTOP = Path(__file__).parent / "clock-desktop" #: Playwright and Puppeteer, which connect to a cdp address from Node. CDP_CLIENTS = Path(__file__).parent / "cdp_clients" @@ -118,6 +121,13 @@ def browser_computer(tmp_path_factory): yield from _composed(tmp_path_factory.mktemp("browser-computer"), parse_bundle(BROWSER_COMPUTER), CHROMIUM_ALONE) +@pytest.fixture(scope="module") +def egress_desktop(tmp_path_factory): + """The egress clients test bundle's desktop, the ubuntu-24.04 base and the reference driver with an apt JDK, an + official Node build and curl, composed as ``desktop`` is.""" + yield from _composed(tmp_path_factory.mktemp("egress-desktop"), parse_bundle(EGRESS_CLIENTS)) + + @pytest.fixture(scope="module") def tls_desktop(tmp_path_factory): """The TLS clients test bundle's desktop, the ubuntu-24.04 base and the reference driver with Node, an apt JDK, a @@ -208,6 +218,8 @@ def run_context(output: str) -> dict: "/usr/bin/python3 /usr/bin/supervisord -c /etc/supervisor/root.conf", "ps -eo user:32=,args=") #: The clock page's writer, which the base installs beside the follower. PAGE_WRITER = str(PurePosixPath(FOLLOWER).parent / "clockpage.py") +#: The base's egress proxy, the user it runs as, and its address. +EGRESS, EGRESS_USER, EGRESS_PROXY = "/usr/local/lib/agentenv-egress/egress", "agentenv-egress", "http://127.0.0.1:3128" def process_owners(name: str) -> dict[str, list[str]]: @@ -219,12 +231,13 @@ def process_owners(name: str) -> dict[str, list[str]]: return owners -def only_session_and_clock(owners: dict[str, list[str]]) -> bool: - """Whether root runs nothing but its init and supervisord, the clock's user nothing but the follower, and every - other process is the session user's.""" - assert set(owners) == {"root", FOLLOWER_USER, "user"}, owners +def only_session_clock_and_egress(owners: dict[str, list[str]]) -> bool: + """Whether root runs nothing but its init and supervisord, the clock's user nothing but the follower, the egress + user nothing but the egress proxy, and every other process is the session user's.""" + assert set(owners) == {"root", FOLLOWER_USER, EGRESS_USER, "user"}, owners assert all(args in ROOTS for args in owners["root"]), owners["root"] assert owners[FOLLOWER_USER] == [f"/usr/bin/python3 -I {FOLLOWER}"], owners[FOLLOWER_USER] + assert owners[EGRESS_USER] == [f"{EGRESS} serve"], owners[EGRESS_USER] return True @@ -232,9 +245,9 @@ def _exec_text(name: str, *argv: str) -> str: return subprocess.run(["docker", "exec", name, *argv], check=True, capture_output=True, text=True).stdout -#: Kills the follower and times it until another runs, inside the container, as root, polling every 20 ms: prints the -#: pid killed, the pid back, and the seconds between. -_KILL_FOLLOWER = """import os, signal, subprocess, sys, time +#: Kills a program of root's supervisord (by its command and user) and times it until another runs, inside the +#: container, as root, polling every 20 ms: prints the pid killed, the pid back, and the seconds between. +_KILL_PROGRAM = """import os, signal, subprocess, sys, time def running(): return subprocess.run(["pgrep", "-u", sys.argv[2], "-f", sys.argv[1]], capture_output=True, text=True).stdout.strip() killed = running() @@ -246,20 +259,23 @@ def running(): """ -#: How soon supervisord has a killed follower back: it notices the exit when it next wakes, up to a second later, -#: and the follower starts Python; on a CI runner busy with other tests that has taken 2.03 s. +#: How soon supervisord has a killed program of its own back: it notices the exit when it next wakes, up to a second +#: later, and the program starts; on a CI runner busy with other tests a follower took 2.03 s. FOLLOWER_BACK_S = 3 -def kill_follower(name: str) -> tuple[str, str, float]: - """The clock follower in the container ``name`` killed with SIGKILL: its pid, the pid of the one back ("-" if - none within 10 s), and how long that took, timed in the container.""" - killed, back, took = subprocess.run(["docker", "exec", "-u", "0", name, "python3", "-c", _KILL_FOLLOWER, FOLLOWER, - FOLLOWER_USER], - check=True, capture_output=True, text=True).stdout.split() +def kill_program(name: str, command: str, user: str) -> tuple[str, str, float]: + """The program running ``command`` as ``user`` in the container ``name`` killed with SIGKILL: its pid, the pid of + the one back ("-" if none within 10 s), and how long that took, timed in the container.""" + killed, back, took = subprocess.run(["docker", "exec", "-u", "0", name, "python3", "-c", _KILL_PROGRAM, command, + user], check=True, capture_output=True, text=True).stdout.split() return killed, back, float(took) +def kill_follower(name: str) -> tuple[str, str, float]: + return kill_program(name, FOLLOWER, FOLLOWER_USER) + + #: A multi env's computer on the local backend in its container form, as its child_envs entry gives it. CONTAINER = {"sandbox_type": "local", "env_params": {"form": "container"}} @@ -279,3 +295,59 @@ def multi_env(items, *computers, websites=()) -> MultiEnv: """A multi env named office of the ``items`` MCP server, ``websites`` and ``computers``, under a local id.""" return MultiEnv.put(id=f"@local/~/office-{uuid.uuid4().hex[:8]}/office", mcp_server_envs=[items], website_envs=list(websites), computer_envs=list(computers), name="office") + + +class UniverseClock: + """A deployed computer's clock page, written as its follower's user, as its follower writes it, and what the + session reads of it.""" + + def __init__(self, deployed): + self.container = computer_container(deployed.sandbox_id) + self.page = None + + def set(self, universe: float, rate: float) -> dict: + """The page written to ``universe`` at ``rate``: the write's own answer, its rate, and when it started.""" + started = time.time() + out = subprocess.run(["docker", "exec", "-u", FOLLOWER_USER, self.container, "python3", PAGE_WRITER, "set", + repr(universe), repr(rate)], capture_output=True, text=True, check=True).stdout + self.page = ast.literal_eval(out.strip()) + self.page.update(rate=rate, started=started) + return self.page + + def advance(self, seconds: float) -> dict: + return self.set(self.now() + seconds, self.page["rate"]) + + def rate(self, rate: float) -> dict: + return self.set(self.now(), rate) + + def now(self, real: float | None = None) -> float: + """The universe at real time ``real`` (now by default), as the page has it.""" + real_ns = (time.time() if real is None else real) * 1e9 + return (self.page["universe_ns"] + max(0.0, real_ns - self.page["anchor_ns"]) * self.page["rate"]) / 1e9 + + async def read(self, deployed) -> float | bool: + """The session's ``date``, when it's the universe between the moments before and after it ran (a few ms either + side for the two machines' reads of the clock); False otherwise.""" + start = time.time() + value = float(await session_run(deployed, "date +%s.%N")) + end = time.time() + rate = self.page["rate"] + return value if self.now(start) - 0.01 * max(rate, 1) <= value <= self.now(end) + 0.01 * max(rate, 1) else False + + +def computer_container(sandbox_id: str) -> str: + """A deployed computer's container, the one of its sandbox's (each named for the sandbox) holding the clock + page's writer.""" + listed = subprocess.run(["docker", "ps", "-q", "--filter", f"name={sandbox_id}"], + capture_output=True, text=True, check=True).stdout.split() + for container in listed: + if subprocess.run(["docker", "exec", container, "test", "-e", PAGE_WRITER]).returncode == 0: + return container + raise AssertionError(f"no container of sandbox {sandbox_id} has {PAGE_WRITER}: {listed}") + + +async def session_run(deployed, command: str) -> str: + """What ``command`` prints, run in the deployed computer's session through its driver; it must succeed.""" + ran = await inside.run(deployed, ["sh", "-c", command], timeout_s=30) + assert ran["exit_code"] == 0, ran + return ran["stdout"].strip() diff --git a/tst/integration/computer/egress/envs/clients/Fetch.java b/tst/integration/computer/egress/envs/clients/Fetch.java new file mode 100644 index 000000000..6038f539d --- /dev/null +++ b/tst/integration/computer/egress/envs/clients/Fetch.java @@ -0,0 +1,9 @@ +public class Fetch { + public static void main(String[] args) throws Exception { + var request = java.net.http.HttpRequest.newBuilder(java.net.URI.create(args[0])) + .header("User-Agent", "agentenv-egress-test").build(); + var response = java.net.http.HttpClient.newHttpClient() + .send(request, java.net.http.HttpResponse.BodyHandlers.discarding()); + System.out.println(response.statusCode()); + } +} diff --git a/tst/integration/computer/egress/envs/clients/app.py b/tst/integration/computer/egress/envs/clients/app.py new file mode 100644 index 000000000..cd683645f --- /dev/null +++ b/tst/integration/computer/egress/envs/clients/app.py @@ -0,0 +1,17 @@ +"""TLS clients for the egress proxy tests: a JDK from Ubuntu's archive, an official Node build, which carries its own +OpenSSL so the TLS shim doesn't reach it, and curl. Each goes out through the egress proxy by the session's settings +alone: Java's HttpClient (Fetch.java) and Node's fetch (fetch.mjs) print the status a URL answers.""" +from agentenv_protocol.computer import ComputerAppEnvironment, Download, Packages, RunScript, computer_app, linux + +NODE = "24.22.2" +_NODE = {"amd64": ("x64", "48cc0e21e048790e54c0cf5749c2725005313d652651c40b7a21feb9697aaf75"), + "arm64": ("arm64", "eb187837a2230d4047161ac0be008584e16b46f726e9c9490ee35dc6b17d9ba8")} + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@linux(packages=Packages(apt=["openjdk-21-jdk-headless", "curl", "xz-utils"]), + downloads=[Download(url=f"https://nodejs.org/dist/v{NODE}/node-v{NODE}-linux-{name}.tar.xz", sha256=sha256, + to="node.tar.xz", arch=arch) for arch, (name, sha256) in _NODE.items()], + run=[RunScript(script="setup.sh")]) +class Clients(ComputerAppEnvironment): + pass diff --git a/tst/integration/computer/egress/envs/clients/env.toml b/tst/integration/computer/egress/envs/clients/env.toml new file mode 100644 index 000000000..d6af807c4 --- /dev/null +++ b/tst/integration/computer/egress/envs/clients/env.toml @@ -0,0 +1,4 @@ +type = "computer_app" +module = "app.py:Clients" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/egress/envs/clients/fetch.mjs b/tst/integration/computer/egress/envs/clients/fetch.mjs new file mode 100644 index 000000000..6e7763877 --- /dev/null +++ b/tst/integration/computer/egress/envs/clients/fetch.mjs @@ -0,0 +1,2 @@ +const response = await fetch(process.argv[2], { headers: { "user-agent": "agentenv-egress-test" } }); +console.log(response.status); diff --git a/tst/integration/computer/egress/envs/clients/setup.sh b/tst/integration/computer/egress/envs/clients/setup.sh new file mode 100644 index 000000000..176ccd14c --- /dev/null +++ b/tst/integration/computer/egress/envs/clients/setup.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Run as root once the app's files, packages and downloads are in: Node's official build under /opt/node. +set -e +cd "$(dirname "$0")" +mkdir -p /opt/node +tar -xJf node.tar.xz -C /opt/node --strip-components=1 +ln -sf /opt/node/bin/node /usr/local/bin/node diff --git a/tst/integration/computer/egress/envs/desktop/env.toml b/tst/integration/computer/egress/envs/desktop/env.toml new file mode 100644 index 000000000..b6181e472 --- /dev/null +++ b/tst/integration/computer/egress/envs/desktop/env.toml @@ -0,0 +1,4 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" +apps = ["clients"] diff --git a/tst/integration/computer/egress/tasks/desktop.json b/tst/integration/computer/egress/tasks/desktop.json new file mode 100644 index 000000000..67f3b3154 --- /dev/null +++ b/tst/integration/computer/egress/tasks/desktop.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"} +] diff --git a/tst/integration/computer/egress_test.py b/tst/integration/computer/egress_test.py new file mode 100644 index 000000000..cf5b09175 --- /dev/null +++ b/tst/integration/computer/egress_test.py @@ -0,0 +1,190 @@ +"""The ubuntu-24.04 base's egress proxy, on real time, which every session program goes out through: the session user +can neither read its CA's key nor stop, replace or retrust it, and a killed one comes back; it refuses every +destination that isn't global; with the universe 182 days back, Java's HttpClient, an official Node build's fetch and +curl reach a live site through it by the session's settings alone, where Java and Node, which check certificates +against the universe's time, don't without it; and fresh handshakes, made all through a run of week-long jumps with +the universe a month on, are none of them aborted. Needs Docker, the local registry on localhost:5000, and the +internet.""" + +import asyncio +import json +import subprocess +import time +import uuid + +import pytest + +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext +from tst.integration.computer.conftest import (EGRESS, EGRESS_USER, FOLLOWER_BACK_S, UniverseClock, computer_container, + docker, kill_program) + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +DAY = 86400 +LIVE = "https://example.com/" +APP = "/opt/agentenv/apps/clients" +CLIENTS = {"java": f"java {APP}/Fetch.java {LIVE}", "node": f"node {APP}/fetch.mjs {LIVE}", + "curl": f"curl -sS -o /dev/null -w '%{{http_code}}' {LIVE}"} +#: What a client that ignores the session's proxy settings would do. +NO_PROXY = "env -u HTTPS_PROXY -u https_proxy -u ALL_PROXY -u all_proxy -u JAVA_TOOL_OPTIONS -u NODE_USE_ENV_PROXY" +#: Private, local and special-purpose destinations, by address, each of which the proxy refuses. +REFUSED = ["10.0.0.1", "10.255.255.254", "172.16.0.1", "172.31.255.1", "192.168.1.1", "127.0.0.1", "127.1.2.3", + "0.0.0.0", "169.254.169.254", "169.254.1.1", "100.64.0.1", "100.127.255.254", "192.0.2.1", "198.18.0.1", + "224.0.0.1", "255.255.255.255", "::1", "::", "fd00::1", "fe80::1", "::ffff:10.0.0.1", "2001:db8::1"] +#: Asks the proxy to CONNECT to each address given, printing the status line it answers each with. +CONNECT_EACH = """import json, socket, sys +out = {} +for address in sys.argv[1:]: + host = f"[{address}]" if ":" in address else address + with socket.create_connection(("127.0.0.1", 3128), timeout=10) as s: + s.sendall(f"CONNECT {host}:443 HTTP/1.1\\r\\nHost: {host}:443\\r\\n\\r\\n".encode()) + out[address] = s.recv(64).split(b"\\r\\n")[0].decode() +print(json.dumps(out)) +""" +#: As the session user, each a change the egress proxy's user or root alone may make. +ATTEMPTS = { + "read the CA's key": "cat /etc/agentenv/egress/ca.key", + "list the CA's folder": "ls /etc/agentenv/egress", + "kill the proxy": "kill -9 $(pgrep -u agentenv-egress -f 'egress serve')", + "stop it through root's supervisord": "supervisorctl -c /etc/supervisor/root.conf stop egress", + "replace the proxy": "cp /bin/true /usr/local/lib/agentenv-egress/egress", + "add to the roots it verifies sites against": "echo x >> /usr/local/lib/agentenv-egress/roots.pem", + "add to agentenv's trust files": "echo x >> /etc/agentenv/ca-bundle.pem", + "replace the public CA": "echo x > /etc/agentenv/egress-ca.pem", +} + + +def _serving(name: str) -> str: + """The egress proxy's pid once it answers on its port.""" + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + up = subprocess.run(["docker", "exec", name, "python3", "-c", + "import socket; socket.create_connection(('127.0.0.1', 3128), timeout=1)"], + capture_output=True) + pid = subprocess.run(["docker", "exec", name, "pgrep", "-u", EGRESS_USER, "-f", f"{EGRESS} serve"], + capture_output=True, text=True).stdout.strip() + if up.returncode == 0 and pid: + return pid + time.sleep(0.5) + raise AssertionError(subprocess.run(["docker", "logs", name], capture_output=True, text=True).stdout[-2000:]) + + +def test_the_session_user_can_neither_read_its_key_nor_stop_replace_or_retrust_it_and_a_killed_one_comes_back(base): + name = f"agentenv-test-egress-{uuid.uuid4().hex[:8]}" + docker("run", "-d", "--name", name, base) + try: + pid = _serving(name) + attempts = {what: subprocess.run(["docker", "exec", "-u", "user", name, "sh", "-c", command], + capture_output=True, text=True).returncode + for what, command in ATTEMPTS.items()} + same = _serving(name) + killed, back, took = kill_program(name, f"{EGRESS} serve", EGRESS_USER) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + assert [what for what, code in attempts.items() if code == 0] == [], attempts + assert same == pid + assert killed == pid and back not in ("-", killed) and took < FOLLOWER_BACK_S, (killed, back, took) + + +def test_every_destination_that_isnt_global_is_refused(base): + name = f"agentenv-test-egress-{uuid.uuid4().hex[:8]}" + docker("run", "-d", "--name", name, base) + try: + _serving(name) + answered = json.loads(subprocess.run(["docker", "exec", "-u", "user", name, "python3", "-c", CONNECT_EACH, + *REFUSED], check=True, capture_output=True, text=True).stdout) + log = subprocess.run(["docker", "exec", name, "cat", "/var/log/agentenv-root/egress.log"], + capture_output=True, text=True).stdout + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + assert len(REFUSED) == 22 + assert {address: status for address, status in answered.items() if "403" not in status} == {}, answered + refused = [json.loads(line) for line in log.splitlines() if '"refused"' in line] + assert len(refused) == 22 and all("isn't public" in row["error"] for row in refused), refused + + +async def _status(deployed, command: str) -> str: + ran = await inside.run(deployed, ["sh", "-c", command], timeout_s=120) + lines = ran["stdout"].strip().splitlines() + return lines[-1] if ran["exit_code"] == 0 and lines else f"exit {ran['exit_code']}: {ran.get('stderr', '')[-300:]}" + + +def _egress_rows(deployed) -> list[dict]: + log = subprocess.run(["docker", "exec", computer_container(deployed.sandbox_id), "cat", + "/var/log/agentenv-root/egress.log"], capture_output=True, text=True).stdout + return [json.loads(line) for line in log.splitlines() if line.startswith("{")] + + +@pytest.mark.asyncio +async def test_java_node_and_curl_reach_a_live_site_through_it_with_the_universe_182_days_back(egress_desktop): + with namespace_routing(): + deployed = await Env.get(egress_desktop.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + UniverseClock(deployed).set(time.time() - 182 * DAY, 1) + await asyncio.sleep(2) # each program maps the page the next time it reads the clock, within a second + through = {client: await _status(deployed, command) for client, command in CLIENTS.items()} + direct = {client: await _status(deployed, f"{NO_PROXY} {command}") for client, command in CLIENTS.items() + if client != "curl"} + rows = _egress_rows(deployed) + finally: + report = await teardown_run(context) + + assert through == {"java": "200", "node": "200", "curl": "200"}, through + # Java and Node check the site's certificate against the universe's time, so without the proxy they refuse it. + assert all(not status.isdigit() for status in direct.values()), direct + intercepted = [row for row in rows if row.get("host") == "example.com" and row.get("mode") == "intercept"] + assert len(intercepted) >= 3 and all(row["verified"] for row in intercepted), rows[-6:] + assert not report.failed + + +#: Fetches each URL given with a fresh Node process, one after another, until ~/.egress-stop appears, printing a line +#: per fetch: the status, or FAIL and the URL. +FETCH_LOOP = f"""while [ ! -e ~/.egress-stop ]; do + for url in "$@"; do + node {APP}/fetch.mjs "$url" 2>>~/.egress-errors || echo "FAIL $url" + done +done > ~/.egress-loop +echo DONE >> ~/.egress-loop""" + + +@pytest.mark.asyncio +async def test_fresh_handshakes_through_week_jumps_with_the_universe_a_month_on_are_none_aborted(egress_desktop): + sites = [LIVE, "https://www.wikipedia.org/"] + with namespace_routing(): + deployed = await Env.get(egress_desktop.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + clock = UniverseClock(deployed) + clock.set(time.time() + 30 * DAY, 1) + await asyncio.sleep(2) + await inside.run(deployed, ["sh", "-c", f"setsid sh -c '{FETCH_LOOP}' loop {' '.join(sites)} " + "> /dev/null 2>&1 < /dev/null &"], timeout_s=30) + for _ in range(12): + await asyncio.sleep(1.5) + clock.advance(7 * DAY) + await asyncio.sleep(3) + await inside.run(deployed, ["sh", "-c", "touch ~/.egress-stop"], timeout_s=30) + loop = "" + for _ in range(60): + loop = (await inside.run(deployed, ["sh", "-c", "cat ~/.egress-loop"], timeout_s=30))["stdout"] + if "DONE" in loop: + break + await asyncio.sleep(1) + errors = (await inside.run(deployed, ["sh", "-c", "cat ~/.egress-errors"], timeout_s=30))["stdout"] + rows = _egress_rows(deployed) + finally: + report = await teardown_run(context) + + fetches = [line for line in loop.splitlines() if line != "DONE"] + assert "DONE" in loop and len(fetches) >= 40, loop[-500:] + assert [line for line in fetches if line != "200"] == [], errors[-1500:] + modes = {row.get("mode") for row in rows if row.get("host") in ("example.com", "www.wikipedia.org")} + assert "intercept" in modes, rows[-6:] + assert not report.failed diff --git a/tst/unit/bundle/shipped_computers_test.py b/tst/unit/bundle/shipped_computers_test.py index 72fc4e626..47860103e 100644 --- a/tst/unit/bundle/shipped_computers_test.py +++ b/tst/unit/bundle/shipped_computers_test.py @@ -47,8 +47,11 @@ def test_a_task_naming_ubuntu_24_04_writes_the_shipped_base_over_its_folders_ima built = next(call["build"] for call in builds if call["build"][2] == image.image_name) assert built[1] == ["Dockerfile", "artifact.toml", *(f"clock/{name}" for name in ( "COPYING", "README.md", "agentenv_clock_page.c", "agentenv_clock_waits.c", "build.sh", "clock-follower", - "clock.conf", "clockpage.py", "libfaketime.patch", "tls_realtime.c")), "root.conf", "start-desktop", - "supervisord.conf", "xfwm4.xml"] + "clock.conf", "clockpage.py", "libfaketime.patch", "tls_realtime.c")), *(f"egress/{name}" for name in ( + "README.md", "agentenv-egress.sh", "boot.go", "boot_test.go", "clock.go", "clock_test.go", "go.mod", "go.sum", + "hello.go", "hello_test.go", "leaves.go", "main.go", "proxy.go", "proxy_test.go", "testdata/page-armed.hex", + "testdata/page-unarmed.hex", "upstream.go")), "root.conf", + "start-desktop", "supervisord.conf", "xfwm4.xml"] assert computer.metadata["validated_images"]["container"]["arm64"]["status"] == "uninspected" diff --git a/tst/unit/env/envs/test_computer_egress_base.py b/tst/unit/env/envs/test_computer_egress_base.py new file mode 100644 index 000000000..2678fbe34 --- /dev/null +++ b/tst/unit/env/envs/test_computer_egress_base.py @@ -0,0 +1,89 @@ +"""The ubuntu-24.04 base's egress proxy agrees with the rest of the base and with site routing: its certificates span +the window routed sites' do, root's supervisord runs it as its own user, init boots it before that supervisord starts, +every session program trusts its CA through agentenv's trust files as the local provider names them, the session's +proxy is the address it listens on, a login shell gets the session's settings, and the clock pages its Go reader is +tested on are the bytes the follower's writer writes.""" +from __future__ import annotations + +import configparser +import importlib.util +import re +import shlex +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlsplit + +import agent_env +from agent_env.env.envs.computer.utils import sites +from agent_env.providers.sandbox_providers.local_docker.provider import LOCAL_TRUST_ENV + +BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" +EGRESS = BASE / "egress" + + +def _go_date(name: str) -> datetime: + match = re.search(rf"{name}\s*=\s*time\.Date\(([^)]*)\)", (EGRESS / "main.go").read_text()) + year, month, day, hour, minute, second, _ns, _zone = (part.strip() for part in match.group(1).split(",")) + return datetime(int(year), int(month), int(day), int(hour), int(minute), int(second), tzinfo=UTC) + + +def _session_environment() -> dict[str, str]: + conf = configparser.ConfigParser(interpolation=None) + conf.read_string((BASE / "supervisord.conf").read_text()) + lexer = shlex.shlex(conf["supervisord"]["environment"], posix=True) + lexer.whitespace += "," + lexer.wordchars += "/.:+-()|=_" + pairs = [token.split("=", 1) for token in lexer] + return {key: value for key, value in pairs} + + +def _profile() -> dict[str, str]: + out = {} + for line in (EGRESS / "agentenv-egress.sh").read_text().splitlines(): + if line.startswith("export "): + for token in shlex.split(line.removeprefix("export ")): + key, _, value = token.partition("=") + out[key] = value + return out + + +def test_the_egress_ca_spans_the_window_routed_sites_certificates_do(): + assert (_go_date("CAFrom"), _go_date("CAUntil")) == (sites.UNIVERSE_FROM, sites.UNIVERSE_UNTIL) + + +def test_roots_supervisord_runs_the_proxy_as_its_own_user_and_init_boots_it_first(): + root = configparser.ConfigParser(interpolation=None) + root.read_string((BASE / "root.conf").read_text()) + init = (BASE / "start-desktop").read_text() + + assert (root["program:egress"]["command"], root["program:egress"]["user"]) == ( + "/usr/local/lib/agentenv-egress/egress serve", "agentenv-egress") + assert init.index("/usr/local/lib/agentenv-egress/egress boot") < init.index("exec /usr/bin/supervisord") + + +def test_a_login_shell_gets_the_proxy_settings_every_session_program_gets(): + session, profile = _session_environment(), _profile() + + assert {key: session[key] for key in profile} == profile + assert session["HTTPS_PROXY"] == "http://127.0.0.1:3128" and session["NODE_USE_ENV_PROXY"] == "1" + assert "-Dhttps.proxyPort=3128" in session["JAVA_TOOL_OPTIONS"] + assert session["NO_PROXY"].split(",") == session["JAVA_TOOL_OPTIONS"].split("nonProxyHosts=")[1].split("|") + assert {key: session[key] for key in LOCAL_TRUST_ENV} == LOCAL_TRUST_ENV + + +def test_the_session_goes_out_through_the_address_the_proxy_listens_on(): + listen = re.search(r'Listen\s*=\s*"([^"]+)"', (EGRESS / "main.go").read_text()).group(1) + + assert urlsplit(_session_environment()["HTTPS_PROXY"]).netloc == listen + + +def test_the_pages_the_proxy_is_tested_on_are_the_ones_the_followers_writer_writes(): + spec = importlib.util.spec_from_file_location("clockpage_written", BASE / "clock" / "clockpage.py") + page = importlib.util.module_from_spec(spec) + spec.loader.exec_module(page) + golden = {name: bytes.fromhex((EGRESS / "testdata" / f"page-{name}.hex").read_text().strip()) + for name in ("armed", "unarmed")} + + assert golden == { + "armed": page.encode(4, 1577836800_000000000, 1794000000_123456789, 2.0, 1, 7, 5_000_000_000, 5_000_000_000), + "unarmed": page.encode(2, 0, 0, 1.0, 0, 3, 0, 0)}