diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4e83760 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Text assets have byte-exact HTTP lengths and content identities on every host. +* text=auto eol=lf diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..fbb1c02 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +ko_fi: execsoftware diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index efac4a2..5f21820 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -29,12 +29,32 @@ jobs: --output scanner.tar.gz printf '%s scanner.tar.gz\n' 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb | sha256sum --check - tar -xzf scanner.tar.gz gitleaks + printf '[extend]\nuseDefault = true\n' > default.toml + : > empty.ignore - name: Scan the working tree and the fetched history shell: bash run: | set -euo pipefail - "$RUNNER_TEMP/source-scanner/gitleaks" dir . --no-banner --redact - "$RUNNER_TEMP/source-scanner/gitleaks" git . --no-banner --redact + flags=(--no-banner --redact=100 --ignore-gitleaks-allow + --config "$RUNNER_TEMP/source-scanner/default.toml" + --gitleaks-ignore-path "$RUNNER_TEMP/source-scanner/empty.ignore" + --max-archive-depth=2 --max-decode-depth=2 --timeout=120) + "$RUNNER_TEMP/source-scanner/gitleaks" dir . "${flags[@]}" + # Git mode already scans all fetched refs; checkout fetched full history. + "$RUNNER_TEMP/source-scanner/gitleaks" git . "${flags[@]}" + - name: Verify path-audit detection and safe diagnostics + run: python3 -B tests/test_audit_paths.py + - name: Verify wire closure and deadline test oracles + run: | + python3 -B - <<'PY' + import json, os, runpy, tempfile + from pathlib import Path + driver = runpy.run_path('scripts/test-platform.py') + env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', PYTHONUTF8='1') + with tempfile.TemporaryDirectory(prefix='nilweb-wire-oracle-') as output: + result = driver['run_wire_oracle'](env=env, evidence=Path(output)) + print(json.dumps(dict(checks=result['checks'], cleanup_verified=result['cleanup_verified']))) + PY - name: Refuse absolute paths, routable addresses and contact details shell: bash run: | @@ -44,6 +64,6 @@ jobs: # Loopback, the unspecified address and the RFC 5737/3849 # documentation ranges are allowed, because a server has to name an # address it binds. See scripts/audit-paths.py for the reasoning. - python3 scripts/audit-paths.py --root . + python3 -B scripts/audit-paths.py --root . # Scan reports are not uploaded. Even redacted findings can expose # filenames, commit messages and surrounding source context. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..bfcc3cb --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,123 @@ +name: Web platform checks + +"on": + push: + branches: [master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: web-platform-${{ github.ref }} + cancel-in-progress: true + +# Build the interpreter and host file boundary from this exact public revision. +# These jobs qualify source consumers, not a packaged SDK or native networking. +env: + NIL_FRAMEWORK_REVISION: 93515c3d4c2b6e0e7c3b870eb283fce26392d613 + PYTHONDONTWRITEBYTECODE: '1' + PYTHONUTF8: '1' + HOMEBREW_NO_AUTO_UPDATE: '1' + +jobs: + web: + name: web-${{ matrix.host.name }} + runs-on: ${{ matrix.host.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + host: + - {name: linux-x86_64, runner: ubuntu-24.04, system: Linux, arch: x86_64, python: python3} + - {name: windows-x86_64, runner: windows-2022, system: Windows, arch: x86_64, python: python} + - {name: macos-x86_64, runner: macos-15-intel, system: Darwin, arch: x86_64, python: python3} + - {name: macos-arm64, runner: macos-15, system: Darwin, arch: arm64, python: python3} + defaults: + run: + shell: bash + env: + NIL_CI_PYTHON: ${{ matrix.host.python }} + NIL_CI_SYSTEM: ${{ matrix.host.system }} + NIL_CI_ARCH: ${{ matrix.host.arch }} + steps: + - name: Check out web + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4, same pin as audit + with: + path: web + persist-credentials: false + - name: Check out pinned public framework + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: exec-software/exec-nil-framework + ref: ${{ env.NIL_FRAMEWORK_REVISION }} + path: framework + persist-credentials: false + - name: Verify framework revision and prepare evidence + timeout-minutes: 2 + run: | + "$NIL_CI_PYTHON" - <<'PY' + import json, os, platform, re, subprocess + from pathlib import Path + evidence = Path(os.environ['RUNNER_TEMP']) / 'web-build-evidence' + evidence.mkdir() + expected = os.environ['NIL_FRAMEWORK_REVISION'] + assert re.fullmatch('[0-9a-f]{40}', expected), 'Framework must be pinned to a full commit SHA' + actual = subprocess.check_output(['git', '-C', 'framework', 'rev-parse', 'HEAD'], text=True, timeout=15).strip() + assert actual == expected + report = dict(framework_revision=actual, web_revision=subprocess.check_output( + ['git', '-C', 'web', 'rev-parse', 'HEAD'], text=True, timeout=15).strip(), + system=platform.system(), machine=platform.machine(), llvm=False, native_database_providers=False) + (evidence/'identity.json').write_text(json.dumps(report, indent=2)+'\n') + PY + - name: Configure focused Release build + timeout-minutes: 8 + run: | + set -euo pipefail + nil_python_executable="$("$NIL_CI_PYTHON" -c 'import sys; print(sys.executable)')" + cmake -S framework -B "$RUNNER_TEMP/web-framework-build" \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DNIL_ENABLE_LLVM=OFF \ + -DNIL_BUILD_EXPERIMENTAL_TOOLS=OFF \ + -DNILCOMPILER_SQLITE_INCLUDE_DIR=OFF -DNILCOMPILER_POSTGRESQL_INCLUDE_DIR=OFF \ + "-DPython3_EXECUTABLE=$nil_python_executable" \ + 2>&1 | tee "$RUNNER_TEMP/web-build-evidence/configure.log" + - name: Build compiler interpreter and host contract + timeout-minutes: 40 + run: | + set -euo pipefail + cmake --build "$RUNNER_TEMP/web-framework-build" --config Release --parallel 2 \ + --target nilcompiler nilinterpreter nil_rooted_file_snapshot_tests \ + 2>&1 | tee "$RUNNER_TEMP/web-build-evidence/build.log" + - name: Run exact web and real loopback contracts + timeout-minutes: 8 + run: | + "$NIL_CI_PYTHON" - <<'PY' + import os, subprocess, sys + from pathlib import Path + build = Path(os.environ['RUNNER_TEMP']) / 'web-framework-build' + binary = build / 'bin' / 'Release' if os.name == 'nt' else build / 'bin' + suffix = '.exe' if os.name == 'nt' else '' + command = [sys.executable, '-B', 'web/scripts/test-platform.py', + '--compiler', str(binary / ('nilcompiler' + suffix)), + '--interpreter', str(binary / ('nilinterpreter' + suffix)), + '--host-contract', str(binary / ('nil_rooted_file_snapshot_tests' + suffix)), + '--nilstd', str(Path('framework/nilstd/src').resolve()), + '--framework-root', str(Path('framework').resolve()), + '--framework-revision', os.environ['NIL_FRAMEWORK_REVISION'], + '--expected-system', os.environ['NIL_CI_SYSTEM'], + '--expected-machine', os.environ['NIL_CI_ARCH'], + '--output', str(Path(os.environ['RUNNER_TEMP']) / 'web-platform-results')] + subprocess.run(command, check=True, timeout=450) + PY + - name: Upload complete or partial evidence only + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: web-${{ matrix.host.name }}-${{ github.sha }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/web-build-evidence/ + ${{ runner.temp }}/web-platform-results/evidence/ + if-no-files-found: error + retention-days: 14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4776487 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,72 @@ +# Contributing to nil-web + +nil-web is a server-side web framework written in Matrix. Start with the +[README](README.md) for its current capabilities, known gaps, and compatible NIL +Framework build. Keep new library behavior in Matrix where the language supports +it, and report any required framework changes separately. + +## Development workflow + +1. Use the SDK described in the README. Set `NIL_SDK` to its directory if it is + not available through the sibling framework checkout. +2. Keep changes focused and add a regression case under `tests/` when changing + behavior. Use synthetic requests and data. +3. Run the test suite from the repository root: + + ```sh + ./scripts/test.sh + ``` + + This compiles the Matrix tests and runs them with `nilinterpreter`. Report + the SDK revision, platform, and checks run; an interpreter pass does not + establish native-route or cross-platform support. +4. Update the public documentation when setup, APIs, or observed behavior + changes. Describe remaining gaps and any skipped or blocked checks in the + pull request. + +Submit changes through a pull request. The public branch requires an approving +review, resolution of review conversations, the GitHub Actions `secrets` check, +and the four `web-*` platform checks. Do not bypass those checks with a direct +push. The platform workflow builds the exact framework revision recorded in +[its configuration](.github/workflows/test.yml), then checks the unit suite, +real loopback HTTP behavior, filesystem boundaries and process cleanup. + +For server changes, also exercise the affected HTTP behavior with bounded +clients against a local instance you own, then stop that instance. Keep +ordinary generated outputs under the ignored `build/` directory. The wire and +platform harnesses require a fresh external `--output` directory so their +fixtures and evidence cannot become tracked source. + +## Public source review + +Do not commit credentials, private configuration, internal notes, real user +requests or session data, developer-specific paths, or generated artifacts. +Use relative paths, explicit configuration, and synthetic examples. Review +history as well as the current files before publishing changes. + +Run the path audit from the repository root after staging the intended files: + +```sh +python3 -B tests/test_audit_paths.py +python3 -B scripts/audit-paths.py --root . +``` + +The path audit checks tracked text. The source-security workflow also scans +the working tree and fetched Git history with checksum-pinned +[Gitleaks 8.30.1](https://github.com/gitleaks/gitleaks/releases/tag/v8.30.1). +With that version installed, basic local scanner commands are: + +```sh +gitleaks dir . --no-banner --redact +gitleaks git . --no-banner --redact +``` + +CI additionally fixes the default rule configuration, rejects inline scanner +suppressions, and bounds archive traversal, decoding and scan time. + +Do not publish scanner reports or unreviewed logs: even redacted output can +expose private context. Scanning supplements source review; it does not replace +it. Report vulnerabilities through the process in [SECURITY.md](SECURITY.md). + +Contributions to original project materials use the repository's +[MIT License](LICENSE). Preserve existing third-party notices. diff --git a/README.md b/README.md index 8bf6ea6..e977781 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,20 @@ much older NIL framework. The design survived the move; almost none of the code did, because the languages differ in ways that reach the architecture. See [Differences from NyWeb](#differences-from-nyweb). +[Contributing](CONTRIBUTING.md) · [Security](SECURITY.md) + +[![Support us on Ko-fi](https://img.shields.io/badge/Support_us-Ko--fi-FF5E5B?logo=kofi&logoColor=white)](https://ko-fi.com/execsoftware) + ## Status -At web revision `41eb756`, all 653 existing checks pass through the NIL -interpreter on Linux with framework SDK `9fa354d`. The server supports concurrent -connections, configuration, virtual hosts, and access and error logs. Request -framing and static-file handling still have the gaps listed below. +The current server passes on Linux, Windows, Intel macOS and Apple Silicon +macOS: 1,128 unit checks, 38 real-wire checks, 15 static HTTP checks and 16 +deadline-oracle controls on each host. The +[four-platform run](https://github.com/exec-software/exec-nil-framework-web/actions/runs/43162279298) +also verifies rooted-file behavior, unchanged inputs and owned-process cleanup. +These checks run through the NIL interpreter against a pinned framework source +build; they do not qualify a published SDK or native networking. All four +platform checks and the source-security audit are required for public changes. | Module | What it does | State | | --- | --- | --- | @@ -28,33 +36,42 @@ framing and static-file handling still have the gaps listed below. | `nilweb.template.context` | the data a template may read | done | | `nilweb.template.render` | `.nilhtml` with layouts, partials, loops, escaping | done | | `nilweb.session.session` | a server-side store, secure ids, cookie attributes | done | -| `nilweb.static.files` | static files, validators and byte ranges | partial; see remaining work | +| `nilweb.static.files` | rooted static reads, content validators and byte ranges | done within the profile below | | `nilweb.middleware.chain` | guards before a handler, finishers after | done | -| `nilweb.server.engine` | the event loop: accepting, framing, keep-alive | partial; see remaining work | +| `nilweb.http.framing` | bounded Content-Length/chunked frames and exact pipeline boundaries | done within the profile below | +| `nilweb.server.engine` | the event loop: accepting, framing, keep-alive | done within the profile below | | `nilweb.time.clock` | HTTP dates, log stamps, ISO instants | done | | `nilweb.log.access` | Combined Log Format access log, error log | done | | `nilweb.config.settings` | a nildata config file, and virtual hosts | done | ```sh ./scripts/serve.sh # a real server on 127.0.0.1:8080 -./scripts/test.sh # 653 checks +./scripts/test.sh # 1,128 checks ./scripts/demo.sh # the same requests, printed instead of served ``` All three take `NIL_SDK` if the framework is not a sibling checkout, and `serve.sh` takes `NILWEB_PORT`. -It needs a framework with the socket, entropy and byte intrinsics — the commit -"Sockets, secure random, and byte access to text" or later. +It needs a framework with `nil.std.io::read_rooted_snapshot` and +`nil.std.text::from_bytes`, as well as the socket and entropy APIs. Older SDKs +without these APIs cannot run this revision. The functional CI workflow pins the +exact compatible framework source. Build that revision with its +`scripts/build-sdk.sh` and point `NIL_SDK` at the resulting SDK. Web remains an +interpreter consumer: native +text does not yet provide the binary-string contract needed by this response +layer. ## Concurrency An event loop, not a connection at a time. `poll` says which sockets can be acted on; the engine frames whole requests out of whatever arrived and the -application answers them. Many connections are open at once and none can hold -up another by sending slowly, reading slowly, or stopping halfway. +application answers them. Socket reads and writes are nonblocking, so a slow +peer does not make the loop wait for its next byte. Application handlers and +filesystem work still run synchronously on that loop. -Measured on the demonstration application: a client that stalls for 1.5 +Historical measurements before content hashing and framing hardening, on the +demonstration application: a client that stalls for 1.5 seconds in the middle of its request headers does not delay 120 requests on 40 other connections, which finish in 0.72s. 64 connections each sending a two-part request with a quarter-second pause are served in about one pause @@ -73,55 +90,71 @@ errors. Per route, at 16 connections: With logging on, each of those drops by roughly a fifth; see Logs below. -That is a tree-walking interpreter rendering templates, so it is not Apache's -number. It is the shape that matters: adding connections does not add latency -until the single core is saturated. +These are historical measurements of the demonstration on one interpreter +build, not a throughput or latency guarantee for the current server. **Keep-alive** is on by default for HTTP/1.1 and off for HTTP/1.0 unless the client asks, which is the way round that avoids holding connections no client will use again. A connection serves up to 1000 requests before being closed anyway, so a long-lived client cannot pin a slot for the life of the process. -**More than one core** means more than one process, because the interpreter -holds one stack, one heap and one set of globals with no lock anywhere near -them. `NILWEB_SHARE_PORT=1` turns on `SO_REUSEPORT` so several copies can share -a port and the kernel balances accepts between them. That is nginx's shape -rather than Apache's prefork, and it is the one this runtime can keep. +**Multiple server processes** can share a listening port on Linux and macOS +with `NILWEB_SHARE_PORT=1`, using `SO_REUSEPORT`. The Windows socket provider +does not support this option; the demonstration reports that sharing failed +and keeps its single listener. This server does not dispatch handlers across +threads or share in-memory sessions between processes. **Three idle deadlines**, because the three cases differ: 10s for a connection that has sent nothing (a scan, or a browser's speculative socket), 20s for a request still arriving (which is what slowloris looks like), and 30s between -keep-alive requests (a real client thinking). Requests are capped at 1 MiB and -the cap stops the reading rather than rejecting afterwards. At most 1024 -connections are open at once; past that, new ones are closed immediately -rather than queued, because a queue there is memory held for a client already -being refused. +keep-alive requests (a real client thinking). Each wire request, including its +headers, is capped at 1 MiB; oversized declarations are rejected before waiting +for the body. One additional socket read of up to 64 KiB can hold bounded +pipeline surplus. The default connection limit is 1024, configurable through +`connectionLimit`; accepted connections above that limit are closed. ### What is still missing for parity -No TLS — it wants an intrinsic family over a TLS library, and until then this -belongs behind something that terminates it. No HTTP/2, no compression (the -standard library's `gzip_compress` is still an unbacked declaration), no +No TLS — use a separate TLS terminator when serving HTTPS. No HTTP/2, no +response compression (the framework's compression prototype is outside the +supported standard library), no `sendfile`, no reverse proxying. `poll` is O(n) per call in the watch set, which starts to matter in the thousands; the interface is shaped so `epoll` can replace it underneath without callers noticing. -Current code review also identifies these remaining boundaries: - -- **Request framing:** malformed `Content-Length` can be treated as a zero-length - body, and `Transfer-Encoding` is not validated. Invalid framing needs an - explicit rejection and connection closure. -- **Pipelining:** requests received together are not separated reliably; clearing - the input after a response can discard the next request. -- **Conditional and range requests:** `Range` is evaluated before conditional - validators, `If-Range` is not handled, and `HEAD` can retain partial-response - status and length. These decisions need to be combined correctly. -- **Static-file containment:** textual traversal checks do not keep filesystem - links inside the public root. Reads and metadata need a shared, root-constrained - filesystem boundary, including intermediate symlinks and Windows junctions. - -The 653-check run verifies the existing suite; these code-review findings were -not re-tested through HTTP network requests in that run. +The current HTTP/1 profile is explicit: + +- Framing requires CRLF. Identical repeated `Content-Length` values are accepted; + malformed or conflicting lengths, and length plus transfer encoding, are + rejected with a final response and connection closure. Chunked bodies are + decoded within the request cap. Extensions and trailers are validated; + trailers are discarded. Other transfer codings receive an explicit rejection. +- Pipelined requests retain exact message boundaries and surplus bytes across + response writes. Partial requests keep a deadline that incoming fragments + cannot extend. Half-closing input still permits complete requests to finish. +- HTTP/1.1 `Expect` requests receive `417` immediately; interim `100 Continue` + negotiation is not implemented. Clients may retry without the expectation. + HTTP/1.0 `100-continue` is ignored as required by HTTP semantics. +- Preconditions run before ranges. `HEAD` ignores `Range`, keeps the full + representation length and sends no body. Strong content-based `If-Range` tags + are supported; date-form `If-Range` conservatively sends the full response. + Invalid date conditions are ignored; parsing accepts canonical IMF-fixdate. +- Static reads reject every symlink or Windows reparse point below the trusted + configured root, including intermediate directories. One file handle supplies + bounded bytes and metadata; SHA-256 identifies those exact bytes. This does + not protect against privileged mount changes or hard links placed in the + public directory. Keep that directory under the application's control. + +The static limit is 2 MiB. The runtime computes the content hash and copies binary +bytes into interpreter text in bulk. A local loopback check served and verified a +1 MiB response in about 10 ms; this is an observed test result, not a throughput +guarantee. The pure Matrix SHA-256 module remains a tested reference implementation. + +`tests/test_static_snapshot.py` verifies binary bodies, conditional/range/HEAD +responses, preserved-timestamp edits, filesystem links and the size cap. +`scripts/test-http-wire.py` checks real framing, backpressure, pipelines, +half-close and deadlines. Both use bounded, owned loopback servers and verify +cleanup; `scripts/test-platform.py` runs the complete platform campaign. ### This used to say the opposite @@ -176,9 +209,9 @@ will not start without a config file is one nobody can try out. A file that is defaults are used, because starting while silently ignoring a file somebody wrote is worse than either refusing or obeying. -**An unknown schema is refused rather than read hopefully.** A file declaring a -version this does not know may mean something different by the same key names, -and guessing is how a server ends up listening somewhere nobody asked for. +**An unknown schema is not applied.** The demonstration reports the rejected +configuration and uses defaults, then applies environment overrides. It does +not abort startup solely because the configuration file was rejected. **Sites are matched on `Host`**, with the port stripped and the case folded, each with its own template and public roots. A site with no `host` is the @@ -201,25 +234,21 @@ a web server's log already reads this one: because a framework that starts writing files into whatever directory it was launched from fills a disk on someone's laptop. -**Buffered, and flushed when the server is idle.** `nil.std.io`'s only -appending call opens, writes and closes, which is three syscalls per request on -the hot path, and the handle API that would avoid them is private and takes raw -byte buffers a Matrix page cannot build. So lines are held and written in -batches — on an idle pass, which the loop already detects, or when 64 have -accumulated. The cost is a window: lines buffered when the process dies are -lost. That is the trade the available I/O imposes, and `flush-now` is there for -anything that would rather pay the syscalls. - -**A log line cannot be forged.** Header values are attacker-controlled, and a -newline in one would end the line and start another of the attacker's -choosing — a fabricated entry in a file someone will later read as evidence. A -quote would close a field early and do the same to a parser; a control byte -would move a terminal's cursor or set a colour. All are replaced with a space, -one space per byte so the length a reader sees is not changed by sanitising. A -malformed request logs a dash where its request line would be, rather than its -bytes. - -**Logging costs about a fifth of the request rate** — 366 req/s becomes 291. +**Buffered, and flushed when the server is idle.** The logger batches lines +before calling `nil.std.io::append_text`, rather than opening the log on every +request. It flushes on an idle pass or when the ordinary access/error buffer +reaches 64 lines. Lines buffered when the process dies are lost; `flush-now` +allows an explicit flush. Failed writes discard the affected batch and increase +`dropped-count` instead of retrying an ever-growing buffer. + +**Log fields are sanitized.** Quotes, backslashes, control bytes and DEL are +replaced with spaces, then surrounding whitespace is trimmed. This prevents +those bytes from injecting extra lines or breaking quoted fields; it does not +preserve the original field length. A malformed request logs a dash in place +of its request line. + +**Historical logging measurements** showed 366 req/s becoming 291, before the +current content hashing and framing changes. One cause was found and fixed: the loop was parsing every request twice, once for the log and once inside `answer`, which was worth 270 → 309 on its own. The remaining fifth is not accounted for. Two likely-looking fixes — a @@ -229,35 +258,47 @@ merits for long inputs rather than credited with a gain they did not produce. ## Static files -Served with the three things a static file server is expected to do about not -re-sending what the client already has. - -**`Last-Modified` and `If-Modified-Since`**, compared at second resolution, -because an HTTP date has no finer one — a file written 300ms after the second -the client was told would otherwise look modified on every request forever. An -unreadable date counts as modified and gets the full representation, which is -always correct; reading a date wrongly and answering 304 serves a stale page, -which is the failure that matters. - -**ETag and `If-None-Match`.** The tag is built from size and modification time -— what Apache does by default, and sound for the same reason: a file whose -length and mtime are both unchanged is, barring deliberate effort, unchanged. -A content hash would be stronger and wants a hash function the standard -library does not have yet. A client holding the current tag gets a 304 with no -body, no `Content-Length` and no `Content-Type`, because the response is not -the representation. +`serve-static` reads a bounded snapshot through `nil.std.io::read_rooted_snapshot`. +That snapshot supplies the bytes, modification time and SHA-256 digest from one +opened file. The 2 MiB cap is checked before the content allocation; oversized +files receive 413. Other read refusals and missing files receive 404. + +**Content ETags.** The strong tag is the quoted lowercase SHA-256 of the exact +returned bytes, including binary content. It changes when content changes even +if the file's size and timestamp are preserved. The runtime computes the digest; +the pure Matrix SHA-256 module is a tested reference, not the static-serving hot +path. Revalidation still reads and hashes the bounded file before deciding +whether a body is needed; this module does not cache snapshots between requests. + +**Preconditions.** `If-Match` uses strong comparison and takes precedence over +`If-Unmodified-Since`. `If-None-Match` uses weak comparison and suppresses +`If-Modified-Since` whenever the tag field is present. A matching +`If-None-Match` produces 304 for GET/HEAD and 412 for other applicable methods. +Failed `If-Match` or `If-Unmodified-Since` conditions produce 412. The 304 response +carries validators and cache policy, with no body, `Content-Length` or +`Content-Type`. + +**Date validators.** `Last-Modified` and date conditions use second resolution. +Only canonical IMF-fixdate is parsed; malformed, obsolete or otherwise +unsupported date values are ignored. A date alone cannot distinguish two +different representations written within the same second; content ETags do. +Ignoring a date does not bypass other preconditions or range handling. **Byte ranges.** `bytes=0-9`, `bytes=500-`, `bytes=-500` for the last 500. A 206 carries `Content-Range`; a first byte past the end is a 416 naming the real -length, so a client resuming a download discovers the file changed size rather -than being quietly handed the wrong bytes. One range per request — a -multi-range request gets the whole file, which RFC 9110 permits and which -avoids a multipart body for a case almost nothing sends. - -**Refusal by shape, before touching the disk** — traversal in any spelling, -absolute names, drive letters, backslashes, doubled separators, null bytes and -dotfiles. An unknown extension is `application/octet-stream` and never -`text/html`, because a browser told something is HTML will run the script in it. +length. Only GET applies ranges, after preconditions. HEAD keeps the full +representation length and sends no body. A matching single strong `If-Range` +tag permits the range; weak tags, mismatches and date-form `If-Range` receive +the whole representation. Multi-range requests and unknown units are ignored; +no multipart response is produced. Empty files also ignore `Range`. + +**Path and file boundaries.** After mapping the URL to a relative name, path +checks reject traversal components, drive syntax, backslashes, doubled +separators, NUL and dotfiles. The framework then opens descendants relative to +the trusted root, rejecting symlinks and Windows reparse points in intermediate +and final components and accepting only regular files. This is not protection +against privileged mount changes or hard links placed inside the public root. +Unknown extensions use `application/octet-stream`. The dates come from `nilweb.time.clock`, which turns epoch milliseconds into HTTP's date format, the Common Log Format stamp and an ISO instant. The @@ -276,35 +317,34 @@ against the file's modification time — which is why `nil.std.io` grew `modified_at`: without it a cache cannot tell whether what it holds is current except by re-reading the file, which is the thing the cache exists to avoid. -Serving freezes the cache, so a page costs no filesystem call at all. Set -`NILWEB_RELOAD=1` to leave it thawed while editing templates. +Serving freezes the cache after each template's first load, so subsequent +cache hits do not recheck timestamps or reread that template. Set +`NILWEB_RELOAD=1` to recheck timestamps while editing; edits that preserve the +timestamp can remain cached. This template cache is separate from static-file +content hashing. Template files are trusted application inputs and do not use +the rooted static-file snapshot boundary. ## Differences from NyWeb -**Code-behind is gone.** NyWeb gave every page a class implementing `IPage` and -the router called it. Matrix has no capturing closures, and an array of class -handles compiles but does not execute, so a callable cannot be stored in a route -table. A route names its handler with text and the application switches on the -name. The cost is one `switch` per application; what it buys is a route table -that is plain data — readable, testable and reorderable without running it. +**Routes name their handlers.** NyWeb used page classes implementing `IPage`. +This framework keeps routes as data: each route names a handler and the +application dispatches on that name. It does not require a stored callback or +capturing closure. This describes the server's design, not a claim that all +class arrays or function values are unsupported by the language. **The template engine is Matrix, not C++.** NyWeb's was 626 lines of C++ reached by FFI. This one is Matrix, because Matrix can do the string work itself, and a template engine written in the language it serves is one the people using it can read. -**Middleware is two lists, not a chain of wrapped calls.** The `next(request)` -shape every other framework uses needs somewhere to capture `next`, and Matrix -has no capturing closures. What replaces it is a list of guards that run before -a handler and may answer instead of it, and a list of finishers applied to -whatever response comes back. A guard cannot wrap a handler in an `attempt` or -carry a local across the handler; what it can do is be printed, which is more -than most middleware stacks manage. An unwired guard fails closed with a 500, -because a configuration mistake in a security check must not be a quiet pass. +**Middleware uses two lists.** Guards run before the handler and may answer +instead; finishers apply to the resulting response. This API does not wrap a +handler in a captured `next` callback or carry a guard's local state through +that callback. An unwired guard produces 500 instead of silently proceeding. -**Everything is a value.** A response is a record; a finisher takes one and -gives back another. Nothing mutates a shared request object, which is what makes -a handler testable without a server. +**Handlers can be tested without a socket.** Requests and responses are records, +and finishers return a response. This does not promise immutable records or +deep-copy semantics; the server engine also uses mutable records internally. **Parsing happens once.** NyWeb re-split the query string on every lookup, so a page reading five parameters parsed the query five times. Here a request is @@ -343,11 +383,10 @@ the server does. `Secure` is the one choice left to the caller, because a ## Templates read; they do not evaluate -`<%= user.name %>` does not mean "evaluate this expression". There is no -runtime evaluator in Matrix — no `eval`, no reflection over locals — so it -means "look this name up in the context the handler published". NyWeb's engine -parsed Ny expressions and the runtime evaluated them; that is not available and -would not be reconstructed here if it were. +`<%= user.name %>` looks up a name in the context the handler published. The +template engine does not evaluate arbitrary Matrix expressions or reflect over +the handler's locals. Includes and layouts use trusted template files; template +values do not gain access to arbitrary application operations. It is a narrower contract and a sturdier one. A template can only read what a handler decided to publish, so it cannot reach into a session, open a file or @@ -382,11 +421,13 @@ src/nilweb/http/headers.matrix header fields src/nilweb/http/uri.matrix percent coding and query strings src/nilweb/http/request.matrix the inbound request src/nilweb/http/response.matrix the outbound response +src/nilweb/http/framing.matrix bounded wire framing and pipeline boundaries src/nilweb/routing/router.matrix route matching src/nilweb/template/context.matrix the data a template may read src/nilweb/template/render.matrix the .nilhtml engine src/nilweb/session/session.matrix sessions and their cookie src/nilweb/static/files.matrix serving a directory +src/nilweb/crypto/sha256.matrix reference SHA-256 implementation and tests src/nilweb/middleware/chain.matrix guards and finishers src/nilweb/server/engine.matrix the event loop src/nilweb/time/clock.matrix HTTP dates, log stamps, ISO instants @@ -402,6 +443,13 @@ tests/nilweb/tests/harness.matrix the checks A book lives at the path its name spells: `book nilweb.http.request` is `src/nilweb/http/request.matrix`. +## Support our work + +You can support Execute Software's work through +[Ko-fi](https://ko-fi.com/execsoftware). Donations are optional and help fund +continued development; they do not change what the project provides or what +its license permits. + ## License -MIT. +nil-web is licensed under the [MIT License](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a2dff8a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,24 @@ +# Reporting security issues + +Use the repository's enabled [private vulnerability reporting](https://github.com/exec-software/exec-nil-framework-web/security/advisories/new). +If that option is unavailable, open a minimal issue requesting a private +contact channel without including exploit details, credentials, or private +data. Ordinary bugs can be reported publicly. + +Include the affected web and NIL Framework SDK revisions, platform, impact, +and a small reproduction using synthetic data. Remove credentials, session +identifiers, private paths, hostnames, and user content from examples and logs. +There is no guaranteed response-time SLA. + +## Current scope + +The [README](README.md#status) records the tested interpreter profile and known +gaps. Treat those limits as part of the current implementation; a passing test +suite is not a claim of production readiness or support for every execution +route and platform. + +When reporting a problem, distinguish request parsing and framing, static-file +access, template rendering, session handling, and application-specific behavior. +Include the relevant configuration without private values. Exercise only local +instances or systems you own or have explicit permission to test, using bounded +requests that do not affect other users. diff --git a/scripts/audit-paths.py b/scripts/audit-paths.py index c8c34e9..6edfd22 100755 --- a/scripts/audit-paths.py +++ b/scripts/audit-paths.py @@ -3,9 +3,9 @@ A framework's source has no business naming a developer's home directory, a machine on anyone's network, or a person. None of that is a secret in the sense -gitleaks looks for, which is exactly why it needs its own check: a path like -/home/someone/work/thing leaks a name and a working habit and no scanner built -for API keys will say a word about it. +gitleaks looks for, which is exactly why it needs its own check. A personal +home-directory path leaks a name and a working habit; a scanner built for API +keys will not necessarily detect it. What is allowed, and why, because a check nobody can satisfy gets deleted: @@ -16,8 +16,13 @@ * The documentation ranges of RFC 5737 and RFC 3849 — 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24, 2001:db8::/32. Reserved precisely so that examples can use them. - * Version numbers and the like that happen to have four dotted parts, which - is why this parses candidates as addresses rather than trusting a regex. + * Numeric strings that do not parse as IP addresses. Four-part version + numbers that also form valid addresses are deliberately reported: context + alone cannot establish that an address is harmless. + +Every tracked file must be a readable, regular UTF-8 text file without NUL +bytes. Symlinks and binary assets require an explicit policy before adding +them; silently skipping a file would leave part of the public source unaudited. Everything else is reported with its file and line, and the line itself is never printed: if a finding is real, printing it puts the thing being @@ -28,20 +33,24 @@ import argparse import ipaddress +import json +import os import re +import stat import subprocess import sys from pathlib import Path # A home directory with a name after it. `/home/` alone is not interesting -- # it appears in prose about Linux -- so a following name is required. -HOME_PATH = re.compile(r"/(?:home|Users)/[A-Za-z0-9._-]+") -WINDOWS_PATH = re.compile(r"[A-Za-z]:\\\\?(?:Users|Documents)", re.IGNORECASE) +HOME_PATH = re.compile(r"/(?:home|Users)/[\w.-]+|/(?:root)(?:/|\b)") +WINDOWS_PATH = re.compile(r"[A-Za-z]:[\\/]+(?:Users|Documents)(?:[\\/]|\b)", re.IGNORECASE) # Deliberately loose: the point is to catch a real address, and the parse below # decides whether a match is one. IPV4 = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b") -IPV6 = re.compile(r"\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b") +# Include compressed and IPv4-mapped forms; ipaddress validates candidates. +IPV6 = re.compile(r"(? list[Path]: - listed = subprocess.run(["git", "ls-files"], cwd=root, - capture_output=True, text=True, check=True) - return [root / line for line in listed.stdout.splitlines() if line.strip()] + listed = subprocess.run(["git", "ls-files", "--cached", "-z"], cwd=root, + capture_output=True, check=True) + return [root / os.fsdecode(name) for name in listed.stdout.split(b"\0") + if name] def is_reportable_address(candidate: str) -> bool: try: - address = ipaddress.ip_address(candidate) + address = ipaddress.ip_address(candidate.rstrip(".")) except ValueError: # Not an address at all -- a version number, a byte sequence, a time. return False + # Mapped loopback/documentation IPv4 addresses retain their classification. + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: + address = address.ipv4_mapped return not any(address in network for network in DOCUMENTATION_NETWORKS) @@ -92,25 +105,42 @@ def audit(root: Path, skip: set[str]) -> list[str]: relative = path.relative_to(root).as_posix() if relative in skip: continue + + def report(number: int, rule: str) -> None: + # Filenames can themselves contain newlines or terminal controls. + # JSON escaping keeps each diagnostic on one line, without content. + findings.append(f"{json.dumps(relative, ensure_ascii=True)}:{number}: {rule}") + + try: + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + report(0, "symlink-file") + continue + if not stat.S_ISREG(mode): + report(0, "non-regular-file") + continue + raw = path.read_bytes() + except OSError: + report(0, "unreadable-file") + continue + if b"\0" in raw: + report(0, "binary-file") + continue try: - body = path.read_text(encoding="utf-8") - except (UnicodeDecodeError, OSError): - continue # not text; gitleaks covers binary content + body = raw.decode("utf-8") + except UnicodeDecodeError: + report(0, "non-utf8-file") + continue for number, line in enumerate(body.splitlines(), start=1): - for found in HOME_PATH.findall(line): - findings.append(f"{relative}:{number}: a home directory path") - break + if HOME_PATH.search(line): + report(number, "home-path") if WINDOWS_PATH.search(line): - findings.append(f"{relative}:{number}: a Windows user path") - for found in IPV4.findall(line) + IPV6.findall(line): - if is_reportable_address(found): - findings.append( - f"{relative}:{number}: a routable address") - break - for found in EMAIL.findall(line): - if is_reportable_email(found): - findings.append(f"{relative}:{number}: an email address") - break + report(number, "windows-user-path") + if any(is_reportable_address(found) + for found in IPV4.findall(line) + IPV6.findall(line)): + report(number, "non-documentation-address") + if any(is_reportable_email(found) for found in EMAIL.findall(line)): + report(number, "contact-address") return findings @@ -121,18 +151,20 @@ def main() -> int: help="a tracked path to leave out, repeatable") args = parser.parse_args() - root = args.root.resolve(strict=True) - # This file names the patterns it looks for, so it would always find them. - skip = set(args.skip) | {"scripts/audit-paths.py"} - - findings = audit(root, skip) + try: + root = args.root.resolve(strict=True) + findings = audit(root, set(args.skip)) + except (OSError, RuntimeError, subprocess.CalledProcessError): + print("Source audit failed: repository files could not be enumerated.", + file=sys.stderr) + return 1 if findings: print("Source audit failures (the matching text is deliberately not shown):") for finding in sorted(set(findings)): print(f" - {finding}") return 1 - print("Source audit: no absolute paths, routable addresses or contact " - "details in tracked text.") + print("Source audit: all selected tracked files are readable text; no " + "personal paths, non-documentation addresses or contact details found.") return 0 diff --git a/scripts/test-http-wire.py b/scripts/test-http-wire.py new file mode 100644 index 0000000..cf2ac72 --- /dev/null +++ b/scripts/test-http-wire.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +"""Bounded loopback HTTP tests using explicit SDK tools and external artifacts. + +Compiles the small Matrix consumer from source for every --nilstd root. Each +owned server has a 90-second internal lifetime; Python also enforces deadlines +and reaps only its own child/process group. With --parent-process-group, an +outer driver owns the group and this harness reaps only its direct server PID. +No developer server is reused. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import select +import signal +import socket +import subprocess +import time +import uuid + +REPO = Path(__file__).resolve().parents[1] + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def pin_tree(root: Path) -> dict[str, str]: + return {str(p.relative_to(root)): digest(p) for p in sorted(root.rglob("*")) if p.is_file()} + + +def request(target: bytes, fields: bytes = b"", body: bytes = b"", method: bytes = b"GET") -> bytes: + return method + b" " + target + b" HTTP/1.1\r\nHost: example.com\r\n" + fields + b"\r\n" + body + + +def windows_connection_aborted(error: OSError) -> bool: + # WSAECONNABORTED is a closed connection, not a successful socket timeout. + # Keep this exception specific to Winsock; other abort/error codes fail. + return (os.name == "nt" and isinstance(error, ConnectionAbortedError) + and getattr(error, "winerror", None) == 10053) + + +class Peer: + def __init__(self, port: int): + self.socket = socket.create_connection(("127.0.0.1", port), timeout=3) + self.socket.settimeout(5) + self.buffer = b"" + + def __enter__(self): + return self + + def __exit__(self, *_): + self.socket.close() + + def send(self, data: bytes): + self.socket.sendall(data) + + def response(self) -> tuple[int, dict[bytes, bytes], bytes]: + while b"\r\n\r\n" not in self.buffer: + self.receive() + head, self.buffer = self.buffer.split(b"\r\n\r\n", 1) + lines = head.split(b"\r\n") + status = int(lines[0].split()[1]) + fields = {} + for line in lines[1:]: + key, value = line.split(b":", 1) + key = key.lower() + if key in fields: + raise AssertionError("duplicate response field") + fields[key] = value.strip() + size = int(fields[b"content-length"]) + if size > 9 * 1024 * 1024: + raise AssertionError("unbounded response") + while len(self.buffer) < size: + self.receive() + body, self.buffer = self.buffer[:size], self.buffer[size:] + return status, fields, body + + def receive(self): + data = self.socket.recv(65536) + if not data: + raise AssertionError("response ended before its boundary") + self.buffer += data + + def closed(self): + if self.buffer: + raise AssertionError("extra response bytes after close") + try: + data = self.socket.recv(1) + except OSError as error: + if not isinstance(error, ConnectionResetError) and not windows_connection_aborted(error): + raise + data = b"" + if data: + raise AssertionError("connection was reused after rejection/close") + + +def check_response(peer: Peer, expected: bytes): + status, _, body = peer.response() + if status != 200 or body != expected: + raise AssertionError("response status or exact payload differs") + + +def check_first_byte_deadline(peer: Peer): + started = time.monotonic() + peer.send(b"POST /slow HTTP/1.1\r\nHost: example.com\r\nX-Slow: ") + next_fragment = started + 3 + until = started + 25 + while True: + now = time.monotonic() + if now >= until: + raise AssertionError("request deadline extended by fragments") + # Observe FIN/reset as soon as it arrives. Sleeping until the next send + # could hide an early close behind buffered successful socket writes. + ready, _, _ = select.select([peer.socket], [], [], min(next_fragment, until) - now) + if ready: + peer.closed() + break + now = time.monotonic() + if now >= next_fragment: + try: + peer.send(b"x") + except OSError as error: + if not isinstance(error, (BrokenPipeError, ConnectionResetError)) and not windows_connection_aborted(error): + raise + peer.closed() + break + next_fragment = now + 3 + elapsed = time.monotonic() - started + if not 19 <= elapsed <= 25: + raise AssertionError("request closed outside the 19–25 second first-byte deadline window") + + +def exercise(port: int, rows: list[dict], real_deadline: bool): + def case(name, fn): + started = time.monotonic() + row = {"name": name, "status": "FAIL"} + rows.append(row) + try: + fn() + row["status"] = "PASS" + finally: + row["elapsed_seconds"] = round(time.monotonic() - started, 3) + + def pipeline(): + with Peer(port) as peer: + peer.send(request(b"/one") + request(b"/two", b"Connection: close\r\n")) + check_response(peer, b"/one|") + check_response(peer, b"/two|") + peer.closed() + case("two requests in one write retain response order", pipeline) + + def fixed_body(): + body = b"\x00\xff\xc3\xa9\r\nGET /not-a-request" + with Peer(port) as peer: + peer.send(request(b"/raw%2fpath?q=%20", b"Content-Length: " + str(len(body)).encode() + b"\r\n", body, b"POST") + + request(b"/after-body", b"Connection: close\r\n")) + check_response(peer, b"/raw%2fpath?q=%20|" + body) + check_response(peer, b"/after-body|") + peer.closed() + case("fixed body preserves binary octets and raw target", fixed_body) + + def fragmented(): + wire = request(b"/fragment", b"Content-Length: 5\r\n", b"abcde", b"POST") + with Peer(port) as peer: + for offset in range(0, len(wire), 3): + peer.send(wire[offset:offset + 3]) + time.sleep(0.002) + check_response(peer, b"/fragment|abcde") + peer.send(request(b"/after-fragment", b"Connection: close\r\n")) + check_response(peer, b"/after-fragment|") + peer.closed() + case("fragmented headers and body finish before dispatch", fragmented) + + def chunked(): + body = b"\x00\xff\xc3\xa9" + chunks = b"2;flag;key=\"a;b\"\r\n" + body[:2] + b"\r\n2\r\n" + body[2:] + b"\r\n0\r\nX-Note: discarded\r\n\r\n" + wire = request(b"/chunks", b"Transfer-Encoding: Chunked\r\nTrailer: X-Note\r\n", chunks, b"POST") + with Peer(port) as peer: + for offset in range(0, len(wire), 7): + peer.send(wire[offset:offset + 7]) + time.sleep(0.001) + peer.send(request(b"/after-chunks", b"Connection: close\r\n")) + check_response(peer, b"/chunks|" + body) + check_response(peer, b"/after-chunks|") + peer.closed() + case("fragmented chunks trailers and pipeline preserve exact octets", chunked) + + def duplicates(): + with Peer(port) as peer: + peer.send(request(b"/same", b"Content-Length: 003, 3\r\ncontent-length: 3\r\nConnection: close\r\n", b"abc", b"POST")) + check_response(peer, b"/same|abc") + peer.closed() + case("identical repeated and comma lengths accepted", duplicates) + + def rejection(fields: bytes, body=b"", status=400): + with Peer(port) as peer: + peer.send(request(b"/must-not-dispatch", fields, body, b"POST") + request(b"/must-not-follow")) + actual, headers, payload = peer.response() + if actual != status or headers.get(b"connection") != b"close" or payload: + raise AssertionError("framing rejection status/close differs") + peer.closed() + + invalid = [ + ("nondigit length", b"Content-Length: banana\r\n", b"", 400), + ("negative length", b"Content-Length: -1\r\n", b"", 400), + ("empty length", b"Content-Length:\r\n", b"", 400), + ("conflicting length fields", b"Content-Length: 0\r\nContent-Length: 3\r\n", b"abc", 400), + ("conflicting list lengths", b"Content-Length: 0, 3\r\n", b"abc", 400), + ("length overflow", b"Content-Length: 999999999999999999999999\r\n", b"", 413), + ("transfer encoding plus length", b"Transfer-Encoding: chunked\r\nContent-Length: 0\r\n", b"0\r\n\r\n", 400), + ("unknown final transfer coding", b"Transfer-Encoding: gzip\r\n", b"", 400), + ("unsupported preceding transfer coding", b"Transfer-Encoding: gzip, chunked\r\n", b"", 501), + ("duplicate chunked coding", b"Transfer-Encoding: chunked, chunked\r\n", b"", 400), + ("chunk size overflow", b"Transfer-Encoding: chunked\r\n", b"FFFFFFFFFFFFFFFFFFFF\r\n", 413), + ("malformed chunk extension", b"Transfer-Encoding: chunked\r\n", b"1;=x\r\n", 400), + ("chunk data missing CRLF", b"Transfer-Encoding: chunked\r\n", b"1\r\nxNO", 400), + ("malformed trailer", b"Transfer-Encoding: chunked\r\n", b"0\r\nBad Field: value\r\n\r\n", 400), + ("whitespace before field colon", b"Content-Length : 0\r\n", b"", 400), + ("obsolete folded field", b"X-Test: value\r\n continued\r\n", b"", 400), + ] + for name, fields, body, status in invalid: + case(name + " rejects and closes before pipeline", lambda f=fields, b=body, s=status: rejection(f, b, s)) + + def expectation(fields: bytes): + with Peer(port) as peer: + peer.socket.settimeout(2) + # Deliberately send headers only: a client awaiting an expectation + # decision has not sent any of its declared body or first chunk. + peer.send(request(b"/expect", fields, method=b"POST")) + status, headers, body = peer.response() + if status != 417 or headers.get(b"connection") != b"close" or body: + raise AssertionError("Expect was not rejected immediately from headers") + peer.closed() + for name, fields in [ + ("100-continue", b"Content-Length: 5\r\nExpect: 100-continue\r\n"), + ("unknown expectation", b"Content-Length: 5\r\nExpect: unsupported\r\n"), + ("mixed expectations", b"Content-Length: 5\r\nExpect: 100-continue, unsupported\r\n"), + ("chunked expectation", b"Transfer-Encoding: chunked\r\nExpect: 100-continue\r\n"), + ("duplicate expectation fields", b"Content-Length: 5\r\nExpect: 100-continue\r\nExpect: 100-CONTINUE, 100-continue\r\n"), + ("empty expectation", b"Content-Length: 5\r\nExpect:\r\n"), + ]: + case(name + " receives417 before any body bytes", lambda f=fields: expectation(f)) + + def legacy_expectation(): + with Peer(port) as peer: + peer.send(b"POST /legacy HTTP/1.0\r\nContent-Length: 5\r\nExpect: 100-continue\r\nExpect: 100-CONTINUE, 100-continue\r\n\r\nabcde") + check_response(peer, b"/legacy|abcde") + peer.closed() + case("HTTP1.0 ignores100-continue while preserving its body", legacy_expectation) + + def retry_without_expectation(): + with Peer(port) as peer: + peer.send(request(b"/expect", b"Content-Length: 5\r\nConnection: close\r\n", b"abcde", b"POST")) + check_response(peer, b"/expect|abcde") + peer.closed() + case("same request succeeds when retried without Expect", retry_without_expectation) + + def half_closed(): + with Peer(port) as peer: + peer.send(request(b"/half-one") + request(b"/half-two")) + peer.socket.shutdown(socket.SHUT_WR) + check_response(peer, b"/half-one|") + check_response(peer, b"/half-two|") + peer.closed() + case("half-close retains two complete buffered requests", half_closed) + + def large_half_closed(): + body = bytes(range(256)) * 600 + with Peer(port) as peer: + peer.send(request(b"/large-body", b"Content-Length: " + str(len(body)).encode() + b"\r\n", body, b"POST")) + peer.socket.shutdown(socket.SHUT_WR) + check_response(peer, b"/large-body|" + body) + peer.closed() + case("half-close drains a body larger than one socket read", large_half_closed) + + def incomplete_eof(): + with Peer(port) as peer: + peer.send(request(b"/incomplete", b"Content-Length: 5\r\n", b"ab", b"POST")) + peer.socket.shutdown(socket.SHUT_WR) + peer.closed() + case("incomplete body on EOF is never dispatched", incomplete_eof) + + def next_partial(): + with Peer(port) as peer: + peer.send(request(b"/first") + request(b"/partial", b"Content-Length: 5\r\n", b"ab", b"POST")) + check_response(peer, b"/first|") + peer.send(b"cde" + request(b"/last", b"Connection: close\r\n")) + check_response(peer, b"/partial|abcde") + check_response(peer, b"/last|") + peer.closed() + case("partial next body survives first reply and accepts remaining bytes", next_partial) + + def expired_next(): + with Peer(port) as peer: + peer.send(request(b"/age-next") + b"GET /incomplete HTTP/1.1\r\nHost:") + check_response(peer, b"/age-next|") + peer.closed() + case("reply and response drain preserve buffered next-request deadline", expired_next) + + def retained_deadline(): + with Peer(port) as peer: + peer.send(request(b"/age-soon") + request(b"/middle") + b"GET /partial HTTP/1.1\r\nHost:") + check_response(peer, b"/age-soon|") + check_response(peer, b"/middle|") + peer.socket.settimeout(3) + peer.closed() + case("consuming another buffered frame does not extend surplus deadline", retained_deadline) + + def backpressure(): + with Peer(port) as peer: + peer.send(request(b"/large") + request(b"/after-large", b"Connection: close\r\n")) + time.sleep(0.2) + check_response(peer, b"x" * (2 ** 23)) + check_response(peer, b"/after-large|") + peer.closed() + case("partial response writes retain buffered next request", backpressure) + + def error_after_backpressure(): + with Peer(port) as peer: + peer.send(request(b"/large") + request(b"/invalid-next", b"Content-Length: bad\r\n", method=b"POST")) + time.sleep(0.2) + check_response(peer, b"x" * (2 ** 23)) + status, headers, body = peer.response() + if status != 400 or headers.get(b"connection") != b"close" or body: + raise AssertionError("rejection overwrote or followed the prior response incorrectly") + peer.closed() + case("framing error follows a partially written response intact", error_after_backpressure) + + if real_deadline: + def deadline(): + with Peer(port) as peer: + check_first_byte_deadline(peer) + case("real first-byte deadline survives continuing fragments", deadline) + + +def stop_owned(process: subprocess.Popen, own_group: bool = True) -> bool: + group_owned = os.name == "posix" and own_group + if process.poll() is None: + try: + if group_owned: + os.killpg(process.pid, signal.SIGTERM) + else: + process.terminate() + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + if group_owned: + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + process.wait(timeout=5) + if group_owned: + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return True + return False + return process.poll() is not None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler", required=True, type=Path) + parser.add_argument("--interpreter", required=True, type=Path) + parser.add_argument("--nilstd", required=True, type=Path, action="append") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--parent-process-group", action="store_true", + help="inherit the outer driver process group; clean up only the owned server PID") + parser.add_argument("--real-deadline", action="store_true", help="also verify the 20-second first-byte deadline per root") + args = parser.parse_args() + args.output = args.output.resolve() + if args.output == REPO or REPO in args.output.parents: + parser.error("--output must be outside the source checkout") + args.output.mkdir(parents=True, exist_ok=False) + sources = [REPO / "src/nilweb/http/framing.matrix", REPO / "src/nilweb/server/engine.matrix", REPO / "tests/wire-server.matrix", Path(__file__).resolve()] + pins = {str(p.relative_to(REPO)): digest(p) for p in sources} + tools = {str(p.resolve()): digest(p) for p in (args.compiler, args.interpreter)} + report = {"schema": "nilweb.http-wire.v1", "status": "FAIL", "source_sha256": pins, "tools_sha256": tools, "roots": [], "parent_process_group": args.parent_process_group} + try: + for number, std in enumerate(args.nilstd): + std = std.resolve(strict=True) + before = pin_tree(std) + output = args.output / str(number) + output.mkdir() + row = {"nilstd": str(std), "nilstd_before": before, "checks": [], "cleanup_verified": False} + report["roots"].append(row) + env = {k: v for k, v in os.environ.items() if not k.startswith(("NIL_", "NY_", "NILWEB_"))} + env.update({"TMPDIR": str(output), "TMP": str(output), "TEMP": str(output)}) + bytecode = output / "wire.nil" + command = [str(args.compiler.resolve()), "--no-bundle", "-I", str(std), "-I", str(REPO / "src"), "--output", str(bytecode), str(REPO / "tests/wire-server.matrix")] + with (output / "compile.stdout").open("wb") as stdout, (output / "compile.stderr").open("wb") as stderr: + result = subprocess.run(command, cwd=output, env=env, stdout=stdout, stderr=stderr, timeout=90) + row["compile_exit"] = result.returncode + if result.returncode: + raise AssertionError("fixture compilation failed; see compiler logs") + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + env["NILWEB_WIRE_PORT"] = str(port) + row["port"] = port + with (output / "server.stdout").open("wb") as stdout, (output / "server.stderr").open("wb") as stderr: + process = subprocess.Popen([str(args.interpreter.resolve()), str(bytecode)], cwd=output, env=env, + stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, start_new_session=os.name == "posix" and not args.parent_process_group) + row["pid"] = process.pid + try: + token = ("/__wire_probe_" + uuid.uuid4().hex).encode() + until = time.monotonic() + 10 + while True: + if process.poll() is not None: + raise AssertionError("owned fixture exited before readiness") + try: + with Peer(port) as peer: + peer.send(request(token, b"Connection: close\r\n")) + check_response(peer, token + b"|") + break + except ConnectionRefusedError: + if time.monotonic() >= until: + raise AssertionError("owned fixture readiness timed out") + time.sleep(0.05) + exercise(port, row["checks"], args.real_deadline) + with Peer(port) as peer: + peer.send(request(b"/__shutdown", b"Connection: close\r\n")) + check_response(peer, b"/__shutdown|") + process.wait(timeout=5) + row["server_exit"] = process.returncode + if process.returncode: + raise AssertionError("owned fixture exit was not clean") + finally: + row["cleanup_verified"] = stop_owned(process, own_group=not args.parent_process_group) + row["nilstd_unchanged"] = before == pin_tree(std) + if not row["cleanup_verified"] or not row["nilstd_unchanged"]: + raise AssertionError("cleanup or nilstd integrity check failed") + report["status"] = "PASS" + except Exception as error: + report["error"] = str(error) + finally: + report["source_unchanged"] = pins == {str(p.relative_to(REPO)): digest(p) for p in sources} + report["tools_unchanged"] = tools == {str(p.resolve()): digest(p) for p in (args.compiler, args.interpreter)} + if not report["source_unchanged"] or not report["tools_unchanged"]: + report["status"] = "FAIL" + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"status": report["status"], "checks": sum(len(r["checks"]) for r in report["roots"]), "report": str(args.output / "report.json"), "error": report.get("error")})) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-platform.py b/scripts/test-platform.py new file mode 100644 index 0000000..f298fa2 --- /dev/null +++ b/scripts/test-platform.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +"""Run exact public-web contracts against explicit built framework tools. + +The output directory must be new and outside this checkout. Only its evidence/ +subdirectory is suitable for upload: work/ contains generated consumer bytecode +and owned server fixtures, never an SDK release. Each loopback harness owns and +reaps its server; this driver adds outer deadlines and verifies their reports. +""" +from __future__ import annotations + +import argparse +import ctypes +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import subprocess +import sys +import time + +REPO = Path(__file__).resolve().parents[1] +UNIT_CHECKS = 1128 +WIRE_CHECKS = 38 # Includes the real first-byte deadline check. +STATIC_CHECKS = 15 +WIRE_ORACLE_CHECKS = 16 +HOST_CHECKS = {"Linux": 87, "Darwin": 87, "Windows": 90} + + +def require(good: bool, message: str): + if not good: + raise AssertionError(message) + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_pin(root: Path) -> dict[str, str]: + return {path.relative_to(root).as_posix(): digest(path) + for path in sorted(root.rglob("*")) if path.is_file()} + + +def source_pin() -> dict[str, str]: + sources = sorted((REPO / "src").rglob("*.matrix")) + sources += [path for path in (REPO / "tests").rglob("*") + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc"] + sources += [REPO / "scripts/test-http-wire.py", Path(__file__).resolve()] + return {path.relative_to(REPO).as_posix(): digest(path) for path in sorted(set(sources))} + + +def normalized_machine() -> str: + raw = platform.machine().lower() + return {"amd64": "x86_64", "aarch64": "arm64"}.get(raw, raw) + + +class WindowsJob: + """An unnamed owned job; descendants inherit it before any command runs.""" + def __init__(self): + from ctypes import wintypes + self.kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel = self.kernel + kernel.CreateJobObjectW.argtypes = [wintypes.LPVOID, wintypes.LPCWSTR] + kernel.CreateJobObjectW.restype = wintypes.HANDLE + kernel.SetInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD] + kernel.SetInformationJobObject.restype = wintypes.BOOL + kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE] + kernel.AssignProcessToJobObject.restype = wintypes.BOOL + kernel.QueryInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, + wintypes.DWORD, ctypes.POINTER(wintypes.DWORD)] + kernel.QueryInformationJobObject.restype = wintypes.BOOL + kernel.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT] + kernel.TerminateJobObject.restype = wintypes.BOOL + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + kernel.CloseHandle.restype = wintypes.BOOL + + class BasicLimits(ctypes.Structure): + _fields_ = [("ProcessTime", ctypes.c_int64), ("JobTime", ctypes.c_int64), + ("Flags", wintypes.DWORD), ("MinWorkingSet", ctypes.c_size_t), + ("MaxWorkingSet", ctypes.c_size_t), ("ActiveLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), ("Priority", wintypes.DWORD), + ("Scheduling", wintypes.DWORD)] + + class IoCounters(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint64) for name in + ("ReadOps", "WriteOps", "OtherOps", "ReadBytes", "WriteBytes", "OtherBytes")] + + class ExtendedLimits(ctypes.Structure): + _fields_ = [("Basic", BasicLimits), ("IO", IoCounters), + ("ProcessMemory", ctypes.c_size_t), ("JobMemory", ctypes.c_size_t), + ("PeakProcessMemory", ctypes.c_size_t), ("PeakJobMemory", ctypes.c_size_t)] + + class Accounting(ctypes.Structure): + _fields_ = [("UserTime", ctypes.c_int64), ("KernelTime", ctypes.c_int64), + ("PeriodUser", ctypes.c_int64), ("PeriodKernel", ctypes.c_int64), + ("PageFaults", wintypes.DWORD), ("TotalProcesses", wintypes.DWORD), + ("ActiveProcesses", wintypes.DWORD), ("TerminatedProcesses", wintypes.DWORD)] + + self.Accounting = Accounting + self.handle = kernel.CreateJobObjectW(None, None) + require(bool(self.handle), "cannot create owned Windows process job") + limits = ExtendedLimits() + limits.Basic.Flags = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if not kernel.SetInformationJobObject(self.handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)): + self.close() + raise AssertionError("cannot set owned Windows job limits") + + def assign(self, process): + # CPython's Popen owns this stable process handle. The bootstrap is + # blocked on stdin until assignment succeeds; no descendant can race it. + require(bool(self.kernel.AssignProcessToJobObject(self.handle, int(process._handle))), + "cannot assign owned Windows bootstrap to its job") + + def active(self): + info = self.Accounting() + returned = ctypes.c_ulong() + require(bool(self.kernel.QueryInformationJobObject(self.handle, 1, ctypes.byref(info), + ctypes.sizeof(info), ctypes.byref(returned))) and returned.value == ctypes.sizeof(info), + "cannot inspect owned Windows job") + return info.ActiveProcesses != 0 + + def terminate(self): + require(bool(self.kernel.TerminateJobObject(self.handle, 124)), "cannot terminate owned Windows job") + + def close(self): + if self.handle: + self.kernel.CloseHandle(self.handle) + self.handle = None + + +def group_active(pid: int) -> bool: + try: + os.killpg(pid, 0) + return True + except ProcessLookupError: + return False + + +def drain(check, seconds: float) -> bool: + until = time.monotonic() + seconds + while check(): + if time.monotonic() >= until: + return False + time.sleep(0.025) + return True + + +def run(command: list[str], *, cwd: Path, env: dict[str, str], evidence: Path, + label: str, timeout: int) -> dict: + started = time.monotonic() + row = {"command": command, "timeout_seconds": timeout, "cleanup_verified": False} + process = None + job = None + assigned = False + problem = None + with (evidence / (label + ".stdout")).open("wb") as stdout, \ + (evidence / (label + ".stderr")).open("wb") as stderr: + try: + if os.name == "nt": + job = WindowsJob() + # Isolated bootstrap cannot spawn the actual command until its + # inherited job has been assigned. It waits for that command so + # Popen's exit status remains the command's status on Windows. + bootstrap = ("import subprocess,sys; " + "go=sys.stdin.buffer.read(1); " + "sys.exit(subprocess.call(sys.argv[1:],stdin=subprocess.DEVNULL) if go==b'G' else 125)") + process = subprocess.Popen([sys.executable, "-I", "-S", "-c", bootstrap, *command], + cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr) + row["pid"] = process.pid + job.assign(process) + assigned = True + process.stdin.write(b"G") + process.stdin.flush() + process.stdin.close() + else: + process = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=stdout, stderr=stderr, start_new_session=True) + row["pid"] = process.pid + row["exit_code"] = process.wait(timeout=timeout) + if job: + # Job accounting may retain terminated process references; + # the exit code is cached, so release Popen's owned handle. + process._handle.Close() + active = job.active if job else lambda: group_active(process.pid) + row["unexpected_descendants"] = not drain(active, 1) + if row["unexpected_descendants"]: + problem = AssertionError(label + " left a descendant process after exit") + except subprocess.TimeoutExpired: + row["timed_out"] = True + problem = AssertionError(label + " exceeded its outer deadline") + except Exception as error: + problem = error + finally: + try: + if process is not None: + if job and assigned: + if job.active(): + job.terminate() + process.wait(timeout=5) + process._handle.Close() + row["cleanup_verified"] = drain(job.active, 5) + elif job: + # Assignment failed while stdin was still a barrier; + # the bootstrap has not been allowed to spawn children. + if process.poll() is None: + process.kill() + process.wait(timeout=5) + process._handle.Close() + row["cleanup_verified"] = True + else: + if group_active(process.pid): + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + if not drain(lambda: group_active(process.pid), 1): + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + row["cleanup_verified"] = drain(lambda: group_active(process.pid), 5) + else: + row["cleanup_verified"] = True + except Exception as error: + row["cleanup_error"] = type(error).__name__ + problem = problem or error + finally: + if job: + job.close() + row["elapsed_seconds"] = round(time.monotonic() - started, 6) + (evidence / (label + ".command.json")).write_text(json.dumps(row, indent=2) + "\n") + if problem: + raise problem + require(row["cleanup_verified"], label + " process cleanup could not be verified") + require(row.get("exit_code") == 0, label + " failed; see preserved logs") + return row + + +def verify_unit_output(text: str): + summaries = re.findall(r"^(\d+) checks, all passed\.$", text, re.MULTILINE) + require(summaries == [str(UNIT_CHECKS)], "full-suite total does not match the exact inventory") + require(len(re.findall(r"^ ok .+$", text, re.MULTILINE)) == UNIT_CHECKS, + "full-suite per-check results do not match the exact inventory") + require(not re.search(r"^\s*(?:FAIL|SKIP(?:PED)?|XFAIL|NOT RUN)\b", text, re.MULTILINE | re.IGNORECASE), + "full suite contains a failure or skipped check") + + +def verify_wire_oracle_output(text: str): + totals = re.findall(r"^Ran (\d+) tests in \d+(?:\.\d+)?s$", text, re.MULTILINE) + require(totals == [str(WIRE_ORACLE_CHECKS)], "wire-oracle total does not match the exact inventory") + passed = re.findall(r"^(test_[^\r\n]+) \.\.\. ok$", text, re.MULTILINE) + require(len(passed) == WIRE_ORACLE_CHECKS and len(set(passed)) == WIRE_ORACLE_CHECKS, + "wire-oracle per-test results include a failure, duplicate or skip") + require(text.strip().splitlines()[-1] == "OK", "wire-oracle summary contains a failure or skip") + + +def run_wire_oracle(*, env: dict[str, str], evidence: Path) -> dict: + result = run([sys.executable, "-B", str(REPO / "tests/test_http_wire_oracle.py"), "-v"], + cwd=REPO, env=env, evidence=evidence, label="wire-oracle", timeout=30) + verify_wire_oracle_output((evidence / "wire-oracle.stderr").read_text()) + result["checks"] = WIRE_ORACLE_CHECKS + return result + + +def verify_wire_report(report: dict, *, schema: str, expected: int, roots: list[Path]): + require(report.get("schema") == schema and report.get("status") == "PASS", "wire report schema or outcome differs") + require(report.get("source_unchanged") is True and report.get("tools_unchanged") is True, + "wire source or tool integrity seal failed") + results = report.get("roots", []) + require(len(results) == len(roots), "wire standard-library root inventory differs") + for row, root in zip(results, roots): + require(Path(row.get("nilstd", "")).resolve() == root, "wire report used an unexpected standard-library root") + checks = row.get("checks", []) + require(len(checks) == expected and len({case.get("name") for case in checks}) == expected, + "wire check count or unique-name inventory differs") + require(all(case.get("status") == "PASS" for case in checks), "wire checks include a failure or skip") + require(row.get("compile_exit") == 0 and row.get("server_exit") == 0, + "wire consumer did not compile and exit cleanly") + require(row.get("cleanup_verified") is True and row.get("nilstd_unchanged") is True, + "wire cleanup or standard-library integrity seal failed") + + +def copy_harness_evidence(work: Path, evidence: Path, label: str, roots: int): + # Explicit file inventory avoids uploading bytecode, binary fixtures, links, + # junction targets, source checkouts, or an accidentally staged SDK. + selected = [Path("report.json")] + for index in range(roots): + selected += [Path(str(index)) / name for name in + ("compile.stdout", "compile.stderr", "server.stdout", "server.stderr")] + for relative in selected: + source = work / label / relative + if source.is_file() and not source.is_symlink(): + target = evidence / label / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler", type=Path, required=True) + parser.add_argument("--interpreter", type=Path, required=True) + parser.add_argument("--host-contract", type=Path, required=True) + parser.add_argument("--nilstd", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--expected-system", choices=tuple(HOST_CHECKS)) + parser.add_argument("--expected-machine", choices=("x86_64", "arm64")) + parser.add_argument("--framework-root", type=Path) + parser.add_argument("--framework-revision") + args = parser.parse_args() + if bool(args.framework_root) != bool(args.framework_revision): + parser.error("--framework-root and --framework-revision must be supplied together") + args.output = args.output.resolve() + if args.output == REPO or REPO in args.output.parents: + parser.error("--output must be outside this source checkout") + args.output.mkdir(parents=True, exist_ok=False) + evidence = args.output / "evidence" + work = args.output / "work" + evidence.mkdir() + work.mkdir() + report = {"schema": "nilweb.platform.v1", "status": "FAIL", "system": platform.system(), + "machine": normalized_machine(), "python": sys.version, "roots": [], + "expected": {"unit_checks_per_root": UNIT_CHECKS, "wire_checks_per_root": WIRE_CHECKS, + "static_checks_per_root": STATIC_CHECKS, "wire_oracle_checks": WIRE_ORACLE_CHECKS}, + "scope": "host file contract and interpreter web consumers; no native networking or installed SDK qualification"} + pins, tools, roots_before = {}, {}, {} + roots = [] + try: + require(platform.system() in HOST_CHECKS, "unsupported host system") + require(not args.expected_system or platform.system() == args.expected_system, "unexpected runner operating system") + require(not args.expected_machine or normalized_machine() == args.expected_machine, "unexpected runner architecture") + for name in ("compiler", "interpreter", "host_contract"): + setattr(args, name, getattr(args, name).resolve(strict=True)) + roots = [path.resolve(strict=True) for path in args.nilstd] + require(len(set(roots)) == len(roots), "duplicate standard-library roots are not independent coverage") + pins = source_pin() + tools = {str(path): digest(path) for path in (args.compiler, args.interpreter, args.host_contract)} + roots_before = {str(root): tree_pin(root) for root in roots} + report.update(source_sha256=pins, tools_sha256=tools, nilstd_before=roots_before) + if args.framework_root: + checkout = args.framework_root.resolve(strict=True) + expected = args.framework_revision + require(bool(re.fullmatch(r"[0-9a-f]{40}", expected)), "framework revision must be a full commit SHA") + actual = subprocess.check_output(["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True, timeout=15).strip() + dirty = subprocess.check_output(["git", "-C", str(checkout), "status", "--porcelain", "--untracked-files=no"], text=True, timeout=15) + require(actual == expected and not dirty, "framework checkout is not the pinned clean revision") + report["framework_revision"] = actual + env = {key: value for key, value in os.environ.items() if not key.startswith(("NIL_", "NY_", "NILWEB_"))} + env.update(PYTHONDONTWRITEBYTECODE="1", PYTHONUTF8="1", TMPDIR=str(work), TMP=str(work), TEMP=str(work)) + report["host"] = run([str(args.host_contract)], cwd=work, env=env, evidence=evidence, + label="host", timeout=60) + expected_host = HOST_CHECKS[platform.system()] + require((evidence / "host.stdout").read_text().strip() == f"{expected_host} rooted snapshot checks passed", + "host contract count does not match this platform inventory") + report["host"]["checks"] = expected_host + (REPO / "build").mkdir(exist_ok=True) + for index, root in enumerate(roots): + bytecode = work / f"unit-{index}.nil" + row = {"nilstd": str(root)} + report["roots"].append(row) + row["compile"] = run([str(args.compiler), "--no-bundle", "-I", str(root), "-I", str(REPO / "src"), + "-I", str(REPO / "tests"), "--output", str(bytecode), str(REPO / "tests/run-tests.matrix")], + cwd=REPO, env=env, evidence=evidence, label=f"unit-{index}-compile", timeout=90) + row["run"] = run([str(args.interpreter), str(bytecode)], cwd=REPO, env=env, evidence=evidence, + label=f"unit-{index}", timeout=90) + verify_unit_output((evidence / f"unit-{index}.stdout").read_text()) + require(not (REPO / "build/static-http-contract-fixture.bin").exists(), "full-suite owned fixture was not cleaned up") + row.update(checks=UNIT_CHECKS, bytecode_sha256=digest(bytecode), cleanup_verified=True) + report["wire_oracle"] = run_wire_oracle(env=env, evidence=evidence) + common = ["--compiler", str(args.compiler), "--interpreter", str(args.interpreter)] + for root in roots: + common += ["--nilstd", str(root)] + for label, script, schema, count, extra in ( + ("wire", REPO / "scripts/test-http-wire.py", "nilweb.http-wire.v1", WIRE_CHECKS, ["--real-deadline"]), + ("static", REPO / "tests/test_static_snapshot.py", "nilweb.static-snapshot-wire.v1", STATIC_CHECKS, [])): + report[label] = run([sys.executable, "-B", str(script), *common, "--output", str(work / label), "--parent-process-group", *extra], + cwd=REPO, env=env, evidence=evidence, label=label + "-runner", timeout=180 * len(roots)) + child_report = json.loads((work / label / "report.json").read_text()) + verify_wire_report(child_report, schema=schema, expected=count, roots=roots) + report[label].update(checks=count * len(roots), report_sha256=digest(work / label / "report.json")) + report["status"] = "PASS" + except Exception as error: + report["error"] = str(error) + finally: + for label in ("wire", "static"): + copy_harness_evidence(work, evidence, label, len(roots)) + checks = { + "source_unchanged": lambda: bool(pins) and pins == source_pin(), + "tools_unchanged": lambda: bool(tools) and all(Path(path).is_file() and digest(Path(path)) == held for path, held in tools.items()), + "nilstd_unchanged": lambda: bool(roots_before) and all(tree_pin(Path(path)) == held for path, held in roots_before.items()), + } + for key, verify in checks.items(): + try: + report[key] = verify() + except Exception as error: + report[key] = False + report.setdefault("integrity_errors", {})[key] = type(error).__name__ + if args.framework_root: + try: + checkout = args.framework_root.resolve() + current = subprocess.check_output(["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True, timeout=15).strip() + dirty = subprocess.check_output(["git", "-C", str(checkout), "status", "--porcelain", "--untracked-files=no"], text=True, timeout=15) + report["framework_unchanged"] = current == args.framework_revision and not dirty + except Exception as error: + report["framework_unchanged"] = False + report.setdefault("integrity_errors", {})["framework_unchanged"] = type(error).__name__ + if not report["framework_unchanged"]: + report["status"] = "FAIL" + if not all(report[key] for key in checks): + report["status"] = "FAIL" + (evidence / "report.json").write_text(json.dumps(report, indent=2) + "\n") + files = {path.relative_to(evidence).as_posix(): {"sha256": digest(path), "bytes": path.stat().st_size} + for path in sorted(evidence.rglob("*")) if path.is_file()} + (evidence / "manifest.json").write_text(json.dumps({"schema": "nilweb.platform-evidence.v1", "files": files}, indent=2) + "\n") + print(json.dumps({"status": report["status"], "report": str(evidence / "report.json"), "error": report.get("error")})) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test.sh b/scripts/test.sh index f9beff5..4a382c0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -17,6 +17,7 @@ else fi OUT="${ROOT}/build" mkdir -p "$OUT" +cd "$ROOT" "$SDK/nilcompiler" --no-bundle \ -I "$STDLIB" -I "$ROOT/src" -I "$ROOT/tests" \ --output "$OUT/tests.nil" "$ROOT/tests/run-tests.matrix" > /dev/null diff --git a/src/nilweb/crypto/sha256.matrix b/src/nilweb/crypto/sha256.matrix new file mode 100644 index 0000000..5e4090d --- /dev/null +++ b/src/nilweb/crypto/sha256.matrix @@ -0,0 +1,158 @@ +!!! SHA-256 over the exact bytes of a text value, including NUL and invalid +!!! UTF-8. FIPS 180-4, sections 4.1.2, 4.2.2, 5.1.1, 5.3.3 and 6.2. +!!! +!!! Matrix numbers are signed 64-bit values. Every SHA word is kept in +!!! 0..0xffffffff, so right shifts are logical and additions are explicitly +!!! reduced modulo 2^32. No native hash provider is involved. +!!! +!!! The input is never padded as a whole: only its final partial block is +!!! copied into a tail of at most 128 bytes. The schedule has 64 words. + +book nilweb.crypto.sha256 + +read nil.std.text + + +public sha256-hex receives data as text returns text begin paragraph + value mask as number is 0xffffffff + value constants as array of number is [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + ] + + variable h0 as number is 0x6a09e667 + variable h1 as number is 0xbb67ae85 + variable h2 as number is 0x3c6ef372 + variable h3 as number is 0xa54ff53a + variable h4 as number is 0x510e527f + variable h5 as number is 0x9b05688c + variable h6 as number is 0x1f83d9ab + variable h7 as number is 0x5be0cd19 + + value width is nil.std.text::length with s being data + value whole-blocks is width over 64 + variable tail is nil.std.text::substring with s being data, + start being whole-blocks times 64, length being width remainder 64 + set tail to tail plus (nil.std.text::from_byte with code being 128) + repeat while (nil.std.text::length with s being tail) remainder 64 does not equal 56 + set tail to tail plus (nil.std.text::from_byte with code being 0) + end repeat + value bit-length as number is width times 8 + variable suffix-byte as number is 7 + repeat while suffix-byte is at least 0 + set tail to tail plus (nil.std.text::from_byte with code being + (bit-length shifted right by (suffix-byte times 8)) bitwise and 255) + set suffix-byte to suffix-byte minus 1 + end repeat + + variable schedule as array of number is [] + repeat while the length of schedule is less than 64 + append 0 to schedule + end repeat + value block-count is whole-blocks plus (nil.std.text::length with s being tail) over 64 + variable block-index as number is 0 + repeat while block-index is less than block-count + variable block as text is "" + when block-index is less than whole-blocks then + set block to nil.std.text::substring with s being data, + start being block-index times 64, length being 64 + otherwise + set block to nil.std.text::substring with s being tail, + start being (block-index minus whole-blocks) times 64, length being 64 + end when + + variable word-index as number is 0 + repeat while word-index is less than 16 + value offset is word-index times 4 + value b0 is nil.std.text::byte_at with s being block, index being offset + value b1 is nil.std.text::byte_at with s being block, index being offset plus 1 + value b2 is nil.std.text::byte_at with s being block, index being offset plus 2 + value b3 is nil.std.text::byte_at with s being block, index being offset plus 3 + set schedule at word-index to (b0 shifted left by 24) + bitwise or (b1 shifted left by 16) bitwise or (b2 shifted left by 8) bitwise or b3 + set word-index to word-index plus 1 + end repeat + repeat while word-index is less than 64 + value x is schedule at (word-index minus 15) + value y is schedule at (word-index minus 2) + value small0 is (((x shifted right by 7) bitwise or (x shifted left by 25)) + bitwise xor ((x shifted right by 18) bitwise or (x shifted left by 14)) + bitwise xor (x shifted right by 3)) bitwise and mask + value small1 is (((y shifted right by 17) bitwise or (y shifted left by 15)) + bitwise xor ((y shifted right by 19) bitwise or (y shifted left by 13)) + bitwise xor (y shifted right by 10)) bitwise and mask + set schedule at word-index to ((schedule at (word-index minus 16)) plus small0 + plus (schedule at (word-index minus 7)) plus small1) bitwise and mask + set word-index to word-index plus 1 + end repeat + + variable a as number is h0 + variable b as number is h1 + variable c as number is h2 + variable d as number is h3 + variable e as number is h4 + variable f as number is h5 + variable g as number is h6 + variable h as number is h7 + variable round-index as number is 0 + repeat while round-index is less than 64 + value large1 is (((e shifted right by 6) bitwise or (e shifted left by 26)) + bitwise xor ((e shifted right by 11) bitwise or (e shifted left by 21)) + bitwise xor ((e shifted right by 25) bitwise or (e shifted left by 7))) bitwise and mask + value choose is (e bitwise and f) bitwise xor ((bitwise not e) bitwise and g) + value first is (h plus large1 plus choose plus (constants at round-index) + plus (schedule at round-index)) bitwise and mask + value large0 is (((a shifted right by 2) bitwise or (a shifted left by 30)) + bitwise xor ((a shifted right by 13) bitwise or (a shifted left by 19)) + bitwise xor ((a shifted right by 22) bitwise or (a shifted left by 10))) bitwise and mask + value majority is (a bitwise and b) bitwise xor (a bitwise and c) bitwise xor (b bitwise and c) + value second is (large0 plus majority) bitwise and mask + set h to g + set g to f + set f to e + set e to (d plus first) bitwise and mask + set d to c + set c to b + set b to a + set a to (first plus second) bitwise and mask + set round-index to round-index plus 1 + end repeat + + set h0 to (h0 plus a) bitwise and mask + set h1 to (h1 plus b) bitwise and mask + set h2 to (h2 plus c) bitwise and mask + set h3 to (h3 plus d) bitwise and mask + set h4 to (h4 plus e) bitwise and mask + set h5 to (h5 plus f) bitwise and mask + set h6 to (h6 plus g) bitwise and mask + set h7 to (h7 plus h) bitwise and mask + set block-index to block-index plus 1 + end repeat + + value digest as array of number is [h0, h1, h2, h3, h4, h5, h6, h7] + value digits as text is "0123456789abcdef" + variable result as text is "" + repeat for each word in digest + variable shift as number is 28 + repeat while shift is at least 0 + value nibble is (word shifted right by shift) bitwise and 15 + set result to result plus (nil.std.text::substring with s being digits, start being nibble, length being 1) + set shift to shift minus 4 + end repeat + end repeat + give result +end paragraph diff --git a/src/nilweb/http/framing.matrix b/src/nilweb/http/framing.matrix new file mode 100644 index 0000000..3a5e851 --- /dev/null +++ b/src/nilweb/http/framing.matrix @@ -0,0 +1,393 @@ +!!! Bounded HTTP/1 wire framing (RFC 9112 sections 5 through 7). +!!! +!!! A frame reports the exact number of wire octets consumed. Any following +!!! request belongs to the connection buffer, never to this request's body. +!!! Chunked content is decoded and normalized to Content-Length for handlers; +!!! trailers are validated and discarded, never merged into trusted headers. +!!! Status zero means either complete or waiting. Every nonzero status must +!!! be answered with Connection: close, without dispatching an application. +!!! +!!! Expect negotiation is deliberately unsupported: a complete HTTP/1.1 +!!! header section containing Expect receives 417 immediately, without waiting +!!! for content or sending an interim 100. A client can retry without Expect. +!!! RFC 9110 section 10.1.1 requires HTTP/1.0's 100-continue expectation to be +!!! ignored; other HTTP/1.0 expectations receive 417 as well. + +book nilweb.http.framing + +read nil.std.core +read nil.std.text +read everything from nilweb.text.scan +read everything from nilweb.http.headers + +public wire-frame begin record + field complete as boolean + field status as number + field consumed as number + field message as text + field keeping as boolean +end record + +counted-size begin record + field valid as boolean + field oversized as boolean + field amount as number +end record + +frame-state receives status as number returns wire-frame begin paragraph + give build wire-frame with complete being false, status being status, + consumed being 0, message being "", keeping being false +end paragraph + +trim-ows receives input as text returns text begin paragraph + variable first as number is 0 + variable after as number is nil.std.text::length with s being input + repeat while first is less than after + value code is nil.std.text::byte_at with s being input, index being first + when not (code equals 32 or code equals 9) then stop repeating end when + set first to first plus 1 + end repeat + repeat while after is greater than first + value code is nil.std.text::byte_at with s being input, index being after minus 1 + when not (code equals 32 or code equals 9) then stop repeating end when + set after to after minus 1 + end repeat + give nil.std.text::substring with s being input, start being first, length being after minus first +end paragraph + +is-token-byte receives code as number returns boolean begin paragraph + when code is at least 48 and code is at most 57 then give true end when + when code is at least 65 and code is at most 90 then give true end when + when code is at least 97 and code is at most 122 then give true end when + give code equals 33 or code equals 35 or code equals 36 or code equals 37 + or code equals 38 or code equals 39 or code equals 42 or code equals 43 + or code equals 45 or code equals 46 or code equals 94 or code equals 95 + or code equals 96 or code equals 124 or code equals 126 +end paragraph + +is-token receives input as text returns boolean begin paragraph + value width is nil.std.text::length with s being input + when width equals 0 then give false end when + variable index as number is 0 + repeat while index is less than width + when not (is-token-byte with code being + (nil.std.text::byte_at with s being input, index being index)) then give false end when + set index to index plus 1 + end repeat + give true +end paragraph + +valid-field-line receives line as text returns boolean begin paragraph + value colon is nil.std.text::index_of with s being line, needle being ":" + when colon is less than 1 then give false end when + value name is nil.std.text::substring with s being line, start being 0, length being colon + when not (is-token with input being name) then give false end when + variable index as number is colon plus 1 + repeat while index is less than (nil.std.text::length with s being line) + value code is nil.std.text::byte_at with s being line, index being index + when (code is less than 32 and not (code equals 9)) or code equals 127 then give false end when + set index to index plus 1 + end repeat + give true +end paragraph + +bounded-size receives digits as text, radix as number, maximum as number returns counted-size begin paragraph + variable total as number is 0 + variable oversized as boolean is false + value width is nil.std.text::length with s being digits + when width equals 0 then + give build counted-size with valid being false, oversized being false, amount being 0 + end when + variable index as number is 0 + repeat while index is less than width + value code is nil.std.text::byte_at with s being digits, index being index + variable digit as number is 0 minus 1 + when code is at least 48 and code is at most 57 then set digit to code minus 48 end when + when radix equals 16 and code is at least 65 and code is at most 70 then set digit to code minus 55 end when + when radix equals 16 and code is at least 97 and code is at most 102 then set digit to code minus 87 end when + when digit is less than 0 then + give build counted-size with valid being false, oversized being false, amount being 0 + end when + when not oversized then + when total is greater than (maximum over radix) + or (total equals (maximum over radix) and digit is greater than (maximum remainder radix)) then + set oversized to true + otherwise + set total to (total times radix) plus digit + end when + end when + set index to index plus 1 + end repeat + give build counted-size with valid being true, oversized being oversized, amount being total +end paragraph + +!!! Validate ignored chunk extensions, including quoted values and escapes. +valid-extensions receives input as text returns boolean begin paragraph + value width is nil.std.text::length with s being input + variable index as number is 0 + repeat while index is less than width + repeat while index is less than width + value code is nil.std.text::byte_at with s being input, index being index + when not (code equals 32 or code equals 9) then stop repeating end when + set index to index plus 1 + end repeat + when index equals width then give false end when + when not ((nil.std.text::byte_at with s being input, index being index) equals 59) then give false end when + set index to index plus 1 + repeat while index is less than width + value code is nil.std.text::byte_at with s being input, index being index + when not (code equals 32 or code equals 9) then stop repeating end when + set index to index plus 1 + end repeat + value name-start is index + repeat while index is less than width + when not (is-token-byte with code being (nil.std.text::byte_at with s being input, index being index)) then stop repeating end when + set index to index plus 1 + end repeat + when index equals name-start then give false end when + repeat while index is less than width + value code is nil.std.text::byte_at with s being input, index being index + when not (code equals 32 or code equals 9) then stop repeating end when + set index to index plus 1 + end repeat + when index is less than width and (nil.std.text::byte_at with s being input, index being index) equals 61 then + set index to index plus 1 + repeat while index is less than width + value code is nil.std.text::byte_at with s being input, index being index + when not (code equals 32 or code equals 9) then stop repeating end when + set index to index plus 1 + end repeat + when index equals width then give false end when + when (nil.std.text::byte_at with s being input, index being index) equals 34 then + set index to index plus 1 + variable closed as boolean is false + repeat while index is less than width + value code is nil.std.text::byte_at with s being input, index being index + set index to index plus 1 + when code equals 34 then + set closed to true + stop repeating + end when + when code equals 92 then + when index equals width then give false end when + value escaped is nil.std.text::byte_at with s being input, index being index + when (escaped is less than 32 and not (escaped equals 9)) or escaped equals 127 then give false end when + set index to index plus 1 + end when + when not (code equals 92) and ((code is less than 32 and not (code equals 9)) or code equals 127) then give false end when + end repeat + when not closed then give false end when + otherwise + value value-start is index + repeat while index is less than width + when not (is-token-byte with code being (nil.std.text::byte_at with s being input, index being index)) then stop repeating end when + set index to index plus 1 + end repeat + when index equals value-start then give false end when + end when + end when + end repeat + give true +end paragraph + +!!! A fragmented line may end in CR, but bare LF or CR followed by another +!!! byte is already invalid. Stop at its first complete CRLF, before payload. +valid-line-prefix receives input as text returns boolean begin paragraph + value width is nil.std.text::length with s being input + variable index as number is 0 + repeat while index is less than width + value code is nil.std.text::byte_at with s being input, index being index + when code equals 10 then give false end when + when code equals 13 then + when index plus 1 equals width then give true end when + give (nil.std.text::byte_at with s being input, index being index plus 1) equals 10 + end when + set index to index plus 1 + end repeat + give true +end paragraph + +chunk-frame receives message as text, head as text, after as number, + maximum as number, keeping as boolean returns wire-frame begin paragraph + value ending is crlf with nothing + value width is nil.std.text::length with s being message + variable offset as number is after + variable body as text is "" + repeat forever + value remaining is nil.std.text::substring with s being message, start being offset, length being width minus offset + when not (valid-line-prefix with input being remaining) then give frame-state with status being 400 end when + value end-line is nil.std.text::index_of with s being remaining, needle being ending + when end-line is less than 0 then + when width is at least maximum or (width minus offset) is greater than 8192 then give frame-state with status being 413 end when + give frame-state with status being 0 + end when + when end-line is greater than 8192 then give frame-state with status being 413 end when + value line is nil.std.text::substring with s being remaining, start being 0, length being end-line + value semicolon is nil.std.text::index_of with s being line, needle being ";" + variable numeral as text is line + when semicolon is at least 0 then + value before is nil.std.text::substring with s being line, start being 0, length being semicolon + set numeral to trim-ows with input being before + value extensions is nil.std.text::substring with s being line, start being (nil.std.text::length with s being numeral), + length being end-line minus (nil.std.text::length with s being numeral) + when not (valid-extensions with input being extensions) then give frame-state with status being 400 end when + end when + value counted is bounded-size with digits being numeral, radix being 16, maximum being maximum + when not counted.valid then give frame-state with status being 400 end when + when counted.oversized then give frame-state with status being 413 end when + set offset to offset plus end-line plus 2 + when offset is greater than maximum then give frame-state with status being 413 end when + when counted.amount equals 0 then + repeat forever + value trailer-rest is nil.std.text::substring with s being message, start being offset, length being width minus offset + when not (valid-line-prefix with input being trailer-rest) then give frame-state with status being 400 end when + value trailer-end is nil.std.text::index_of with s being trailer-rest, needle being ending + when trailer-end is less than 0 then + when width is at least maximum then give frame-state with status being 413 end when + give frame-state with status being 0 + end when + value trailer is nil.std.text::substring with s being trailer-rest, start being 0, length being trailer-end + set offset to offset plus trailer-end plus 2 + when offset is greater than maximum then give frame-state with status being 413 end when + when trailer-end equals 0 then + value normalized is head plus "Content-Length: " plus ((nil.std.text::length with s being body) becomes text) + plus ending plus ending plus body + give build wire-frame with complete being true, status being 0, consumed being offset, + message being normalized, keeping being keeping + end when + when not (valid-field-line with line being trailer) then give frame-state with status being 400 end when + end repeat + end when + when counted.amount is greater than maximum minus offset minus 2 then give frame-state with status being 413 end when + value chunk-after is offset plus counted.amount + when width is less than chunk-after plus 2 then give frame-state with status being 0 end when + when not ((nil.std.text::substring with s being message, start being chunk-after, length being 2) equals ending) then + give frame-state with status being 400 + end when + set body to body plus (nil.std.text::substring with s being message, start being offset, length being counted.amount) + set offset to chunk-after plus 2 + end repeat + give frame-state with status being 400 +end paragraph + +public inspect-frame receives message as text, maximum as number returns wire-frame begin paragraph + value ending is crlf with nothing + value width is nil.std.text::length with s being message + value cut is nil.std.text::index_of with s being message, needle being ending plus ending + when cut is less than 0 then + !!! Reject bare LF and stray CR as soon as they are unambiguous; a + !!! final CR may still receive its LF in the next socket fragment. + variable index as number is 0 + repeat while index is less than width + value code is nil.std.text::byte_at with s being message, index being index + when code equals 10 then + when index equals 0 then give frame-state with status being 400 end when + when not ((nil.std.text::byte_at with s being message, index being index minus 1) equals 13) then give frame-state with status being 400 end when + end when + when code equals 13 and index plus 1 is less than width then + when not ((nil.std.text::byte_at with s being message, index being index plus 1) equals 10) then give frame-state with status being 400 end when + end when + set index to index plus 1 + end repeat + when width is at least maximum then give frame-state with status being 413 end when + give frame-state with status being 0 + end when + value after is cut plus 4 + when after is greater than maximum then give frame-state with status being 413 end when + value raw-head is nil.std.text::substring with s being message, start being 0, length being cut + value lines is split-on with input being raw-head, separator being ending + value first-line is lines at 0 + value words is split-on with input being first-line, separator being " " + when not ((the length of words) equals 3) then give frame-state with status being 400 end when + when not (is-token with input being words at 0) or (words at 1) equals "" then give frame-state with status being 400 end when + variable byte-index as number is 0 + repeat while byte-index is less than (nil.std.text::length with s being first-line) + value code is nil.std.text::byte_at with s being first-line, index being byte-index + when code is less than 32 or code equals 127 then give frame-state with status being 400 end when + set byte-index to byte-index plus 1 + end repeat + value version is words at 2 + when not (version equals "HTTP/1.1" or version equals "HTTP/1.0") then give frame-state with status being 505 end when + variable normalized as text is first-line plus ending + variable declared as number is 0 minus 1 + variable has-length as boolean is false + variable too-large as boolean is false + variable codings as array of text is [] + variable has-coding as boolean is false + variable has-expectation as boolean is false + variable unsupported-expectation as boolean is false + variable wants-close as boolean is false + variable wants-keep as boolean is false + variable line-index as number is 1 + repeat while line-index is less than the length of lines + value line is lines at line-index + when not (valid-field-line with line being line) then give frame-state with status being 400 end when + value name is nil.std.text::to_lower with s being (before-first with input being line, separator being ":") + value content is trim-ows with input being (after-first with input being line, separator being ":") + when name equals "content-length" then + set has-length to true + value counts is split-on with input being content, separator being "," + repeat for each count-text in counts + value count is bounded-size with digits being (trim-ows with input being count-text), radix being 10, maximum being maximum + when not count.valid then give frame-state with status being 400 end when + when count.oversized then set too-large to true end when + when declared is at least 0 and not (declared equals count.amount) then give frame-state with status being 400 end when + set declared to count.amount + end repeat + end when + when name equals "transfer-encoding" then + set has-coding to true + value pieces is split-on with input being content, separator being "," + repeat for each piece in pieces + value coding is nil.std.text::to_lower with s being (trim-ows with input being piece) + when not (is-token with input being coding) then give frame-state with status being 400 end when + append coding to codings + end repeat + end when + when not (name equals "content-length" or name equals "transfer-encoding" or name equals "trailer") then + set normalized to normalized plus line plus ending + end when + when name equals "expect" then + set has-expectation to true + value expectations is split-on with input being content, separator being "," + repeat for each expectation in expectations + value wanted is nil.std.text::to_lower with s being (trim-ows with input being expectation) + when not (wanted equals "100-continue") then set unsupported-expectation to true end when + end repeat + end when + when name equals "connection" then + value tokens is split-on with input being content, separator being "," + repeat for each token in tokens + value directive is nil.std.text::to_lower with s being (trim-ows with input being token) + when directive equals "close" then set wants-close to true end when + when directive equals "keep-alive" then set wants-keep to true end when + end repeat + end when + set line-index to line-index plus 1 + end repeat + when has-length and has-coding then give frame-state with status being 400 end when + when has-expectation and (version equals "HTTP/1.1" or unsupported-expectation) then + give frame-state with status being 417 + end when + value keeping is not wants-close and (version equals "HTTP/1.1" or wants-keep) + when has-coding then + when version equals "HTTP/1.0" then give frame-state with status being 400 end when + when not ((codings at ((the length of codings) minus 1)) equals "chunked") then give frame-state with status being 400 end when + variable chunks as number is 0 + repeat for each coding in codings + when coding equals "chunked" then set chunks to chunks plus 1 end when + end repeat + when chunks is greater than 1 then give frame-state with status being 400 end when + when (the length of codings) is greater than 1 then give frame-state with status being 501 end when + give chunk-frame with message being message, head being normalized, after being after, maximum being maximum, keeping being keeping + end when + when too-large then give frame-state with status being 413 end when + when not has-length then set declared to 0 end when + when declared is greater than maximum minus after then give frame-state with status being 413 end when + value consumed is after plus declared + when width is less than consumed then give frame-state with status being 0 end when + when has-length then set normalized to normalized plus "Content-Length: " plus (declared becomes text) plus ending end when + set normalized to normalized plus ending plus (nil.std.text::substring with s being message, start being after, length being declared) + give build wire-frame with complete being true, status being 0, consumed being consumed, + message being normalized, keeping being keeping +end paragraph diff --git a/src/nilweb/http/status.matrix b/src/nilweb/http/status.matrix index e7af611..d314cb0 100644 --- a/src/nilweb/http/status.matrix +++ b/src/nilweb/http/status.matrix @@ -29,6 +29,7 @@ public constant status-method-not-allowed as number is 405 public constant status-not-acceptable as number is 406 public constant status-request-timeout as number is 408 public constant status-conflict as number is 409 +public constant status-precondition-failed as number is 412 public constant status-payload-too-large as number is 413 public constant status-uri-too-long as number is 414 public constant status-unsupported-media-type as number is 415 @@ -70,6 +71,7 @@ public reason-phrase receives code as number returns text begin paragraph case 406 then stop switching with "Not Acceptable" case 408 then stop switching with "Request Timeout" case 409 then stop switching with "Conflict" + case 412 then stop switching with "Precondition Failed" case 413 then stop switching with "Payload Too Large" case 414 then stop switching with "URI Too Long" case 415 then stop switching with "Unsupported Media Type" diff --git a/src/nilweb/server/engine.matrix b/src/nilweb/server/engine.matrix index eeb77c3..bf5be05 100644 --- a/src/nilweb/server/engine.matrix +++ b/src/nilweb/server/engine.matrix @@ -35,6 +35,7 @@ read nil.std.text read nil.std.text.string_utils read everything from nilweb.text.scan read everything from nilweb.http.headers +read everything from nilweb.http.framing !!! How long a connection may sit idle before it is closed. @@ -52,8 +53,9 @@ constant between-requests-limit-ms as number is 30000 !!! slot indefinitely -- which is what slowloris is. constant request-deadline-ms as number is 20000 -!!! The most a request may be, header and body together. The cap stops the -!!! reading rather than rejecting afterwards. +!!! The most one wire request may be, header and body together. Framing +!!! rejects an oversized declaration before waiting for its body. One socket +!!! read of bounded pipeline surplus is retained separately. constant request-limit as number is 1048576 !!! One read per pass. Large enough that an ordinary request arrives whole. @@ -79,9 +81,12 @@ public connection begin record field live as boolean !!! Whether to keep the connection after the response in flight. field keeping as boolean - !!! When the current request started arriving, and when anything last did. + !!! Oldest buffered input time (conservative across a pipeline), and the + !!! most recent input/output progress. New fragments never extend a deadline. field started-at as number field touched-at as number + field input-closed as boolean + field serving as boolean !!! Captured at accept, because by the time a response is logged the !!! socket may be closed and getpeername would fail. For the log only: !!! an address is not an identity, and behind a proxy it is the proxy's. @@ -202,6 +207,8 @@ public pump receives server as engine, timeout as number returns pumped begin pa give nothing-found with nothing end when + evaluate sweep-idle with server being server + !!! Something framed on an earlier pass and not yet handed over. value already is first-framed with server being server when already.found then @@ -233,6 +240,7 @@ public pump receives server as engine, timeout as number returns pumped begin pa end consider end repeat + evaluate sweep-idle with server being server give first-framed with server being server end paragraph @@ -261,11 +269,9 @@ act-on receives server as engine, ready as number returns nothing begin paragrap when nil.std.net::is_readable with packed being ready then evaluate pull-incoming with server being server, slot being slot end when - when nil.std.net::is_finished with packed being ready then - value held is server.connections at slot - when held.live and (nil.std.text::length with s being held.outgoing) equals held.sent then - evaluate drop with server being server, slot being slot - end when + when (nil.std.net::is_finished with packed being ready) + and not (nil.std.net::is_readable with packed being ready) then + evaluate finish-input with server being server, slot being slot end when give nothing end paragraph @@ -325,7 +331,7 @@ admit receives server as engine, client as number returns nothing begin paragrap value fresh is build connection with handle being client, gathered being "", outgoing being "", sent being 0, served being 0, live being true, keeping being true, started-at being now-at, touched-at being now-at, - peer being (peer-of with handle being client) + input-closed being false, serving being false, peer being (peer-of with handle being client) when free-slot is at least 0 then set server.connections at free-slot to fresh otherwise @@ -342,6 +348,10 @@ pull-incoming receives server as engine, slot as number returns nothing begin pa give nothing end when + !!! A readiness batch can contain both flags. Do not receive more input + !!! after a writable event left a response partially sent. + when (nil.std.text::length with s being held.outgoing) is greater than held.sent then give nothing end when + value counted is nil.std.net::receive with handle being held.handle, maximum_bytes being chunk-size consider counted when Result.Error holding why then begin block @@ -356,7 +366,7 @@ pull-incoming receives server as engine, slot as number returns nothing begin pa give nothing end when when amount equals 0 then - evaluate drop with server being server, slot being slot + evaluate finish-input with server being server, slot being slot give nothing end when value arrived is nil.std.net::received with nothing @@ -373,45 +383,97 @@ pull-incoming receives server as engine, slot as number returns nothing begin pa end paragraph +finish-input receives server as engine, slot as number returns nothing begin paragraph + value held is server.connections at slot + when not held.live then give nothing end when + set held.input-closed to true + evaluate nil.std.net::unwatch with handle being held.handle + when (nil.std.text::length with s being held.outgoing) is greater than held.sent then + evaluate nil.std.net::watch with handle being held.handle, flags being nil.std.net::ready_writable + end when + when held.gathered equals "" and not held.serving + and (nil.std.text::length with s being held.outgoing) equals held.sent then + evaluate drop with server being server, slot being slot + end when + give nothing +end paragraph + absorb receives server as engine, slot as number, piece as text returns nothing begin paragraph value held is server.connections at slot value whole is held.gathered plus piece value now-at is now in milliseconds - when (nil.std.text::length with s being whole) is greater than request-limit then - !!! Refused by closing rather than by answering 413: the request is - !!! still arriving, and a response written into a stream the client is - !!! still filling is a response it may never read. - evaluate drop with server being server, slot being slot + !!! One additional read may contain the next request. Bound that surplus + !!! independently; the pure framer applies request-limit to this request. + when (nil.std.text::length with s being whole) is greater than request-limit plus chunk-size then + evaluate reject-frame with server being server, slot being slot, status being 413 give nothing end when - set server.connections at slot to build connection with handle being held.handle, - gathered being whole, outgoing being held.outgoing, sent being held.sent, - served being held.served, live being true, keeping being held.keeping, - started-at being held.started-at, touched-at being now-at, peer being held.peer + when held.gathered equals "" then set held.started-at to now-at end when + set held.gathered to whole + set held.touched-at to now-at give nothing end paragraph - -!!! --- The first connection holding a complete request. - +!!! --- Consume exactly one frame. Dispatch waits for the previous response; +!!! the remaining bytes retain their arrival time and survive response writes. first-framed receives server as engine returns pumped begin paragraph variable index as number is 0 repeat while index is less than the length of server.connections value held is server.connections at index - value waiting is held.live and (nil.std.text::length with s being held.outgoing) equals held.sent + value waiting is held.live and not held.serving + and (nil.std.text::length with s being held.outgoing) equals held.sent when waiting and not (held.gathered equals "") then - when request-is-complete with message being held.gathered then + value framed is inspect-frame with message being held.gathered, maximum being request-limit + when framed.status is greater than 0 then + evaluate reject-frame with server being server, slot being index, status being framed.status + end when + when framed.complete then + value remaining is (nil.std.text::length with s being held.gathered) minus framed.consumed + set held.gathered to nil.std.text::substring with s being held.gathered, + start being framed.consumed, length being remaining + !!! Conservatively retain the first buffered byte time for + !!! surplus; only an empty-buffer absorb starts a new deadline. + set held.serving to true give build pumped with found being true, handle being held.handle, - request being held.gathered, - keep-alive being (wants-keep-alive with message being held.gathered), + request being framed.message, keep-alive being framed.keeping, peer being held.peer end when + when framed.status equals 0 and not framed.complete and held.input-closed then + evaluate drop with server being server, slot being index + end when end when set index to index plus 1 end repeat give nothing-found with nothing end paragraph +reject-frame receives server as engine, slot as number, status as number returns nothing begin paragraph + value held is server.connections at slot + when not held.live then give nothing end when + value reason is switch status + case 413 then stop switching with "Content Too Large" + case 417 then stop switching with "Expectation Failed" + case 501 then stop switching with "Not Implemented" + case 505 then stop switching with "HTTP Version Not Supported" + otherwise stop switching with "Bad Request" + end switch + value ending is crlf with nothing + value response is "HTTP/1.1 " plus (status becomes text) plus " " plus reason plus ending + plus "Content-Length: 0" plus ending plus "Connection: close" plus ending plus ending + set held.gathered to "" + set held.serving to false + set server.rejected-total to server.rejected-total plus 1 + when (nil.std.text::length with s being held.outgoing) is greater than held.sent then + !!! Preserve any preceding status/body already on the wire. Queue this + !!! rejection after it and close when both responses have drained. + set held.outgoing to held.outgoing plus response + set held.keeping to false + evaluate push-outgoing with server being server, slot being slot + give nothing + end when + evaluate reply with server being server, handle being held.handle, response being response, keep-open being false + give nothing +end paragraph !!! --- Answer a request and either keep the connection or close it. !!! @@ -427,11 +489,12 @@ public reply receives server as engine, handle as number, response as text, keep value held is server.connections at slot value now-at is now in milliseconds value more-allowed is held.served plus 1 is less than keep-alive-limit - set server.connections at slot to build connection with handle being handle, - gathered being "", outgoing being response, sent being 0, - served being held.served plus 1, live being true, - keeping being keep-open and more-allowed, - started-at being now-at, touched-at being now-at, peer being held.peer + set held.outgoing to response + set held.sent to 0 + set held.served to held.served plus 1 + set held.keeping to keep-open and more-allowed + set held.serving to false + set held.touched-at to now-at set server.served-total to server.served-total plus 1 evaluate push-outgoing with server being server, slot being slot give nothing @@ -463,10 +526,8 @@ push-outgoing receives server as engine, slot as number returns nothing begin pa value moved is when amount is less than 0 give 0 otherwise give amount end when value now-sent is held.sent plus moved value now-at is now in milliseconds - set server.connections at slot to build connection with handle being held.handle, - gathered being held.gathered, outgoing being held.outgoing, sent being now-sent, - served being held.served, live being true, keeping being held.keeping, - started-at being held.started-at, touched-at being now-at, peer being held.peer + set held.sent to now-sent + set held.touched-at to now-at when now-sent is less than width then !!! More to go. Watch for writability so the next pass carries !!! on, rather than spinning on a socket that is full. @@ -484,17 +545,17 @@ end paragraph finish-response receives server as engine, slot as number returns nothing begin paragraph value held is server.connections at slot - when not held.keeping then + when not held.keeping or (held.input-closed and held.gathered equals "") then evaluate drop with server being server, slot being slot give nothing end when - !!! Kept open for another request: clear the response, read again. - value now-at is now in milliseconds - set server.connections at slot to build connection with handle being held.handle, - gathered being "", outgoing being "", sent being 0, served being held.served, - live being true, keeping being true, started-at being now-at, touched-at being now-at, - peer being held.peer - evaluate nil.std.net::watch with handle being held.handle, flags being nil.std.net::ready_readable + set held.outgoing to "" + set held.sent to 0 + !!! Do not reset started-at: a partial pipelined request may already have + !!! waited while the preceding response was being drained. + when not held.input-closed then + evaluate nil.std.net::watch with handle being held.handle, flags being nil.std.net::ready_readable + end when give nothing end paragraph @@ -508,7 +569,8 @@ drop receives server as engine, slot as number returns nothing begin paragraph evaluate nil.std.net::close with handle being held.handle set server.connections at slot to build connection with handle being 0, gathered being "", outgoing being "", sent being 0, served being 0, live being false, keeping being false, - started-at being 0, touched-at being 0, peer being "" + started-at being 0, touched-at being 0, + input-closed being true, serving being false, peer being "" set server.open-count to server.open-count minus 1 give nothing end paragraph @@ -554,97 +616,15 @@ end paragraph !!! server ends up holding connections no client will ever use again. public wants-keep-alive receives message as text returns boolean begin paragraph - value head is head-of with message being message - value stated is nil.std.text::to_lower with s being - (field-in with head being head, name being "Connection") - value first-line is before-first with input being head, separator being newline - value old-version is nil.std.text::contains with s being first-line, needle being "HTTP/1.0" - when nil.std.text::contains with s being stated, needle being "close" then - give false - end when - when old-version then - give nil.std.text::contains with s being stated, needle being "keep-alive" - end when - give true -end paragraph - - -field-in receives head as text, name as text returns text begin paragraph - value lines is split-on with input being head, separator being newline - repeat for each line in lines - value trimmed is nil.std.text::trim with s being line - value found is nil.std.text::trim with s being (before-first with input being trimmed, separator being ":") - when same-folded with left being found, right being name then - give nil.std.text::trim with s being (after-first with input being trimmed, separator being ":") - end when - end repeat - give "" -end paragraph - - -head-of receives message as text returns text begin paragraph - value marker is (crlf with nothing) plus (crlf with nothing) - value cut is nil.std.text::index_of with s being message, needle being marker - when cut is at least 0 then - give nil.std.text::substring with s being message, start being 0, length being cut - end when - value loose is nil.std.text::index_of with s being message, needle being newline plus newline - when loose is at least 0 then - give nil.std.text::substring with s being message, start being 0, length being loose - end when - give message + value framed is inspect-frame with message being message, maximum being request-limit + give framed.complete and framed.keeping end paragraph - -!!! --- Whether enough of the message has arrived to answer it. -!!! -!!! The header section ends at the first blank line. After that a body is -!!! expected only if Content-Length says so. Chunked transfer encoding is not -!!! accepted, and a request asking for it is answered by the layer above -!!! rather than waited on here. - +!!! Compatibility query: invalid framing is never a complete request. +!!! Call inspect-frame when the rejection status or exact boundary is needed. public request-is-complete receives message as text returns boolean begin paragraph - value marker is (crlf with nothing) plus (crlf with nothing) - variable head-length as number is nil.std.text::index_of with s being message, needle being marker - variable separator as number is 4 - when head-length is less than 0 then - set head-length to nil.std.text::index_of with s being message, needle being newline plus newline - set separator to 2 - end when - when head-length is less than 0 then - give false - end when - - value head is nil.std.text::substring with s being message, start being 0, length being head-length - value declared is digits-value with digits being (field-in with head being head, name being "Content-Length") - when declared is at most 0 then - give true - end when - value body-start is head-length plus separator - give ((nil.std.text::length with s being message) minus body-start) is at least declared -end paragraph - - -!!! --- A decimal count, without borrowing a parser that raises on bad input. -!!! Anything that is not all digits is zero: a Content-Length a client made up -!!! must not be able to stop the server with a fault. - -digits-value receives digits as text returns number begin paragraph - value width is nil.std.text::length with s being digits - when width is less than 1 or width is greater than 18 then - give 0 - end when - variable total as number is 0 - variable index as number is 0 - repeat while index is less than width - value code is nil.std.text::byte_at with s being digits, index being index - when code is less than 48 or code is greater than 57 then - give 0 - end when - set total to (total times 10) plus (code minus 48) - set index to index plus 1 - end repeat - give total + value framed is inspect-frame with message being message, maximum being request-limit + give framed.complete end paragraph diff --git a/src/nilweb/static/files.matrix b/src/nilweb/static/files.matrix index f251b7d..c0201f1 100644 --- a/src/nilweb/static/files.matrix +++ b/src/nilweb/static/files.matrix @@ -4,12 +4,10 @@ !!! path is allowed to name — and the ways that decision goes wrong are well !!! known enough to enumerate. Each check below is one of them. !!! -!!! The file is read whole into text. That is fine for the stylesheets and -!!! scripts a page needs and wrong for anything large, and it is what the -!!! runtime offers: `file.read_text` reads a whole file, and the streaming -!!! handles exist but there is nothing to stream *to* until there is a socket. -!!! `maximum-bytes` keeps a large file from being read at all rather than -!!! pretending it can be served. +!!! One framework snapshot reads bytes, metadata and a content digest through +!!! directory-relative handles. No component below the configured root may be +!!! a symlink or Windows reparse point. The bounded bytes become interpreter +!!! text in one copy; the response layer is still an interpreter-only API. book nilweb.static.files @@ -21,6 +19,7 @@ read everything from nilweb.text.scan read everything from nilweb.http.status read everything from nilweb.http.headers read everything from nilweb.http.request +read everything from nilweb.http.method read everything from nilweb.time.clock read everything from nilweb.http.response @@ -38,46 +37,49 @@ constant maximum-bytes as number is 2097152 !!! things that exist — a probe must not be able to tell a file it may not see !!! from a file that is not there. -public serve-file receives root as text, path as text returns http-response begin paragraph - value relative is without-leading-slash with path being path - when not (is-safe-name with name being relative) then - give error-response with code being status-not-found, explanation being "No such file" - end when - when relative equals "" then - give error-response with code being status-not-found, explanation being "No such file" - end when +static-read begin record + field answer as http-response + field marks as validators +end record - value full is when root equals "" give relative otherwise give root plus "/" plus relative end when - when not (nil.std.io::exists with path being full) then - give error-response with code being status-not-found, explanation being "No such file" - end when +failed-read receives code as number returns static-read begin paragraph + value explanation is when code equals status-payload-too-large + give "That file is too large to serve" otherwise give "No such file" end when + give build static-read with + answer being (error-response with code being code, explanation being explanation), + marks being (build validators with found being false, width being 0, stamped-at being 0, tag being "") +end paragraph - value measured is nil.std.io::size with path being full - consider measured - !!! `begin block` because this arm holds its own `when`: without it - !!! the nested conditional reads as the next pattern arm. - when Result.Ok holding width then begin block - when width is greater than maximum-bytes then - give error-response with code being status-payload-too-large, - explanation being "That file is too large to serve" +read-static receives root as text, path as text returns static-read begin paragraph + value relative is without-leading-slash with path being path + when relative equals "" or not (is-safe-name with name being relative) then + give failed-read with code being status-not-found + end when + value directory is when root equals "" give "." otherwise give root end when + value outcome is nil.std.io::read_rooted_snapshot with root being directory, + relative being relative, maximum_bytes being maximum-bytes + consider outcome + when Result.Ok holding snapshot then begin block + value body is nil.std.text::from_bytes with data being snapshot.content + give build static-read with + answer being (make-response with code being status-ok, + kind being (media-type-of with name being relative), body being body), + marks being (build validators with found being true, + width being (nil.std.text::length with s being body), + stamped-at being snapshot.modified_ms, tag being quote plus snapshot.sha256 plus quote) + end block + when Result.Error holding why then begin block + when why equals "file.read_rooted_snapshot: input exceeds maximum_bytes" then + give failed-read with code being status-payload-too-large end when + give failed-read with code being status-not-found end block - when Result.Error holding why then - give error-response with code being status-not-found, explanation being "No such file" - otherwise - give error-response with code being status-not-found, explanation being "No such file" + otherwise give failed-read with code being status-not-found end consider +end paragraph - value outcome is nil.std.io::read_text with path being full - consider outcome - when Result.Ok holding body then - give make-response with code being status-ok, - kind being (media-type-of with name being relative), body being body - when Result.Error holding why then - give error-response with code being status-not-found, explanation being "No such file" - otherwise - give error-response with code being status-not-found, explanation being "No such file" - end consider +public serve-file receives root as text, path as text returns http-response begin paragraph + give (read-static with root being root, path being path).answer end paragraph @@ -214,151 +216,173 @@ public validators begin record end record -!!! --- Read a file's size and timestamp without reading the file. -!!! -!!! The ETag is built from both, which is what Apache does by default and is -!!! sound for the same reason: a file whose length and modification time are -!!! both unchanged is, barring deliberate effort, unchanged. A content hash -!!! would be stronger and needs a hash function the standard library does not -!!! have yet. +!!! --- Identity of the bytes actually served. !!! -!!! Quoted and not weak-prefixed, because it compares byte-for-byte: two -!!! responses with this tag really are the same bytes, so it is usable for -!!! ranges as well as for revalidation, which a weak tag is not. +!!! Hashing the representation prevents equal-length edits with preserved +!!! timestamps from reusing an ETag. A request reads content once and hashes +!!! that same body; Last-Modified is supplemental, second-resolution metadata. public validators-of receives root as text, path as text returns validators begin paragraph - value relative is without-leading-slash with path being path - when not (is-safe-name with name being relative) or relative equals "" then - give build validators with found being false, width being 0, stamped-at being 0, tag being "" - end when - value full is when root equals "" give relative otherwise give root plus "/" plus relative end when - when not (nil.std.io::exists with path being full) then - give build validators with found being false, width being 0, stamped-at being 0, tag being "" - end when - - variable width as number is 0 minus 1 - value measured is nil.std.io::size with path being full - consider measured - when Result.Ok holding found-width then set width to found-width - otherwise set width to 0 minus 1 - end consider - variable stamped as number is 0 minus 1 - value timed is nil.std.io::modified_at with path being full - consider timed - when Result.Ok holding at-ms then set stamped to at-ms - otherwise set stamped to 0 minus 1 - end consider - when width is less than 0 or stamped is less than 0 then - give build validators with found being false, width being 0, stamped-at being 0, tag being "" - end when - - give build validators with found being true, width being width, stamped-at being stamped, - tag being quote plus (base-sixteen with amount being stamped) plus "-" - plus (base-sixteen with amount being width) plus quote + give (read-static with root being root, path being path).marks end paragraph -!!! --- Serve a file, answering 304 when the client's copy is still good. -!!! -!!! The 304 carries the ETag and the caching headers and no body, which is the -!!! whole point: the client keeps what it has and the server sends a few dozen -!!! bytes instead of a few kilobytes. -!!! -!!! A 304 must not carry Content-Type or Content-Length — the response is not -!!! the representation, and a client told otherwise may believe the -!!! representation is now empty. render-response drops the body for a 304 and -!!! omits Content-Length; Content-Type is never set because this builds the -!!! response directly rather than through make-response. +!!! --- Ordinary failures precede conditions; conditions precede Range. +!!! RFC 9110 sections 13.2.1 and 13.2.2. A present empty tag field still +!!! suppresses its date fallback. If-Match compares strongly; If-None-Match +!!! compares weakly and returns 304 only for GET/HEAD, otherwise 412. public serve-revalidated receives root as text, path as text, request as http-request, seconds as number returns http-response begin paragraph - value marks is validators-of with root being root, path being path - when not marks.found then - give error-response with code being status-not-found, explanation being "No such file" + value loaded is read-static with root being root, path being path + value answer is loaded.answer + when answer.code does not equal status-ok then give answer end when + value marks is loaded.marks + value full is with-validators with response being answer, marks being marks, seconds being seconds + when (method-name with verb being request.verb) equals "OPTIONS" then give full end when + + value has-match is has-field with fields being request.fields, name being "If-Match" + when has-match then + when not (tag-matches with offered being (list-field with fields being request.fields, + name being "If-Match"), tag being marks.tag, strong being true) then + give precondition-failed with nothing + end when + otherwise + value unmodified is valid-date with stated being (single-field with fields being request.fields, + name being "If-Unmodified-Since") + when marks.stamped-at is at least 0 and unmodified is at least 0 + and (marks.stamped-at over 1000) is greater than (unmodified over 1000) then + give precondition-failed with nothing + end when end when - !!! The tag is checked before the date, as RFC 9110 requires: a client - !!! sending both means the tag to win, because it is the precise one. - value offered is header-of with request being request, name being "If-None-Match" - value since is header-of with request being request, name being "If-Modified-Since" - value by-tag is tag-matches with offered being offered, tag being marks.tag - value by-date is offered equals "" - and (not-modified-since with stated being since, stamped-at being marks.stamped-at) - when by-tag or by-date then - variable fields is no-headers with nothing - set fields to set-field with fields being fields, name being "ETag", content being marks.tag - set fields to set-field with fields being fields, name being "Last-Modified", - content being (http-date with at-ms being marks.stamped-at) - set fields to set-field with fields being fields, name being "Cache-Control", - content being "public, max-age=" plus (seconds becomes text) - give build http-response with code being status-not-modified, fields being fields, body being "" + value retrieval is (method-name with verb being request.verb) equals "GET" or (method-name with verb being request.verb) equals "HEAD" + value has-none-match is has-field with fields being request.fields, name being "If-None-Match" + when has-none-match then + when tag-matches with offered being (list-field with fields being request.fields, + name being "If-None-Match"), tag being marks.tag, strong being false then + when retrieval then + give with-validators with response being (empty-response with code being status-not-modified), + marks being marks, seconds being seconds + end when + give precondition-failed with nothing + end when end when + when not has-none-match and retrieval and (not-modified-since with stated being (single-field with fields being request.fields, + name being "If-Modified-Since"), stamped-at being marks.stamped-at) then + give with-validators with response being (empty-response with code being status-not-modified), + marks being marks, seconds being seconds + end when + give full +end paragraph - value answer is serve-file with root being root, path being path - when answer.code does not equal status-ok then - give answer + +with-validators receives response as http-response, marks as validators, + seconds as number returns http-response begin paragraph + variable answer is with-header with response being response, name being "ETag", content being marks.tag + when marks.stamped-at is at least 0 then + set answer to with-header with response being answer, name being "Last-Modified", + content being (http-date with at-ms being marks.stamped-at) end when - variable full is with-header with response being answer, name being "ETag", content being marks.tag - set full to with-header with response being full, name being "Last-Modified", - content being (http-date with at-ms being marks.stamped-at) - give with-header with response being full, name being "Cache-Control", + give with-header with response being answer, name being "Cache-Control", content being "public, max-age=" plus (seconds becomes text) end paragraph -!!! --- Whether the file is unchanged since the date a client holds. -!!! -!!! Compared in seconds, because an HTTP date has no finer resolution: a file -!!! written 300ms after the second the client was told would otherwise look -!!! modified on every request, forever, and the client would never get a 304. -!!! Truncating both sides to the second is what the specification's own -!!! comparison means. -!!! -!!! An unreadable date is "modified" — the full representation, which is -!!! always correct. Reading it wrongly and answering 304 would serve a stale -!!! page, which is the failure that matters. +precondition-failed returns http-response begin paragraph + give error-response with code being status-precondition-failed, explanation being "The file does not meet the request conditions" +end paragraph + + +!!! The current clock reader supports IMF-fixdate. Canonical round-tripping +!!! rejects malformed separators, impossible calendar dates and wrong weekdays. +!!! Unrecognised dates, including obsolete HTTP-date spellings, are ignored. + +valid-date receives stated as text returns number begin paragraph + value trimmed is nil.std.text::trim with s being stated + value parsed is ms-from-http-date with stated being trimmed + when parsed is less than 0 then give 0 minus 1 end when + when (http-date with at-ms being parsed) does not equal trimmed then give 0 minus 1 end when + give parsed +end paragraph + not-modified-since receives stated as text, stamped-at as number returns boolean begin paragraph - when stated equals "" then - give false - end when - value held is ms-from-http-date with stated being stated - when held is less than 0 then - give false - end when + value held is valid-date with stated being stated + when held is less than 0 or stamped-at is less than 0 then give false end when give (stamped-at over 1000) is at most (held over 1000) end paragraph -!!! --- Whether an If-None-Match names this representation. -!!! -!!! `*` matches anything that exists, which is what a conditional request uses -!!! to mean "if there is one at all". Otherwise the field is a comma-separated -!!! list and a match is a match; the weak prefix is accepted and ignored, -!!! because a client that sends `W/"x"` for a tag it was given as `"x"` is -!!! being imprecise rather than asking about a different thing. - -tag-matches receives offered as text, tag as text returns boolean begin paragraph - when offered equals "" then - give false - end when - value trimmed is nil.std.text::trim with s being offered - when trimmed equals "*" then - give true - end when - value parts is split-on with input being trimmed, separator being "," - repeat for each part in parts - value one is nil.std.text::trim with s being part - value bare is when nil.std.text::starts_with with s being one, prefix being "W/" - give nil.std.text::substring with s being one, start being 2, - length being (nil.std.text::length with s being one) minus 2 - otherwise give one - end when - when bare equals tag then - give true +!!! List fields may be repeated; singleton date/If-Range fields may not. + +list-field receives fields as array of header-field, name as text returns text begin paragraph + value held is field-values with fields being fields, name being name + variable joined as text is "" + variable first as boolean is true + repeat for each one in held + when not first then set joined to joined plus "," end when + set joined to joined plus one + set first to false + end repeat + give joined +end paragraph + + +single-field receives fields as array of header-field, name as text returns text begin paragraph + value held is field-values with fields being fields, name being name + when the length of held does not equal 1 then give "" end when + give held at 0 +end paragraph + + +!!! Parse entity-tag lists without splitting commas inside opaque quoted tags. +!!! Validate the whole list before accepting a match, and only permit the weak +!!! prefix for weak comparison. Backslashes have no escape meaning in an ETag. + +tag-matches receives offered as text, tag as text, strong as boolean returns boolean begin paragraph + value input is nil.std.text::trim with s being offered + when input equals "*" then give true end when + value width is nil.std.text::length with s being input + variable index as number is 0 + variable matched as boolean is false + repeat while index is less than width + variable code as number is nil.std.text::byte_at with s being input, index being index + when code equals 32 or code equals 9 or code equals 44 then + set index to index plus 1 + otherwise + variable weak as boolean is false + when (nil.std.text::substring with s being input, start being index, length being 2) equals "W/" then + set weak to true + set index to index plus 2 + end when + when index is at least width then give false end when + when (nil.std.text::byte_at with s being input, index being index) does not equal 34 then give false end when + value first is index + set index to index plus 1 + variable closed as boolean is false + repeat while index is less than width and not closed + set code to nil.std.text::byte_at with s being input, index being index + when code equals 34 then + set closed to true + end when + when code is less than 33 or code equals 127 then give false end when + set index to index plus 1 + end repeat + when not closed then give false end when + value bare is nil.std.text::substring with s being input, start being first, length being index minus first + when bare equals tag and (not strong or not weak) then set matched to true end when + repeat while index is less than width + set code to nil.std.text::byte_at with s being input, index being index + when code does not equal 32 and code does not equal 9 then stop repeating end when + set index to index plus 1 + end repeat + when index is less than width then + when (nil.std.text::byte_at with s being input, index being index) does not equal 44 then give false end when + end when end when end repeat - give false + give matched end paragraph @@ -386,11 +410,12 @@ end record public range-of receives stated as text, width as number returns asked-range begin paragraph value absent is build asked-range with present being false, satisfiable being false, first-byte being 0, last-byte being 0 - when stated equals "" then + when stated equals "" or width is at most 0 then + !!! RFC 9110 permits ignoring Range for an empty representation. give absent end when value trimmed is nil.std.text::trim with s being stated - when not (nil.std.text::starts_with with s being trimmed, prefix being "bytes=") then + when not (nil.std.text::starts_with with s being (nil.std.text::to_lower with s being trimmed), prefix being "bytes=") then !!! An unknown unit is ignored, as RFC 9110 requires, rather than !!! refused: the whole representation is a valid answer to it. give absent @@ -402,8 +427,8 @@ public range-of receives stated as text, width as number returns asked-range beg give absent end when - value first-text is nil.std.text::trim with s being (before-first with input being spec, separator being "-") - value last-text is nil.std.text::trim with s being (after-first with input being spec, separator being "-") + value first-text is before-first with input being spec, separator being "-" + value last-text is after-first with input being spec, separator being "-" when not (nil.std.text::contains with s being spec, needle being "-") then give unsatisfiable with nothing end when @@ -450,7 +475,7 @@ end paragraph count-of receives digits as text returns number begin paragraph value width is nil.std.text::length with s being digits - when width is less than 1 or width is greater than 18 then + when width is less than 1 then give 0 minus 1 end when variable total as number is 0 @@ -460,97 +485,57 @@ count-of receives digits as text returns number begin paragraph when code is less than 48 or code is greater than 57 then give 0 minus 1 end when - set total to (total times 10) plus (code minus 48) + !!! Saturate while continuing to validate every digit. A huge suffix + !!! or end offset is valid and clamps; a huge first offset cannot fit. + when total is greater than 922337203685477580 + or (total equals 922337203685477580 and code is greater than 55) then + set total to 9223372036854775807 + otherwise + set total to (total times 10) plus (code minus 48) + end when set index to index plus 1 end repeat give total end paragraph -!!! --- Serve a file, honouring Range and If-None-Match. -!!! -!!! What a browser fetching a video or a client resuming a download expects. -!!! Accept-Ranges is advertised on the full response too, because a client -!!! only asks for a range if it has been told ranges are available. +!!! --- Apply ranges only to a GET whose ordinary response would be 200. +!!! HEAD keeps the full body internally so render-response can calculate the +!!! GET Content-Length while suppressing its wire body. If-Range compares a +!!! single strong content ETag. Date validators are not known strong for files +!!! that can be edited more than once a second, so date-form If-Range falls +!!! back to the whole representation (RFC 9110 Last-Modified comparison and If-Range rules). public serve-static receives root as text, path as text, request as http-request, seconds as number returns http-response begin paragraph - value marks is validators-of with root being root, path being path - when not marks.found then - give error-response with code being status-not-found, explanation being "No such file" - end when - - value wanted is range-of with stated being - (header-of with request being request, name being "Range"), width being marks.width - when not wanted.present then - value whole is serve-revalidated with root being root, path being path, - request being request, seconds being seconds - when whole.code does not equal status-ok then - give whole - end when - give with-header with response being whole, name being "Accept-Ranges", content being "bytes" + value answer is serve-revalidated with root being root, path being path, + request being request, seconds being seconds + when answer.code does not equal status-ok then give answer end when + value whole is with-header with response being answer, name being "Accept-Ranges", content being "bytes" + when (method-name with verb being request.verb) does not equal "GET" then give whole end when + + when has-field with fields being request.fields, name being "If-Range" then + value offered is nil.std.text::trim with s being (single-field with fields being request.fields, name being "If-Range") + value tag is field-value with fields being whole.fields, name being "ETag" + when offered does not equal tag then give whole end when end when - + value width is nil.std.text::length with s being whole.body + value wanted is range-of with stated being (single-field with fields being request.fields, name being "Range"), + width being width + when not wanted.present then give whole end when when not wanted.satisfiable then - !!! 416 carries Content-Range naming the real length, which is how a - !!! client that was resuming discovers the file is a different size. value refusal is error-response with code being status-range-not-satisfiable, explanation being "That range is outside the file" value ranged is with-header with response being refusal, name being "Content-Range", - content being "bytes */" plus (marks.width becomes text) + content being "bytes */" plus (width becomes text) give with-header with response being ranged, name being "Accept-Ranges", content being "bytes" end when - value answer is serve-file with root being root, path being path - when answer.code does not equal status-ok then - give answer - end when - value taken is nil.std.text::substring with s being answer.body, start being wanted.first-byte, + value taken is nil.std.text::substring with s being whole.body, start being wanted.first-byte, length being wanted.last-byte minus wanted.first-byte plus 1 - variable partial is with-body with response being (with-status with response being answer, + value partial is with-body with response being (with-status with response being whole, code being status-partial-content), body being taken - set partial to with-header with response being partial, name being "Content-Range", + give with-header with response being partial, name being "Content-Range", content being "bytes " plus (wanted.first-byte becomes text) plus "-" - plus (wanted.last-byte becomes text) plus "/" plus (marks.width becomes text) - set partial to with-header with response being partial, name being "Accept-Ranges", content being "bytes" - set partial to with-header with response being partial, name being "ETag", content being marks.tag - give partial -end paragraph - - -!!! --- A whole number in lower-case hexadecimal, for the ETag. - -base-sixteen receives amount as number returns text begin paragraph - when amount is at most 0 then - give "0" - end when - variable rest as number is amount - variable out as text is "" - repeat while rest is greater than 0 - set out to (hex-glyph with amount being (rest remainder 16)) plus out - set rest to rest over 16 - end repeat - give out -end paragraph - - -hex-glyph receives amount as number returns text begin paragraph - give switch amount - case 0 then stop switching with "0" - case 1 then stop switching with "1" - case 2 then stop switching with "2" - case 3 then stop switching with "3" - case 4 then stop switching with "4" - case 5 then stop switching with "5" - case 6 then stop switching with "6" - case 7 then stop switching with "7" - case 8 then stop switching with "8" - case 9 then stop switching with "9" - case 10 then stop switching with "a" - case 11 then stop switching with "b" - case 12 then stop switching with "c" - case 13 then stop switching with "d" - case 14 then stop switching with "e" - otherwise stop switching with "f" - end switch + plus (wanted.last-byte becomes text) plus "/" plus (width becomes text) end paragraph diff --git a/tests/nilweb/tests/framing.matrix b/tests/nilweb/tests/framing.matrix new file mode 100644 index 0000000..64d7578 --- /dev/null +++ b/tests/nilweb/tests/framing.matrix @@ -0,0 +1,145 @@ +!!! Pure framing regressions, including every byte boundary in chunk syntax. +book nilweb.tests.framing + +read nil.std.core +read nil.std.text +read everything from nilweb.http.headers +read everything from nilweb.http.request +read everything from nilweb.http.framing +read everything from nilweb.tests.harness + +framing-head receives fields as text returns text begin paragraph + value ending is crlf with nothing + give "POST /raw%2fpath?q=raw%20query HTTP/1.1" plus ending plus "Host: example.com" plus ending + plus fields plus ending +end paragraph + +rejected-frame receives label as text, message as text, status as number returns nothing begin paragraph + value framed is inspect-frame with message being message, maximum being 1048576 + evaluate same-number with label being label plus " status", actual being framed.status, expected being status + evaluate is-true with label being label plus " is not dispatched", actual being not framed.complete + give nothing +end paragraph + +public run-framing-tests returns nothing begin paragraph + evaluate section with title being "nilweb.http.framing" + value ending is crlf with nothing + value no-body is framing-head with fields being "" + value next-request is "GET /second HTTP/1.1" plus ending plus "Host: example.com" plus ending plus ending + value empty-frame is inspect-frame with message being no-body plus next-request, maximum being 1048576 + evaluate same-number with label being "bodyless frame consumes just its headers", + actual being empty-frame.consumed, expected being (nil.std.text::length with s being no-body) + evaluate same-text with label being "bodyless frame does not capture pipeline bytes", + actual being (parse-request with raw being empty-frame.message).body, expected being "" + value body-head is framing-head with fields being "Content-Length: 5" plus ending + value body-wire is body-head plus "abcde" + value body-frame is inspect-frame with message being body-wire plus next-request, maximum being 1048576 + value body-request is parse-request with raw being body-frame.message + evaluate same-text with label being "fixed body is exactly declared length", actual being body-request.body, expected being "abcde" + evaluate same-text with label being "raw target remains unchanged", actual being body-request.target, expected being "/raw%2fpath?q=raw%20query" + evaluate same-number with label being "fixed frame consumes no following request", actual being body-frame.consumed, + expected being (nil.std.text::length with s being body-wire) + variable offset as number is 0 + repeat while offset is less than (nil.std.text::length with s being body-wire) + value prefix is nil.std.text::substring with s being body-wire, start being 0, length being offset + value partial is inspect-frame with message being prefix, maximum being 1048576 + evaluate is-true with label being "every fixed-body prefix waits", actual being not partial.complete and partial.status equals 0 + set offset to offset plus 1 + end repeat + value matching is inspect-frame with message being (framing-head with fields being + "Content-Length: 005, 5" plus ending plus "content-length: 5" plus ending) plus "abcde", maximum being 1048576 + evaluate is-true with label being "identical repeated decimal lengths are accepted", actual being matching.complete + evaluate same-text with label being "duplicate lengths are normalized", actual being (header-of with request being + (parse-request with raw being matching.message), name being "Content-Length"), expected being "5" + variable bad-lengths as array of text is ["", "banana", "-1", "+1", "1 2", "1.0", "1,", ",1", "1,2"] + repeat for each bad in bad-lengths + evaluate rejected-frame with label being "invalid decimal length", message being + (framing-head with fields being "Content-Length: " plus bad plus ending), status being 400 + end repeat + evaluate rejected-frame with label being "conflicting repeated lengths", message being (framing-head with fields being + "Content-Length: 1" plus ending plus "Content-Length: 2" plus ending), status being 400 + evaluate rejected-frame with label being "huge decimal length is bounded", message being (framing-head with fields being + "Content-Length: 999999999999999999999999999999999999" plus ending), status being 413 + evaluate rejected-frame with label being "transfer encoding and length conflict", message being (framing-head with fields being + "Content-Length: 0" plus ending plus "Transfer-Encoding: chunked" plus ending), status being 400 + evaluate rejected-frame with label being "unsupported coding before chunked", message being (framing-head with fields being + "Transfer-Encoding: gzip, chunked" plus ending), status being 501 + variable bad-codings as array of text is ["", "gzip", "chunked, gzip", "chunked, chunked", "chunked;foo=bar", "chunked,"] + repeat for each bad in bad-codings + evaluate rejected-frame with label being "invalid transfer framing", message being + (framing-head with fields being "Transfer-Encoding: " plus bad plus ending), status being 400 + end repeat + evaluate rejected-frame with label being "HTTP1.0 cannot use transfer coding", message being + "POST / HTTP/1.0" plus ending plus "Transfer-Encoding: chunked" plus ending plus ending, status being 400 + variable bad-headers as array of text is ["Content-Length : 0", " Content-Length: 0", "NoColon", "Bad(Name): 0"] + repeat for each bad in bad-headers + evaluate rejected-frame with label being "malformed field syntax", message being + (framing-head with fields being bad plus ending), status being 400 + end repeat + evaluate rejected-frame with label being "bare LF is not wire framing", message being + "GET / HTTP/1.1" plus newline plus "Host: example.com" plus newline plus newline, status being 400 + variable expectations as array of text is ["100-continue", "100-CONTINUE", "unsupported", "", "100-continue, unsupported"] + repeat for each expectation in expectations + evaluate rejected-frame with label being "Expect rejected from headers before body", message being + (framing-head with fields being "Content-Length: 5" plus ending plus "Expect: " plus expectation plus ending), status being 417 + end repeat + evaluate rejected-frame with label being "Expect on chunked request does not wait for chunks", message being + (framing-head with fields being "Transfer-Encoding: chunked" plus ending plus "Expect: 100-continue" plus ending), status being 417 + evaluate rejected-frame with label being "Expect policy also applies when body already arrived", message being + (framing-head with fields being "Content-Length: 5" plus ending plus "Expect: 100-continue" plus ending) plus "abcde", status being 417 + evaluate rejected-frame with label being "repeated Expect fields share the same HTTP1.1 policy", message being + (framing-head with fields being "Content-Length: 5" plus ending plus "Expect: 100-continue" plus ending + plus "Expect: 100-CONTINUE, 100-continue" plus ending), status being 417 + value legacy-head is "POST /legacy HTTP/1.0" plus ending plus "Content-Length: 5" plus ending + plus "Expect: 100-continue" plus ending plus "Expect: 100-CONTINUE, 100-continue" plus ending plus ending + value legacy-waiting is inspect-frame with message being legacy-head, maximum being 1048576 + evaluate is-true with label being "HTTP1.0 ignores 100-continue and awaits its body", actual being + legacy-waiting.status equals 0 and not legacy-waiting.complete + value legacy-complete is inspect-frame with message being legacy-head plus "abcde", maximum being 1048576 + evaluate is-true with label being "HTTP1.0 ignored expectation preserves body framing", actual being legacy-complete.complete + evaluate rejected-frame with label being "HTTP1.0 unsupported expectation fails", message being + "POST /legacy HTTP/1.0" plus ending plus "Content-Length: 5" plus ending + plus "Expect: unsupported" plus ending plus ending, status being 417 + evaluate rejected-frame with label being "HTTP1.0 mixed repeated expectations still fail", message being + "POST /legacy HTTP/1.0" plus ending plus "Content-Length: 5" plus ending + plus "Expect: 100-continue" plus ending plus "Expect: 100-continue, unsupported" plus ending plus ending, status being 417 + value chunk-head is framing-head with fields being "Transfer-Encoding: Chunked" plus ending plus "Trailer: X-Note" plus ending + value chunks is "2;flag;name=" plus quote plus "a;b" plus quote plus ending plus "ab" plus ending + plus "3" plus ending plus "cde" plus ending plus "0" plus ending plus "X-Note: discarded" plus ending plus ending + value chunk-wire is chunk-head plus chunks + value chunked is inspect-frame with message being chunk-wire plus next-request, maximum being 1048576 + value decoded is parse-request with raw being chunked.message + evaluate is-true with label being "chunked frame is complete", actual being chunked.complete + evaluate same-text with label being "chunked body is decoded", actual being decoded.body, expected being "abcde" + evaluate same-number with label being "chunked framing consumes its trailers only", actual being chunked.consumed, + expected being (nil.std.text::length with s being chunk-wire) + evaluate same-text with label being "chunked transfer header is removed", actual being (header-of with request being decoded, name being "Transfer-Encoding"), expected being "" + evaluate same-text with label being "trailer declaration is removed", actual being (header-of with request being decoded, name being "Trailer"), expected being "" + evaluate same-text with label being "trailer is not merged into application headers", actual being (header-of with request being decoded, name being "X-Note"), expected being "" + evaluate same-text with label being "decoded content length is synthesized", actual being (header-of with request being decoded, name being "Content-Length"), expected being "5" + set offset to 0 + repeat while offset is less than (nil.std.text::length with s being chunk-wire) + value partial is inspect-frame with message being (nil.std.text::substring with s being chunk-wire, start being 0, length being offset), maximum being 1048576 + evaluate is-true with label being "every chunked byte boundary waits", actual being not partial.complete and partial.status equals 0 + set offset to offset plus 1 + end repeat + variable bad-chunks as array of text is ["Z", "-1", "+1", "1.0", "1;", "1;=bad", "1;name=", "1;name=" plus quote plus "open"] + repeat for each bad in bad-chunks + evaluate rejected-frame with label being "invalid chunk size or extension", message being chunk-head plus bad plus ending, status being 400 + end repeat + evaluate rejected-frame with label being "huge hexadecimal chunk is bounded", message being chunk-head plus "FFFFFFFFFFFFFFFFFFFFFFFF" plus ending, status being 413 + evaluate rejected-frame with label being "chunk payload requires CRLF", message being chunk-head plus "1" plus ending plus "xNO", status being 400 + evaluate rejected-frame with label being "trailers use strict field syntax", message being chunk-head plus "0" plus ending plus "Bad Field: value" plus ending plus ending, status being 400 + evaluate rejected-frame with label being "bare LF chunk line is rejected immediately", message being chunk-head plus "1" plus newline, status being 400 + evaluate rejected-frame with label being "bare LF trailer is rejected immediately", message being chunk-head plus "0" plus ending plus "X-Note: discarded" plus newline, status being 400 + value capped is inspect-frame with message being body-wire, maximum being (nil.std.text::length with s being body-wire) minus 1 + evaluate same-number with label being "request cap counts headers plus body", actual being capped.status, expected being 413 + value exact is inspect-frame with message being body-wire plus next-request, maximum being (nil.std.text::length with s being body-wire) + evaluate is-true with label being "pipeline surplus does not count against first request cap", actual being exact.complete + value closing is inspect-frame with message being (framing-head with fields being "Connection: keep-alive" plus ending + plus "Connection: upgrade, close" plus ending), maximum being 1048576 + evaluate is-true with label being "close token wins across repeated connection fields", actual being closing.complete and not closing.keeping + value not-closing is inspect-frame with message being (framing-head with fields being "Connection: disclose" plus ending), maximum being 1048576 + evaluate is-true with label being "connection tokens are not substring matches", actual being not-closing.keeping + give nothing +end paragraph diff --git a/tests/nilweb/tests/sha256.matrix b/tests/nilweb/tests/sha256.matrix new file mode 100644 index 0000000..63b7634 --- /dev/null +++ b/tests/nilweb/tests/sha256.matrix @@ -0,0 +1,85 @@ +!!! Independent SHA-256 known answers. The two NIST message examples are at +!!! https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Standards-and-Guidelines/documents/examples/SHA256.pdf +!!! Other expected digests were computed with Python hashlib.sha256, including +!!! binary messages whose bytes are (index * 73 + 19) modulo 256. + +book nilweb.tests.sha256 + +read nil.std.text +read everything from nilweb.crypto.sha256 +read everything from nilweb.tests.harness + + +public run-sha256-tests returns nothing begin paragraph + evaluate section with title being "nilweb.crypto.sha256" + evaluate same-text with label being "SHA-256 empty input", + actual being (sha256-hex with data being ""), + expected being "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + evaluate same-text with label being "SHA-256 NIST one-block message", + actual being (sha256-hex with data being "abc"), + expected being "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + evaluate same-text with label being "SHA-256 NIST two-block message", + actual being (sha256-hex with data being "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + expected being "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + evaluate same-text with label being "SHA-256 112-byte multi-block message", + actual being (sha256-hex with data being "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"), + expected being "cf5b16a778af8380036ce59e7b0492370b249b11e8f07a51afac45037afee9d1" + evaluate same-text with label being "SHA-256 NUL is a byte, not a terminator", + actual being (sha256-hex with data being (nil.std.text::from_byte with code being 0)), + expected being "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d" + evaluate same-text with label being "SHA-256 high byte is not reencoded as Unicode", + actual being (sha256-hex with data being (nil.std.text::from_byte with code being 255)), + expected being "a8100ae6aa1940d0b663bb31cd466142ebbdbd5187131b92d93818987832eb89" + evaluate same-text with label being "SHA-256 hashes UTF-8 bytes exactly", + actual being (sha256-hex with data being "café 🌍"), + expected being "2a51ea11819c3ccea2fb3cf397b809b1fb333b8d191698a5fb47d04cb40db1e4" + evaluate same-text with label being "SHA-256 same-length changed contents", + actual being (sha256-hex with data being "abd"), + expected being "a52d159f262b2c6ddb724a61840befc36eb30c88877a4030b65cbe86298449c9" + + value widths as array of number is [1, 2, 3, 31, 32, 55, 56, 57, 63, 64, 65, + 119, 120, 121, 127, 128, 129, 255, 256, 257, 1024] + value expected as array of text is [ + "ab897fbdedfa502b2d839b6a56100887dccdc507555c282e59589e06300a62e2", + "c756bd104d1406ec2c380c268dbef88fd2082819c15a21c8644bcd8e8de4bc96", + "2b8c60738b844cd8716c5134363930fbc133c1f41cc0e568dfd0b4106a7b65e1", + "30726c77ff641e29a2633c4689164298376cf322d8da842d5e2fab43bee5aeae", + "c28f0ff0408b4f778c5c9dc1ebc973b9686862167ecbccac22962e436b325db9", + "0c43a30970ee9525a6603d5656803c325bded148fa65bbd363df54cb3ac3dee5", + "f4dcdc09c846d1b3621701516b300055c1e7875e216ba019dd4423f5a420f2d0", + "579e08b0aa0a7cb68eed325a658ecc20557e144ffd5c5d9addaa84055fc51566", + "ba3bb62c4369fe74e7b94392d0f04ed284a7b5d7295bfa48fabdb25c01d85ab7", + "cb6862268ca7c02e7b6af558f0e3d236ce9ec7ff41cc746b2848c04c582da94e", + "b68af55df9ce8ee60870e45d10e36b2e8363c9896f6e62478dcb6f593a4d7133", + "175624f0f95b0a47a1b3b52f5b149f88802d3822b184325d0c20a1a7c132a687", + "0598a3d8b5a82710dda54f05c4aa11fdd1c0aa5fafb216840fccbae323f07520", + "22cd13d4f3251bfe1d793f401ddc6d489ee3421958920e873e7e0138c576a136", + "a8365b6e6ed1daf56c41bae89ab089c03b671240ae103070480a7f271bdbad57", + "d399f065c9f21e2fd51e2aeadb7768eab7e6e45e5150f31227c9711934a4d1d3", + "d672408e0571636ea21092304eb31ba8fc1fbffc5d5ea6531d62b072714599db", + "6d59f1b3a886e8cb813e77d1f0ca0ea63ed8aa8f52511c4a438efb6a4a51d1ff", + "8e5ed1e6474d39bf1594411085e7a50c15aabaa768e633ebf0960ce20e84e336", + "dd31e55fd954d9edc9e29f8e078387f0dd38e8ea832376311820de6322cdb0d1", + "e051d1007607de494c073da3c29903d6c0abfee7a4c0609f560a340a1947b470" + ] + variable index as number is 0 + repeat while index is less than the length of widths + value width is widths at index + evaluate same-text with label being "SHA-256 binary pattern length " plus (nilweb.tests.harness::as-text with amount being width), + actual being (sha256-hex with data being (patterned-bytes with width being width)), + expected being expected at index + set index to index plus 1 + end repeat + give nothing +end paragraph + + +patterned-bytes receives width as number returns text begin paragraph + variable data as text is "" + variable index as number is 0 + repeat while index is less than width + set data to data plus (nil.std.text::from_byte with code being (index times 73 plus 19) remainder 256) + set index to index plus 1 + end repeat + give data +end paragraph diff --git a/tests/nilweb/tests/static_http.matrix b/tests/nilweb/tests/static_http.matrix new file mode 100644 index 0000000..28bd1b7 --- /dev/null +++ b/tests/nilweb/tests/static_http.matrix @@ -0,0 +1,303 @@ +!!! Conditional and range semantics exercised through complete static responses. +book nilweb.tests.static_http + +read nil.std.text +read nil.std.io.file +read nil.std.core +read everything from nilweb.tests.harness +read everything from nilweb.http.headers +read everything from nilweb.http.method +read everything from nilweb.http.request +read everything from nilweb.http.response +read everything from nilweb.http.status +read everything from nilweb.static.files +read everything from nilweb.time.clock +read everything from nilweb.text.scan + + +asset receives fields as array of header-field, verb as request-method returns http-response begin paragraph + value request is make-request with verb being verb, target being "/assets/app.css", fields being fields, body being "" + give serve-static with root being "tests/public", path being request.path, request being request, seconds being 60 +end paragraph + + +one-field receives name as text, content as text returns array of header-field begin paragraph + give set-field with fields being (no-headers with nothing), name being name, content being content +end paragraph + + +expect-code receives label as text, fields as array of header-field, verb as request-method, + expected as number returns nothing begin paragraph + evaluate same-number with label being label, actual being (asset with fields being fields, verb being verb).code, + expected being expected + give nothing +end paragraph + + +public run-static-http-tests returns nothing begin paragraph + evaluate section with title being "nilweb.static HTTP preconditions, ranges and HEAD" + value whole is asset with fields being (no-headers with nothing), verb being request-method.Get + value tag is field-value with fields being whole.fields, name being "ETag" + value modified is field-value with fields being whole.fields, name being "Last-Modified" + value old-date is http-date with at-ms being 0 + value future-date is "Fri, 31 Dec 9999 23:59:59 GMT" + value stale is quote plus "old-copy" plus quote + value range is one-field with name being "Range", content being "bytes=0-5" + value past-end is one-field with name being "Range", content being "bytes=9999-" + evaluate same-text with label being "412 has its registered reason phrase", + actual being (reason-phrase with code being status-precondition-failed), expected being "Precondition Failed" + evaluate same-number with label being "strong SHA-256 ETag has 64 digits and two quotes", + actual being (nil.std.text::length with s being tag), expected being 66 + evaluate same-text with label being "ETag identifies the exact CSS bytes", + actual being tag, expected being quote plus "a8755988aba6a204ea234649ac4bb76aaef7d2b5b1aa32083e835fa31ed5b61a" plus quote + + variable fields is set-field with fields being range, name being "If-None-Match", content being tag + value cached-range is asset with fields being fields, verb being request-method.Get + evaluate same-number with label being "If-None-Match precedes a satisfiable Range", actual being cached-range.code, expected being 304 + evaluate same-text with label being "304 has no partial content metadata", + actual being (field-value with fields being cached-range.fields, name being "Content-Range"), expected being "" + evaluate same-text with label being "304 retains the validator", + actual being (field-value with fields being cached-range.fields, name being "ETag"), expected being tag + evaluate same-text with label being "304 retains cache policy", + actual being (field-value with fields being cached-range.fields, name being "Cache-Control"), expected being "public, max-age=60" + evaluate same-text with label being "304 retains modification time", + actual being (field-value with fields being cached-range.fields, name being "Last-Modified"), expected being modified + set fields to set-field with fields being past-end, name being "If-None-Match", content being tag + evaluate expect-code with label being "If-None-Match precedes an unsatisfiable Range", fields being fields, + verb being request-method.Get, expected being 304 + set fields to set-field with fields being past-end, name being "If-Modified-Since", content being modified + evaluate expect-code with label being "If-Modified-Since precedes an unsatisfiable Range", fields being fields, + verb being request-method.Get, expected being 304 + + set fields to set-field with fields being range, name being "If-Match", content being stale + evaluate expect-code with label being "If-Match failure precedes Range", fields being fields, + verb being request-method.Get, expected being 412 + set fields to set-field with fields being fields, name being "If-None-Match", content being tag + evaluate expect-code with label being "If-Match failure precedes matching If-None-Match", fields being fields, + verb being request-method.Get, expected being 412 + evaluate expect-code with label being "If-Match uses strong comparison", fields being + (one-field with name being "If-Match", content being "W/" plus tag), verb being request-method.Get, expected being 412 + evaluate expect-code with label being "If-Match wildcard accepts the existing representation", fields being + (one-field with name being "If-Match", content being "*"), verb being request-method.Get, expected being 200 + set fields to set-field with fields being (one-field with name being "If-Match", content being tag), + name being "If-Unmodified-Since", content being old-date + evaluate expect-code with label being "If-Match suppresses If-Unmodified-Since", fields being fields, + verb being request-method.Get, expected being 200 + set fields to set-field with fields being range, name being "If-Unmodified-Since", content being old-date + evaluate expect-code with label being "If-Unmodified-Since failure precedes Range", fields being fields, + verb being request-method.Get, expected being 412 + evaluate expect-code with label being "equal If-Unmodified-Since succeeds", fields being + (one-field with name being "If-Unmodified-Since", content being modified), verb being request-method.Get, expected being 200 + evaluate expect-code with label being "invalid If-Unmodified-Since is ignored", fields being + (one-field with name being "If-Unmodified-Since", content being "not a date"), verb being request-method.Get, expected being 200 + evaluate expect-code with label being "empty If-Match cannot accidentally match", fields being + (one-field with name being "If-Match", content being ""), verb being request-method.Get, expected being 412 + + set fields to set-field with fields being (one-field with name being "If-None-Match", content being stale), + name being "If-Modified-Since", content being future-date + evaluate expect-code with label being "stale If-None-Match suppresses a matching date", fields being fields, + verb being request-method.Get, expected being 200 + set fields to set-field with fields being fields, name being "If-None-Match", content being "" + evaluate expect-code with label being "present empty If-None-Match suppresses date fallback", fields being fields, + verb being request-method.Get, expected being 200 + evaluate expect-code with label being "If-None-Match weak comparison matches", fields being + (one-field with name being "If-None-Match", content being "W/" plus tag), verb being request-method.Get, expected being 304 + evaluate expect-code with label being "a matching tag on POST returns 412 rather than 304", fields being + (one-field with name being "If-None-Match", content being tag), verb being request-method.Post, expected being 412 + evaluate expect-code with label being "If-Modified-Since is ignored on POST", fields being + (one-field with name being "If-Modified-Since", content being future-date), verb being request-method.Post, expected being 200 + evaluate expect-code with label being "preconditions are ignored on OPTIONS", fields being + (one-field with name being "If-Match", content being stale), verb being request-method.Options, expected being 200 + set fields to add-field with fields being (one-field with name being "If-None-Match", content being stale), + name being "if-none-match", content being tag + evaluate expect-code with label being "repeated entity-tag fields combine as a list", fields being fields, + verb being request-method.Get, expected being 304 + evaluate expect-code with label being "commas inside an opaque tag do not split the list", fields being + (one-field with name being "If-None-Match", content being quote plus "other,copy" plus quote plus ", " plus tag), + verb being request-method.Get, expected being 304 + evaluate expect-code with label being "invalid list suffix does not partially match", fields being + (one-field with name being "If-None-Match", content being tag plus "garbage"), verb being request-method.Get, expected being 200 + evaluate expect-code with label being "an unclosed quoted entity tag is not a match", fields being + (one-field with name being "If-None-Match", content being quote plus "other, " plus tag), + verb being request-method.Get, expected being 200 + evaluate expect-code with label being "a wildcard mixed with a list is not valid If-Match", fields being + (one-field with name being "If-Match", content being "*, " plus tag), verb being request-method.Get, expected being 412 + + evaluate expect-code with label being "matching HTTP date revalidates", fields being + (one-field with name being "If-Modified-Since", content being modified), verb being request-method.Get, expected being 304 + evaluate expect-code with label being "invalid date separators cannot revalidate", fields being + (one-field with name being "If-Modified-Since", content being "Fri; 31 Dec 9999 23:59:59 GMT"), + verb being request-method.Get, expected being 200 + evaluate expect-code with label being "impossible calendar dates cannot revalidate", fields being + (one-field with name being "If-Modified-Since", content being "Fri, 31 Feb 9999 23:59:59 GMT"), + verb being request-method.Get, expected being 200 + set fields to add-field with fields being (one-field with name being "If-Modified-Since", content being modified), + name being "If-Modified-Since", content being future-date + evaluate expect-code with label being "repeated singleton dates are ignored", fields being fields, + verb being request-method.Get, expected being 200 + + set fields to set-field with fields being range, name being "If-Range", content being tag + value partial is asset with fields being fields, verb being request-method.Get + evaluate same-number with label being "matching strong If-Range permits partial content", actual being partial.code, expected being 206 + evaluate same-text with label being "partial body is the exact requested slice", actual being partial.body, + expected being (nil.std.text::substring with s being whole.body, start being 0, length being 6) + evaluate same-text with label being "206 retains the full representation ETag", + actual being (field-value with fields being partial.fields, name being "ETag"), expected being tag + evaluate same-text with label being "206 retains Last-Modified", + actual being (field-value with fields being partial.fields, name being "Last-Modified"), expected being modified + evaluate same-text with label being "206 retains cache policy", + actual being (field-value with fields being partial.fields, name being "Cache-Control"), expected being "public, max-age=60" + evaluate same-text with label being "206 states its representation width", + actual being (field-value with fields being partial.fields, name being "Content-Range"), expected being "bytes 0-5/31" + set fields to set-field with fields being range, name being "If-Range", content being stale + evaluate expect-code with label being "stale If-Range sends the full representation", fields being fields, + verb being request-method.Get, expected being 200 + set fields to set-field with fields being past-end, name being "If-Range", content being stale + evaluate expect-code with label being "stale If-Range suppresses even an unsatisfiable Range", fields being fields, + verb being request-method.Get, expected being 200 + set fields to set-field with fields being range, name being "If-Range", content being "W/" plus tag + evaluate expect-code with label being "weak If-Range cannot authorize a partial response", fields being fields, + verb being request-method.Get, expected being 200 + set fields to set-field with fields being range, name being "If-Range", content being modified + evaluate expect-code with label being "second-resolution file dates cannot prove strong If-Range identity", fields being fields, + verb being request-method.Get, expected being 200 + set fields to set-field with fields being range, name being "If-Range", content being "" + evaluate expect-code with label being "present empty If-Range falls back to full content", fields being fields, + verb being request-method.Get, expected being 200 + set fields to add-field with fields being (set-field with fields being range, name being "If-Range", content being tag), + name being "If-Range", content being tag + evaluate expect-code with label being "repeated If-Range cannot authorize a partial response", fields being fields, + verb being request-method.Get, expected being 200 + evaluate expect-code with label being "If-Range alone has no effect", fields being + (one-field with name being "If-Range", content being stale), verb being request-method.Get, expected being 200 + + value head is asset with fields being past-end, verb being request-method.Head + evaluate same-number with label being "HEAD ignores an unsatisfiable Range", actual being head.code, expected being 200 + evaluate same-text with label being "HEAD has no Content-Range", + actual being (field-value with fields being head.fields, name being "Content-Range"), expected being "" + evaluate same-text with label being "HEAD uses the GET ETag", + actual being (field-value with fields being head.fields, name being "ETag"), expected being tag + value wire is render-response with response being head, verb being request-method.Head + evaluate is-true with label being "HEAD advertises the complete GET byte count", + actual being (nil.std.text::contains with s being wire, needle being "Content-Length: 31" plus (crlf with nothing)) + evaluate is-true with label being "HEAD ends at the header boundary without content", + actual being (nil.std.text::ends_with with s being wire, suffix being (crlf with nothing) plus (crlf with nothing)) + evaluate expect-code with label being "HEAD ignores a satisfiable Range", fields being range, + verb being request-method.Head, expected being 200 + set fields to set-field with fields being range, name being "If-None-Match", content being tag + value cached-head is asset with fields being fields, verb being request-method.Head + evaluate same-number with label being "HEAD still evaluates preconditions", actual being cached-head.code, expected being 304 + evaluate is-true with label being "conditional HEAD 304 omits Content-Length", + actual being not (nil.std.text::contains with s being (render-response with response being cached-head, + verb being request-method.Head), needle being "Content-Length") + evaluate expect-code with label being "POST ignores Range", fields being past-end, + verb being request-method.Post, expected being 200 + + evaluate expect-code with label being "range-unit matching is case insensitive", fields being + (one-field with name being "Range", content being "ByTeS=0-5"), verb being request-method.Get, expected being 206 + set fields to add-field with fields being range, name being "Range", content being "bytes=6-9" + evaluate expect-code with label being "multiple Range fields fall back to full content", fields being fields, + verb being request-method.Get, expected being 200 + evaluate is-true with label being "a suffix on an empty representation is ignored", + actual being not (range-of with stated being "bytes=-5", width being 0).present + evaluate is-true with label being "an explicit range on an empty representation is ignored", + actual being not (range-of with stated being "bytes=0-0", width being 0).present + evaluate same-number with label being "huge valid suffix clamps without decimal overflow", + actual being (range-of with stated being "bytes=-9999999999999999999999999999999999", width being 31).first-byte, expected being 0 + evaluate same-number with label being "huge valid last offset clamps without decimal overflow", + actual being (range-of with stated being "bytes=5-9999999999999999999999999999999999", width being 31).last-byte, expected being 30 + evaluate is-true with label being "huge first offset remains unsatisfiable", + actual being not (range-of with stated being "bytes=9999999999999999999999999999999999-", width being 31).satisfiable + evaluate is-true with label being "digits following saturation are still validated", + actual being not (range-of with stated being "bytes=-9999999999999999999999999999999999x", width being 31).satisfiable + evaluate is-true with label being "whitespace inside a byte interval is invalid", + actual being not (range-of with stated being "bytes=0 - 5", width being 31).satisfiable + + value missing-request is make-request with verb being request-method.Get, target being "/missing.css", + fields being (one-field with name being "If-None-Match", content being "*"), body being "" + evaluate same-number with label being "ordinary missing-file errors precede conditionals", + actual being (serve-static with root being "tests/public", path being missing-request.path, + request being missing-request, seconds being 60).code, expected being 404 + evaluate file-body-tests with nothing + give nothing +end paragraph + + +!!! Own one scratch file only, refuse a pre-existing path, and remove it again. +!!! These are body/framing checks, separate from the containment contract. +file-body-tests returns nothing begin paragraph + value scratch is "build/static-http-contract-fixture.bin" + when nil.std.io::exists with path being scratch then + evaluate is-true with label being "static HTTP scratch fixture is not already in use", actual being false + give nothing + end when + value binary is "A" plus (nil.std.text::from_byte with code being 0) + plus (nil.std.text::from_byte with code being 255) plus (nil.std.text::from_byte with code being 128) plus "Z" + value wrote is write-fixture with path being scratch, body being binary + evaluate is-true with label being "create an owned binary static fixture", actual being wrote + when not wrote then give nothing end when + value plain is make-request with verb being request-method.Get, target being "/static-http-contract-fixture.bin", + fields being (no-headers with nothing), body being "" + value whole is serve-static with root being "build", path being plain.path, request being plain, seconds being 60 + evaluate same-number with label being "binary file serves successfully", actual being whole.code, expected being 200 + evaluate same-text with label being "binary file body preserves NUL and high bytes", actual being whole.body, expected being binary + evaluate same-text with label being "binary ETag hashes bytes without UTF-8 reinterpretation", + actual being (field-value with fields being whole.fields, name being "ETag"), + expected being quote plus "2d013a390aa422ff459b7fcf793fd75df0d46f6cb7e4312a7f0b5cfc568250dd" plus quote + value ranged is make-request with verb being request-method.Get, target being plain.path, + fields being (one-field with name being "Range", content being "bytes=1-3"), body being "" + value partial is serve-static with root being "build", path being plain.path, request being ranged, seconds being 60 + evaluate same-number with label being "binary Range returns partial content", actual being partial.code, expected being 206 + evaluate same-text with label being "binary Range slices bytes including NUL", actual being partial.body, + expected being (nil.std.text::substring with s being binary, start being 1, length being 3) + evaluate is-true with label being "binary partial Content-Length counts bytes", + actual being (nil.std.text::contains with s being (render-for-get with response being partial), + needle being "Content-Length: 3" plus (crlf with nothing)) + + value replaced is write-fixture with path being scratch, body being "B" plus + (nil.std.text::substring with s being binary, start being 1, length being 4) + evaluate is-true with label being "rewrite fixture without changing its length", actual being replaced + value changed is serve-static with root being "build", path being plain.path, request being plain, seconds being 60 + evaluate is-true with label being "same-size file edit changes the strong ETag", + actual being (field-value with fields being changed.fields, name being "ETag") + does not equal (field-value with fields being whole.fields, name being "ETag") + + value emptied is write-fixture with path being scratch, body being "" + evaluate is-true with label being "empty the owned static fixture", actual being emptied + value suffix is make-request with verb being request-method.Get, target being plain.path, + fields being (one-field with name being "Range", content being "bytes=-5"), body being "" + value empty is serve-static with root being "build", path being plain.path, request being suffix, seconds being 60 + evaluate same-number with label being "empty file ignores suffix range with a full 200", actual being empty.code, expected being 200 + evaluate same-text with label being "empty file has no invalid Content-Range", + actual being (field-value with fields being empty.fields, name being "Content-Range"), expected being "" + evaluate is-true with label being "empty response has zero Content-Length", + actual being (nil.std.text::contains with s being (render-for-get with response being empty), + needle being "Content-Length: 0" plus (crlf with nothing)) + + variable large-body as text is "x" + variable doubling as number is 0 + repeat while doubling is less than 21 + set large-body to large-body plus large-body + set doubling to doubling plus 1 + end repeat + value enlarged is write-fixture with path being scratch, body being large-body plus "x" + evaluate is-true with label being "create the bounded oversized fixture", actual being enlarged + value star is make-request with verb being request-method.Get, target being plain.path, + fields being (one-field with name being "If-None-Match", content being "*"), body being "" + evaluate same-number with label being "ordinary size refusal precedes a matching wildcard", + actual being (serve-static with root being "build", path being plain.path, request being star, seconds being 60).code, + expected being 413 + value removed is nil.std.io::delete with path being scratch + evaluate is-true with label being "remove the owned static HTTP fixture", + actual being not (nil.std.io::exists with path being scratch) + give nothing +end paragraph + + +write-fixture receives path as text, body as text returns boolean begin paragraph + value result is nil.std.io::write_text with path being path, content being body + consider result + when Result.Ok holding written then give written + otherwise give false + end consider +end paragraph diff --git a/tests/run-tests.matrix b/tests/run-tests.matrix index 818e37d..7a960cb 100644 --- a/tests/run-tests.matrix +++ b/tests/run-tests.matrix @@ -4,6 +4,9 @@ book nilweb.tests.run read everything from nilweb.tests.harness +read everything from nilweb.tests.sha256 +read everything from nilweb.tests.static_http +read everything from nilweb.tests.framing read nil.std.core read nil.std.text read everything from nilweb.text.scan @@ -738,7 +741,7 @@ static-tests returns nothing begin paragraph actual being ((field-value with fields being (serve-cached with root being "tests/public", path being "/nope.css", seconds being 60).fields, name being "Cache-Control")) equals "" - !!! Validators: size and timestamp, without reading the file. + !!! Validators identify the bytes served. value marks is validators-of with root being "tests/public", path being "/assets/app.css" evaluate is-true with label being "a present file has validators", actual being marks.found evaluate same-number with label being "the width matches the file", actual being marks.width, expected being 31 @@ -917,8 +920,8 @@ listener-tests returns nothing begin paragraph actual being not (request-is-complete with message being "GET / HTTP/1.1" plus (crlf with nothing) plus "Host: x") evaluate is-true with label being "nothing at all is incomplete", actual being not (request-is-complete with message being "") - evaluate is-true with label being "a bare-LF request is recognised", - actual being (request-is-complete with message being "GET / HTTP/1.1" plus newline plus "Host: x" + evaluate is-true with label being "a bare-LF request is rejected", + actual being not (request-is-complete with message being "GET / HTTP/1.1" plus newline plus "Host: x" plus newline plus newline) value header-part is "POST /f HTTP/1.1" plus (crlf with nothing) plus "Content-Length: 9" plus marker @@ -931,14 +934,14 @@ listener-tests returns nothing begin paragraph evaluate is-true with label being "Content-Length folds case like any field", actual being (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) plus "content-length: 2" plus marker plus "ab") - evaluate is-true with label being "a Content-Length that is not a number does not stall the loop", - actual being (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) + evaluate is-true with label being "a nonnumeric Content-Length is not a valid complete frame", + actual being not (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) plus "Content-Length: banana" plus marker) - evaluate is-true with label being "a negative Content-Length does not stall the loop", - actual being (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) + evaluate is-true with label being "a negative Content-Length is not a valid complete frame", + actual being not (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) plus "Content-Length: -5" plus marker) - evaluate is-true with label being "an absurd Content-Length does not stall the loop", - actual being (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) + evaluate is-true with label being "an oversized Content-Length is not a valid complete frame", + actual being not (request-is-complete with message being "POST /f HTTP/1.1" plus (crlf with nothing) plus "Content-Length: 99999999999999999999" plus marker) evaluate same-text with label being "the startup line names a loopback address", @@ -1517,8 +1520,11 @@ main returns number begin paragraph evaluate template-tests with nothing evaluate session-tests with nothing evaluate static-tests with nothing + evaluate run-sha256-tests with nothing + evaluate run-static-http-tests with nothing evaluate middleware-tests with nothing evaluate listener-tests with nothing + evaluate run-framing-tests with nothing evaluate clock-tests with nothing evaluate log-tests with nothing evaluate config-tests with nothing diff --git a/tests/static-server.matrix b/tests/static-server.matrix new file mode 100644 index 0000000..1be6b46 --- /dev/null +++ b/tests/static-server.matrix @@ -0,0 +1,53 @@ +!!! Bounded owned fixture for test_static_snapshot.py; never a demo server. +book nilweb.static.fixture + +read nil.std.core +read nil.std.system.environment +read everything from nilweb.http.request +read everything from nilweb.http.response +read everything from nilweb.http.headers +read everything from nilweb.server.engine +read everything from nilweb.static.files + +main returns number begin paragraph + variable port as number is 0 + variable root as text is "" + variable token as text is "" + value configured-port is nil.std.system::get_environment_variable with name being "NILWEB_STATIC_PORT" + consider configured-port + when Option.Some holding digits then set port to digits becomes number + otherwise give 2 + end consider + value configured-root is nil.std.system::get_environment_variable with name being "NILWEB_STATIC_ROOT" + consider configured-root + when Option.Some holding path then set root to path + otherwise give 2 + end consider + value configured-token is nil.std.system::get_environment_variable with name being "NILWEB_STATIC_TOKEN" + consider configured-token + when Option.Some holding secret then set token to secret + otherwise give 2 + end consider + when root equals "" or token equals "" then give 2 end when + value server is start-engine with host being "127.0.0.1", port being port, limit being 8 + when not server.live then give 3 end when + value started is now in milliseconds + variable running as boolean is true + repeat while running and ((now in milliseconds) minus started) is less than 90000 + value ready is pump with server being server, timeout being 10 + when ready.found then + value request is parse-request with raw being ready.request + value control is request.path equals "/__probe_" plus token or request.path equals "/__shutdown_" plus token + variable answer is text-response with body being token + when not control then + set answer to serve-static with root being root, path being request.path, request being request, seconds being 60 + end when + set answer to with-header with response being answer, name being "Connection", content being "close" + evaluate reply with server being server, handle being ready.handle, + response being (render-response with response being answer, verb being request.verb), keep-open being false + when request.path equals "/__shutdown_" plus token then set running to false end when + end when + end repeat + evaluate stop-engine with server being server + give 0 +end paragraph diff --git a/tests/test_audit_paths.py b/tests/test_audit_paths.py new file mode 100644 index 0000000..ea9e2e2 --- /dev/null +++ b/tests/test_audit_paths.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Synthetic public-source audit controls; no real personal data or credentials.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "audit-paths.py" +SPEC = importlib.util.spec_from_file_location("audit_paths", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +AUDIT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(AUDIT) + + +def ipv4(*parts: int) -> str: + return ".".join(map(str, parts)) + + +def ipv6(*parts: str) -> str: + return ":".join(parts) + + +class AuditTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(prefix="nil-web-audit-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.file = self.root / "fixture.txt" + + def scan(self, body: str | bytes, name: str = "fixture.txt") -> list[str]: + self.file = self.root / name + self.file.parent.mkdir(parents=True, exist_ok=True) + self.file.write_bytes(body.encode() if isinstance(body, str) else body) + with patch.object(AUDIT, "tracked_files", return_value=[self.file]): + return AUDIT.audit(self.root, set()) + + def rule(self, body: str | bytes, rule: str) -> None: + findings = self.scan(body) + self.assertTrue(any(item.endswith(": " + rule) for item in findings)) + for item in findings: + self.assertEqual(len(item.splitlines()), 1) + + def test_posix_and_root_homes(self) -> None: + for parent in ("home", "Users", "root"): + with self.subTest(parent=parent): + self.rule("/" + parent + "/audit-fixture/project", "home-path") + + def test_unicode_home(self) -> None: + self.rule("/" + "home/\u00e9quipe/project", "home-path") + + def test_windows_user_paths(self) -> None: + for directory in ("Users", "Documents"): + for separator in ("/", "\\", "\\\\"): + with self.subTest(directory=directory, separator=separator): + self.rule("C:" + separator + directory + separator + "fixture", + "windows-user-path") + + def test_private_and_global_ipv4(self) -> None: + for parts in ((10, 73, 91, 17), (172, 19, 4, 8), (192, 168, 47, 8), + (8, 8, 4, 4)): + self.rule(ipv4(*parts), "non-documentation-address") + + def test_compressed_ipv6_and_sentence_punctuation(self) -> None: + for address in (ipv6("fd73", "91ab", "", "17"), + ipv6("2606", "4700", "", "1111")): + for decorated in (address, "[" + address + "]:443", address + "."): + self.rule(decorated, "non-documentation-address") + + def test_full_and_scoped_ipv6(self) -> None: + self.rule(ipv6("fd73", "91ab", "0", "0", "0", "0", "0", "17"), + "non-documentation-address") + self.rule(ipv6("fe80", "", "17") + "%fixture", "non-documentation-address") + + def test_mapped_ipv4_classification(self) -> None: + self.rule("::ffff:" + ipv4(10, 73, 91, 17), "non-documentation-address") + for address in ("127.0.0.1", "192.0.2.8"): + self.assertEqual(self.scan("::ffff:" + address), []) + + def test_documentation_and_binding_addresses(self) -> None: + for address in ("127.0.0.1", "127.23.0.9", "0.0.0.0", "255.255.255.255", + "192.0.2.8", "198.51.100.8", "203.0.113.8", "::", "::1", + "2001:db8::8", "[::1]:8080"): + with self.subTest(address=address): + self.assertEqual(self.scan(address), []) + + def test_public_documentation_urls_and_templates(self) -> None: + body = "https://example.com/docs contact@example.org name@type.matrix name@file.nilhtml" + self.assertEqual(self.scan(body), []) + + def test_nonreserved_contact(self) -> None: + self.rule("fixture" + "@" + "audit.invalid", "contact-address") + + def test_invalid_ip_like_text(self) -> None: + self.assertEqual(self.scan("999.888.777.666 release 1.2.3 12:34:56"), []) + + def test_ambiguous_four_part_version_is_conservatively_reported(self) -> None: + self.rule("version " + ipv4(1, 2, 3, 4), "non-documentation-address") + + def test_diagnostics_do_not_include_matching_content(self) -> None: + payload = "/" + "home/audit-fixture/private-name" + findings = self.scan("ordinary line\n" + payload) + self.assertEqual(findings, ['"fixture.txt":2: home-path']) + self.assertNotIn(payload, "\n".join(findings)) + + def test_non_utf8_fails_closed(self) -> None: + self.rule(b"\xff synthetic bytes", "non-utf8-file") + + def test_nul_bytes_fail_closed(self) -> None: + self.rule(b"text\x00synthetic bytes", "binary-file") + + def test_unreadable_file_fails_closed_without_exception_details(self) -> None: + self.file.write_text("safe") + with patch.object(AUDIT, "tracked_files", return_value=[self.file]), \ + patch.object(Path, "read_bytes", side_effect=PermissionError("private-detail")): + self.assertEqual(AUDIT.audit(self.root, set()), + ['"fixture.txt":0: unreadable-file']) + + def test_missing_tracked_file_fails_closed(self) -> None: + with patch.object(AUDIT, "tracked_files", return_value=[self.file]): + self.assertEqual(AUDIT.audit(self.root, set()), + ['"fixture.txt":0: unreadable-file']) + + def test_directory_is_not_silently_skipped(self) -> None: + self.file.mkdir() + with patch.object(AUDIT, "tracked_files", return_value=[self.file]): + self.assertEqual(AUDIT.audit(self.root, set()), + ['"fixture.txt":0: non-regular-file']) + + @unittest.skipUnless(hasattr(os, "symlink"), "platform has no symlink support") + def test_symlink_target_is_never_read(self) -> None: + target = self.root / "target.txt" + target.write_text("safe") + try: + self.file.symlink_to(target) + except OSError: + self.skipTest("symlink creation is unavailable for this account") + with patch.object(AUDIT, "tracked_files", return_value=[self.file]), \ + patch.object(Path, "read_bytes", side_effect=AssertionError("target read")): + self.assertEqual(AUDIT.audit(self.root, set()), + ['"fixture.txt":0: symlink-file']) + + def test_real_git_unicode_and_control_filenames(self) -> None: + subprocess.run(["git", "init", "--quiet", str(self.root)], check=True, + capture_output=True) + names = ["unicode-\u00e9.txt", "space name.txt"] + if os.name != "nt": + names += ["line\nbreak.txt", "control\x1b.txt", "tab\tname.txt"] + for name in names: + (self.root / name).write_text("/" + "home/audit-fixture/project") + subprocess.run(["git", "add", "--all"], cwd=self.root, check=True, + capture_output=True) + self.assertEqual(set(AUDIT.tracked_files(self.root)), + {self.root / name for name in names}) + self.assertEqual(set(AUDIT.audit(self.root, set())), + {f"{json.dumps(name, ensure_ascii=True)}:1: home-path" + for name in names}) + + def test_auditor_path_has_no_implicit_exemption(self) -> None: + findings = self.scan("/" + "home/audit-fixture/project", "scripts/audit-paths.py") + self.assertEqual(findings, ['"scripts/audit-paths.py":1: home-path']) + with patch.object(AUDIT, "tracked_files", return_value=[self.file]), \ + patch.object(sys, "argv", ["audit-paths", "--root", str(self.root)]), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(AUDIT.main(), 1) + + def test_explicit_skip_is_exact(self) -> None: + self.file.write_bytes(b"\xff") + with patch.object(AUDIT, "tracked_files", return_value=[self.file]): + self.assertEqual(AUDIT.audit(self.root, {"fixture.txt"}), []) + self.assertNotEqual(AUDIT.audit(self.root, {"fixture"}), []) + + def test_git_failure_has_no_command_output(self) -> None: + with patch.object(AUDIT, "tracked_files", side_effect=subprocess.CalledProcessError( + 1, ["git"], stderr=b"private-detail")), \ + patch.object(sys, "argv", ["audit-paths", "--root", str(self.root)]), \ + contextlib.redirect_stderr(io.StringIO()) as output: + self.assertEqual(AUDIT.main(), 1) + self.assertNotIn("private-detail", output.getvalue()) + + def test_audit_script_and_controls_pass_their_own_policy(self) -> None: + paths = [SCRIPT, Path(__file__).resolve()] + with patch.object(AUDIT, "tracked_files", return_value=paths): + self.assertEqual(AUDIT.audit(SCRIPT.parents[1], set()), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_http_wire_oracle.py b/tests/test_http_wire_oracle.py new file mode 100644 index 0000000..c8469fc --- /dev/null +++ b/tests/test_http_wire_oracle.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Deterministic socket/clock controls for the real HTTP wire test's oracle.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path +import socket +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location( + "http_wire_oracle", Path(__file__).resolve().parents[1] / "scripts/test-http-wire.py") +assert SPEC is not None and SPEC.loader is not None +WIRE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(WIRE) + + +def aborted(code=10053): + error = ConnectionAbortedError("synthetic Winsock abort") + error.winerror = code + return error + + +class SocketFixture: + def __init__(self, outcome=b"", send_error=None, send_at=21): + self.outcome = outcome + self.send_error = send_error + self.send_at = send_at + self.sent = [] + self.clock = 0.0 + + def recv(self, _): + if isinstance(self.outcome, BaseException): + raise self.outcome + return self.outcome + + def sendall(self, data): + self.sent.append((self.clock, data)) + if self.send_error is not None and self.clock >= self.send_at: + raise self.send_error + + +def peer_for(sock): + peer = object.__new__(WIRE.Peer) + peer.socket = sock + peer.buffer = b"" + return peer + + +class WireOracleTests(unittest.TestCase): + def closed(self, outcome, host="nt", buffered=b""): + peer = peer_for(SocketFixture(outcome)) + peer.buffer = buffered + with patch.object(WIRE.os, "name", host): + peer.closed() + + def deadline(self, *, close_at=20.0, outcome=b"", host="nt", send_error=None, + send_at=21, selection_error=None): + sock = SocketFixture(outcome, send_error, send_at) + + def select(read, write, errors, timeout): + self.assertEqual(read, [sock]) + self.assertEqual(write, []) + self.assertEqual(errors, []) + self.assertGreaterEqual(timeout, 0) + if selection_error is not None: + raise selection_error + until = sock.clock + timeout + if close_at is not None and close_at <= until: + sock.clock = close_at + return [sock], [], [] + sock.clock = until + return [], [], [] + + with patch.object(WIRE.os, "name", host), \ + patch.object(WIRE.time, "monotonic", side_effect=lambda: sock.clock), \ + patch.object(WIRE.select, "select", side_effect=select): + WIRE.check_first_byte_deadline(peer_for(sock)) + return sock + + def test_eof_and_reset_are_closed_on_each_host(self): + for host in ("nt", "posix"): + for outcome in (b"", ConnectionResetError("synthetic reset")): + with self.subTest(host=host, outcome=type(outcome).__name__): + self.closed(outcome, host) + + def test_exact_windows_abort_is_closed(self): + self.closed(aborted()) + + def test_windows_abort_is_not_accepted_on_posix(self): + with self.assertRaises(ConnectionAbortedError): + self.closed(aborted(), "posix") + + def test_different_abort_codes_and_generic_errors_propagate(self): + for error in (aborted(10052), ConnectionAbortedError("no Winsock code"), + OSError(10053, "generic code is insufficient"), OSError("unrelated")): + with self.subTest(error=repr(error)), self.assertRaises(type(error)): + self.closed(error) + + def test_receive_timeout_is_not_a_close(self): + with self.assertRaises(TimeoutError): + self.closed(socket.timeout("still open")) + + def test_extra_wire_or_buffered_bytes_are_not_a_close(self): + for received, buffered in ((b"x", b""), (b"", b"previous response bytes")): + with self.subTest(buffered=bool(buffered)), self.assertRaises(AssertionError): + self.closed(received, buffered=buffered) + + def test_deadline_accepts_eof_with_continuing_fragments(self): + sock = self.deadline() + self.assertEqual([when for when, _ in sock.sent], [0, 3, 6, 9, 12, 15, 18]) + self.assertEqual([body for _, body in sock.sent[1:]], [b"x"] * 6) + self.assertEqual(sock.clock, 20) + + def test_deadline_accepts_reset_and_windows_abort_at_read(self): + for outcome in (ConnectionResetError("synthetic reset"), aborted()): + with self.subTest(outcome=type(outcome).__name__): + self.assertEqual(self.deadline(outcome=outcome).clock, 20) + + def test_deadline_accepts_closed_send_and_confirms_closed_receive(self): + for error in (BrokenPipeError("synthetic pipe"), ConnectionResetError("reset"), aborted()): + with self.subTest(error=type(error).__name__): + self.assertEqual(self.deadline(close_at=None, send_error=error, + outcome=aborted()).clock, 21) + + def test_deadline_send_does_not_hide_unrelated_errors_or_timeouts(self): + for error in (aborted(10052), OSError("unrelated"), socket.timeout("send timed out")): + with self.subTest(error=repr(error)), self.assertRaises(type(error)): + self.deadline(close_at=None, send_error=error) + + def test_deadline_does_not_accept_windows_abort_on_posix(self): + for send in (False, True): + with self.subTest(send=send), self.assertRaises(ConnectionAbortedError): + self.deadline(close_at=None if send else 20, host="posix", + send_error=aborted() if send else None, outcome=aborted()) + + def test_deadline_rejects_early_eof_reset_abort_and_send_failure(self): + for outcome in (b"", ConnectionResetError("reset"), aborted()): + with self.subTest(outcome=type(outcome).__name__), self.assertRaisesRegex(AssertionError, "19–25"): + self.deadline(close_at=18.9, outcome=outcome) + with self.assertRaisesRegex(AssertionError, "19–25"): + self.deadline(close_at=None, send_error=aborted(), send_at=3) + + def test_deadline_accepts_lower_and_upper_bound(self): + for close_at in (19, 25): + with self.subTest(close_at=close_at): + self.assertEqual(self.deadline(close_at=close_at).clock, close_at) + + def test_deadline_rejects_open_or_late_connection(self): + for close_at in (None, 25.1): + with self.subTest(close_at=close_at), self.assertRaisesRegex(AssertionError, "extended"): + self.deadline(close_at=close_at) + + def test_deadline_rejects_response_data_and_receive_timeout(self): + with self.assertRaises(AssertionError): + self.deadline(outcome=b"unexpected HTTP response") + with self.assertRaises(TimeoutError): + self.deadline(outcome=socket.timeout("readiness was not EOF")) + with self.assertRaises(TimeoutError): + self.deadline(close_at=None, send_error=aborted(), outcome=socket.timeout("still open")) + + def test_selection_error_is_not_a_close(self): + with self.assertRaises(OSError): + self.deadline(selection_error=OSError("select failed")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_static_snapshot.py b/tests/test_static_snapshot.py new file mode 100644 index 0000000..8f23f44 --- /dev/null +++ b/tests/test_static_snapshot.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Real bounded loopback static responses from explicit candidate SDK tools. + +Each consumer is compiled independently for every --nilstd root. All files, +links, ports and subprocesses belong to a newly created external artifact +folder. No existing server or installed payload is modified. With +--parent-process-group, an outer driver owns the process group and this harness +reaps only its direct server PID. +""" +from __future__ import annotations + +import argparse +import ctypes +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import socket +import struct +import subprocess +import sys +import time +import uuid +from urllib.parse import quote + +REPO = Path(__file__).resolve().parents[1] +sys.dont_write_bytecode = True +_SPEC = importlib.util.spec_from_file_location("nilweb_wire_helpers", REPO / "scripts/test-http-wire.py") +assert _SPEC and _SPEC.loader +WIRE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(WIRE) + + +class StaticPeer(WIRE.Peer): + deadline: float | None = None + + def remaining(self): + if self.deadline is not None: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("absolute response deadline exceeded") + self.socket.settimeout(remaining) + + def receive(self): + self.remaining() + super().receive() + + def closed(self): + self.remaining() + super().closed() + + def response(self, head: bool = False) -> tuple[int, dict[bytes, bytes], bytes]: + while b"\r\n\r\n" not in self.buffer: + if len(self.buffer) > 65536: + raise AssertionError("response header bound exceeded") + self.receive() + header, self.buffer = self.buffer.split(b"\r\n\r\n", 1) + lines = header.split(b"\r\n") + status = int(lines[0].split()[1]) + fields = {} + for line in lines[1:]: + name, value = line.split(b":", 1) + name = name.lower() + if name in fields: + raise AssertionError("duplicate response header") + fields[name] = value.strip() + no_body = head or status in (204, 304) or 100 <= status < 200 + size = 0 if no_body else int(fields[b"content-length"]) + if not 0 <= size <= 3 * 1024 * 1024: + raise AssertionError("response body bound exceeded") + while len(self.buffer) < size: + self.receive() + payload, self.buffer = self.buffer[:size], self.buffer[size:] + return status, fields, payload + + +def exchange(port: int, target: str, *, fields: dict[str, str] | None = None, + method: str = "GET", deadline: float = 5): + header = b"Connection: close\r\n" + b"".join( + name.encode("ascii") + b": " + value.encode("ascii") + b"\r\n" + for name, value in (fields or {}).items()) + with StaticPeer(port) as peer: + peer.deadline = time.monotonic() + deadline + peer.socket.settimeout(deadline) + peer.send(WIRE.request(quote(target, safe="/%").encode("ascii"), header, method=method.encode("ascii"))) + answer = peer.response(head=method == "HEAD") + peer.closed() + return answer + + +def entity_tag(data: bytes) -> bytes: + return b'"' + hashlib.sha256(data).hexdigest().encode("ascii") + b'"' + + +def require(good: bool, message: str): + if not good: + raise AssertionError(message) + + +def directory_link(path: Path, target: Path): + if os.name != "nt": + path.symlink_to(target, target_is_directory=True) + return + # A junction is an actual reparse point and requires no symlink privilege. + # Every target and link is inside this harness's fresh artifact directory. + from ctypes import wintypes + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, + wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE] + kernel.CreateFileW.restype = wintypes.HANDLE + kernel.DeviceIoControl.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.LPVOID, + wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD, ctypes.POINTER(wintypes.DWORD), wintypes.LPVOID] + kernel.DeviceIoControl.restype = wintypes.BOOL + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + kernel.CloseHandle.restype = wintypes.BOOL + shown = str(target.resolve()).encode("utf-16-le") + substitute = ("\\??\\" + str(target.resolve())).encode("utf-16-le") + names = substitute + b"\0\0" + shown + b"\0\0" + raw = struct.pack(" int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler", required=True, type=Path) + parser.add_argument("--interpreter", required=True, type=Path) + parser.add_argument("--nilstd", required=True, type=Path, action="append") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--parent-process-group", action="store_true", + help="inherit the outer driver process group; clean up only the owned server PID") + parser.add_argument("--benchmark-deadline", type=float, default=10) + args = parser.parse_args() + if not 0 < args.benchmark_deadline <= 30: + parser.error("--benchmark-deadline must be positive and at most 30 seconds") + args.output = args.output.resolve() + if args.output == REPO or REPO in args.output.parents: + parser.error("--output must be outside the source checkout") + args.output.mkdir(parents=True, exist_ok=False) + sources = sorted((REPO / "src").rglob("*.matrix")) + [Path(__file__).resolve(), REPO / "tests/static-server.matrix", REPO / "scripts/test-http-wire.py"] + pins = {str(path.relative_to(REPO)): WIRE.digest(path) for path in sources} + tools = {str(path.resolve()): WIRE.digest(path) for path in (args.compiler, args.interpreter)} + report = {"schema": "nilweb.static-snapshot-wire.v1", "status": "FAIL", "source_sha256": pins, + "tools_sha256": tools, "roots": [], "parent_process_group": args.parent_process_group} + try: + for number, std in enumerate(args.nilstd): + std = std.resolve(strict=True) + before = WIRE.pin_tree(std) + output = args.output / str(number) + output.mkdir() + row = {"nilstd": str(std), "nilstd_before": before, "checks": [], "cleanup_verified": False} + report["roots"].append(row) + root, binary, large, stamp = prepare(output) + row["link_kind"] = "junction" if os.name == "nt" else "symlink" + env = {key: value for key, value in os.environ.items() if not key.startswith(("NIL_", "NY_", "NILWEB_"))} + env.update({"TMPDIR": str(output), "TMP": str(output), "TEMP": str(output)}) + bytecode = output / "static.nil" + command = [str(args.compiler.resolve()), "--no-bundle", "-I", str(std), "-I", str(REPO / "src"), + "--output", str(bytecode), str(REPO / "tests/static-server.matrix")] + with (output / "compile.stdout").open("wb") as stdout, (output / "compile.stderr").open("wb") as stderr: + result = subprocess.run(command, cwd=output, env=env, stdout=stdout, stderr=stderr, timeout=90) + row["compile_exit"] = result.returncode + if result.returncode: + raise AssertionError("static fixture compilation failed; see compiler logs") + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + token = uuid.uuid4().hex + env.update({"NILWEB_STATIC_PORT": str(port), "NILWEB_STATIC_ROOT": str(root), "NILWEB_STATIC_TOKEN": token}) + row["port"] = port + with (output / "server.stdout").open("wb") as stdout, (output / "server.stderr").open("wb") as stderr: + process = subprocess.Popen([str(args.interpreter.resolve()), str(bytecode)], cwd=output, env=env, + stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, start_new_session=os.name == "posix" and not args.parent_process_group) + row["pid"] = process.pid + try: + until = time.monotonic() + 10 + while True: + if process.poll() is not None: + raise AssertionError("owned static fixture exited before readiness") + try: + status, _, body = exchange(port, "/__probe_" + token) + require(status == 200 and body == token.encode(), "readiness did not identify this owned fixture") + break + except ConnectionRefusedError: + if time.monotonic() >= until: + raise AssertionError("owned static fixture readiness timed out") + time.sleep(0.05) + exercise(port, output, row, binary, large, stamp, args.benchmark_deadline) + status, _, body = exchange(port, "/__shutdown_" + token) + require(status == 200 and body == token.encode(), "owned fixture shutdown acknowledgement differs") + process.wait(timeout=5) + row["server_exit"] = process.returncode + require(process.returncode == 0, "owned fixture exit was not clean") + finally: + row["cleanup_verified"] = WIRE.stop_owned(process, own_group=not args.parent_process_group) + row["nilstd_unchanged"] = before == WIRE.pin_tree(std) + require(row["cleanup_verified"] and row["nilstd_unchanged"], "child cleanup or standard-library integrity failed") + report["status"] = "PASS" + except Exception as error: + report["error"] = str(error) + finally: + for row in report["roots"]: + row["nilstd_unchanged"] = row["nilstd_before"] == WIRE.pin_tree(Path(row["nilstd"])) + if not row["nilstd_unchanged"]: + report["status"] = "FAIL" + report["source_unchanged"] = pins == {str(path.relative_to(REPO)): WIRE.digest(path) for path in sources} + report["tools_unchanged"] = tools == {str(path.resolve()): WIRE.digest(path) for path in (args.compiler, args.interpreter)} + if not report["source_unchanged"] or not report["tools_unchanged"]: + report["status"] = "FAIL" + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"status": report["status"], "checks": sum(len(row["checks"]) for row in report["roots"]), + "report": str(args.output / "report.json"), "error": report.get("error")})) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/wire-server.matrix b/tests/wire-server.matrix new file mode 100644 index 0000000..1cbc050 --- /dev/null +++ b/tests/wire-server.matrix @@ -0,0 +1,58 @@ +!!! Owned loopback fixture for scripts/test-http-wire.py. Never a demo server. +book nilweb.wire.fixture + +read nil.std.core +read nil.std.text +read nil.std.system.environment +read everything from nilweb.http.headers +read everything from nilweb.http.request +read everything from nilweb.server.engine + +main returns number begin paragraph + variable port as number is 0 + value configured is nil.std.system::get_environment_variable with name being "NILWEB_WIRE_PORT" + consider configured + when Option.Some holding digits then set port to digits becomes number + otherwise give 2 + end consider + variable server is start-engine with host being "127.0.0.1", port being port, limit being 32 + when not server.live then give 3 end when + value started is now in milliseconds + variable running as boolean is true + repeat while running and ((now in milliseconds) minus started) is less than 90000 + value ready is pump with server being server, timeout being 10 + when ready.found then + value request is parse-request with raw being ready.request + value ending is crlf with nothing + variable body as text is request.target plus "|" plus request.body + when request.target equals "/large" then + set body to "x" + variable doubling as number is 0 + repeat while doubling is less than 23 + set body to body plus body + set doubling to doubling plus 1 + end repeat + end when + !!! Age only the buffered next request, after its bytes have arrived. + !!! The reply/drain path must preserve this timestamp, and pump must + !!! close it before dispatch. No real clock or process is changed. + when request.target equals "/age-next" or request.target equals "/age-soon" then + repeat for each held in server.connections + when held.handle equals ready.handle and not (held.gathered equals "") then + value age is when request.target equals "/age-soon" give 19000 otherwise give 21000 end when + set held.started-at to (now in milliseconds) minus age + end when + end repeat + end when + value closing is when ready.keep-alive give "keep-alive" otherwise give "close" end when + value response is "HTTP/1.1 200 OK" plus ending plus "Content-Length: " + plus ((nil.std.text::length with s being body) becomes text) plus ending + plus "Connection: " plus closing plus ending plus ending plus body + evaluate reply with server being server, handle being ready.handle, + response being response, keep-open being ready.keep-alive + when request.target equals "/__shutdown" then set running to false end when + end when + end repeat + evaluate stop-engine with server being server + give 0 +end paragraph