From 93515c3d4c2b6e0e7c3b870eb283fce26392d613 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Wed, 11 Nov 2026 23:58:39 -0500 Subject: [PATCH 1/4] Add anchored file snapshots with binary-safe content identities --- .github/workflows/web-runtime-parity.yml | 10 +- CMakeLists.txt | 14 + docs/standard-library/index.md | 3 + .../standard-library/rooted-file-snapshots.md | 52 +++ docs/standard-library/text-spans.md | 14 +- nilcompiler/src/llvm_backend.cpp | 106 ++++++ nilcompiler/src/native_runtime.cpp | 43 +++ nilcompiler/src/native_runtime.hpp | 6 + nilcompiler/src/secure_file_io.cpp | 254 ++++++++++++++ nilcompiler/src/secure_file_io.hpp | 20 ++ nilcompiler/src/vm.cpp | 90 +++++ nilstd/src/io/file.ny | 22 ++ nilstd/src/text/string_utils.ny | 10 +- .../rooted-file-snapshot-invalid-schema.ny | 22 ++ .../rooted-file-snapshot-invalid-signature.ny | 12 + tests/language/rooted-file-snapshot.ny | 82 +++++ .../language/text-from-bytes-native-reject.ny | 7 + tests/language/text-from-bytes.ny | 33 ++ tests/rooted_file_snapshot_tests.cpp | 315 ++++++++++++++++++ tests/tools/rooted_file_snapshot_contract.py | 246 ++++++++++++++ 20 files changed, 1354 insertions(+), 7 deletions(-) create mode 100644 docs/standard-library/rooted-file-snapshots.md create mode 100644 tests/language/rooted-file-snapshot-invalid-schema.ny create mode 100644 tests/language/rooted-file-snapshot-invalid-signature.ny create mode 100644 tests/language/rooted-file-snapshot.ny create mode 100644 tests/language/text-from-bytes-native-reject.ny create mode 100644 tests/language/text-from-bytes.ny create mode 100644 tests/rooted_file_snapshot_tests.cpp create mode 100644 tests/tools/rooted_file_snapshot_contract.py diff --git a/.github/workflows/web-runtime-parity.yml b/.github/workflows/web-runtime-parity.yml index ae4dba20..ef10f0ba 100644 --- a/.github/workflows/web-runtime-parity.yml +++ b/.github/workflows/web-runtime-parity.yml @@ -9,8 +9,12 @@ name: Web runtime platform checks - 'nilstd/**' - 'tests/net_*' - 'tests/file_time_support_tests.cpp' + - 'tests/rooted_file_snapshot_tests.cpp' - 'tests/tools/net_socket_contract.py' - 'tests/tools/file_modified_at_contract.py' + - 'tests/tools/rooted_file_snapshot_contract.py' + - 'tests/language/rooted-file-snapshot.ny' + - 'tests/language/text-from-bytes*.ny' - 'tests/matrix/**' workflow_dispatch: @@ -82,6 +86,7 @@ jobs: cmake --build build/web-parity --config Release --parallel 2 \ --target nilcompiler nilinterpreter nil_net_support_tests \ nil_net_lifecycle_tests nil_file_time_support_tests \ + nil_rooted_file_snapshot_tests \ 2>&1 | tee build/web-parity-evidence/build.log - name: Run real peer and language contracts timeout-minutes: 8 @@ -89,13 +94,14 @@ jobs: "$NIL_CI_PYTHON" - <<'PY' import json, subprocess from pathlib import Path - pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract)|matrix[.]execution)$' + pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract|rooted-file-snapshot|rooted-file-snapshot-contract)|matrix[.]execution)$' base = ['ctest', '--test-dir', 'build/web-parity', '-C', 'Release', '-R', pattern] root = Path('build/web-parity-evidence').resolve() inventory = subprocess.check_output(base+['--show-only=json-v1'], timeout=30) (root/'inventory.json').write_bytes(inventory) expected = {'net.support-contract', 'net.lifecycle-contract', 'net.socket-contract', - 'nil.file-time-support', 'nil.file-modified-at-contract', 'matrix.execution'} + 'nil.file-time-support', 'nil.file-modified-at-contract', 'matrix.execution', + 'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract'} assert {test['name'] for test in json.loads(inventory)['tests']} == expected with (root/'ctest.log').open('wb') as output: result = subprocess.run(base+['--no-tests=error', '--output-on-failure', diff --git a/CMakeLists.txt b/CMakeLists.txt index e6927f1f..bf26728e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,12 @@ if(BUILD_TESTING) LABELS "tools;windows;rc" TIMEOUT 30) endif() + add_executable(nil_rooted_file_snapshot_tests tests/rooted_file_snapshot_tests.cpp) + target_link_libraries(nil_rooted_file_snapshot_tests PRIVATE nilruntime) + target_include_directories(nil_rooted_file_snapshot_tests PRIVATE nilcompiler/src) + add_test(NAME nil.rooted-file-snapshot COMMAND $) + set_tests_properties(nil.rooted-file-snapshot PROPERTIES LABELS "storage;runtime;rc" TIMEOUT 60) + add_executable(nil_utf8_file_paths_tests tests/utf8_file_paths_tests.cpp) target_link_libraries(nil_utf8_file_paths_tests PRIVATE nilruntime) target_include_directories(nil_utf8_file_paths_tests PRIVATE nilcompiler/src) @@ -1216,6 +1222,14 @@ if(BUILD_TESTING) --llvm "${NIL_ENABLE_LLVM}" --output-parent "${CMAKE_BINARY_DIR}/test-evidence") set_tests_properties(nil.file-modified-at-contract PROPERTIES LABELS "io;routes;rc" TIMEOUT 180) + add_test(NAME nil.rooted-file-snapshot-contract + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/tools/rooted_file_snapshot_contract.py" + --compiler $ + --interpreter $ + --nilstd "${CMAKE_CURRENT_SOURCE_DIR}/nilstd/src" + --llvm "${NIL_ENABLE_LLVM}" + --output-parent "${CMAKE_BINARY_DIR}/test-evidence") + set_tests_properties(nil.rooted-file-snapshot-contract PROPERTIES LABELS "io;routes;rc" TIMEOUT 360) add_test(NAME nillsp.semantic-protocol COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/lsp_semantic_protocol.py" --lsp $) diff --git a/docs/standard-library/index.md b/docs/standard-library/index.md index 2f10ab4a..a7dd34c0 100644 --- a/docs/standard-library/index.md +++ b/docs/standard-library/index.md @@ -128,6 +128,9 @@ the runtime and included in Aperture's text allocation measurements. For untrusted file/container input, `nil.std.io.read_regular_bytes` accepts an explicit byte ceiling and rejects the final-component symlink, special file, short/growing read, and observed metadata change before returning bytes. +For paths below a trusted directory, [`read_rooted_snapshot`](rooted-file-snapshots.md) +also rejects links in descendant directories and returns the exact byte array, +descriptor-observed modification time, and SHA-256 together. `write_bytes_atomic` stages and flushes bytes in the destination directory, then atomically publishes them; `replace=false` cannot clobber an existing entry (including a dangling symlink), while `replace=true` only replaces a diff --git a/docs/standard-library/rooted-file-snapshots.md b/docs/standard-library/rooted-file-snapshots.md new file mode 100644 index 00000000..7d0e52b1 --- /dev/null +++ b/docs/standard-library/rooted-file-snapshots.md @@ -0,0 +1,52 @@ +# Rooted file snapshots + +`nil.std.io.read_rooted_snapshot(root, relative, maximum_bytes)` reads one +bounded regular file below a trusted directory. It returns +`Result`: + +| Field | Type | Meaning | +| --- | --- | --- | +| `content` | `array` | The exact bytes read, including NUL and arbitrary high bytes. | +| `modified_ms` | `i64` | Modification time observed on the final file handle, in Unix milliseconds. | +| `sha256` | `text` | The 64 lowercase hexadecimal digits of SHA-256 over `content`. | + +The caller supplies a limit from zero through 512 MiB. A zero limit accepts +an empty file. Oversized input is rejected before allocating its contents. +SHA-256 is computed over the loaded bytes, so replacing content while keeping +its size and timestamp does not preserve its digest. + +```ny +import nil.std.io.file + +fn inspect_asset(root: text) + match nil::std::io::read_rooted_snapshot(root, "assets/site.css", 1048576) + Result.Ok(snapshot) -> + println(snapshot.content.len()) + println(snapshot.modified_ms) + println(snapshot.sha256) + Result.Error(message) -> println(message) + end match +end fn +``` + +The `relative` path must be nonempty strict UTF-8, with `/` separating its +components. Empty components, `.` and `..`, absolute paths, backslashes, +colons, and NUL bytes are rejected. The root selects the trusted directory; +every descendant component is opened relative to an already-open directory. +Descendant symlinks and Windows reparse points are rejected, including the +final component. Directories and other special files cannot be returned as +content. Errors do not include caller-supplied paths. + +The final handle supplies both bytes and metadata, and the runtime checks for +observed changes around the read. This does not make a file continuously +immutable or prevent privileged mount changes, hard links already placed in +the root, or every concurrent write. The returned digest describes the bytes +actually returned; it is not a separate path lookup or a metadata hash. + +The snapshot record uses the same byte-array representation for interpreter, +LLVM JIT, and native executable routes. Code using the interpreter's raw-byte +text representation can convert `snapshot.content` with +[`nil.std.text.from_bytes`](text-spans.md#byte-values). That conversion remains +interpreter-only: native text has a NUL-terminated representation and must not +silently truncate binary content. Native consumers can inspect, hash-check, or +write the byte array without converting it to text. diff --git a/docs/standard-library/text-spans.md b/docs/standard-library/text-spans.md index 0b0d2ba1..9b6b2f17 100644 --- a/docs/standard-library/text-spans.md +++ b/docs/standard-library/text-spans.md @@ -67,14 +67,20 @@ assembler/linker execution, and installed source/binary libraries. ## Byte values -`nil.std.text` measures and slices by byte — `length` of `"café"` is 5 — but -until these two existed there was no way to read a byte *as a number* or write -one as text. +`nil.std.text` measures and slices by byte — `length` of `"café"` is 5. +Use explicit byte operations to inspect or construct raw content. | Function | Result | | --- | --- | | `byte_at(s: text, index: i64) -> i64` | The byte at an index, `0..255`, or minus one when the index is outside the text. | | `from_byte(code: i64) -> text` | One byte as text; empty text for a code outside `0..255`. | +| `from_bytes(data: array) -> text` | An independent text copy of every array byte, including NUL and invalid UTF-8. | + +These raw-byte operations are interpreter-only. Native text currently requires +NUL-free values; native compilation rejects the unsupported intrinsics rather +than silently truncating or replacing bytes. `from_bytes` copies the array in +one operation and does not change when the source array is later mutated. +Use a byte array directly for binary data shared with native routes. The conversion that looks like it would do this is `nil.std.core`'s `convert.char_to_i32`, and it will not: it takes a whole @@ -85,7 +91,7 @@ isolated in order to inspect it whole, because isolating it needed the same test. Percent-encoding anything outside ASCII was unreachable because of this, and so was reading arbitrary bytes off a socket. -These two are about bytes and say so in their names. They know nothing about +These operations are about bytes and say so in their names. They know nothing about UTF-8; assembling characters from bytes is the caller's job. `from_byte(195)` is the single byte `0xC3`, where `convert.i32_to_char(195)` is diff --git a/nilcompiler/src/llvm_backend.cpp b/nilcompiler/src/llvm_backend.cpp index 268c779b..36d29a09 100644 --- a/nilcompiler/src/llvm_backend.cpp +++ b/nilcompiler/src/llvm_backend.cpp @@ -3388,6 +3388,111 @@ bool NativeCodegen::generateModule(const nil_module &mod) { return false; impl_->builder->CreateRet(error_variant); annotation_handled = true; + } else if (dispatch_name == "@intrinsic" && + dispatch_provider == "file.read_rooted_snapshot") { + const auto result_arguments = + native_generic_arguments(fn.return_type, "result"); + const auto *definition = result_arguments && result_arguments->size() == 2 + ? find_native_aggregate_definition(mod, (*result_arguments)[0]) + : nullptr; + const auto layout_it = definition ? native_struct_layouts.find(definition) + : native_struct_layouts.end(); + bool valid_schema = definition && definition->fields.size() == 3 && + compact_nil_type(definition->name) == "nil::std::io::FileSnapshot" && + layout_it != native_struct_layouts.end() && + layout_it->second.hidden_fields == 0; + const std::pair required_fields[] = { + {"content", "array"}, {"modified_ms", "i64"}, {"sha256", "text"}}; + if (valid_schema) { + for (const auto &[name, type] : required_fields) { + const auto field = layout_it->second.field_indices.find(name); + if (field == layout_it->second.field_indices.end()) { + valid_schema = false; + break; + } + const auto &actual_type = layout_it->second.field_nil_types[field->second]; + valid_schema = valid_schema && + (std::string(type) == "text" + ? native_handle_kind_for_nil_type(actual_type) == native_handle_kind::text + : normalize_nil_type(actual_type) == type); + } + } + if (!valid_schema || fn.params.size() != 3 || + native_handle_kind_for_nil_type(nil_parameter_type(fn.params[0])) != + native_handle_kind::text || + native_handle_kind_for_nil_type(nil_parameter_type(fn.params[1])) != + native_handle_kind::text || + normalize_nil_type(nil_parameter_type(fn.params[2])) != "i64" || + native_handle_kind_for_nil_type((*result_arguments)[1]) != + native_handle_kind::text || + impl_->current_function->arg_size() != 3 || + !impl_->current_function->getReturnType()->isIntegerTy(64)) { + std::cerr << "[NativeCodegen] file.read_rooted_snapshot requires " + "(text,text,i64) -> Result with the " + "exact content/modified_ms/sha256 field schema\n"; + return false; + } + auto *pointer_type = llvm::PointerType::getUnqual(ctx); + auto *word_type = llvm::Type::getInt64Ty(ctx); + auto *bytes_out = impl_->builder->CreateAlloca(pointer_type); + auto *modified_out = impl_->builder->CreateAlloca(word_type); + auto *digest_out = impl_->builder->CreateAlloca(pointer_type); + auto *error_out = impl_->builder->CreateAlloca(pointer_type); + auto *runtime_type = llvm::FunctionType::get( + llvm::Type::getInt32Ty(ctx), + {pointer_type, pointer_type, word_type, pointer_type, pointer_type, + pointer_type, pointer_type}, false); + auto runtime = impl_->module->getOrInsertFunction( + "nyrt_read_rooted_snapshot", runtime_type); + auto *status = impl_->builder->CreateCall(runtime_type, runtime.getCallee(), + {impl_->current_function->getArg(0), impl_->current_function->getArg(1), + impl_->current_function->getArg(2), bytes_out, modified_out, + digest_out, error_out}); + auto *success = llvm::BasicBlock::Create(ctx, "snapshot.success", + impl_->current_function); + auto *failure = llvm::BasicBlock::Create(ctx, "snapshot.failure", + impl_->current_function); + impl_->builder->CreateCondBr(impl_->builder->CreateICmpNE( + status, llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), 0)), + success, failure); + impl_->builder->SetInsertPoint(success); + const auto allocation_size = impl_->module->getDataLayout() + .getTypeAllocSize(layout_it->second.llvm_type).getFixedValue(); + llvm::Value *record = nullptr; + if (impl_->memory_mode == MemoryMode::SAFE_GC) { + auto *allocate_type = llvm::FunctionType::get(pointer_type, {word_type}, false); + auto allocate = impl_->module->getOrInsertFunction("nyrt_alloc", allocate_type); + record = impl_->builder->CreateCall(allocate_type, allocate.getCallee(), + {llvm::ConstantInt::get(word_type, allocation_size)}); + } else { + auto *allocate_type = llvm::FunctionType::get(pointer_type, {native_size_type}, false); + auto allocate = impl_->module->getOrInsertFunction("malloc", allocate_type); + record = impl_->builder->CreateCall(allocate_type, allocate.getCallee(), + {native_size_constant(allocation_size)}); + } + emit_allocation_null_check(record, "FileSnapshot"); + impl_->setNilType(record, definition->name); + llvm::Value *values[] = { + impl_->builder->CreateLoad(pointer_type, bytes_out), + impl_->builder->CreateLoad(word_type, modified_out), + impl_->builder->CreateLoad(pointer_type, digest_out)}; + for (std::size_t index = 0; index < 3; ++index) { + const auto field_index = layout_it->second.field_indices.at(required_fields[index].first); + auto *field_pointer = impl_->builder->CreateStructGEP( + layout_it->second.llvm_type, record, field_index); + impl_->builder->CreateStore(values[index], field_pointer); + } + auto *ok = emit_single_payload_variant("Result::Ok", record, + fn.return_type, definition->name); + if (!ok) return false; + impl_->builder->CreateRet(ok); + impl_->builder->SetInsertPoint(failure); + auto *error = impl_->builder->CreateLoad(pointer_type, error_out); + auto *failed = emit_single_payload_variant("Result::Error", error, + fn.return_type, "text"); + if (!failed) return false; + impl_->builder->CreateRet(failed); + annotation_handled = true; } else if (ann.find("@intrinsic(\"file.read_regular_bytes\")") != std::string::npos || ann.find("@intrinsic(file.read_regular_bytes)") != @@ -9380,6 +9485,7 @@ int NativeCodegen::executeJIT(const std::vector &program_args) { expose_runtime_symbol("nyrt_buffer_get", &nyrt_buffer_get); expose_runtime_symbol("nyrt_buffer_set", &nyrt_buffer_set); expose_runtime_symbol("nyrt_read_regular_bytes", &nyrt_read_regular_bytes); + expose_runtime_symbol("nyrt_read_rooted_snapshot", &nyrt_read_rooted_snapshot); expose_runtime_symbol("nyrt_write_bytes_atomic", &nyrt_write_bytes_atomic); expose_runtime_symbol("nyrt_io_v1", &nyrt_io_v1); expose_runtime_symbol("nyrt_try_i32", &nyrt_try_i32); diff --git a/nilcompiler/src/native_runtime.cpp b/nilcompiler/src/native_runtime.cpp index 2a1a0d5c..7c20629b 100644 --- a/nilcompiler/src/native_runtime.cpp +++ b/nilcompiler/src/native_runtime.cpp @@ -1,6 +1,7 @@ #include "native_runtime.hpp" #include "../../common/standard_streams.hpp" #include "../../common/strict_json.hpp" +#include "../../common/content_digest.hpp" #include "../../common/windows_environment.hpp" #include "ordinary_io_intrinsic.hpp" #include "dynamic_type.hpp" @@ -1955,6 +1956,48 @@ std::int32_t nyrt_io_v1(const char *operation, std::uint64_t first, } } +std::int32_t nyrt_read_rooted_snapshot( + const char *root, const char *relative, std::int64_t maximum_bytes, + void **bytes_out, std::int64_t *modified_ms_out, const char **sha256_out, + const char **error_out) noexcept { + if (bytes_out) *bytes_out = nullptr; + if (modified_ms_out) *modified_ms_out = 0; + if (sha256_out) *sha256_out = nullptr; + if (error_out) *error_out = nullptr; + if (!root || !relative || !bytes_out || !modified_ms_out || !sha256_out || + maximum_bytes < 0 || + static_cast(maximum_bytes) > maximum_file_bytes) { + set_error(error_out, "file.read_rooted_snapshot: invalid arguments"); + return 0; + } + try { + auto loaded = ny::detail::read_rooted_file_snapshot_utf8( + root, relative, static_cast(maximum_bytes)); + if (!loaded.ok) { + set_error(error_out, loaded.error); + return 0; + } + nil::detail::sha256 hash; + hash.update(loaded.bytes.data(), loaded.bytes.size()); + const auto digest = hash.finish(); + auto *array = make_array(NYRT_ARRAY_U8, loaded.bytes.size()); + const auto *digest_text = own_string(digest); + if (!array || !digest_text) { + set_error(error_out, "file.read_rooted_snapshot: allocation failed"); + return 0; + } + if (!loaded.bytes.empty()) + std::memcpy(array->data, loaded.bytes.data(), loaded.bytes.size()); + *bytes_out = array; + *modified_ms_out = loaded.modified_ms; + *sha256_out = digest_text; + return 1; + } catch (...) { + set_error(error_out, "file.read_rooted_snapshot: host operation failed"); + return 0; + } +} + std::int32_t nyrt_read_regular_bytes(const char *path, std::int64_t maximum_bytes, void **bytes_out, diff --git a/nilcompiler/src/native_runtime.hpp b/nilcompiler/src/native_runtime.hpp index c5ee085b..5435df50 100644 --- a/nilcompiler/src/native_runtime.hpp +++ b/nilcompiler/src/native_runtime.hpp @@ -210,6 +210,12 @@ std::int32_t nyrt_read_regular_bytes(const char *path, std::int64_t maximum_bytes, void **bytes_out, const char **error_out) noexcept; +// All outputs describe the same bounded, rooted file read. The compiler owns +// the language record layout; this ABI only returns its independent fields. +std::int32_t nyrt_read_rooted_snapshot( + const char *root, const char *relative, std::int64_t maximum_bytes, + void **bytes_out, std::int64_t *modified_ms_out, const char **sha256_out, + const char **error_out) noexcept; std::int32_t nyrt_write_bytes_atomic(const char *path, const void *bytes, std::int32_t replace, const char **error_out) noexcept; diff --git a/nilcompiler/src/secure_file_io.cpp b/nilcompiler/src/secure_file_io.cpp index 076cda64..dda23a81 100644 --- a/nilcompiler/src/secure_file_io.cpp +++ b/nilcompiler/src/secure_file_io.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -39,6 +41,7 @@ #define NOMINMAX #endif #include +#include #endif namespace ny::detail { @@ -119,6 +122,50 @@ path_from_api_string(const std::string &path) { } } +constexpr std::uint64_t rooted_snapshot_maximum = 512ULL * 1024 * 1024; + +std::optional +rooted_directory_from_utf8(const std::string &root) { + auto path = path_from_utf8_string(root); + // A trailing separator or '/.' would make O_NOFOLLOW / OPEN_REPARSE_POINT + // inspect the directory reached through a root symlink instead of the link. + // Remove only these trailing spellings, never resolve '..' or ancestors. + while (path && (path->filename().empty() || path->filename() == ".")) { + auto parent = path->parent_path(); + if (parent.empty() || parent == *path) + break; + *path = std::move(parent); + } + return path; +} + +RootedFileSnapshot rooted_failure(const char *reason) { + return {false, {}, 0, std::string("file.read_rooted_snapshot: ") + reason}; +} + +std::optional> +rooted_components(const std::string &relative) { + if (!path_from_utf8_string(relative) || relative.front() == '/' || + relative.find('\\') != std::string::npos || + relative.find(':') != std::string::npos) + return std::nullopt; + std::vector components; + std::size_t start = 0; + for (;;) { + const auto slash = relative.find('/', start); + const std::string component = relative.substr(start, slash - start); + if (component.empty() || component == "." || component == "..") + return std::nullopt; + const auto native = path_from_utf8_string(component); + if (!native) + return std::nullopt; + components.push_back(*native); + if (slash == std::string::npos) + return components; + start = slash + 1; + } +} + } // namespace #ifndef _WIN32 @@ -240,6 +287,81 @@ BoundedFileRead read_regular_file_bounded(const std::filesystem::path &path, return {true, std::move(bytes), {}}; } +RootedFileSnapshot read_rooted_file_snapshot_utf8(const std::string &root, + const std::string &relative, + std::uint64_t maximum_bytes) { + if (maximum_bytes > rooted_snapshot_maximum) + return rooted_failure("invalid maximum_bytes"); + const auto native_root = rooted_directory_from_utf8(root); + const auto components = rooted_components(relative); + if (!native_root || !components) + return rooted_failure("invalid path"); + + const int directory_flags = O_RDONLY | O_CLOEXEC | O_DIRECTORY | O_NOFOLLOW; + UniqueFileDescriptor directory(::open(native_root->c_str(), directory_flags)); + if (directory.get() < 0) + return rooted_failure("cannot open root directory"); + for (std::size_t index = 0; index + 1 < components->size(); ++index) { + UniqueFileDescriptor child(::openat(directory.get(), (*components)[index].c_str(), + directory_flags)); + if (child.get() < 0) + return rooted_failure("cannot open descendant directory"); + directory = std::move(child); + } + UniqueFileDescriptor input(::openat( + directory.get(), components->back().c_str(), + O_RDONLY | O_CLOEXEC | O_NONBLOCK | O_NOFOLLOW)); + if (input.get() < 0) + return rooted_failure("cannot open regular file"); + + struct stat before {}; + if (::fstat(input.get(), &before) != 0 || !S_ISREG(before.st_mode)) + return rooted_failure("input is not a regular file"); + if (before.st_size < 0 || static_cast(before.st_size) > maximum_bytes) + return rooted_failure("input exceeds maximum_bytes"); +#if defined(__APPLE__) + const auto seconds = before.st_mtimespec.tv_sec; + const auto nanos = before.st_mtimespec.tv_nsec; +#else + const auto seconds = before.st_mtim.tv_sec; + const auto nanos = before.st_mtim.tv_nsec; +#endif + if (seconds < (std::numeric_limits::min)() / 1000 || + seconds > (std::numeric_limits::max)() / 1000 || + nanos < 0 || nanos >= 1000000000) + return rooted_failure("invalid modification timestamp"); + const auto millis = static_cast(seconds) * 1000; + const auto fraction = static_cast(nanos) / 1000000; + if (millis > (std::numeric_limits::max)() - fraction) + return rooted_failure("invalid modification timestamp"); + + std::vector bytes; + try { + bytes.resize(static_cast(before.st_size)); + } catch (const std::bad_alloc &) { + return rooted_failure("cannot allocate snapshot"); + } + std::size_t offset = 0; + while (offset < bytes.size()) { + const ssize_t count = ::read(input.get(), bytes.data() + offset, bytes.size() - offset); + if (count < 0 && errno == EINTR) + continue; + if (count <= 0) + return rooted_failure("input changed or read failed"); + offset += static_cast(count); + } + char extra = 0; + ssize_t count = -1; + do { + count = ::read(input.get(), &extra, 1); + } while (count < 0 && errno == EINTR); + struct stat after {}; + if (count != 0 || ::fstat(input.get(), &after) != 0 || + !S_ISREG(after.st_mode) || !same_regular_file_snapshot(before, after)) + return rooted_failure("input changed or read failed"); + return {true, std::move(bytes), millis + fraction, {}}; +} + AtomicFileWrite write_file_atomic(const std::filesystem::path &path, const std::vector &bytes, bool replace) { @@ -470,6 +592,138 @@ BoundedFileRead read_regular_file_bounded(const std::filesystem::path &path, return {true, std::move(bytes), {}}; } +namespace { + +using RootedNtCreateFile = NTSTATUS (NTAPI *)( + PHANDLE, ACCESS_MASK, POBJECT_ATTRIBUTES, PIO_STATUS_BLOCK, PLARGE_INTEGER, + ULONG, ULONG, ULONG, ULONG, PVOID, ULONG); + +RootedNtCreateFile rooted_nt_create_file() { + static const auto function = []() -> RootedNtCreateFile { + const HMODULE module = GetModuleHandleW(L"ntdll.dll"); + const FARPROC symbol = module ? GetProcAddress(module, "NtCreateFile") : nullptr; + RootedNtCreateFile function = nullptr; + static_assert(sizeof(function) == sizeof(symbol), "Windows function pointer size"); + std::memcpy(&function, &symbol, sizeof(function)); + return function; + }(); + return function; +} + +UniqueWindowsHandle open_rooted_child(HANDLE directory, + const std::filesystem::path &component, + bool is_directory) { + const auto create = rooted_nt_create_file(); + const auto &name = component.native(); + if (!create || name.empty() || name.size() > 32767) + return UniqueWindowsHandle(); + UNICODE_STRING string {}; + string.Buffer = const_cast(name.data()); + string.Length = static_cast(name.size() * sizeof(wchar_t)); + string.MaximumLength = string.Length; + OBJECT_ATTRIBUTES attributes {}; + attributes.Length = sizeof(attributes); + attributes.RootDirectory = directory; + attributes.ObjectName = &string; + attributes.Attributes = 0x40; // OBJ_CASE_INSENSITIVE; do not inherit handles. + IO_STATUS_BLOCK io {}; + HANDLE child = INVALID_HANDLE_VALUE; + constexpr ULONG open_existing = 1; // FILE_OPEN + constexpr ULONG open_reparse = 0x00200000; // FILE_OPEN_REPARSE_POINT + constexpr ULONG synchronous = 0x20; // FILE_SYNCHRONOUS_IO_NONALERT + constexpr ULONG directory_only = 1; // FILE_DIRECTORY_FILE + constexpr ULONG non_directory_only = 0x40; // FILE_NON_DIRECTORY_FILE + const ULONG sharing = is_directory + ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ; + const NTSTATUS status = create( + &child, GENERIC_READ | SYNCHRONIZE, &attributes, &io, nullptr, 0, + sharing, open_existing, + open_reparse | synchronous | (is_directory ? directory_only : non_directory_only), + nullptr, 0); + if (status != 0 || io.Status != 0 || io.Information != 1 || + child == nullptr || child == INVALID_HANDLE_VALUE) { + if (child != nullptr && child != INVALID_HANDLE_VALUE) + CloseHandle(child); + return UniqueWindowsHandle(); + } + UniqueWindowsHandle held(child); + BY_HANDLE_FILE_INFORMATION information {}; + if (GetFileType(held.get()) != FILE_TYPE_DISK || + !GetFileInformationByHandle(held.get(), &information) || + (information.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 || + ((information.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) != is_directory) + return UniqueWindowsHandle(); + return held; +} + +} // namespace + +RootedFileSnapshot read_rooted_file_snapshot_utf8(const std::string &root, + const std::string &relative, + std::uint64_t maximum_bytes) { + if (maximum_bytes > rooted_snapshot_maximum) + return rooted_failure("invalid maximum_bytes"); + const auto native_root = rooted_directory_from_utf8(root); + const auto components = rooted_components(relative); + if (!native_root || !components) + return rooted_failure("invalid path"); + UniqueWindowsHandle directory(CreateFileW( + native_root->c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, nullptr)); + BY_HANDLE_FILE_INFORMATION root_information {}; + if (directory.get() == INVALID_HANDLE_VALUE || + GetFileType(directory.get()) != FILE_TYPE_DISK || + !GetFileInformationByHandle(directory.get(), &root_information) || + (root_information.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 || + (root_information.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) + return rooted_failure("cannot open root directory"); + for (std::size_t index = 0; index + 1 < components->size(); ++index) { + auto child = open_rooted_child(directory.get(), (*components)[index], true); + if (child.get() == INVALID_HANDLE_VALUE) + return rooted_failure("cannot open descendant directory"); + directory = std::move(child); + } + auto input = open_rooted_child(directory.get(), components->back(), false); + if (input.get() == INVALID_HANDLE_VALUE) + return rooted_failure("cannot open regular file"); + BY_HANDLE_FILE_INFORMATION before {}; + if (!GetFileInformationByHandle(input.get(), &before)) + return rooted_failure("cannot inspect regular file"); + const std::uint64_t size = (static_cast(before.nFileSizeHigh) << 32U) | + before.nFileSizeLow; + if (size > maximum_bytes) + return rooted_failure("input exceeds maximum_bytes"); + const std::uint64_t ticks = (static_cast(before.ftLastWriteTime.dwHighDateTime) << 32U) | + before.ftLastWriteTime.dwLowDateTime; + // Divide before converting: even the largest FILETIME quotient fits i64. + const std::int64_t modified = static_cast(ticks / 10000) - 11644473600000LL; + std::vector bytes; + try { + bytes.resize(static_cast(size)); + } catch (const std::bad_alloc &) { + return rooted_failure("cannot allocate snapshot"); + } + std::size_t offset = 0; + while (offset < bytes.size()) { + const DWORD requested = static_cast(std::min( + bytes.size() - offset, (std::numeric_limits::max)())); + DWORD count = 0; + if (!ReadFile(input.get(), bytes.data() + offset, requested, &count, nullptr) || + count == 0 || count > requested) + return rooted_failure("input changed or read failed"); + offset += count; + } + char extra = 0; + DWORD count = 0; + BY_HANDLE_FILE_INFORMATION after {}; + if (!ReadFile(input.get(), &extra, 1, &count, nullptr) || count != 0 || + !GetFileInformationByHandle(input.get(), &after) || + !same_regular_file_snapshot(before, after)) + return rooted_failure("input changed or read failed"); + return {true, std::move(bytes), modified, {}}; +} + AtomicFileWrite write_file_atomic(const std::filesystem::path &path, const std::vector &bytes, bool replace) { diff --git a/nilcompiler/src/secure_file_io.hpp b/nilcompiler/src/secure_file_io.hpp index 1db9bd2b..52e5f33f 100644 --- a/nilcompiler/src/secure_file_io.hpp +++ b/nilcompiler/src/secure_file_io.hpp @@ -14,6 +14,26 @@ struct BoundedFileRead { std::string error; }; +struct RootedFileSnapshot { + bool ok = false; + std::vector bytes; + std::int64_t modified_ms = 0; + std::string error; +}; + +// The trusted root selector is opened once. Its ancestors are outside the +// relative-path trust boundary. Every relative component is opened from the +// preceding directory handle, rejecting links/reparse points. The relative +// name is strict UTF-8, slash-separated, with no empty, dot, parent, colon or +// backslash components. Only regular files up to maximum_bytes are read; +// maximum_bytes itself cannot exceed 512 MiB. Bytes and mtime are inspected +// through the same final handle, rejecting observed changes during the read. +// This is an anchored namespace read, not a filesystem transaction: privileged +// mounts, hard links and changes after the snapshot are outside its contract. +RootedFileSnapshot read_rooted_file_snapshot_utf8(const std::string &root, + const std::string &relative, + std::uint64_t maximum_bytes); + struct AtomicFileWrite { bool ok = false; std::string error; diff --git a/nilcompiler/src/vm.cpp b/nilcompiler/src/vm.cpp index 5231c1f9..50c7620b 100644 --- a/nilcompiler/src/vm.cpp +++ b/nilcompiler/src/vm.cpp @@ -19,6 +19,7 @@ #include "time_support.hpp" #include "file_time_support.hpp" #include "../../common/windows_environment.hpp" +#include "../../common/content_digest.hpp" #include #include #include @@ -1678,6 +1679,13 @@ void VM::executeInstruction(const Instruction &instr) { const auto *handle = std::get_if(&value); if (!handle) reject(); const auto found = heap_.find(handle->address); + // File/random byte arrays use compact byte storage instead of scalar + // Value slots. They retain the same exact array language identity. + // Do not admit raw u8 pointer allocations or any other array element. + if (found != heap_.end() && expected == "array" && + found->second.type_name == "bytes" && found->second.data.empty() && + found->second.element_count == found->second.byte_data.size()) + return; if (found == heap_.end() || found->second.type_name != "array" || "array<" + canonical_array_type(found->second.element_type) + ">" != expected) reject(); return; @@ -4411,6 +4419,37 @@ void VM::executeInstruction(const Instruction &instr) { return; } + if (fn_name == "text.from_bytes") { + if (args.size() != 1 || !std::holds_alternative(args[0])) + throw std::runtime_error("text.from_bytes: expected a live byte array"); + const auto found = heap_.find(std::get(args[0]).address); + if (found == heap_.end()) + throw std::runtime_error("text.from_bytes: expected a live byte array"); + const auto &block = found->second; + std::string result; + if (block.type_name == "bytes" && block.data.empty() && + block.element_count == block.byte_data.size()) { + if (block.byte_data.size() > max_intrinsic_file_bytes) + throw std::runtime_error("text.from_bytes: input exceeds size limit"); + result.assign(block.byte_data.begin(), block.byte_data.end()); + } else if (block.type_name == "array" && + canonical_array_scalar(block.element_type) == "u8") { + if (block.data.size() > max_intrinsic_file_bytes) + throw std::runtime_error("text.from_bytes: input exceeds size limit"); + result.reserve(block.data.size()); + for (const auto &item : block.data) { + const auto *byte = std::get_if(&item); + if (!byte || *byte < 0 || *byte > 255) + throw std::runtime_error("text.from_bytes: invalid byte element"); + result.push_back(static_cast(static_cast(*byte))); + } + } else { + throw std::runtime_error("text.from_bytes: expected a live byte array"); + } + pushStack(std::move(result)); + return; + } + if (fn_name == "text.from_byte") { std::int64_t code = -1; if (!args.empty()) { @@ -5079,6 +5118,57 @@ void VM::executeInstruction(const Instruction &instr) { pushStack(result); return; } + // Return bytes, timestamp and content identity from one anchored read. + if (fn_name == "file.read_rooted_snapshot") { + auto push_error = [&](const std::string &message) { + EnumValue result; + result.variant_name = "Result::Error"; + result.payloads.push_back(message); + pushStack(result); + }; + if (args.size() != 3 || !std::holds_alternative(args[0]) || + !std::holds_alternative(args[1]) || + !std::holds_alternative(args[2]) || + std::get(args[2]) < 0 || + static_cast(std::get(args[2])) > + max_intrinsic_file_bytes) { + push_error("file.read_rooted_snapshot: invalid arguments"); + return; + } + try { + auto loaded = detail::read_rooted_file_snapshot_utf8( + std::get(args[0]), std::get(args[1]), + static_cast(std::get(args[2]))); + if (!loaded.ok) { + push_error(loaded.error); + return; + } + nil::detail::sha256 hash; + hash.update(loaded.bytes.data(), loaded.bytes.size()); + StructValue snapshot; + snapshot.type_name = "nil::std::io::FileSnapshot"; + snapshot.fields()["modified_ms"] = loaded.modified_ms; + snapshot.string_fields()["sha256"] = hash.finish(); + HeapBlock block; + block.type_name = "bytes"; + block.element_count = loaded.bytes.size(); + block.byte_data = std::move(loaded.bytes); + const std::int64_t handle = next_ptr_; + const std::size_t storage_bytes = + std::max(block.byte_data.size(), 1U); + next_ptr_ += static_cast( + (storage_bytes + 7U) & ~std::size_t(7U)); + storeHeapBlock(handle, std::move(block)); + snapshot.array_fields()["content"] = ArrayHandle{handle}; + EnumValue result; + result.variant_name = "Result::Ok"; + result.payloads.push_back(std::move(snapshot)); + pushStack(result); + } catch (...) { + push_error("file.read_rooted_snapshot: host operation failed"); + } + return; + } // file.read_regular_bytes(path, maximum_bytes) // -> result, text> if (fn_name == "file.read_regular_bytes") { diff --git a/nilstd/src/io/file.ny b/nilstd/src/io/file.ny index c1058272..dcc70a84 100644 --- a/nilstd/src/io/file.ny +++ b/nilstd/src/io/file.ny @@ -36,6 +36,28 @@ pub fn read_bytes(path: text) -> Result, text> @intrinsic("file.read_regular_bytes") pub fn read_regular_bytes(path: text, maximum_bytes: i64) -> Result, text> +# The exact bytes read, their descriptor-observed modification time in Unix +# milliseconds, and lowercase SHA-256 of those returned bytes. Binary content +# remains an array on every execution route; no text encoding is assumed. +pub struct FileSnapshot + content: array + modified_ms: i64 + sha256: text +end struct + +# Read a bounded regular file below a trusted root directory. The relative +# path uses strict UTF-8 slash-separated components; empty, dot, dot-dot, +# absolute, backslash, colon and NUL spellings are rejected. Every descendant +# component is opened relative to an already-open directory without following +# symlinks or Windows reparse points. The final handle supplies the bytes and +# metadata, with observed changes checked before returning. maximum_bytes is +# 0..512 MiB; zero admits an empty file. Errors contain no caller path values. +# This is a directory-handle containment boundary, not protection against +# privileged mount changes, hard links placed in the root, or all concurrent +# writes. sha256 always describes content, rather than a size/timestamp key. +@intrinsic("file.read_rooted_snapshot") +pub fn read_rooted_snapshot(root: text, relative: text, maximum_bytes: i64) -> Result + # Write text to a file (creates or overwrites) @intrinsic("file.write_text") pub fn write_text(path: text, content: text) -> Result diff --git a/nilstd/src/text/string_utils.ny b/nilstd/src/text/string_utils.ny index 7c59ee50..8275de70 100644 --- a/nilstd/src/text/string_utils.ny +++ b/nilstd/src/text/string_utils.ny @@ -800,7 +800,7 @@ end fn # outside ASCII was unreachable because of this, and so was reading the bytes # off a socket. # -# These two are about bytes and say so in their names. They know nothing about +# These operations are about bytes and say so in their names. They know nothing about # UTF-8; assembling characters from bytes is the caller's job, and now it is a # job that can be done. @@ -821,3 +821,11 @@ pub fn byte_at(s: text, index: i64) -> i64 # A code outside 0..255 gives empty text. @intrinsic("text.from_byte") pub fn from_byte(code: i64) -> text + +# Copy a byte array into interpreter text in one operation. Every byte is +# preserved, including NUL and invalid UTF-8; no decoding or normalization is +# performed, and later array mutation does not change the returned text. +# Interpreter-only, like byte_at/from_byte: native text currently requires +# NUL-free values, so native compilation rejects this intrinsic explicitly. +@intrinsic("text.from_bytes") +pub fn from_bytes(data: array) -> text diff --git a/tests/language/rooted-file-snapshot-invalid-schema.ny b/tests/language/rooted-file-snapshot-invalid-schema.ny new file mode 100644 index 00000000..86ecbfc1 --- /dev/null +++ b/tests/language/rooted-file-snapshot-invalid-schema.ny @@ -0,0 +1,22 @@ +module nil.std.io + +import nil.std.core + +# Exact canonical type name, deliberately incompatible timestamp field. The +# native bridge must validate the schema rather than trust its spelling. +pub struct FileSnapshot + content: array + modified_ms: text + sha256: text +end struct + +@intrinsic("file.read_rooted_snapshot") +fn invalid_snapshot(root: text, relative: text, maximum_bytes: i64) -> Result + +fn main() -> i32 + match invalid_snapshot(".", "never-read.bin", 256) + Result.Ok(snapshot) -> println(snapshot.sha256) + Result.Error(message) -> println(message) + end match + return 0 +end fn diff --git a/tests/language/rooted-file-snapshot-invalid-signature.ny b/tests/language/rooted-file-snapshot-invalid-signature.ny new file mode 100644 index 00000000..bfd265c3 --- /dev/null +++ b/tests/language/rooted-file-snapshot-invalid-signature.ny @@ -0,0 +1,12 @@ +import nil.std.io.file + +@intrinsic("file.read_rooted_snapshot") +fn invalid_snapshot(root: text, relative: text, maximum_bytes: i32) -> Result + +fn main() -> i32 + match invalid_snapshot(".", "never-read.bin", 256) + Result.Ok(snapshot) -> println(snapshot.sha256) + Result.Error(message) -> println(message) + end match + return 0 +end fn diff --git a/tests/language/rooted-file-snapshot.ny b/tests/language/rooted-file-snapshot.ny new file mode 100644 index 00000000..1e70a04f --- /dev/null +++ b/tests/language/rooted-file-snapshot.ny @@ -0,0 +1,82 @@ +module nil.tests.rooted_file_snapshot + +import nil.std.io.file +import nil.std.system.std_sys +import nil.std.text + +fn inspect(snapshot: nil::std::io::FileSnapshot, stamp: i64, digest: text, seed: i64) -> bool + let boxed: anything = snapshot + if not anything_is(boxed) then return false end if + let recovered = boxed as nil::std::io::FileSnapshot + let boxed_content: anything = recovered.content + if not anything_is>(boxed_content) or anything_is>(boxed_content) then return false end if + let recovered_content = boxed_content as array + if recovered.content.len() != 256 then return false end if + if recovered.modified_ms != stamp then return false end if + if recovered.sha256 != digest then return false end if + let mut index: i64 = 0 + while index < 256 + if recovered_content[index] != (((index * 73 + seed) % 256) as u8) then return false end if + index = index + 1 + end while + return true +end fn + +fn reject(root: text, relative: text, maximum: i64, expected: text) -> bool + match nil::std::io::read_rooted_snapshot(root, relative, maximum) + Result.Ok(_) -> return false + Result.Error(message) -> + if not nil::std::text::starts_with(message, "file.read_rooted_snapshot:") then return false end if + if nil::std::text::index_of(message, root) >= 0 then return false end if + if expected != "" and message != expected then return false end if + end match + return true +end fn + +fn main() -> i32 + if nil::std::system::arg_count() != 4 and nil::std::system::arg_count() != 5 then return 60 end if + let root = nil::std::system::arg(0) + let stamp = nil::std::core::Convert.try_to_i64(nil::std::system::arg(1)).unwrap() + let digest = nil::std::system::arg(2) + let seed = nil::std::core::Convert.try_to_i64(nil::std::system::arg(3)).unwrap() + match nil::std::io::read_rooted_snapshot(root, "nested/café-λ.bin", 256) + Result.Error(message) -> + println(message) + return 61 + Result.Ok(snapshot) -> + if not inspect(snapshot, stamp, digest, seed) then return 62 end if + if nil::std::system::arg_count() == 5 then + if nil::std::system::arg(4) != "wrong-element" then return 74 end if + let boxed_content: anything = snapshot.content + let invalid_content = boxed_content as array + println(invalid_content.len()) + return 75 + end if + end match + match nil::std::io::read_rooted_snapshot(root, "empty.bin", 0) + Result.Error(message) -> + println(message) + return 63 + Result.Ok(snapshot) -> + if snapshot.content.len() != 0 or snapshot.modified_ms != stamp then return 64 end if + if snapshot.sha256 != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" then return 65 end if + end match + println("snapshot-content-ok") + if not reject(root, "nested/café-λ.bin", 255, + "file.read_rooted_snapshot: input exceeds maximum_bytes") then return 66 end if + if not reject(root, "empty.bin", -1, "") then return 67 end if + if not reject(root, "empty.bin", 536870913, "") then return 68 end if + let invalid: array = ["", ".", "..", "../outside/sentinel.bin", "nested/../empty.bin", + "/empty.bin", "nested//café-λ.bin", "nested/", "nested\\café-λ.bin", "C:empty.bin", + "missing.bin", "nested"] + for path in invalid + if not reject(root, path, 256, "") then return 69 end if + end for + if not reject(root + "/missing-root", "empty.bin", 256, "") then return 70 end if + if not reject(root + "/empty.bin", "empty.bin", 256, "") then return 71 end if + println("snapshot-errors-ok") + if not reject(root, "linked.bin", 256, "") then return 72 end if + if not reject(root, "via-link/sentinel.bin", 256, "") then return 73 end if + println("snapshot-links-ok") + return 0 +end fn diff --git a/tests/language/text-from-bytes-native-reject.ny b/tests/language/text-from-bytes-native-reject.ny new file mode 100644 index 00000000..9c49ea80 --- /dev/null +++ b/tests/language/text-from-bytes-native-reject.ny @@ -0,0 +1,7 @@ +import nil.std.text + +fn main() -> i32 + let bytes: array = [65 as u8, 66 as u8] + println(nil::std::text::from_bytes(bytes)) + return 0 +end fn diff --git a/tests/language/text-from-bytes.ny b/tests/language/text-from-bytes.ny new file mode 100644 index 00000000..a9b67c16 --- /dev/null +++ b/tests/language/text-from-bytes.ny @@ -0,0 +1,33 @@ +module nil.tests.text_from_bytes + +import nil.std.text +import nil.std.collections.array + +fn main() -> i32 + let mut bytes: array = [] + let mut index: i64 = 0 + while index < 256 + bytes.push(index as u8) + index = index + 1 + end while + let text = nil::std::text::from_bytes(bytes) + if nil::std::text::length(text) != 256 then return 61 end if + bytes[0] = 17 + index = 0 + while index < 256 + if nil::std::text::byte_at(text, index) != index then return 62 end if + index = index + 1 + end while + let empty: array = [] + if nil::std::text::from_bytes(empty) != "" then return 63 end if + let mut large = nil::std::collections::new_u8(1048576, 255 as u8) + large[524288] = 0 + large[1048575] = 128 + let copied = nil::std::text::from_bytes(large) + if nil::std::text::length(copied) != 1048576 then return 64 end if + if nil::std::text::byte_at(copied, 0) != 255 then return 65 end if + if nil::std::text::byte_at(copied, 524288) != 0 then return 66 end if + if nil::std::text::byte_at(copied, 1048575) != 128 then return 67 end if + println("from-bytes-ok") + return 0 +end fn diff --git a/tests/rooted_file_snapshot_tests.cpp b/tests/rooted_file_snapshot_tests.cpp new file mode 100644 index 00000000..489bf938 --- /dev/null +++ b/tests/rooted_file_snapshot_tests.cpp @@ -0,0 +1,315 @@ +// All paths and concurrent namespace changes belong to this private fixture. +#include "../nilcompiler/src/secure_file_io.hpp" +#include "../common/private_directory.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#endif + +namespace { +namespace fs = std::filesystem; +int checks = 0; + +void check(bool good, const char *label) { + ++checks; + if (!good) throw std::runtime_error(label); +} + +class Fixture { +public: + Fixture() : path(nil::detail::create_private_directory(fs::temp_directory_path(), "nil-rooted-snapshot-")) {} + ~Fixture() { std::error_code ignored; fs::remove_all(path, ignored); } + fs::path path; +}; + +void write(const fs::path &path, const std::string &bytes) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output.write(bytes.data(), static_cast(bytes.size())); + output.close(); + check(!output.fail(), "write owned fixture"); +} + +ny::detail::RootedFileSnapshot read(const fs::path &root, const std::string &relative, + std::uint64_t maximum = 1048576) { + return ny::detail::read_rooted_file_snapshot_utf8(root.u8string(), relative, maximum); +} + +std::string body(const ny::detail::RootedFileSnapshot &result) { + return std::string(result.bytes.begin(), result.bytes.end()); +} + +void refused(const ny::detail::RootedFileSnapshot &result, const char *label, + const fs::path &private_root) { + check(!result.ok && result.bytes.empty(), label); + check(result.error.find(private_root.u8string()) == std::string::npos && + result.error.rfind("file.read_rooted_snapshot: ", 0) == 0, + "refusal has a fixed diagnostic without private paths"); +} + +void timestamp(const fs::path &path, std::int64_t milliseconds) { +#ifdef _WIN32 + HANDLE file = CreateFileW(path.c_str(), FILE_WRITE_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + check(file != INVALID_HANDLE_VALUE, "open owned timestamp fixture"); + const auto ticks = static_cast(milliseconds + 11644473600000LL) * 10000; + FILETIME held {static_cast(ticks), static_cast(ticks >> 32U)}; + const bool okay = SetFileTime(file, nullptr, nullptr, &held) != 0; + CloseHandle(file); + check(okay, "set owned timestamp fixture"); +#else + timespec times[2] {}; + times[0].tv_sec = times[1].tv_sec = milliseconds / 1000; + times[0].tv_nsec = times[1].tv_nsec = (milliseconds % 1000) * 1000000; + check(::utimensat(AT_FDCWD, path.c_str(), times, 0) == 0, "set owned timestamp fixture"); +#endif +} + +void directory_link(const fs::path &link, const fs::path &target) { +#ifdef _WIN32 + // Junctions exercise reparse refusal without requiring symlink privilege. + check(fs::create_directory(link), "create owned junction directory"); + const std::wstring shown = fs::absolute(target).wstring(); + const std::wstring substitute = L"\\??\\" + shown; + struct Junction { + DWORD tag; + USHORT data_length; + USHORT reserved; + USHORT substitute_offset; + USHORT substitute_length; + USHORT print_offset; + USHORT print_length; + WCHAR names[1]; + }; + const auto name_bytes = (substitute.size() + shown.size() + 2) * sizeof(WCHAR); + std::vector storage(offsetof(Junction, names) + name_bytes, 0); + auto *data = reinterpret_cast(storage.data()); + data->tag = IO_REPARSE_TAG_MOUNT_POINT; + data->data_length = static_cast(storage.size() - 8); + data->substitute_length = static_cast(substitute.size() * sizeof(WCHAR)); + data->print_offset = static_cast((substitute.size() + 1) * sizeof(WCHAR)); + data->print_length = static_cast(shown.size() * sizeof(WCHAR)); + std::memcpy(data->names, substitute.data(), data->substitute_length); + std::memcpy(reinterpret_cast(data->names) + data->print_offset, + shown.data(), data->print_length); + HANDLE held = CreateFileW(link.c_str(), GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, nullptr); + check(held != INVALID_HANDLE_VALUE, "open owned junction directory"); + DWORD returned = 0; + const bool okay = DeviceIoControl(held, FSCTL_SET_REPARSE_POINT, data, + static_cast(storage.size()), nullptr, 0, &returned, nullptr) != 0; + CloseHandle(held); + check(okay, "set owned junction reparse data"); +#else + fs::create_directory_symlink(target, link); +#endif +} + +void contracts(const fs::path &base) { + const auto root = base / "public"; + const auto outside = base / "outside"; + fs::create_directories(root / "nested"); + fs::create_directory(outside); + const std::string binary("A\0\xff\x80Z", 5); + write(root / "nested" / "asset.bin", binary); + write(outside / "asset.bin", "outside-owned-marker"); + timestamp(root / "nested" / "asset.bin", 1700000000123LL); + const auto exact = read(root, "nested/asset.bin", binary.size()); + check(exact.ok && body(exact) == binary && exact.error.empty(), "read exact bounded binary payload"); + check(exact.modified_ms == 1700000000123LL, "return same-handle milliseconds without clock sampling"); + const auto short_limit = read(root, "nested/asset.bin", binary.size() - 1); + refused(short_limit, "reject oversized before allocation", base); + check(short_limit.error == "file.read_rooted_snapshot: input exceeds maximum_bytes", + "oversized refusal is distinct for HTTP mapping"); + refused(read(root, "nested/asset.bin", 512ULL * 1024 * 1024 + 1), "reject excessive requested limit", base); + write(root / "empty", ""); + const auto empty = read(root, "empty", 0); + check(empty.ok && empty.bytes.empty(), "zero limit accepts empty regular file"); + refused(read(root, "nested/asset.bin", 0), "zero limit rejects nonempty file", base); + refused(read(root, "absent"), "missing file refused", base); + refused(read(root, "nested"), "directory cannot be served as a regular file", base); + refused(read(root / "empty", "asset"), "regular file cannot be a root directory", base); + + const std::vector bad_names = { + "", ".", "..", "/nested/asset.bin", "nested/../empty", "nested/./asset.bin", + "nested//asset.bin", "nested/asset.bin/", "nested\\asset.bin", "C:asset.bin", + "empty:stream", std::string("empty\0ignored", 13), std::string("\xc0\xaf", 2), + std::string("\xed\xa0\x80", 3), std::string("\xf4\x90\x80\x80", 4)}; + for (const auto &name : bad_names) + refused(read(root, name), "invalid relative spelling refused", base); + refused(ny::detail::read_rooted_file_snapshot_utf8("", "empty", 10), "empty root refused", base); + refused(ny::detail::read_rooted_file_snapshot_utf8(std::string("\xc0\xaf", 2), "empty", 10), + "malformed UTF-8 root refused before filesystem access", base); + const auto unicode = fs::u8path(u8"space λ 🧪"); + fs::create_directory(root / unicode); + write(root / unicode / "asset.bin", binary); + check(body(read(root, unicode.u8string() + "/asset.bin")) == binary, "UTF-8 descendant names round-trip"); + directory_link(root / "linked", outside); + // This control intentionally demonstrates the old API's documented final- + // component-only boundary, using only a marker owned by the test fixture. + const auto legacy = ny::detail::read_regular_file_bounded(root / "linked" / "asset.bin", 100); + check(legacy.ok && std::string(legacy.bytes.begin(), legacy.bytes.end()) == "outside-owned-marker", + "negative control: final-component-only API follows intermediate link"); + refused(read(root, "linked/asset.bin"), "intermediate directory link refused", base); + refused(read(root, "linked"), "final directory reparse/link refused", base); + refused(read(root / "linked", "asset.bin"), "root final reparse/link refused", base); + refused(read(root / "linked" / "", "asset.bin"), "trailing slash cannot disguise a root link", base); + refused(read(root / "linked" / ".", "asset.bin"), "trailing dot cannot disguise a root link", base); + check(body(read(root / "", "nested/asset.bin")) == binary, + "trailing slash on an ordinary root remains valid"); + check(body(read(root / ".", "nested/asset.bin")) == binary, + "trailing dot on an ordinary root remains valid"); +#ifndef _WIN32 + fs::create_symlink(outside / "asset.bin", root / "final-link"); + refused(read(root, "final-link"), "final file symlink refused", base); + check(::mkfifo((root / "pipe").c_str(), 0600) == 0, "create owned FIFO"); + const auto began = std::chrono::steady_clock::now(); + refused(read(root, "pipe"), "FIFO is refused without opening a writer", base); + check(std::chrono::steady_clock::now() - began < std::chrono::seconds(2), "FIFO refusal is nonblocking"); +#endif +} + +void namespace_race(const fs::path &base) { + const auto root = base / "race-root"; + const auto outside = base / "race-outside"; + fs::create_directories(root / "live"); + fs::create_directory(outside); + write(root / "live" / "asset", "inside-owned-marker"); + write(outside / "asset", "outside-owned-marker"); + directory_link(root / "link-spare", outside); + std::atomic stop{false}; + std::atomic swaps{0}; + std::atomic mutator_failed{false}; + std::thread mutator([&] { + // Windows can briefly reject a directory rename while a descendant file + // handle denies delete sharing. Complete each owned swap before stopping. + const auto finish_rename = [&](const fs::path &from, const fs::path &to) { + const auto bound = std::chrono::steady_clock::now() + std::chrono::seconds(2); + std::error_code error; + do { + fs::rename(from, to, error); + if (!error) return true; + std::this_thread::yield(); + } while (std::chrono::steady_clock::now() < bound); + return false; + }; + while (!stop.load()) { + std::error_code error; + fs::rename(root / "live", root / "safe-spare", error); + if (error) { std::this_thread::yield(); continue; } + if (!finish_rename(root / "link-spare", root / "live")) { mutator_failed = true; break; } + ++swaps; + std::this_thread::yield(); + if (!finish_rename(root / "live", root / "link-spare") || + !finish_rename(root / "safe-spare", root / "live")) { mutator_failed = true; break; } + } + }); + bool escaped = false; + int reads = 0; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + try { + while (reads < 500 && std::chrono::steady_clock::now() < deadline) { + const auto result = read(root, "live/asset"); + ++reads; + if (result.ok && body(result) != "inside-owned-marker") escaped = true; + } + } catch (...) { + stop = true; + mutator.join(); + throw; + } + stop = true; + mutator.join(); + check(!mutator_failed.load(), "owned namespace mutator completed its swaps"); + check(swaps.load() > 0 && reads > 0, "concurrent namespace changes actually occurred"); + check(!escaped, "anchored opens never follow replacement link outside root"); +} + +void coupled_snapshot_race(const fs::path &base) { + const auto root = base / "snapshot-race"; + fs::create_directory(root); + const std::string first(65536, 'a'), second(65536, 'b'); + write(root / "live", first); + write(root / "spare", second); + timestamp(root / "live", 1700000000123LL); + timestamp(root / "spare", 1700000001567LL); + std::atomic stop{false}; + std::atomic swaps{0}; + std::atomic failed{false}; + std::thread publisher([&] { + const auto finish = [&](const char *from, const char *to) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + std::error_code error; + do { + fs::rename(root / from, root / to, error); + if (!error) return true; + std::this_thread::yield(); + } while (std::chrono::steady_clock::now() < deadline); + return false; + }; + while (!stop.load()) { + std::error_code error; + fs::rename(root / "live", root / "park", error); + if (error) { std::this_thread::yield(); continue; } + if (!finish("spare", "live") || !finish("park", "spare")) { + failed = true; + break; + } + ++swaps; + } + }); + bool mixed = false; + int succeeded = 0; + try { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + for (int index = 0; index < 500 && std::chrono::steady_clock::now() < deadline; ++index) { + const auto result = read(root, "live"); + if (result.ok) { + ++succeeded; + const auto bytes = body(result); + if (!((bytes == first && result.modified_ms == 1700000000123LL) || + (bytes == second && result.modified_ms == 1700000001567LL))) mixed = true; + } + } + } catch (...) { + stop = true; + publisher.join(); + throw; + } + stop = true; + publisher.join(); + check(!failed.load() && swaps.load() > 0, "same-handle fixture published distinct file versions"); + check(succeeded > 0, "same-handle fixture observed successful snapshots"); + check(!mixed, "bytes and timestamp always identify the same opened file version"); +} + +} // namespace + +int main() { + try { + Fixture fixture; + contracts(fixture.path); + namespace_race(fixture.path); + coupled_snapshot_race(fixture.path); + std::cout << checks << " rooted snapshot checks passed\n"; + return 0; + } catch (const std::exception &error) { + std::cerr << "rooted snapshot contract failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/tools/rooted_file_snapshot_contract.py b/tests/tools/rooted_file_snapshot_contract.py new file mode 100644 index 00000000..d1cf8c03 --- /dev/null +++ b/tests/tools/rooted_file_snapshot_contract.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Bounded public snapshot/byte-conversion routes with independent host oracles. + +This is a source-library regression, not installed SDK or release qualification. +No server is started. Raw process evidence and the owned fixture are retained. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile +import time + +import ordinary_io_parity_contract as support + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from native_output_staging import native_output_path + +SNAPSHOT_OUTPUT = b'snapshot-content-ok\nsnapshot-errors-ok\nsnapshot-links-ok\n' +CONVERSION_OUTPUT = b'from-bytes-ok\n' +NATIVE_REJECTION = b"unsupported native annotation '@intrinsic(\"text.from_bytes\")'" +SNAPSHOT_REJECTION = b'file.read_rooted_snapshot requires (text,text,i64) -> Result' +SOURCES = ('rooted-file-snapshot.ny', 'text-from-bytes.ny', 'text-from-bytes-native-reject.ny', + 'rooted-file-snapshot-invalid-schema.ny', 'rooted-file-snapshot-invalid-signature.ny') + + +def snapshot_signature(root: Path) -> dict: + result = support.inventory(root, maximum_entries=32, maximum_file=1024) + for name, entry in result.items(): + path = root / name + if entry['kind'] == 'file': + entry['modified_ns'] = path.stat().st_mtime_ns + elif entry['kind'] == 'link-or-reparse': + entry['target'] = os.readlink(path) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + for name in ('compiler', 'interpreter', 'nilstd'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--llvm', choices=('ON', 'OFF'), required=True) + destination = parser.add_mutually_exclusive_group() + destination.add_argument('--output', type=Path, help='Fresh exact evidence directory') + destination.add_argument('--output-parent', type=Path, help='Retain a fresh child here') + args = parser.parse_args() + compiler = args.compiler.resolve(strict=True) + interpreter = args.interpreter.resolve(strict=True) + nilstd = args.nilstd.resolve(strict=True) + if args.output_parent: + parent = args.output_parent.resolve() + parent.mkdir(parents=True, exist_ok=True) + output = Path(tempfile.mkdtemp(prefix='rooted-file-snapshot-', dir=parent)) + elif args.output: + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + else: + output = Path(tempfile.mkdtemp(prefix='rooted-file-snapshot-')) + work, logs = output / 'work', output / 'logs' + work.mkdir() + logs.mkdir() + environment = {key: value for key, value in os.environ.items() + if not key.startswith(('NY_', 'NIL_'))} + ownership = support.load_ownership(ROOT) + deadline = time.monotonic() + 300 + inputs = {name: {'path': str(path), **support.file_record(path)} for name, path in + [('compiler', compiler), ('interpreter', interpreter), + *[(name, ROOT / 'tests/language' / name) for name in SOURCES], + ('file_library', nilstd / 'io/file.ny'), + ('text_library', nilstd / 'text/string_utils.ny'), + ('contract', Path(__file__).resolve()), + ('process_owner', ROOT / 'validation/ny-3d/scripts/debugger_process.py'), + ('process_support', ROOT / 'tests/tools/ordinary_io_parity_contract.py')]} + report = {'schema': 'nil.rooted-file-snapshot.v1', 'status': 'FAIL', + 'inputs': inputs, 'nilstd': str(nilstd), 'commands': [], + 'snapshot_routes': [], 'conversion_routes': [], 'native_rejections': [], + 'unbox_rejections': [], + 'llvm_enabled': args.llvm == 'ON', 'whole_seconds': 300, + 'scope': 'Source-library interpreter/native regression. FileSnapshot content is a byte array on every route; text.from_bytes is interpreter-only.', + 'link_cases': 'required; POSIX file/directory symlinks, Windows directory junctions (no symlink privilege required)', + 'work_directory_retained': True} + + def run(name, command, expected=b'', limit=60, rejection=None): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AssertionError('whole snapshot contract deadline exhausted') + row = support.run_process(ownership, [str(value) for value in command], + work, environment, logs, name, min(limit, remaining)) + report['commands'].append(row) + stdout = (logs / row['stdout']).read_bytes() + stderr = (logs / row['stderr']).read_bytes() + if not row['cleanup_verified'] or row.get('timed_out') or row.get('error'): + raise AssertionError(name + ': bounded process cleanup failed') + if rejection: + if row.get('exit_code', 0) <= 0 or stdout or rejection not in stderr: + raise AssertionError(name + ': native compilation did not fail for the exact expected contract') + elif row['status'] != 'PASS' or stderr or (expected is not None and stdout != expected): + raise AssertionError(name + ': exit/stdout/stderr contract failed') + return row + + try: + fixture = work / 'fixtures' + fixture.mkdir() + rooted = fixture / 'root café λ' + rooted.mkdir() + (rooted / 'nested').mkdir() + outside = fixture / 'outside' + outside.mkdir() + sentinel = outside / 'sentinel.bin' + sentinel.write_bytes(b'outside fixture must never be returned\x00\xff') + if os.name == 'nt': + for name in ('linked.bin', 'via-link'): + link = rooted / name + run('create-junction-' + name, + ['cmd.exe', '/d', '/c', 'mklink', '/J', link, outside], expected=None) + if not getattr(link.lstat(), 'st_file_attributes', 0) & 0x400: + raise AssertionError('Windows fixture is not a reparse point') + report['link_fixture_kind'] = 'directory junctions: final reparse entry and intermediate directory escape' + else: + (rooted / 'linked.bin').symlink_to(sentinel) + (rooted / 'via-link').symlink_to(outside, target_is_directory=True) + report['link_fixture_kind'] = 'final file symlink and intermediate directory symlink' + payload = rooted / 'nested/café-λ.bin' + empty = rooted / 'empty.bin' + empty.write_bytes(b'') + fixed_time = 1_600_000_000_123_000_000 + os.utime(empty, ns=(fixed_time, fixed_time)) + sources = {} + for name in SOURCES: + source = work / name + source.write_bytes((ROOT / 'tests/language' / name).read_bytes()) + sources[name] = source + base = [compiler, '--quiet', '--no-stdlib', '--sdk-path', nilstd] + routes = [] + for encoding in ('text', 'binary'): + flags = [] if encoding == 'text' else ['--emit-binary-nil'] + nil = work / ('snapshot-' + encoding + '.nil') + run('compile-snapshot-' + encoding, [*base, '--no-bundle', *flags, '-o', nil, + sources['rooted-file-snapshot.ny']]) + if nil.read_bytes().startswith(b'NIL\0') != (encoding == 'binary'): + raise AssertionError('compiled snapshot encoding differs') + routes.append((encoding + '-interpreter', [interpreter, '--no-ai', nil, '--'])) + if args.llvm == 'ON': + native = work / 'snapshot-native' + run('compile-snapshot-native', [*base, '--native', '-o', native, + sources['rooted-file-snapshot.ny']]) + routes.append(('llvm-aot', [native_output_path(native)])) + routes.append(('llvm-jit', [*base, '--native', '--jit', '-o', work / 'snapshot-jit', + sources['rooted-file-snapshot.ny'], '--'])) + + initial_digest = None + for stage, seed in (('initial', 19), ('same-size-time-replacement', 21)): + data = bytes((index * 73 + seed) % 256 for index in range(256)) + payload.write_bytes(data) + os.utime(payload, ns=(fixed_time, fixed_time)) + stamp = payload.stat().st_mtime_ns // 1_000_000 + if empty.stat().st_mtime_ns // 1_000_000 != stamp: + raise AssertionError('host did not preserve equal fixture timestamps') + digest = hashlib.sha256(data).hexdigest() + if initial_digest is None: + initial_digest = digest + elif digest == initial_digest: + raise AssertionError('same-size/time replacement failed to change content digest') + before = snapshot_signature(fixture) + arguments = [rooted, str(stamp), digest, str(seed)] + for route, command in routes: + row = run(stage + '-' + route, [*command, *arguments], SNAPSHOT_OUTPUT) + if snapshot_signature(fixture) != before: + raise AssertionError('snapshot read changed the owned fixture') + report['snapshot_routes'].append({'stage': stage, 'route': route, 'status': 'PASS', + 'bytes': len(data), 'sha256': digest, 'modified_ms': stamp, + 'exact_payload_bytes_checked': 256, 'empty_file_checked': True, + 'descendant_and_final_links_rejected': True, 'namespace_unchanged': True, + 'elapsed_seconds': row['elapsed_seconds']}) + report[stage + '_namespace'] = before + + for route, command in routes: + expected = (b"anything type mismatch: expected 'array', got 'array'" + if route.startswith('llvm-') else + b"anything contains 'array', cannot extract 'array'") + run('wrong-element-' + route, [*command, *arguments, 'wrong-element'], rejection=expected) + if snapshot_signature(fixture) != before: + raise AssertionError('rejected extraction changed the owned fixture') + report['unbox_rejections'].append({'route': route, 'status': 'PASS', + 'reason': expected.decode()}) + + for encoding in ('text', 'binary'): + flags = [] if encoding == 'text' else ['--emit-binary-nil'] + nil = work / ('conversion-' + encoding + '.nil') + run('compile-conversion-' + encoding, [*base, '--no-bundle', *flags, '-o', nil, + sources['text-from-bytes.ny']]) + if nil.read_bytes().startswith(b'NIL\0') != (encoding == 'binary'): + raise AssertionError('compiled conversion encoding differs') + row = run('conversion-' + encoding, [interpreter, '--no-ai', nil], CONVERSION_OUTPUT) + report['conversion_routes'].append({'route': encoding + '-interpreter', 'status': 'PASS', + 'all_256_byte_values': True, 'copy_independent_of_source_mutation': True, + 'empty_and_1MiB_payloads': True, 'elapsed_seconds': row['elapsed_seconds']}) + if args.llvm == 'ON': + for route, flags in (('llvm-aot', []), ('llvm-jit', ['--jit'])): + target = work / ('conversion-rejected-' + route) + run('conversion-rejected-' + route, [*base, '--native', *flags, '-o', target, + sources['text-from-bytes-native-reject.ny']], + rejection=NATIVE_REJECTION) + if target.exists() or native_output_path(target).exists(): + raise AssertionError('rejected binary text conversion published an executable') + report['native_rejections'].append({'route': route, 'status': 'PASS', + 'case': 'text.from_bytes', + 'reason': NATIVE_REJECTION.decode()}) + for case in ('invalid-schema', 'invalid-signature'): + for route, flags in (('llvm-aot', []), ('llvm-jit', ['--jit'])): + target = work / ('snapshot-rejected-' + case + '-' + route) + run('snapshot-rejected-' + case + '-' + route, + [*base, '--native', *flags, '-o', target, + sources['rooted-file-snapshot-' + case + '.ny']], + rejection=SNAPSHOT_REJECTION) + if target.exists() or native_output_path(target).exists(): + raise AssertionError('invalid snapshot ABI published an executable') + report['native_rejections'].append({'route': route, 'case': case, + 'status': 'PASS', 'reason': SNAPSHOT_REJECTION.decode()}) + if len(report['snapshot_routes']) != (8 if args.llvm == 'ON' else 4): + raise AssertionError('snapshot route inventory incomplete') + if len(report['unbox_rejections']) != (4 if args.llvm == 'ON' else 2): + raise AssertionError('wrong-element extraction route inventory incomplete') + if len(report['conversion_routes']) != 2 or len(report['native_rejections']) != (6 if args.llvm == 'ON' else 0): + raise AssertionError('conversion route inventory incomplete') + for item in inputs.values(): + if support.file_record(Path(item['path'])) != {key: item[key] for key in ('bytes', 'sha256')}: + raise AssertionError('input changed during contract') + report['status'] = 'PASS' + except Exception as error: + report['error'] = str(error) + report['cleanup_verified'] = all(row.get('cleanup_verified') is True for row in report['commands']) + (output / 'report.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(report['status'] + ': ' + str(output / 'report.json')) + if report['status'] != 'PASS': + print(report.get('error', 'snapshot contract failed'), file=sys.stderr) + return 0 if report['status'] == 'PASS' else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) From 92dd8909da84c13304db7e0d8b0ed870ab871ea9 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 00:35:14 -0500 Subject: [PATCH 2/4] Qualify rooted file native contracts on both Mac architectures --- .../workflows/rooted-file-native-parity.yml | 326 ++++++++++++++++++ notebook/2026-11-12-web-server-boundaries.md | 90 +++++ notebook/README.md | 5 + 3 files changed, 421 insertions(+) create mode 100644 .github/workflows/rooted-file-native-parity.yml create mode 100644 notebook/2026-11-12-web-server-boundaries.md diff --git a/.github/workflows/rooted-file-native-parity.yml b/.github/workflows/rooted-file-native-parity.yml new file mode 100644 index 00000000..94c23989 --- /dev/null +++ b/.github/workflows/rooted-file-native-parity.yml @@ -0,0 +1,326 @@ +name: Rooted file native macOS checks + +"on": + workflow_dispatch: + +permissions: + contents: read + +# This campaign tests a fixed source revision. The workflow revision is recorded +# separately; neither an installed SDK nor any release artifact is replaced. +jobs: + native: + name: macOS ${{ matrix.host.arch }} rooted file native contract + runs-on: ${{ matrix.host.runner }} + timeout-minutes: 65 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + host: + - {runner: macos-15-intel, arch: x86_64} + - {runner: macos-15, arch: arm64} + defaults: + run: + shell: bash + env: + HOMEBREW_NO_AUTO_UPDATE: '1' + PYTHONDONTWRITEBYTECODE: '1' + PYTHONUTF8: '1' + NIL_TESTED_SOURCE: 93515c3d4c2b6e0e7c3b870eb283fce26392d613 + NIL_EXPECTED_ARCH: ${{ matrix.host.arch }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: 93515c3d4c2b6e0e7c3b870eb283fce26392d613 + path: source + persist-credentials: false + - name: Record identities and prepare bounded commands + timeout-minutes: 2 + run: | + mkdir -p build/rooted-native-evidence + cat > build/rooted_native_campaign.py <<'PY' + import hashlib, json, os, stat, subprocess, sys + from pathlib import Path + + ROOT = Path(os.environ['GITHUB_WORKSPACE']).resolve() + SOURCE = ROOT / 'source' + BUILD = ROOT / 'build/rooted-native' + EVIDENCE = ROOT / 'build/rooted-native-evidence' + sys.path.insert(0, str(SOURCE / 'tests/tools')) + import ordinary_io_parity_contract as support + + def save(name, value): + (EVIDENCE / name).write_text(json.dumps(value, indent=2) + '\n') + + def record(path): + return {**support.file_record(path), 'mode': stat.S_IMODE(path.stat().st_mode)} + + def source_identity(): + revision = subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], + text=True, timeout=15).strip() + dirty = subprocess.check_output(['git', '-C', str(SOURCE), 'status', '--porcelain', + '--untracked-files=no'], text=True, timeout=15) + assert revision == os.environ['NIL_TESTED_SOURCE'] and not dirty + return revision + + def run(name, command, timeout, environment=None): + # The canonical owner retains/reaps the process group and caps raw + # pipes. Keep full bounded logs, but print only their final 64 KiB. + support.MAX_OUTPUT = 8 * 1024**2 + row = support.run_process(support.load_ownership(SOURCE), command, ROOT, + environment or dict(os.environ), EVIDENCE, name, timeout) + save(name + '.json', row) + for key in ('stdout', 'stderr'): + path = EVIDENCE / row[key] + if path.is_file(): + data = path.read_bytes() + print(key + ': ' + str(len(data)) + ' bytes retained') + print(data[-65536:].decode('utf-8', errors='replace')) + assert row['status'] == 'PASS' and row['exit_code'] == 0 + assert row['cleanup_verified'] is True and not row.get('timed_out') and not row.get('error') + return row + + def seal(): + files = [BUILD / 'bin' / name for name in + ('nilcompiler', 'nilinterpreter', 'nil_rooted_file_snapshot_tests')] + files.append(BUILD / 'lib/libnil.a') + files.extend(path for path in (SOURCE / 'nilstd/src').rglob('*') if path.is_file()) + files.extend(SOURCE / name for name in ( + 'tests/tools/rooted_file_snapshot_contract.py', + 'tests/tools/ordinary_io_parity_contract.py', 'tests/native_output_staging.py', + 'validation/ny-3d/scripts/debugger_process.py', 'tests/rooted_file_snapshot_tests.cpp', + 'tests/language/rooted-file-snapshot.ny', 'tests/language/text-from-bytes.ny', + 'tests/language/text-from-bytes-native-reject.ny', + 'tests/language/rooted-file-snapshot-invalid-schema.ny', + 'tests/language/rooted-file-snapshot-invalid-signature.ny')) + return {str(path.relative_to(ROOT)): record(path) for path in sorted(files)} + PY + python3 - <<'PY' + import os, platform, re, subprocess, sys + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + assert platform.system() == 'Darwin' and platform.machine() == os.environ['NIL_EXPECTED_ARCH'] + workflow_revision = os.environ['GITHUB_WORKFLOW_SHA'] + assert re.fullmatch(r'[0-9a-f]{40}', workflow_revision) + campaign.save('identity.json', dict(tested_source_revision=campaign.source_identity(), + workflow_revision=workflow_revision, dispatch_revision=os.environ['GITHUB_SHA'], + workflow_ref=os.environ['GITHUB_WORKFLOW_REF'], run_id=os.environ['GITHUB_RUN_ID'], + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'], system=platform.system(), + machine=platform.machine(), python=sys.version, + macos=subprocess.check_output(['sw_vers'], text=True, timeout=15), + llvm=True, scope='Pinned source-library host, interpreter, LLVM AOT/JIT contracts; no installed SDK or release activation')) + PY + - name: Configure LLVM Release build + timeout-minutes: 8 + run: | + python3 - <<'PY' + import subprocess, sys + from pathlib import Path + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + prefix = Path(subprocess.check_output(['brew', '--prefix', 'llvm@18'], text=True, timeout=30).strip()) + version = subprocess.check_output([str(prefix/'bin/llvm-config'), '--version'], text=True, timeout=15).strip() + assert version.startswith('18.') + campaign.save('toolchain.json', dict(llvm_prefix=str(prefix), llvm_version=version, + sdk_path=subprocess.check_output(['xcrun', '--show-sdk-path'], text=True, timeout=15).strip(), + xcode=subprocess.check_output(['xcodebuild', '-version'], text=True, timeout=15))) + campaign.run('configure', ['cmake', '-S', str(campaign.SOURCE), '-B', str(campaign.BUILD), + '-DCMAKE_BUILD_TYPE=Release', '-DBUILD_TESTING=ON', '-DNIL_ENABLE_LLVM=ON', + '-DLLVM_DIR=' + str(prefix/'lib/cmake/llvm'), '-DPython3_EXECUTABLE=' + sys.executable, + '-DNILCOMPILER_SQLITE_INCLUDE_DIR=OFF', '-DNILCOMPILER_POSTGRESQL_INCLUDE_DIR=OFF'], 390) + PY + - name: Build compiler interpreter and host fixture + timeout-minutes: 40 + run: | + python3 - <<'PY' + import sys + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + campaign.run('build', ['cmake', '--build', str(campaign.BUILD), '--config', 'Release', + '--parallel', '2', '--target', 'nilcompiler', 'nilinterpreter', 'nil_rooted_file_snapshot_tests'], 2350) + campaign.save('seal-before.json', campaign.seal()) + PY + - name: Run exactly two rooted file contracts + timeout-minutes: 8 + run: | + python3 - <<'PY' + import json, os, subprocess, sys, xml.etree.ElementTree as ET + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + expected = {'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract'} + base = ['ctest', '--test-dir', str(campaign.BUILD), '-C', 'Release', '-R', + r'^nil[.]rooted-file-snapshot(-contract)?$'] + inventory = subprocess.check_output(base + ['--show-only=json-v1'], timeout=30) + (campaign.EVIDENCE/'inventory.json').write_bytes(inventory) + tests = json.loads(inventory)['tests'] + assert len(tests) == 2 and {test['name'] for test in tests} == expected + language = next(test for test in tests if test['name'].endswith('-contract'))['command'] + assert language[language.index('--llvm') + 1] == 'ON' + assert language[language.index('--compiler') + 1] == str(campaign.BUILD/'bin/nilcompiler') + assert language[language.index('--interpreter') + 1] == str(campaign.BUILD/'bin/nilinterpreter') + assert language[language.index('--nilstd') + 1] == str(campaign.SOURCE/'nilstd/src') + temporary = campaign.BUILD/'owned-test-temp' + temporary.mkdir(mode=0o700) + environment = dict(os.environ, TMPDIR=str(temporary) + os.sep) + campaign.run('ctest', base + ['--parallel', '1', '--no-tests=error', '--output-on-failure', + '--test-output-size-passed', '65536', '--test-output-size-failed', '65536', + '--output-junit', str(campaign.EVIDENCE/'results.xml')], 440, environment) + cases = list(ET.parse(campaign.EVIDENCE/'results.xml').getroot().iter('testcase')) + assert len(cases) == 2 and {case.get('name') for case in cases} == expected + assert all(case.get('status') == 'run' and all(case.find(kind) is None + for kind in ('failure', 'error', 'skipped')) for case in cases) + host = next(case for case in cases if case.get('name') == 'nil.rooted-file-snapshot') + assert host.findtext('system-out', '').strip() == '87 rooted snapshot checks passed' + assert not list(temporary.iterdir()), 'Owned host/compiler temporary files remain' + PY + - name: Independently verify route inventories raw output and seals + timeout-minutes: 3 + run: | + python3 - <<'PY' + import hashlib, json, sys + from pathlib import Path + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + roots = list((campaign.BUILD/'test-evidence').glob('rooted-file-snapshot-*')) + assert len(roots) == 1 and roots[0].is_dir() and not roots[0].is_symlink() + root = roots[0] + report = json.loads((root/'report.json').read_text()) + assert report['schema'] == 'nil.rooted-file-snapshot.v1' and report['status'] == 'PASS' + assert report['llvm_enabled'] is True and report['cleanup_verified'] is True + routes = {'text-interpreter', 'binary-interpreter', 'llvm-aot', 'llvm-jit'} + stages = {'initial', 'same-size-time-replacement'} + snapshot = report['snapshot_routes'] + assert len(snapshot) == 8 and {(row['stage'], row['route']) for row in snapshot} == { + (stage, route) for stage in stages for route in routes} + for row in snapshot: + seed = 19 if row['stage'] == 'initial' else 21 + digest = hashlib.sha256(bytes((index * 73 + seed) % 256 for index in range(256))).hexdigest() + assert row['status'] == 'PASS' and row['sha256'] == digest and row['bytes'] == 256 + assert row['exact_payload_bytes_checked'] == 256 and row['modified_ms'] == 1600000000123 + assert all(row[key] is True for key in ('empty_file_checked', + 'descendant_and_final_links_rejected', 'namespace_unchanged')) + unbox = report['unbox_rejections'] + assert len(unbox) == 4 and {row['route'] for row in unbox} == routes + conversions = report['conversion_routes'] + assert len(conversions) == 2 and {row['route'] for row in conversions} == {'text-interpreter', 'binary-interpreter'} + for row in conversions: + assert row['status'] == 'PASS' and all(row[key] is True for key in + ('all_256_byte_values', 'copy_independent_of_source_mutation', 'empty_and_1MiB_payloads')) + rejections = report['native_rejections'] + assert len(rejections) == 6 and {(row['case'], row['route']) for row in rejections} == { + (case, route) for case in ('text.from_bytes', 'invalid-schema', 'invalid-signature') + for route in ('llvm-aot', 'llvm-jit')} + expected_output = {'compile-snapshot-' + encoding: b'' for encoding in ('text', 'binary', 'native')} + for stage in stages: + for route in routes: + expected_output[stage + '-' + route] = b'snapshot-content-ok\nsnapshot-errors-ok\nsnapshot-links-ok\n' + expected_errors = {} + for row in unbox: + reason = ("anything type mismatch: expected 'array', got 'array'" + if row['route'].startswith('llvm-') else + "anything contains 'array', cannot extract 'array'") + assert row['status'] == 'PASS' and row['reason'] == reason + expected_errors['wrong-element-' + row['route']] = reason.encode() + for encoding in ('text', 'binary'): + expected_output['compile-conversion-' + encoding] = b'' + expected_output['conversion-' + encoding] = b'from-bytes-ok\n' + for row in rejections: + if row['case'] == 'text.from_bytes': + name = 'conversion-rejected-' + row['route'] + reason = "unsupported native annotation '@intrinsic(\"text.from_bytes\")'" + else: + name = 'snapshot-rejected-' + row['case'] + '-' + row['route'] + reason = 'file.read_rooted_snapshot requires (text,text,i64) -> Result' + assert row['status'] == 'PASS' and row['reason'] == reason + expected_errors[name] = reason.encode() + commands = report['commands'] + assert len(commands) == 25 and {row['name'] for row in commands} == set(expected_output) | set(expected_errors) + for row in commands: + assert row['cleanup_verified'] is True and not row.get('timed_out') and not row.get('error') + streams = {} + for stream in ('stdout', 'stderr'): + path = root/'logs'/row[stream] + assert path.parent == root/'logs' and path.name == row['name'] + '.' + stream + assert campaign.support.file_record(path, 1024**2) == row[stream + '_record'] + streams[stream] = path.read_bytes() + if row['name'] in expected_errors: + assert row['status'] == 'FAIL' and row['exit_code'] > 0 and not streams['stdout'] + assert expected_errors[row['name']] in streams['stderr'] + if '-rejected-' in row['name']: + target = Path(row['argv'][row['argv'].index('-o') + 1]) + assert not target.exists() and not target.is_symlink() + else: + assert row['status'] == 'PASS' and row['exit_code'] == 0 and not streams['stderr'] + assert streams['stdout'] == expected_output[row['name']] + paths = {name: campaign.SOURCE/'tests/language'/name for name in ( + 'rooted-file-snapshot.ny', 'text-from-bytes.ny', 'text-from-bytes-native-reject.ny', + 'rooted-file-snapshot-invalid-schema.ny', 'rooted-file-snapshot-invalid-signature.ny')} + paths.update(compiler=campaign.BUILD/'bin/nilcompiler', interpreter=campaign.BUILD/'bin/nilinterpreter', + file_library=campaign.SOURCE/'nilstd/src/io/file.ny', text_library=campaign.SOURCE/'nilstd/src/text/string_utils.ny', + contract=campaign.SOURCE/'tests/tools/rooted_file_snapshot_contract.py', + process_owner=campaign.SOURCE/'validation/ny-3d/scripts/debugger_process.py', + process_support=campaign.SOURCE/'tests/tools/ordinary_io_parity_contract.py') + assert set(report['inputs']) == set(paths) and report['nilstd'] == str(campaign.SOURCE/'nilstd/src') + for name, path in paths.items(): + item = report['inputs'][name] + assert item['path'] == str(path) and campaign.support.file_record(path) == { + key: item[key] for key in ('bytes', 'sha256')} + after = campaign.seal() + campaign.save('seal-after.json', after) + assert after == json.loads((campaign.EVIDENCE/'seal-before.json').read_text()) + assert not list((campaign.BUILD/'owned-test-temp').iterdir()) + campaign.save('replay.json', dict(status='PASS', tested_source_revision=campaign.source_identity(), + host_checks=87, snapshot_observations=8, wrong_element_rejections=4, + byte_conversions=2, native_compile_rejections=6, commands=25, raw_streams=50, + source_std_tools_unchanged=True, process_cleanup_verified=True, owned_temp_empty=True, + report=campaign.support.file_record(root/'report.json'))) + print('PASS: 87 host checks; 8 snapshot, 4 unbox, 2 conversion, 6 native rejection observations; 25 owned commands') + PY + - name: Preserve complete or partial evidence without product binaries + if: always() + timeout-minutes: 3 + run: | + python3 - <<'PY' + import hashlib, json, shutil, stat, sys + from pathlib import Path + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + destination = campaign.EVIDENCE/'contracts' + destination.mkdir(exist_ok=True) + for root in sorted((campaign.BUILD/'test-evidence').glob('rooted-file-snapshot-*')): + assert root.is_dir() and not root.is_symlink() + output = destination/root.name + output.mkdir() + paths = [root/'report.json', *sorted((root/'logs').glob('*'))] + for path in paths: + if not path.exists(): + continue + assert stat.S_ISREG(path.lstat().st_mode) and not path.is_symlink() + assert path.stat().st_size <= 8 * 1024**2 + target = output/path.relative_to(root) + target.parent.mkdir(exist_ok=True) + shutil.copyfile(path, target) + files = {} + total = 0 + for path in sorted(campaign.EVIDENCE.rglob('*')): + if path.is_dir(): + continue + assert stat.S_ISREG(path.lstat().st_mode) and not path.is_symlink() + assert path.stat().st_size <= 8 * 1024**2 + total += path.stat().st_size + assert total <= 64 * 1024**2 + files[str(path.relative_to(campaign.EVIDENCE))] = campaign.support.file_record(path, 8 * 1024**2) + campaign.save('evidence.json', dict(tested_source_revision=campaign.source_identity(), + workflow_revision=__import__('os').environ['GITHUB_WORKFLOW_SHA'], + scope='Evidence and bounded logs only; generated executables, objects, SDKs and fixture data excluded', + total_bytes=total, files=files)) + PY + - name: Upload complete or partial results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rooted-file-native-${{ matrix.host.arch }}-source-93515c3-${{ github.sha }}-${{ github.run_attempt }} + path: build/rooted-native-evidence/ + if-no-files-found: error + retention-days: 14 diff --git a/notebook/2026-11-12-web-server-boundaries.md b/notebook/2026-11-12-web-server-boundaries.md new file mode 100644 index 00000000..18280faa --- /dev/null +++ b/notebook/2026-11-12-web-server-boundaries.md @@ -0,0 +1,90 @@ +# Web server boundaries — 2026-11-12 + +## Status and question + +Implemented in framework `93515c3d4c2b6e0e7c3b870eb283fce26392d613` +([PR 39](https://github.com/exec-software/exec-nil-framework/pull/39)) and the +[web server PR](https://github.com/exec-software/exec-nil-framework-web/pull/1). +The application exposed gaps that isolated language examples did not: request +boundaries, file identity, filesystem containment and binary-value storage. + +The question was how to keep protocol policy in Matrix while giving it one +reliable filesystem observation on Linux, Windows and both macOS architectures. + +## Decision + +`read_rooted_snapshot(root, relative, maximum_bytes)` returns a typed +`FileSnapshot`: byte-array content, same-handle modification time and SHA-256 of +those exact bytes. It opens each descendant relative to an already-open +directory and rejects symlinks or Windows reparse points. Root spellings ending +in a separator or `/.` cannot disguise a final root link. Ancestors of the +trusted root, hard links and privileged mount changes remain outside this +boundary; the API is not a filesystem sandbox. + +Matrix retains HTTP framing, precondition ordering, range decisions and response +construction. A native content digest and one bulk byte-to-text copy avoid a +per-byte interpreter loop on every request. The pure Matrix SHA-256 code remains +a reference checked against independent vectors. The observed local 1 MiB +loopback response took about 10 ms; this is a measurement, not a service SLA. + +Raw `text.from_bytes` is explicitly interpreter-only until native text can +represent embedded NUL. The snapshot itself remains binary-safe on native +routes because its content is an array. Native compilation rejects unsupported +raw-text conversion instead of truncating data or silently changing its type. + +The work also corrected compact byte-array validation inside boxed records. +Exact `array` values from file operations are valid storage; raw pointer +allocations do not become arrays merely because their bytes look compatible. +Forced-GC and function-return controls verify that a boxed snapshot retains its +content and digest on the supported execution routes. + +## Evidence + +- Linux: 87 host filesystem checks; five focused CTest contracts, including + existing UTF-8, dynamic-value and native-runtime regressions. +- The rooted-file language contract checks textual and binary NIL, LLVM JIT + and AOT before and after an equal-size edit with its timestamp restored: + eight positive observations, four wrong-type extraction refusals, two raw + byte-text interpreter cases and six native declaration/provider refusals. +- [Four-host runtime run 43158309597](https://github.com/exec-software/exec-nil-framework/actions/runs/43158309597) + passed the exact eight-contract inventory on Linux, Windows, Intel macOS + and Apple Silicon macOS. These jobs use LLVM OFF; they are not native-route + or installed-SDK qualification. +- A fresh Windows LLVM build of `93515c3` passed 90 host checks and the + complete AOT/JIT snapshot contract. Existing checkouts and accepted SDKs + remained unchanged. Both macOS application/editor jobs also passed; the new + native snapshot contract is qualified separately on both Mac architectures. +- The web platform campaign requires 1,128 unit checks, 38 real-wire checks + including the actual first-byte deadline, and 15 static HTTP checks. Its + Windows run exposed CRLF checkout conversion of a byte-exact CSS fixture. + `.gitattributes` now requires LF text rather than weakening length or digest + expectations. The original failure remains in the CI record. A second run + found a test-oracle gap: Winsock reports deadline closure as error 10053. + The corrected oracle accepts that exact Windows closure, observes readiness + continuously, and rejects early disconnects outside the 19–25 second window. + Sixteen synthetic controls and 76 actual Windows wire checks passed across + two library locations, with closure observed at 20.004 and 20.027 seconds. +- Owned-process timeout controls passed on Linux and Windows, including a real + listening Matrix child, nested Windows Jobs and failed Job assignment before + command release. Test cleanup is part of the pass condition. + +The test implementations live in `tests/rooted_file_snapshot_tests.cpp` and +`tests/tools/rooted_file_snapshot_contract.py`; web consumer tests live in its +own repository. No accepted SDK or release deployment is replaced by this work. + +## Follow-up + +Complete native snapshot qualification on both macOS architectures, then record the corrected web campaign's four-host results. Keep the platform +checks required for public changes alongside the secret/path audit and review. + +The next project is a clean Matrix WebView package. The archived implementation +has no macOS provider and advertises some Linux-only functions through a shared +wrapper. Its replacement needs a checked provider ABI, bounded event queues and +observable asynchronous completion; generic legacy FFI and provider hot reload +do not establish those guarantees. Browser-origin provenance must come from +native evidence or be marked unavailable, never inferred from a page message. + +For future AI diagnostics, content identities can correlate a file observation +with an action without duplicating its bytes. This is a proposed use, not an +automatic logging or executable-context feature; fingerprints can themselves +be sensitive and need the same deliberate disclosure policy as other context. diff --git a/notebook/README.md b/notebook/README.md index ddf6fc5f..8c1b0d7a 100644 --- a/notebook/README.md +++ b/notebook/README.md @@ -11,6 +11,11 @@ and explicit about evidence, uncertainty, and implementation status. ## Contents +- [`2026-11-12-web-server-boundaries.md`](2026-11-12-web-server-boundaries.md) — + application-driven file snapshots, protocol boundaries, binary values and + the checked provider boundary needed by the planned WebView rewrite. +- [`2026-11-11-web-runtime-parity.md`](2026-11-11-web-runtime-parity.md) — + current-source and installed-SDK platform evidence for the web runtime. - [`2026-11-04-macos-parity-plan.md`](2026-11-04-macos-parity-plan.md) — initial Darwin/Apple Silicon gap audit, hosted and VM testing options, and the Windows-first sequence for establishing macOS parity. From aad707761460951c2e42401d3548faec226a8e79 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 00:35:24 -0500 Subject: [PATCH 3/4] Run the new native Mac campaign during workflow review --- .github/workflows/rooted-file-native-parity.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/rooted-file-native-parity.yml b/.github/workflows/rooted-file-native-parity.yml index 94c23989..978f7811 100644 --- a/.github/workflows/rooted-file-native-parity.yml +++ b/.github/workflows/rooted-file-native-parity.yml @@ -2,6 +2,12 @@ name: Rooted file native macOS checks "on": workflow_dispatch: + # A new workflow cannot be dispatched until it exists on the default branch. + # This scoped PR trigger qualifies it before review/merge without changing + # the fixed source revision under test. + pull_request: + paths: + - .github/workflows/rooted-file-native-parity.yml permissions: contents: read From 6923566732a38d7c2917466cc1befb6a5b4eee1f Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 00:56:06 -0500 Subject: [PATCH 4/4] Record successful server and native Mac qualification --- notebook/2026-11-12-web-server-boundaries.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/notebook/2026-11-12-web-server-boundaries.md b/notebook/2026-11-12-web-server-boundaries.md index 18280faa..51475ad7 100644 --- a/notebook/2026-11-12-web-server-boundaries.md +++ b/notebook/2026-11-12-web-server-boundaries.md @@ -74,8 +74,13 @@ own repository. No accepted SDK or release deployment is replaced by this work. ## Follow-up -Complete native snapshot qualification on both macOS architectures, then record the corrected web campaign's four-host results. Keep the platform -checks required for public changes alongside the secret/path audit and review. +Native snapshot qualification passed on both Mac architectures in +[run 43162404951](https://github.com/exec-software/exec-nil-framework/actions/runs/43162404951), +with the exact 87 host and 8/4/2/6 language-route inventories against fixed +source `93515c3`. The corrected [web run 43162279298](https://github.com/exec-software/exec-nil-framework-web/actions/runs/43162279298) +passed all four hosts at web head `4bcb43f`: 1,128 unit, 16 oracle, 38 wire +and 15 static checks each. The four `web-*` checks now join `secrets` as +required checks; review and the other existing branch protections are unchanged. The next project is a clean Matrix WebView package. The archived implementation has no macOS provider and advertises some Linux-only functions through a shared