From 93515c3d4c2b6e0e7c3b870eb283fce26392d613 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Wed, 11 Nov 2026 23:58:39 -0500 Subject: [PATCH 01/12] Add anchored file snapshots with binary-safe content identities --- .github/workflows/web-runtime-parity.yml | 10 +- CMakeLists.txt | 14 + docs/standard-library/index.md | 3 + .../standard-library/rooted-file-snapshots.md | 52 +++ docs/standard-library/text-spans.md | 14 +- nilcompiler/src/llvm_backend.cpp | 106 ++++++ nilcompiler/src/native_runtime.cpp | 43 +++ nilcompiler/src/native_runtime.hpp | 6 + nilcompiler/src/secure_file_io.cpp | 254 ++++++++++++++ nilcompiler/src/secure_file_io.hpp | 20 ++ nilcompiler/src/vm.cpp | 90 +++++ nilstd/src/io/file.ny | 22 ++ nilstd/src/text/string_utils.ny | 10 +- .../rooted-file-snapshot-invalid-schema.ny | 22 ++ .../rooted-file-snapshot-invalid-signature.ny | 12 + tests/language/rooted-file-snapshot.ny | 82 +++++ .../language/text-from-bytes-native-reject.ny | 7 + tests/language/text-from-bytes.ny | 33 ++ tests/rooted_file_snapshot_tests.cpp | 315 ++++++++++++++++++ tests/tools/rooted_file_snapshot_contract.py | 246 ++++++++++++++ 20 files changed, 1354 insertions(+), 7 deletions(-) create mode 100644 docs/standard-library/rooted-file-snapshots.md create mode 100644 tests/language/rooted-file-snapshot-invalid-schema.ny create mode 100644 tests/language/rooted-file-snapshot-invalid-signature.ny create mode 100644 tests/language/rooted-file-snapshot.ny create mode 100644 tests/language/text-from-bytes-native-reject.ny create mode 100644 tests/language/text-from-bytes.ny create mode 100644 tests/rooted_file_snapshot_tests.cpp create mode 100644 tests/tools/rooted_file_snapshot_contract.py diff --git a/.github/workflows/web-runtime-parity.yml b/.github/workflows/web-runtime-parity.yml index ae4dba20..ef10f0ba 100644 --- a/.github/workflows/web-runtime-parity.yml +++ b/.github/workflows/web-runtime-parity.yml @@ -9,8 +9,12 @@ name: Web runtime platform checks - 'nilstd/**' - 'tests/net_*' - 'tests/file_time_support_tests.cpp' + - 'tests/rooted_file_snapshot_tests.cpp' - 'tests/tools/net_socket_contract.py' - 'tests/tools/file_modified_at_contract.py' + - 'tests/tools/rooted_file_snapshot_contract.py' + - 'tests/language/rooted-file-snapshot.ny' + - 'tests/language/text-from-bytes*.ny' - 'tests/matrix/**' workflow_dispatch: @@ -82,6 +86,7 @@ jobs: cmake --build build/web-parity --config Release --parallel 2 \ --target nilcompiler nilinterpreter nil_net_support_tests \ nil_net_lifecycle_tests nil_file_time_support_tests \ + nil_rooted_file_snapshot_tests \ 2>&1 | tee build/web-parity-evidence/build.log - name: Run real peer and language contracts timeout-minutes: 8 @@ -89,13 +94,14 @@ jobs: "$NIL_CI_PYTHON" - <<'PY' import json, subprocess from pathlib import Path - pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract)|matrix[.]execution)$' + pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract|rooted-file-snapshot|rooted-file-snapshot-contract)|matrix[.]execution)$' base = ['ctest', '--test-dir', 'build/web-parity', '-C', 'Release', '-R', pattern] root = Path('build/web-parity-evidence').resolve() inventory = subprocess.check_output(base+['--show-only=json-v1'], timeout=30) (root/'inventory.json').write_bytes(inventory) expected = {'net.support-contract', 'net.lifecycle-contract', 'net.socket-contract', - 'nil.file-time-support', 'nil.file-modified-at-contract', 'matrix.execution'} + 'nil.file-time-support', 'nil.file-modified-at-contract', 'matrix.execution', + 'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract'} assert {test['name'] for test in json.loads(inventory)['tests']} == expected with (root/'ctest.log').open('wb') as output: result = subprocess.run(base+['--no-tests=error', '--output-on-failure', diff --git a/CMakeLists.txt b/CMakeLists.txt index e6927f1f..bf26728e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,12 @@ if(BUILD_TESTING) LABELS "tools;windows;rc" TIMEOUT 30) endif() + add_executable(nil_rooted_file_snapshot_tests tests/rooted_file_snapshot_tests.cpp) + target_link_libraries(nil_rooted_file_snapshot_tests PRIVATE nilruntime) + target_include_directories(nil_rooted_file_snapshot_tests PRIVATE nilcompiler/src) + add_test(NAME nil.rooted-file-snapshot COMMAND $) + set_tests_properties(nil.rooted-file-snapshot PROPERTIES LABELS "storage;runtime;rc" TIMEOUT 60) + add_executable(nil_utf8_file_paths_tests tests/utf8_file_paths_tests.cpp) target_link_libraries(nil_utf8_file_paths_tests PRIVATE nilruntime) target_include_directories(nil_utf8_file_paths_tests PRIVATE nilcompiler/src) @@ -1216,6 +1222,14 @@ if(BUILD_TESTING) --llvm "${NIL_ENABLE_LLVM}" --output-parent "${CMAKE_BINARY_DIR}/test-evidence") set_tests_properties(nil.file-modified-at-contract PROPERTIES LABELS "io;routes;rc" TIMEOUT 180) + add_test(NAME nil.rooted-file-snapshot-contract + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/tools/rooted_file_snapshot_contract.py" + --compiler $ + --interpreter $ + --nilstd "${CMAKE_CURRENT_SOURCE_DIR}/nilstd/src" + --llvm "${NIL_ENABLE_LLVM}" + --output-parent "${CMAKE_BINARY_DIR}/test-evidence") + set_tests_properties(nil.rooted-file-snapshot-contract PROPERTIES LABELS "io;routes;rc" TIMEOUT 360) add_test(NAME nillsp.semantic-protocol COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/lsp_semantic_protocol.py" --lsp $) diff --git a/docs/standard-library/index.md b/docs/standard-library/index.md index 2f10ab4a..a7dd34c0 100644 --- a/docs/standard-library/index.md +++ b/docs/standard-library/index.md @@ -128,6 +128,9 @@ the runtime and included in Aperture's text allocation measurements. For untrusted file/container input, `nil.std.io.read_regular_bytes` accepts an explicit byte ceiling and rejects the final-component symlink, special file, short/growing read, and observed metadata change before returning bytes. +For paths below a trusted directory, [`read_rooted_snapshot`](rooted-file-snapshots.md) +also rejects links in descendant directories and returns the exact byte array, +descriptor-observed modification time, and SHA-256 together. `write_bytes_atomic` stages and flushes bytes in the destination directory, then atomically publishes them; `replace=false` cannot clobber an existing entry (including a dangling symlink), while `replace=true` only replaces a diff --git a/docs/standard-library/rooted-file-snapshots.md b/docs/standard-library/rooted-file-snapshots.md new file mode 100644 index 00000000..7d0e52b1 --- /dev/null +++ b/docs/standard-library/rooted-file-snapshots.md @@ -0,0 +1,52 @@ +# Rooted file snapshots + +`nil.std.io.read_rooted_snapshot(root, relative, maximum_bytes)` reads one +bounded regular file below a trusted directory. It returns +`Result`: + +| Field | Type | Meaning | +| --- | --- | --- | +| `content` | `array` | The exact bytes read, including NUL and arbitrary high bytes. | +| `modified_ms` | `i64` | Modification time observed on the final file handle, in Unix milliseconds. | +| `sha256` | `text` | The 64 lowercase hexadecimal digits of SHA-256 over `content`. | + +The caller supplies a limit from zero through 512 MiB. A zero limit accepts +an empty file. Oversized input is rejected before allocating its contents. +SHA-256 is computed over the loaded bytes, so replacing content while keeping +its size and timestamp does not preserve its digest. + +```ny +import nil.std.io.file + +fn inspect_asset(root: text) + match nil::std::io::read_rooted_snapshot(root, "assets/site.css", 1048576) + Result.Ok(snapshot) -> + println(snapshot.content.len()) + println(snapshot.modified_ms) + println(snapshot.sha256) + Result.Error(message) -> println(message) + end match +end fn +``` + +The `relative` path must be nonempty strict UTF-8, with `/` separating its +components. Empty components, `.` and `..`, absolute paths, backslashes, +colons, and NUL bytes are rejected. The root selects the trusted directory; +every descendant component is opened relative to an already-open directory. +Descendant symlinks and Windows reparse points are rejected, including the +final component. Directories and other special files cannot be returned as +content. Errors do not include caller-supplied paths. + +The final handle supplies both bytes and metadata, and the runtime checks for +observed changes around the read. This does not make a file continuously +immutable or prevent privileged mount changes, hard links already placed in +the root, or every concurrent write. The returned digest describes the bytes +actually returned; it is not a separate path lookup or a metadata hash. + +The snapshot record uses the same byte-array representation for interpreter, +LLVM JIT, and native executable routes. Code using the interpreter's raw-byte +text representation can convert `snapshot.content` with +[`nil.std.text.from_bytes`](text-spans.md#byte-values). That conversion remains +interpreter-only: native text has a NUL-terminated representation and must not +silently truncate binary content. Native consumers can inspect, hash-check, or +write the byte array without converting it to text. diff --git a/docs/standard-library/text-spans.md b/docs/standard-library/text-spans.md index 0b0d2ba1..9b6b2f17 100644 --- a/docs/standard-library/text-spans.md +++ b/docs/standard-library/text-spans.md @@ -67,14 +67,20 @@ assembler/linker execution, and installed source/binary libraries. ## Byte values -`nil.std.text` measures and slices by byte — `length` of `"café"` is 5 — but -until these two existed there was no way to read a byte *as a number* or write -one as text. +`nil.std.text` measures and slices by byte — `length` of `"café"` is 5. +Use explicit byte operations to inspect or construct raw content. | Function | Result | | --- | --- | | `byte_at(s: text, index: i64) -> i64` | The byte at an index, `0..255`, or minus one when the index is outside the text. | | `from_byte(code: i64) -> text` | One byte as text; empty text for a code outside `0..255`. | +| `from_bytes(data: array) -> text` | An independent text copy of every array byte, including NUL and invalid UTF-8. | + +These raw-byte operations are interpreter-only. Native text currently requires +NUL-free values; native compilation rejects the unsupported intrinsics rather +than silently truncating or replacing bytes. `from_bytes` copies the array in +one operation and does not change when the source array is later mutated. +Use a byte array directly for binary data shared with native routes. The conversion that looks like it would do this is `nil.std.core`'s `convert.char_to_i32`, and it will not: it takes a whole @@ -85,7 +91,7 @@ isolated in order to inspect it whole, because isolating it needed the same test. Percent-encoding anything outside ASCII was unreachable because of this, and so was reading arbitrary bytes off a socket. -These two are about bytes and say so in their names. They know nothing about +These operations are about bytes and say so in their names. They know nothing about UTF-8; assembling characters from bytes is the caller's job. `from_byte(195)` is the single byte `0xC3`, where `convert.i32_to_char(195)` is diff --git a/nilcompiler/src/llvm_backend.cpp b/nilcompiler/src/llvm_backend.cpp index 268c779b..36d29a09 100644 --- a/nilcompiler/src/llvm_backend.cpp +++ b/nilcompiler/src/llvm_backend.cpp @@ -3388,6 +3388,111 @@ bool NativeCodegen::generateModule(const nil_module &mod) { return false; impl_->builder->CreateRet(error_variant); annotation_handled = true; + } else if (dispatch_name == "@intrinsic" && + dispatch_provider == "file.read_rooted_snapshot") { + const auto result_arguments = + native_generic_arguments(fn.return_type, "result"); + const auto *definition = result_arguments && result_arguments->size() == 2 + ? find_native_aggregate_definition(mod, (*result_arguments)[0]) + : nullptr; + const auto layout_it = definition ? native_struct_layouts.find(definition) + : native_struct_layouts.end(); + bool valid_schema = definition && definition->fields.size() == 3 && + compact_nil_type(definition->name) == "nil::std::io::FileSnapshot" && + layout_it != native_struct_layouts.end() && + layout_it->second.hidden_fields == 0; + const std::pair required_fields[] = { + {"content", "array"}, {"modified_ms", "i64"}, {"sha256", "text"}}; + if (valid_schema) { + for (const auto &[name, type] : required_fields) { + const auto field = layout_it->second.field_indices.find(name); + if (field == layout_it->second.field_indices.end()) { + valid_schema = false; + break; + } + const auto &actual_type = layout_it->second.field_nil_types[field->second]; + valid_schema = valid_schema && + (std::string(type) == "text" + ? native_handle_kind_for_nil_type(actual_type) == native_handle_kind::text + : normalize_nil_type(actual_type) == type); + } + } + if (!valid_schema || fn.params.size() != 3 || + native_handle_kind_for_nil_type(nil_parameter_type(fn.params[0])) != + native_handle_kind::text || + native_handle_kind_for_nil_type(nil_parameter_type(fn.params[1])) != + native_handle_kind::text || + normalize_nil_type(nil_parameter_type(fn.params[2])) != "i64" || + native_handle_kind_for_nil_type((*result_arguments)[1]) != + native_handle_kind::text || + impl_->current_function->arg_size() != 3 || + !impl_->current_function->getReturnType()->isIntegerTy(64)) { + std::cerr << "[NativeCodegen] file.read_rooted_snapshot requires " + "(text,text,i64) -> Result with the " + "exact content/modified_ms/sha256 field schema\n"; + return false; + } + auto *pointer_type = llvm::PointerType::getUnqual(ctx); + auto *word_type = llvm::Type::getInt64Ty(ctx); + auto *bytes_out = impl_->builder->CreateAlloca(pointer_type); + auto *modified_out = impl_->builder->CreateAlloca(word_type); + auto *digest_out = impl_->builder->CreateAlloca(pointer_type); + auto *error_out = impl_->builder->CreateAlloca(pointer_type); + auto *runtime_type = llvm::FunctionType::get( + llvm::Type::getInt32Ty(ctx), + {pointer_type, pointer_type, word_type, pointer_type, pointer_type, + pointer_type, pointer_type}, false); + auto runtime = impl_->module->getOrInsertFunction( + "nyrt_read_rooted_snapshot", runtime_type); + auto *status = impl_->builder->CreateCall(runtime_type, runtime.getCallee(), + {impl_->current_function->getArg(0), impl_->current_function->getArg(1), + impl_->current_function->getArg(2), bytes_out, modified_out, + digest_out, error_out}); + auto *success = llvm::BasicBlock::Create(ctx, "snapshot.success", + impl_->current_function); + auto *failure = llvm::BasicBlock::Create(ctx, "snapshot.failure", + impl_->current_function); + impl_->builder->CreateCondBr(impl_->builder->CreateICmpNE( + status, llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), 0)), + success, failure); + impl_->builder->SetInsertPoint(success); + const auto allocation_size = impl_->module->getDataLayout() + .getTypeAllocSize(layout_it->second.llvm_type).getFixedValue(); + llvm::Value *record = nullptr; + if (impl_->memory_mode == MemoryMode::SAFE_GC) { + auto *allocate_type = llvm::FunctionType::get(pointer_type, {word_type}, false); + auto allocate = impl_->module->getOrInsertFunction("nyrt_alloc", allocate_type); + record = impl_->builder->CreateCall(allocate_type, allocate.getCallee(), + {llvm::ConstantInt::get(word_type, allocation_size)}); + } else { + auto *allocate_type = llvm::FunctionType::get(pointer_type, {native_size_type}, false); + auto allocate = impl_->module->getOrInsertFunction("malloc", allocate_type); + record = impl_->builder->CreateCall(allocate_type, allocate.getCallee(), + {native_size_constant(allocation_size)}); + } + emit_allocation_null_check(record, "FileSnapshot"); + impl_->setNilType(record, definition->name); + llvm::Value *values[] = { + impl_->builder->CreateLoad(pointer_type, bytes_out), + impl_->builder->CreateLoad(word_type, modified_out), + impl_->builder->CreateLoad(pointer_type, digest_out)}; + for (std::size_t index = 0; index < 3; ++index) { + const auto field_index = layout_it->second.field_indices.at(required_fields[index].first); + auto *field_pointer = impl_->builder->CreateStructGEP( + layout_it->second.llvm_type, record, field_index); + impl_->builder->CreateStore(values[index], field_pointer); + } + auto *ok = emit_single_payload_variant("Result::Ok", record, + fn.return_type, definition->name); + if (!ok) return false; + impl_->builder->CreateRet(ok); + impl_->builder->SetInsertPoint(failure); + auto *error = impl_->builder->CreateLoad(pointer_type, error_out); + auto *failed = emit_single_payload_variant("Result::Error", error, + fn.return_type, "text"); + if (!failed) return false; + impl_->builder->CreateRet(failed); + annotation_handled = true; } else if (ann.find("@intrinsic(\"file.read_regular_bytes\")") != std::string::npos || ann.find("@intrinsic(file.read_regular_bytes)") != @@ -9380,6 +9485,7 @@ int NativeCodegen::executeJIT(const std::vector &program_args) { expose_runtime_symbol("nyrt_buffer_get", &nyrt_buffer_get); expose_runtime_symbol("nyrt_buffer_set", &nyrt_buffer_set); expose_runtime_symbol("nyrt_read_regular_bytes", &nyrt_read_regular_bytes); + expose_runtime_symbol("nyrt_read_rooted_snapshot", &nyrt_read_rooted_snapshot); expose_runtime_symbol("nyrt_write_bytes_atomic", &nyrt_write_bytes_atomic); expose_runtime_symbol("nyrt_io_v1", &nyrt_io_v1); expose_runtime_symbol("nyrt_try_i32", &nyrt_try_i32); diff --git a/nilcompiler/src/native_runtime.cpp b/nilcompiler/src/native_runtime.cpp index 2a1a0d5c..7c20629b 100644 --- a/nilcompiler/src/native_runtime.cpp +++ b/nilcompiler/src/native_runtime.cpp @@ -1,6 +1,7 @@ #include "native_runtime.hpp" #include "../../common/standard_streams.hpp" #include "../../common/strict_json.hpp" +#include "../../common/content_digest.hpp" #include "../../common/windows_environment.hpp" #include "ordinary_io_intrinsic.hpp" #include "dynamic_type.hpp" @@ -1955,6 +1956,48 @@ std::int32_t nyrt_io_v1(const char *operation, std::uint64_t first, } } +std::int32_t nyrt_read_rooted_snapshot( + const char *root, const char *relative, std::int64_t maximum_bytes, + void **bytes_out, std::int64_t *modified_ms_out, const char **sha256_out, + const char **error_out) noexcept { + if (bytes_out) *bytes_out = nullptr; + if (modified_ms_out) *modified_ms_out = 0; + if (sha256_out) *sha256_out = nullptr; + if (error_out) *error_out = nullptr; + if (!root || !relative || !bytes_out || !modified_ms_out || !sha256_out || + maximum_bytes < 0 || + static_cast(maximum_bytes) > maximum_file_bytes) { + set_error(error_out, "file.read_rooted_snapshot: invalid arguments"); + return 0; + } + try { + auto loaded = ny::detail::read_rooted_file_snapshot_utf8( + root, relative, static_cast(maximum_bytes)); + if (!loaded.ok) { + set_error(error_out, loaded.error); + return 0; + } + nil::detail::sha256 hash; + hash.update(loaded.bytes.data(), loaded.bytes.size()); + const auto digest = hash.finish(); + auto *array = make_array(NYRT_ARRAY_U8, loaded.bytes.size()); + const auto *digest_text = own_string(digest); + if (!array || !digest_text) { + set_error(error_out, "file.read_rooted_snapshot: allocation failed"); + return 0; + } + if (!loaded.bytes.empty()) + std::memcpy(array->data, loaded.bytes.data(), loaded.bytes.size()); + *bytes_out = array; + *modified_ms_out = loaded.modified_ms; + *sha256_out = digest_text; + return 1; + } catch (...) { + set_error(error_out, "file.read_rooted_snapshot: host operation failed"); + return 0; + } +} + std::int32_t nyrt_read_regular_bytes(const char *path, std::int64_t maximum_bytes, void **bytes_out, diff --git a/nilcompiler/src/native_runtime.hpp b/nilcompiler/src/native_runtime.hpp index c5ee085b..5435df50 100644 --- a/nilcompiler/src/native_runtime.hpp +++ b/nilcompiler/src/native_runtime.hpp @@ -210,6 +210,12 @@ std::int32_t nyrt_read_regular_bytes(const char *path, std::int64_t maximum_bytes, void **bytes_out, const char **error_out) noexcept; +// All outputs describe the same bounded, rooted file read. The compiler owns +// the language record layout; this ABI only returns its independent fields. +std::int32_t nyrt_read_rooted_snapshot( + const char *root, const char *relative, std::int64_t maximum_bytes, + void **bytes_out, std::int64_t *modified_ms_out, const char **sha256_out, + const char **error_out) noexcept; std::int32_t nyrt_write_bytes_atomic(const char *path, const void *bytes, std::int32_t replace, const char **error_out) noexcept; diff --git a/nilcompiler/src/secure_file_io.cpp b/nilcompiler/src/secure_file_io.cpp index 076cda64..dda23a81 100644 --- a/nilcompiler/src/secure_file_io.cpp +++ b/nilcompiler/src/secure_file_io.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -39,6 +41,7 @@ #define NOMINMAX #endif #include +#include #endif namespace ny::detail { @@ -119,6 +122,50 @@ path_from_api_string(const std::string &path) { } } +constexpr std::uint64_t rooted_snapshot_maximum = 512ULL * 1024 * 1024; + +std::optional +rooted_directory_from_utf8(const std::string &root) { + auto path = path_from_utf8_string(root); + // A trailing separator or '/.' would make O_NOFOLLOW / OPEN_REPARSE_POINT + // inspect the directory reached through a root symlink instead of the link. + // Remove only these trailing spellings, never resolve '..' or ancestors. + while (path && (path->filename().empty() || path->filename() == ".")) { + auto parent = path->parent_path(); + if (parent.empty() || parent == *path) + break; + *path = std::move(parent); + } + return path; +} + +RootedFileSnapshot rooted_failure(const char *reason) { + return {false, {}, 0, std::string("file.read_rooted_snapshot: ") + reason}; +} + +std::optional> +rooted_components(const std::string &relative) { + if (!path_from_utf8_string(relative) || relative.front() == '/' || + relative.find('\\') != std::string::npos || + relative.find(':') != std::string::npos) + return std::nullopt; + std::vector components; + std::size_t start = 0; + for (;;) { + const auto slash = relative.find('/', start); + const std::string component = relative.substr(start, slash - start); + if (component.empty() || component == "." || component == "..") + return std::nullopt; + const auto native = path_from_utf8_string(component); + if (!native) + return std::nullopt; + components.push_back(*native); + if (slash == std::string::npos) + return components; + start = slash + 1; + } +} + } // namespace #ifndef _WIN32 @@ -240,6 +287,81 @@ BoundedFileRead read_regular_file_bounded(const std::filesystem::path &path, return {true, std::move(bytes), {}}; } +RootedFileSnapshot read_rooted_file_snapshot_utf8(const std::string &root, + const std::string &relative, + std::uint64_t maximum_bytes) { + if (maximum_bytes > rooted_snapshot_maximum) + return rooted_failure("invalid maximum_bytes"); + const auto native_root = rooted_directory_from_utf8(root); + const auto components = rooted_components(relative); + if (!native_root || !components) + return rooted_failure("invalid path"); + + const int directory_flags = O_RDONLY | O_CLOEXEC | O_DIRECTORY | O_NOFOLLOW; + UniqueFileDescriptor directory(::open(native_root->c_str(), directory_flags)); + if (directory.get() < 0) + return rooted_failure("cannot open root directory"); + for (std::size_t index = 0; index + 1 < components->size(); ++index) { + UniqueFileDescriptor child(::openat(directory.get(), (*components)[index].c_str(), + directory_flags)); + if (child.get() < 0) + return rooted_failure("cannot open descendant directory"); + directory = std::move(child); + } + UniqueFileDescriptor input(::openat( + directory.get(), components->back().c_str(), + O_RDONLY | O_CLOEXEC | O_NONBLOCK | O_NOFOLLOW)); + if (input.get() < 0) + return rooted_failure("cannot open regular file"); + + struct stat before {}; + if (::fstat(input.get(), &before) != 0 || !S_ISREG(before.st_mode)) + return rooted_failure("input is not a regular file"); + if (before.st_size < 0 || static_cast(before.st_size) > maximum_bytes) + return rooted_failure("input exceeds maximum_bytes"); +#if defined(__APPLE__) + const auto seconds = before.st_mtimespec.tv_sec; + const auto nanos = before.st_mtimespec.tv_nsec; +#else + const auto seconds = before.st_mtim.tv_sec; + const auto nanos = before.st_mtim.tv_nsec; +#endif + if (seconds < (std::numeric_limits::min)() / 1000 || + seconds > (std::numeric_limits::max)() / 1000 || + nanos < 0 || nanos >= 1000000000) + return rooted_failure("invalid modification timestamp"); + const auto millis = static_cast(seconds) * 1000; + const auto fraction = static_cast(nanos) / 1000000; + if (millis > (std::numeric_limits::max)() - fraction) + return rooted_failure("invalid modification timestamp"); + + std::vector bytes; + try { + bytes.resize(static_cast(before.st_size)); + } catch (const std::bad_alloc &) { + return rooted_failure("cannot allocate snapshot"); + } + std::size_t offset = 0; + while (offset < bytes.size()) { + const ssize_t count = ::read(input.get(), bytes.data() + offset, bytes.size() - offset); + if (count < 0 && errno == EINTR) + continue; + if (count <= 0) + return rooted_failure("input changed or read failed"); + offset += static_cast(count); + } + char extra = 0; + ssize_t count = -1; + do { + count = ::read(input.get(), &extra, 1); + } while (count < 0 && errno == EINTR); + struct stat after {}; + if (count != 0 || ::fstat(input.get(), &after) != 0 || + !S_ISREG(after.st_mode) || !same_regular_file_snapshot(before, after)) + return rooted_failure("input changed or read failed"); + return {true, std::move(bytes), millis + fraction, {}}; +} + AtomicFileWrite write_file_atomic(const std::filesystem::path &path, const std::vector &bytes, bool replace) { @@ -470,6 +592,138 @@ BoundedFileRead read_regular_file_bounded(const std::filesystem::path &path, return {true, std::move(bytes), {}}; } +namespace { + +using RootedNtCreateFile = NTSTATUS (NTAPI *)( + PHANDLE, ACCESS_MASK, POBJECT_ATTRIBUTES, PIO_STATUS_BLOCK, PLARGE_INTEGER, + ULONG, ULONG, ULONG, ULONG, PVOID, ULONG); + +RootedNtCreateFile rooted_nt_create_file() { + static const auto function = []() -> RootedNtCreateFile { + const HMODULE module = GetModuleHandleW(L"ntdll.dll"); + const FARPROC symbol = module ? GetProcAddress(module, "NtCreateFile") : nullptr; + RootedNtCreateFile function = nullptr; + static_assert(sizeof(function) == sizeof(symbol), "Windows function pointer size"); + std::memcpy(&function, &symbol, sizeof(function)); + return function; + }(); + return function; +} + +UniqueWindowsHandle open_rooted_child(HANDLE directory, + const std::filesystem::path &component, + bool is_directory) { + const auto create = rooted_nt_create_file(); + const auto &name = component.native(); + if (!create || name.empty() || name.size() > 32767) + return UniqueWindowsHandle(); + UNICODE_STRING string {}; + string.Buffer = const_cast(name.data()); + string.Length = static_cast(name.size() * sizeof(wchar_t)); + string.MaximumLength = string.Length; + OBJECT_ATTRIBUTES attributes {}; + attributes.Length = sizeof(attributes); + attributes.RootDirectory = directory; + attributes.ObjectName = &string; + attributes.Attributes = 0x40; // OBJ_CASE_INSENSITIVE; do not inherit handles. + IO_STATUS_BLOCK io {}; + HANDLE child = INVALID_HANDLE_VALUE; + constexpr ULONG open_existing = 1; // FILE_OPEN + constexpr ULONG open_reparse = 0x00200000; // FILE_OPEN_REPARSE_POINT + constexpr ULONG synchronous = 0x20; // FILE_SYNCHRONOUS_IO_NONALERT + constexpr ULONG directory_only = 1; // FILE_DIRECTORY_FILE + constexpr ULONG non_directory_only = 0x40; // FILE_NON_DIRECTORY_FILE + const ULONG sharing = is_directory + ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ; + const NTSTATUS status = create( + &child, GENERIC_READ | SYNCHRONIZE, &attributes, &io, nullptr, 0, + sharing, open_existing, + open_reparse | synchronous | (is_directory ? directory_only : non_directory_only), + nullptr, 0); + if (status != 0 || io.Status != 0 || io.Information != 1 || + child == nullptr || child == INVALID_HANDLE_VALUE) { + if (child != nullptr && child != INVALID_HANDLE_VALUE) + CloseHandle(child); + return UniqueWindowsHandle(); + } + UniqueWindowsHandle held(child); + BY_HANDLE_FILE_INFORMATION information {}; + if (GetFileType(held.get()) != FILE_TYPE_DISK || + !GetFileInformationByHandle(held.get(), &information) || + (information.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 || + ((information.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) != is_directory) + return UniqueWindowsHandle(); + return held; +} + +} // namespace + +RootedFileSnapshot read_rooted_file_snapshot_utf8(const std::string &root, + const std::string &relative, + std::uint64_t maximum_bytes) { + if (maximum_bytes > rooted_snapshot_maximum) + return rooted_failure("invalid maximum_bytes"); + const auto native_root = rooted_directory_from_utf8(root); + const auto components = rooted_components(relative); + if (!native_root || !components) + return rooted_failure("invalid path"); + UniqueWindowsHandle directory(CreateFileW( + native_root->c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, nullptr)); + BY_HANDLE_FILE_INFORMATION root_information {}; + if (directory.get() == INVALID_HANDLE_VALUE || + GetFileType(directory.get()) != FILE_TYPE_DISK || + !GetFileInformationByHandle(directory.get(), &root_information) || + (root_information.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 || + (root_information.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) + return rooted_failure("cannot open root directory"); + for (std::size_t index = 0; index + 1 < components->size(); ++index) { + auto child = open_rooted_child(directory.get(), (*components)[index], true); + if (child.get() == INVALID_HANDLE_VALUE) + return rooted_failure("cannot open descendant directory"); + directory = std::move(child); + } + auto input = open_rooted_child(directory.get(), components->back(), false); + if (input.get() == INVALID_HANDLE_VALUE) + return rooted_failure("cannot open regular file"); + BY_HANDLE_FILE_INFORMATION before {}; + if (!GetFileInformationByHandle(input.get(), &before)) + return rooted_failure("cannot inspect regular file"); + const std::uint64_t size = (static_cast(before.nFileSizeHigh) << 32U) | + before.nFileSizeLow; + if (size > maximum_bytes) + return rooted_failure("input exceeds maximum_bytes"); + const std::uint64_t ticks = (static_cast(before.ftLastWriteTime.dwHighDateTime) << 32U) | + before.ftLastWriteTime.dwLowDateTime; + // Divide before converting: even the largest FILETIME quotient fits i64. + const std::int64_t modified = static_cast(ticks / 10000) - 11644473600000LL; + std::vector bytes; + try { + bytes.resize(static_cast(size)); + } catch (const std::bad_alloc &) { + return rooted_failure("cannot allocate snapshot"); + } + std::size_t offset = 0; + while (offset < bytes.size()) { + const DWORD requested = static_cast(std::min( + bytes.size() - offset, (std::numeric_limits::max)())); + DWORD count = 0; + if (!ReadFile(input.get(), bytes.data() + offset, requested, &count, nullptr) || + count == 0 || count > requested) + return rooted_failure("input changed or read failed"); + offset += count; + } + char extra = 0; + DWORD count = 0; + BY_HANDLE_FILE_INFORMATION after {}; + if (!ReadFile(input.get(), &extra, 1, &count, nullptr) || count != 0 || + !GetFileInformationByHandle(input.get(), &after) || + !same_regular_file_snapshot(before, after)) + return rooted_failure("input changed or read failed"); + return {true, std::move(bytes), modified, {}}; +} + AtomicFileWrite write_file_atomic(const std::filesystem::path &path, const std::vector &bytes, bool replace) { diff --git a/nilcompiler/src/secure_file_io.hpp b/nilcompiler/src/secure_file_io.hpp index 1db9bd2b..52e5f33f 100644 --- a/nilcompiler/src/secure_file_io.hpp +++ b/nilcompiler/src/secure_file_io.hpp @@ -14,6 +14,26 @@ struct BoundedFileRead { std::string error; }; +struct RootedFileSnapshot { + bool ok = false; + std::vector bytes; + std::int64_t modified_ms = 0; + std::string error; +}; + +// The trusted root selector is opened once. Its ancestors are outside the +// relative-path trust boundary. Every relative component is opened from the +// preceding directory handle, rejecting links/reparse points. The relative +// name is strict UTF-8, slash-separated, with no empty, dot, parent, colon or +// backslash components. Only regular files up to maximum_bytes are read; +// maximum_bytes itself cannot exceed 512 MiB. Bytes and mtime are inspected +// through the same final handle, rejecting observed changes during the read. +// This is an anchored namespace read, not a filesystem transaction: privileged +// mounts, hard links and changes after the snapshot are outside its contract. +RootedFileSnapshot read_rooted_file_snapshot_utf8(const std::string &root, + const std::string &relative, + std::uint64_t maximum_bytes); + struct AtomicFileWrite { bool ok = false; std::string error; diff --git a/nilcompiler/src/vm.cpp b/nilcompiler/src/vm.cpp index 5231c1f9..50c7620b 100644 --- a/nilcompiler/src/vm.cpp +++ b/nilcompiler/src/vm.cpp @@ -19,6 +19,7 @@ #include "time_support.hpp" #include "file_time_support.hpp" #include "../../common/windows_environment.hpp" +#include "../../common/content_digest.hpp" #include #include #include @@ -1678,6 +1679,13 @@ void VM::executeInstruction(const Instruction &instr) { const auto *handle = std::get_if(&value); if (!handle) reject(); const auto found = heap_.find(handle->address); + // File/random byte arrays use compact byte storage instead of scalar + // Value slots. They retain the same exact array language identity. + // Do not admit raw u8 pointer allocations or any other array element. + if (found != heap_.end() && expected == "array" && + found->second.type_name == "bytes" && found->second.data.empty() && + found->second.element_count == found->second.byte_data.size()) + return; if (found == heap_.end() || found->second.type_name != "array" || "array<" + canonical_array_type(found->second.element_type) + ">" != expected) reject(); return; @@ -4411,6 +4419,37 @@ void VM::executeInstruction(const Instruction &instr) { return; } + if (fn_name == "text.from_bytes") { + if (args.size() != 1 || !std::holds_alternative(args[0])) + throw std::runtime_error("text.from_bytes: expected a live byte array"); + const auto found = heap_.find(std::get(args[0]).address); + if (found == heap_.end()) + throw std::runtime_error("text.from_bytes: expected a live byte array"); + const auto &block = found->second; + std::string result; + if (block.type_name == "bytes" && block.data.empty() && + block.element_count == block.byte_data.size()) { + if (block.byte_data.size() > max_intrinsic_file_bytes) + throw std::runtime_error("text.from_bytes: input exceeds size limit"); + result.assign(block.byte_data.begin(), block.byte_data.end()); + } else if (block.type_name == "array" && + canonical_array_scalar(block.element_type) == "u8") { + if (block.data.size() > max_intrinsic_file_bytes) + throw std::runtime_error("text.from_bytes: input exceeds size limit"); + result.reserve(block.data.size()); + for (const auto &item : block.data) { + const auto *byte = std::get_if(&item); + if (!byte || *byte < 0 || *byte > 255) + throw std::runtime_error("text.from_bytes: invalid byte element"); + result.push_back(static_cast(static_cast(*byte))); + } + } else { + throw std::runtime_error("text.from_bytes: expected a live byte array"); + } + pushStack(std::move(result)); + return; + } + if (fn_name == "text.from_byte") { std::int64_t code = -1; if (!args.empty()) { @@ -5079,6 +5118,57 @@ void VM::executeInstruction(const Instruction &instr) { pushStack(result); return; } + // Return bytes, timestamp and content identity from one anchored read. + if (fn_name == "file.read_rooted_snapshot") { + auto push_error = [&](const std::string &message) { + EnumValue result; + result.variant_name = "Result::Error"; + result.payloads.push_back(message); + pushStack(result); + }; + if (args.size() != 3 || !std::holds_alternative(args[0]) || + !std::holds_alternative(args[1]) || + !std::holds_alternative(args[2]) || + std::get(args[2]) < 0 || + static_cast(std::get(args[2])) > + max_intrinsic_file_bytes) { + push_error("file.read_rooted_snapshot: invalid arguments"); + return; + } + try { + auto loaded = detail::read_rooted_file_snapshot_utf8( + std::get(args[0]), std::get(args[1]), + static_cast(std::get(args[2]))); + if (!loaded.ok) { + push_error(loaded.error); + return; + } + nil::detail::sha256 hash; + hash.update(loaded.bytes.data(), loaded.bytes.size()); + StructValue snapshot; + snapshot.type_name = "nil::std::io::FileSnapshot"; + snapshot.fields()["modified_ms"] = loaded.modified_ms; + snapshot.string_fields()["sha256"] = hash.finish(); + HeapBlock block; + block.type_name = "bytes"; + block.element_count = loaded.bytes.size(); + block.byte_data = std::move(loaded.bytes); + const std::int64_t handle = next_ptr_; + const std::size_t storage_bytes = + std::max(block.byte_data.size(), 1U); + next_ptr_ += static_cast( + (storage_bytes + 7U) & ~std::size_t(7U)); + storeHeapBlock(handle, std::move(block)); + snapshot.array_fields()["content"] = ArrayHandle{handle}; + EnumValue result; + result.variant_name = "Result::Ok"; + result.payloads.push_back(std::move(snapshot)); + pushStack(result); + } catch (...) { + push_error("file.read_rooted_snapshot: host operation failed"); + } + return; + } // file.read_regular_bytes(path, maximum_bytes) // -> result, text> if (fn_name == "file.read_regular_bytes") { diff --git a/nilstd/src/io/file.ny b/nilstd/src/io/file.ny index c1058272..dcc70a84 100644 --- a/nilstd/src/io/file.ny +++ b/nilstd/src/io/file.ny @@ -36,6 +36,28 @@ pub fn read_bytes(path: text) -> Result, text> @intrinsic("file.read_regular_bytes") pub fn read_regular_bytes(path: text, maximum_bytes: i64) -> Result, text> +# The exact bytes read, their descriptor-observed modification time in Unix +# milliseconds, and lowercase SHA-256 of those returned bytes. Binary content +# remains an array on every execution route; no text encoding is assumed. +pub struct FileSnapshot + content: array + modified_ms: i64 + sha256: text +end struct + +# Read a bounded regular file below a trusted root directory. The relative +# path uses strict UTF-8 slash-separated components; empty, dot, dot-dot, +# absolute, backslash, colon and NUL spellings are rejected. Every descendant +# component is opened relative to an already-open directory without following +# symlinks or Windows reparse points. The final handle supplies the bytes and +# metadata, with observed changes checked before returning. maximum_bytes is +# 0..512 MiB; zero admits an empty file. Errors contain no caller path values. +# This is a directory-handle containment boundary, not protection against +# privileged mount changes, hard links placed in the root, or all concurrent +# writes. sha256 always describes content, rather than a size/timestamp key. +@intrinsic("file.read_rooted_snapshot") +pub fn read_rooted_snapshot(root: text, relative: text, maximum_bytes: i64) -> Result + # Write text to a file (creates or overwrites) @intrinsic("file.write_text") pub fn write_text(path: text, content: text) -> Result diff --git a/nilstd/src/text/string_utils.ny b/nilstd/src/text/string_utils.ny index 7c59ee50..8275de70 100644 --- a/nilstd/src/text/string_utils.ny +++ b/nilstd/src/text/string_utils.ny @@ -800,7 +800,7 @@ end fn # outside ASCII was unreachable because of this, and so was reading the bytes # off a socket. # -# These two are about bytes and say so in their names. They know nothing about +# These operations are about bytes and say so in their names. They know nothing about # UTF-8; assembling characters from bytes is the caller's job, and now it is a # job that can be done. @@ -821,3 +821,11 @@ pub fn byte_at(s: text, index: i64) -> i64 # A code outside 0..255 gives empty text. @intrinsic("text.from_byte") pub fn from_byte(code: i64) -> text + +# Copy a byte array into interpreter text in one operation. Every byte is +# preserved, including NUL and invalid UTF-8; no decoding or normalization is +# performed, and later array mutation does not change the returned text. +# Interpreter-only, like byte_at/from_byte: native text currently requires +# NUL-free values, so native compilation rejects this intrinsic explicitly. +@intrinsic("text.from_bytes") +pub fn from_bytes(data: array) -> text diff --git a/tests/language/rooted-file-snapshot-invalid-schema.ny b/tests/language/rooted-file-snapshot-invalid-schema.ny new file mode 100644 index 00000000..86ecbfc1 --- /dev/null +++ b/tests/language/rooted-file-snapshot-invalid-schema.ny @@ -0,0 +1,22 @@ +module nil.std.io + +import nil.std.core + +# Exact canonical type name, deliberately incompatible timestamp field. The +# native bridge must validate the schema rather than trust its spelling. +pub struct FileSnapshot + content: array + modified_ms: text + sha256: text +end struct + +@intrinsic("file.read_rooted_snapshot") +fn invalid_snapshot(root: text, relative: text, maximum_bytes: i64) -> Result + +fn main() -> i32 + match invalid_snapshot(".", "never-read.bin", 256) + Result.Ok(snapshot) -> println(snapshot.sha256) + Result.Error(message) -> println(message) + end match + return 0 +end fn diff --git a/tests/language/rooted-file-snapshot-invalid-signature.ny b/tests/language/rooted-file-snapshot-invalid-signature.ny new file mode 100644 index 00000000..bfd265c3 --- /dev/null +++ b/tests/language/rooted-file-snapshot-invalid-signature.ny @@ -0,0 +1,12 @@ +import nil.std.io.file + +@intrinsic("file.read_rooted_snapshot") +fn invalid_snapshot(root: text, relative: text, maximum_bytes: i32) -> Result + +fn main() -> i32 + match invalid_snapshot(".", "never-read.bin", 256) + Result.Ok(snapshot) -> println(snapshot.sha256) + Result.Error(message) -> println(message) + end match + return 0 +end fn diff --git a/tests/language/rooted-file-snapshot.ny b/tests/language/rooted-file-snapshot.ny new file mode 100644 index 00000000..1e70a04f --- /dev/null +++ b/tests/language/rooted-file-snapshot.ny @@ -0,0 +1,82 @@ +module nil.tests.rooted_file_snapshot + +import nil.std.io.file +import nil.std.system.std_sys +import nil.std.text + +fn inspect(snapshot: nil::std::io::FileSnapshot, stamp: i64, digest: text, seed: i64) -> bool + let boxed: anything = snapshot + if not anything_is(boxed) then return false end if + let recovered = boxed as nil::std::io::FileSnapshot + let boxed_content: anything = recovered.content + if not anything_is>(boxed_content) or anything_is>(boxed_content) then return false end if + let recovered_content = boxed_content as array + if recovered.content.len() != 256 then return false end if + if recovered.modified_ms != stamp then return false end if + if recovered.sha256 != digest then return false end if + let mut index: i64 = 0 + while index < 256 + if recovered_content[index] != (((index * 73 + seed) % 256) as u8) then return false end if + index = index + 1 + end while + return true +end fn + +fn reject(root: text, relative: text, maximum: i64, expected: text) -> bool + match nil::std::io::read_rooted_snapshot(root, relative, maximum) + Result.Ok(_) -> return false + Result.Error(message) -> + if not nil::std::text::starts_with(message, "file.read_rooted_snapshot:") then return false end if + if nil::std::text::index_of(message, root) >= 0 then return false end if + if expected != "" and message != expected then return false end if + end match + return true +end fn + +fn main() -> i32 + if nil::std::system::arg_count() != 4 and nil::std::system::arg_count() != 5 then return 60 end if + let root = nil::std::system::arg(0) + let stamp = nil::std::core::Convert.try_to_i64(nil::std::system::arg(1)).unwrap() + let digest = nil::std::system::arg(2) + let seed = nil::std::core::Convert.try_to_i64(nil::std::system::arg(3)).unwrap() + match nil::std::io::read_rooted_snapshot(root, "nested/café-λ.bin", 256) + Result.Error(message) -> + println(message) + return 61 + Result.Ok(snapshot) -> + if not inspect(snapshot, stamp, digest, seed) then return 62 end if + if nil::std::system::arg_count() == 5 then + if nil::std::system::arg(4) != "wrong-element" then return 74 end if + let boxed_content: anything = snapshot.content + let invalid_content = boxed_content as array + println(invalid_content.len()) + return 75 + end if + end match + match nil::std::io::read_rooted_snapshot(root, "empty.bin", 0) + Result.Error(message) -> + println(message) + return 63 + Result.Ok(snapshot) -> + if snapshot.content.len() != 0 or snapshot.modified_ms != stamp then return 64 end if + if snapshot.sha256 != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" then return 65 end if + end match + println("snapshot-content-ok") + if not reject(root, "nested/café-λ.bin", 255, + "file.read_rooted_snapshot: input exceeds maximum_bytes") then return 66 end if + if not reject(root, "empty.bin", -1, "") then return 67 end if + if not reject(root, "empty.bin", 536870913, "") then return 68 end if + let invalid: array = ["", ".", "..", "../outside/sentinel.bin", "nested/../empty.bin", + "/empty.bin", "nested//café-λ.bin", "nested/", "nested\\café-λ.bin", "C:empty.bin", + "missing.bin", "nested"] + for path in invalid + if not reject(root, path, 256, "") then return 69 end if + end for + if not reject(root + "/missing-root", "empty.bin", 256, "") then return 70 end if + if not reject(root + "/empty.bin", "empty.bin", 256, "") then return 71 end if + println("snapshot-errors-ok") + if not reject(root, "linked.bin", 256, "") then return 72 end if + if not reject(root, "via-link/sentinel.bin", 256, "") then return 73 end if + println("snapshot-links-ok") + return 0 +end fn diff --git a/tests/language/text-from-bytes-native-reject.ny b/tests/language/text-from-bytes-native-reject.ny new file mode 100644 index 00000000..9c49ea80 --- /dev/null +++ b/tests/language/text-from-bytes-native-reject.ny @@ -0,0 +1,7 @@ +import nil.std.text + +fn main() -> i32 + let bytes: array = [65 as u8, 66 as u8] + println(nil::std::text::from_bytes(bytes)) + return 0 +end fn diff --git a/tests/language/text-from-bytes.ny b/tests/language/text-from-bytes.ny new file mode 100644 index 00000000..a9b67c16 --- /dev/null +++ b/tests/language/text-from-bytes.ny @@ -0,0 +1,33 @@ +module nil.tests.text_from_bytes + +import nil.std.text +import nil.std.collections.array + +fn main() -> i32 + let mut bytes: array = [] + let mut index: i64 = 0 + while index < 256 + bytes.push(index as u8) + index = index + 1 + end while + let text = nil::std::text::from_bytes(bytes) + if nil::std::text::length(text) != 256 then return 61 end if + bytes[0] = 17 + index = 0 + while index < 256 + if nil::std::text::byte_at(text, index) != index then return 62 end if + index = index + 1 + end while + let empty: array = [] + if nil::std::text::from_bytes(empty) != "" then return 63 end if + let mut large = nil::std::collections::new_u8(1048576, 255 as u8) + large[524288] = 0 + large[1048575] = 128 + let copied = nil::std::text::from_bytes(large) + if nil::std::text::length(copied) != 1048576 then return 64 end if + if nil::std::text::byte_at(copied, 0) != 255 then return 65 end if + if nil::std::text::byte_at(copied, 524288) != 0 then return 66 end if + if nil::std::text::byte_at(copied, 1048575) != 128 then return 67 end if + println("from-bytes-ok") + return 0 +end fn diff --git a/tests/rooted_file_snapshot_tests.cpp b/tests/rooted_file_snapshot_tests.cpp new file mode 100644 index 00000000..489bf938 --- /dev/null +++ b/tests/rooted_file_snapshot_tests.cpp @@ -0,0 +1,315 @@ +// All paths and concurrent namespace changes belong to this private fixture. +#include "../nilcompiler/src/secure_file_io.hpp" +#include "../common/private_directory.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#endif + +namespace { +namespace fs = std::filesystem; +int checks = 0; + +void check(bool good, const char *label) { + ++checks; + if (!good) throw std::runtime_error(label); +} + +class Fixture { +public: + Fixture() : path(nil::detail::create_private_directory(fs::temp_directory_path(), "nil-rooted-snapshot-")) {} + ~Fixture() { std::error_code ignored; fs::remove_all(path, ignored); } + fs::path path; +}; + +void write(const fs::path &path, const std::string &bytes) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output.write(bytes.data(), static_cast(bytes.size())); + output.close(); + check(!output.fail(), "write owned fixture"); +} + +ny::detail::RootedFileSnapshot read(const fs::path &root, const std::string &relative, + std::uint64_t maximum = 1048576) { + return ny::detail::read_rooted_file_snapshot_utf8(root.u8string(), relative, maximum); +} + +std::string body(const ny::detail::RootedFileSnapshot &result) { + return std::string(result.bytes.begin(), result.bytes.end()); +} + +void refused(const ny::detail::RootedFileSnapshot &result, const char *label, + const fs::path &private_root) { + check(!result.ok && result.bytes.empty(), label); + check(result.error.find(private_root.u8string()) == std::string::npos && + result.error.rfind("file.read_rooted_snapshot: ", 0) == 0, + "refusal has a fixed diagnostic without private paths"); +} + +void timestamp(const fs::path &path, std::int64_t milliseconds) { +#ifdef _WIN32 + HANDLE file = CreateFileW(path.c_str(), FILE_WRITE_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + check(file != INVALID_HANDLE_VALUE, "open owned timestamp fixture"); + const auto ticks = static_cast(milliseconds + 11644473600000LL) * 10000; + FILETIME held {static_cast(ticks), static_cast(ticks >> 32U)}; + const bool okay = SetFileTime(file, nullptr, nullptr, &held) != 0; + CloseHandle(file); + check(okay, "set owned timestamp fixture"); +#else + timespec times[2] {}; + times[0].tv_sec = times[1].tv_sec = milliseconds / 1000; + times[0].tv_nsec = times[1].tv_nsec = (milliseconds % 1000) * 1000000; + check(::utimensat(AT_FDCWD, path.c_str(), times, 0) == 0, "set owned timestamp fixture"); +#endif +} + +void directory_link(const fs::path &link, const fs::path &target) { +#ifdef _WIN32 + // Junctions exercise reparse refusal without requiring symlink privilege. + check(fs::create_directory(link), "create owned junction directory"); + const std::wstring shown = fs::absolute(target).wstring(); + const std::wstring substitute = L"\\??\\" + shown; + struct Junction { + DWORD tag; + USHORT data_length; + USHORT reserved; + USHORT substitute_offset; + USHORT substitute_length; + USHORT print_offset; + USHORT print_length; + WCHAR names[1]; + }; + const auto name_bytes = (substitute.size() + shown.size() + 2) * sizeof(WCHAR); + std::vector storage(offsetof(Junction, names) + name_bytes, 0); + auto *data = reinterpret_cast(storage.data()); + data->tag = IO_REPARSE_TAG_MOUNT_POINT; + data->data_length = static_cast(storage.size() - 8); + data->substitute_length = static_cast(substitute.size() * sizeof(WCHAR)); + data->print_offset = static_cast((substitute.size() + 1) * sizeof(WCHAR)); + data->print_length = static_cast(shown.size() * sizeof(WCHAR)); + std::memcpy(data->names, substitute.data(), data->substitute_length); + std::memcpy(reinterpret_cast(data->names) + data->print_offset, + shown.data(), data->print_length); + HANDLE held = CreateFileW(link.c_str(), GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, nullptr); + check(held != INVALID_HANDLE_VALUE, "open owned junction directory"); + DWORD returned = 0; + const bool okay = DeviceIoControl(held, FSCTL_SET_REPARSE_POINT, data, + static_cast(storage.size()), nullptr, 0, &returned, nullptr) != 0; + CloseHandle(held); + check(okay, "set owned junction reparse data"); +#else + fs::create_directory_symlink(target, link); +#endif +} + +void contracts(const fs::path &base) { + const auto root = base / "public"; + const auto outside = base / "outside"; + fs::create_directories(root / "nested"); + fs::create_directory(outside); + const std::string binary("A\0\xff\x80Z", 5); + write(root / "nested" / "asset.bin", binary); + write(outside / "asset.bin", "outside-owned-marker"); + timestamp(root / "nested" / "asset.bin", 1700000000123LL); + const auto exact = read(root, "nested/asset.bin", binary.size()); + check(exact.ok && body(exact) == binary && exact.error.empty(), "read exact bounded binary payload"); + check(exact.modified_ms == 1700000000123LL, "return same-handle milliseconds without clock sampling"); + const auto short_limit = read(root, "nested/asset.bin", binary.size() - 1); + refused(short_limit, "reject oversized before allocation", base); + check(short_limit.error == "file.read_rooted_snapshot: input exceeds maximum_bytes", + "oversized refusal is distinct for HTTP mapping"); + refused(read(root, "nested/asset.bin", 512ULL * 1024 * 1024 + 1), "reject excessive requested limit", base); + write(root / "empty", ""); + const auto empty = read(root, "empty", 0); + check(empty.ok && empty.bytes.empty(), "zero limit accepts empty regular file"); + refused(read(root, "nested/asset.bin", 0), "zero limit rejects nonempty file", base); + refused(read(root, "absent"), "missing file refused", base); + refused(read(root, "nested"), "directory cannot be served as a regular file", base); + refused(read(root / "empty", "asset"), "regular file cannot be a root directory", base); + + const std::vector bad_names = { + "", ".", "..", "/nested/asset.bin", "nested/../empty", "nested/./asset.bin", + "nested//asset.bin", "nested/asset.bin/", "nested\\asset.bin", "C:asset.bin", + "empty:stream", std::string("empty\0ignored", 13), std::string("\xc0\xaf", 2), + std::string("\xed\xa0\x80", 3), std::string("\xf4\x90\x80\x80", 4)}; + for (const auto &name : bad_names) + refused(read(root, name), "invalid relative spelling refused", base); + refused(ny::detail::read_rooted_file_snapshot_utf8("", "empty", 10), "empty root refused", base); + refused(ny::detail::read_rooted_file_snapshot_utf8(std::string("\xc0\xaf", 2), "empty", 10), + "malformed UTF-8 root refused before filesystem access", base); + const auto unicode = fs::u8path(u8"space λ 🧪"); + fs::create_directory(root / unicode); + write(root / unicode / "asset.bin", binary); + check(body(read(root, unicode.u8string() + "/asset.bin")) == binary, "UTF-8 descendant names round-trip"); + directory_link(root / "linked", outside); + // This control intentionally demonstrates the old API's documented final- + // component-only boundary, using only a marker owned by the test fixture. + const auto legacy = ny::detail::read_regular_file_bounded(root / "linked" / "asset.bin", 100); + check(legacy.ok && std::string(legacy.bytes.begin(), legacy.bytes.end()) == "outside-owned-marker", + "negative control: final-component-only API follows intermediate link"); + refused(read(root, "linked/asset.bin"), "intermediate directory link refused", base); + refused(read(root, "linked"), "final directory reparse/link refused", base); + refused(read(root / "linked", "asset.bin"), "root final reparse/link refused", base); + refused(read(root / "linked" / "", "asset.bin"), "trailing slash cannot disguise a root link", base); + refused(read(root / "linked" / ".", "asset.bin"), "trailing dot cannot disguise a root link", base); + check(body(read(root / "", "nested/asset.bin")) == binary, + "trailing slash on an ordinary root remains valid"); + check(body(read(root / ".", "nested/asset.bin")) == binary, + "trailing dot on an ordinary root remains valid"); +#ifndef _WIN32 + fs::create_symlink(outside / "asset.bin", root / "final-link"); + refused(read(root, "final-link"), "final file symlink refused", base); + check(::mkfifo((root / "pipe").c_str(), 0600) == 0, "create owned FIFO"); + const auto began = std::chrono::steady_clock::now(); + refused(read(root, "pipe"), "FIFO is refused without opening a writer", base); + check(std::chrono::steady_clock::now() - began < std::chrono::seconds(2), "FIFO refusal is nonblocking"); +#endif +} + +void namespace_race(const fs::path &base) { + const auto root = base / "race-root"; + const auto outside = base / "race-outside"; + fs::create_directories(root / "live"); + fs::create_directory(outside); + write(root / "live" / "asset", "inside-owned-marker"); + write(outside / "asset", "outside-owned-marker"); + directory_link(root / "link-spare", outside); + std::atomic stop{false}; + std::atomic swaps{0}; + std::atomic mutator_failed{false}; + std::thread mutator([&] { + // Windows can briefly reject a directory rename while a descendant file + // handle denies delete sharing. Complete each owned swap before stopping. + const auto finish_rename = [&](const fs::path &from, const fs::path &to) { + const auto bound = std::chrono::steady_clock::now() + std::chrono::seconds(2); + std::error_code error; + do { + fs::rename(from, to, error); + if (!error) return true; + std::this_thread::yield(); + } while (std::chrono::steady_clock::now() < bound); + return false; + }; + while (!stop.load()) { + std::error_code error; + fs::rename(root / "live", root / "safe-spare", error); + if (error) { std::this_thread::yield(); continue; } + if (!finish_rename(root / "link-spare", root / "live")) { mutator_failed = true; break; } + ++swaps; + std::this_thread::yield(); + if (!finish_rename(root / "live", root / "link-spare") || + !finish_rename(root / "safe-spare", root / "live")) { mutator_failed = true; break; } + } + }); + bool escaped = false; + int reads = 0; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + try { + while (reads < 500 && std::chrono::steady_clock::now() < deadline) { + const auto result = read(root, "live/asset"); + ++reads; + if (result.ok && body(result) != "inside-owned-marker") escaped = true; + } + } catch (...) { + stop = true; + mutator.join(); + throw; + } + stop = true; + mutator.join(); + check(!mutator_failed.load(), "owned namespace mutator completed its swaps"); + check(swaps.load() > 0 && reads > 0, "concurrent namespace changes actually occurred"); + check(!escaped, "anchored opens never follow replacement link outside root"); +} + +void coupled_snapshot_race(const fs::path &base) { + const auto root = base / "snapshot-race"; + fs::create_directory(root); + const std::string first(65536, 'a'), second(65536, 'b'); + write(root / "live", first); + write(root / "spare", second); + timestamp(root / "live", 1700000000123LL); + timestamp(root / "spare", 1700000001567LL); + std::atomic stop{false}; + std::atomic swaps{0}; + std::atomic failed{false}; + std::thread publisher([&] { + const auto finish = [&](const char *from, const char *to) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + std::error_code error; + do { + fs::rename(root / from, root / to, error); + if (!error) return true; + std::this_thread::yield(); + } while (std::chrono::steady_clock::now() < deadline); + return false; + }; + while (!stop.load()) { + std::error_code error; + fs::rename(root / "live", root / "park", error); + if (error) { std::this_thread::yield(); continue; } + if (!finish("spare", "live") || !finish("park", "spare")) { + failed = true; + break; + } + ++swaps; + } + }); + bool mixed = false; + int succeeded = 0; + try { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + for (int index = 0; index < 500 && std::chrono::steady_clock::now() < deadline; ++index) { + const auto result = read(root, "live"); + if (result.ok) { + ++succeeded; + const auto bytes = body(result); + if (!((bytes == first && result.modified_ms == 1700000000123LL) || + (bytes == second && result.modified_ms == 1700000001567LL))) mixed = true; + } + } + } catch (...) { + stop = true; + publisher.join(); + throw; + } + stop = true; + publisher.join(); + check(!failed.load() && swaps.load() > 0, "same-handle fixture published distinct file versions"); + check(succeeded > 0, "same-handle fixture observed successful snapshots"); + check(!mixed, "bytes and timestamp always identify the same opened file version"); +} + +} // namespace + +int main() { + try { + Fixture fixture; + contracts(fixture.path); + namespace_race(fixture.path); + coupled_snapshot_race(fixture.path); + std::cout << checks << " rooted snapshot checks passed\n"; + return 0; + } catch (const std::exception &error) { + std::cerr << "rooted snapshot contract failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/tools/rooted_file_snapshot_contract.py b/tests/tools/rooted_file_snapshot_contract.py new file mode 100644 index 00000000..d1cf8c03 --- /dev/null +++ b/tests/tools/rooted_file_snapshot_contract.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Bounded public snapshot/byte-conversion routes with independent host oracles. + +This is a source-library regression, not installed SDK or release qualification. +No server is started. Raw process evidence and the owned fixture are retained. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile +import time + +import ordinary_io_parity_contract as support + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from native_output_staging import native_output_path + +SNAPSHOT_OUTPUT = b'snapshot-content-ok\nsnapshot-errors-ok\nsnapshot-links-ok\n' +CONVERSION_OUTPUT = b'from-bytes-ok\n' +NATIVE_REJECTION = b"unsupported native annotation '@intrinsic(\"text.from_bytes\")'" +SNAPSHOT_REJECTION = b'file.read_rooted_snapshot requires (text,text,i64) -> Result' +SOURCES = ('rooted-file-snapshot.ny', 'text-from-bytes.ny', 'text-from-bytes-native-reject.ny', + 'rooted-file-snapshot-invalid-schema.ny', 'rooted-file-snapshot-invalid-signature.ny') + + +def snapshot_signature(root: Path) -> dict: + result = support.inventory(root, maximum_entries=32, maximum_file=1024) + for name, entry in result.items(): + path = root / name + if entry['kind'] == 'file': + entry['modified_ns'] = path.stat().st_mtime_ns + elif entry['kind'] == 'link-or-reparse': + entry['target'] = os.readlink(path) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + for name in ('compiler', 'interpreter', 'nilstd'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--llvm', choices=('ON', 'OFF'), required=True) + destination = parser.add_mutually_exclusive_group() + destination.add_argument('--output', type=Path, help='Fresh exact evidence directory') + destination.add_argument('--output-parent', type=Path, help='Retain a fresh child here') + args = parser.parse_args() + compiler = args.compiler.resolve(strict=True) + interpreter = args.interpreter.resolve(strict=True) + nilstd = args.nilstd.resolve(strict=True) + if args.output_parent: + parent = args.output_parent.resolve() + parent.mkdir(parents=True, exist_ok=True) + output = Path(tempfile.mkdtemp(prefix='rooted-file-snapshot-', dir=parent)) + elif args.output: + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + else: + output = Path(tempfile.mkdtemp(prefix='rooted-file-snapshot-')) + work, logs = output / 'work', output / 'logs' + work.mkdir() + logs.mkdir() + environment = {key: value for key, value in os.environ.items() + if not key.startswith(('NY_', 'NIL_'))} + ownership = support.load_ownership(ROOT) + deadline = time.monotonic() + 300 + inputs = {name: {'path': str(path), **support.file_record(path)} for name, path in + [('compiler', compiler), ('interpreter', interpreter), + *[(name, ROOT / 'tests/language' / name) for name in SOURCES], + ('file_library', nilstd / 'io/file.ny'), + ('text_library', nilstd / 'text/string_utils.ny'), + ('contract', Path(__file__).resolve()), + ('process_owner', ROOT / 'validation/ny-3d/scripts/debugger_process.py'), + ('process_support', ROOT / 'tests/tools/ordinary_io_parity_contract.py')]} + report = {'schema': 'nil.rooted-file-snapshot.v1', 'status': 'FAIL', + 'inputs': inputs, 'nilstd': str(nilstd), 'commands': [], + 'snapshot_routes': [], 'conversion_routes': [], 'native_rejections': [], + 'unbox_rejections': [], + 'llvm_enabled': args.llvm == 'ON', 'whole_seconds': 300, + 'scope': 'Source-library interpreter/native regression. FileSnapshot content is a byte array on every route; text.from_bytes is interpreter-only.', + 'link_cases': 'required; POSIX file/directory symlinks, Windows directory junctions (no symlink privilege required)', + 'work_directory_retained': True} + + def run(name, command, expected=b'', limit=60, rejection=None): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AssertionError('whole snapshot contract deadline exhausted') + row = support.run_process(ownership, [str(value) for value in command], + work, environment, logs, name, min(limit, remaining)) + report['commands'].append(row) + stdout = (logs / row['stdout']).read_bytes() + stderr = (logs / row['stderr']).read_bytes() + if not row['cleanup_verified'] or row.get('timed_out') or row.get('error'): + raise AssertionError(name + ': bounded process cleanup failed') + if rejection: + if row.get('exit_code', 0) <= 0 or stdout or rejection not in stderr: + raise AssertionError(name + ': native compilation did not fail for the exact expected contract') + elif row['status'] != 'PASS' or stderr or (expected is not None and stdout != expected): + raise AssertionError(name + ': exit/stdout/stderr contract failed') + return row + + try: + fixture = work / 'fixtures' + fixture.mkdir() + rooted = fixture / 'root café λ' + rooted.mkdir() + (rooted / 'nested').mkdir() + outside = fixture / 'outside' + outside.mkdir() + sentinel = outside / 'sentinel.bin' + sentinel.write_bytes(b'outside fixture must never be returned\x00\xff') + if os.name == 'nt': + for name in ('linked.bin', 'via-link'): + link = rooted / name + run('create-junction-' + name, + ['cmd.exe', '/d', '/c', 'mklink', '/J', link, outside], expected=None) + if not getattr(link.lstat(), 'st_file_attributes', 0) & 0x400: + raise AssertionError('Windows fixture is not a reparse point') + report['link_fixture_kind'] = 'directory junctions: final reparse entry and intermediate directory escape' + else: + (rooted / 'linked.bin').symlink_to(sentinel) + (rooted / 'via-link').symlink_to(outside, target_is_directory=True) + report['link_fixture_kind'] = 'final file symlink and intermediate directory symlink' + payload = rooted / 'nested/café-λ.bin' + empty = rooted / 'empty.bin' + empty.write_bytes(b'') + fixed_time = 1_600_000_000_123_000_000 + os.utime(empty, ns=(fixed_time, fixed_time)) + sources = {} + for name in SOURCES: + source = work / name + source.write_bytes((ROOT / 'tests/language' / name).read_bytes()) + sources[name] = source + base = [compiler, '--quiet', '--no-stdlib', '--sdk-path', nilstd] + routes = [] + for encoding in ('text', 'binary'): + flags = [] if encoding == 'text' else ['--emit-binary-nil'] + nil = work / ('snapshot-' + encoding + '.nil') + run('compile-snapshot-' + encoding, [*base, '--no-bundle', *flags, '-o', nil, + sources['rooted-file-snapshot.ny']]) + if nil.read_bytes().startswith(b'NIL\0') != (encoding == 'binary'): + raise AssertionError('compiled snapshot encoding differs') + routes.append((encoding + '-interpreter', [interpreter, '--no-ai', nil, '--'])) + if args.llvm == 'ON': + native = work / 'snapshot-native' + run('compile-snapshot-native', [*base, '--native', '-o', native, + sources['rooted-file-snapshot.ny']]) + routes.append(('llvm-aot', [native_output_path(native)])) + routes.append(('llvm-jit', [*base, '--native', '--jit', '-o', work / 'snapshot-jit', + sources['rooted-file-snapshot.ny'], '--'])) + + initial_digest = None + for stage, seed in (('initial', 19), ('same-size-time-replacement', 21)): + data = bytes((index * 73 + seed) % 256 for index in range(256)) + payload.write_bytes(data) + os.utime(payload, ns=(fixed_time, fixed_time)) + stamp = payload.stat().st_mtime_ns // 1_000_000 + if empty.stat().st_mtime_ns // 1_000_000 != stamp: + raise AssertionError('host did not preserve equal fixture timestamps') + digest = hashlib.sha256(data).hexdigest() + if initial_digest is None: + initial_digest = digest + elif digest == initial_digest: + raise AssertionError('same-size/time replacement failed to change content digest') + before = snapshot_signature(fixture) + arguments = [rooted, str(stamp), digest, str(seed)] + for route, command in routes: + row = run(stage + '-' + route, [*command, *arguments], SNAPSHOT_OUTPUT) + if snapshot_signature(fixture) != before: + raise AssertionError('snapshot read changed the owned fixture') + report['snapshot_routes'].append({'stage': stage, 'route': route, 'status': 'PASS', + 'bytes': len(data), 'sha256': digest, 'modified_ms': stamp, + 'exact_payload_bytes_checked': 256, 'empty_file_checked': True, + 'descendant_and_final_links_rejected': True, 'namespace_unchanged': True, + 'elapsed_seconds': row['elapsed_seconds']}) + report[stage + '_namespace'] = before + + for route, command in routes: + expected = (b"anything type mismatch: expected 'array', got 'array'" + if route.startswith('llvm-') else + b"anything contains 'array', cannot extract 'array'") + run('wrong-element-' + route, [*command, *arguments, 'wrong-element'], rejection=expected) + if snapshot_signature(fixture) != before: + raise AssertionError('rejected extraction changed the owned fixture') + report['unbox_rejections'].append({'route': route, 'status': 'PASS', + 'reason': expected.decode()}) + + for encoding in ('text', 'binary'): + flags = [] if encoding == 'text' else ['--emit-binary-nil'] + nil = work / ('conversion-' + encoding + '.nil') + run('compile-conversion-' + encoding, [*base, '--no-bundle', *flags, '-o', nil, + sources['text-from-bytes.ny']]) + if nil.read_bytes().startswith(b'NIL\0') != (encoding == 'binary'): + raise AssertionError('compiled conversion encoding differs') + row = run('conversion-' + encoding, [interpreter, '--no-ai', nil], CONVERSION_OUTPUT) + report['conversion_routes'].append({'route': encoding + '-interpreter', 'status': 'PASS', + 'all_256_byte_values': True, 'copy_independent_of_source_mutation': True, + 'empty_and_1MiB_payloads': True, 'elapsed_seconds': row['elapsed_seconds']}) + if args.llvm == 'ON': + for route, flags in (('llvm-aot', []), ('llvm-jit', ['--jit'])): + target = work / ('conversion-rejected-' + route) + run('conversion-rejected-' + route, [*base, '--native', *flags, '-o', target, + sources['text-from-bytes-native-reject.ny']], + rejection=NATIVE_REJECTION) + if target.exists() or native_output_path(target).exists(): + raise AssertionError('rejected binary text conversion published an executable') + report['native_rejections'].append({'route': route, 'status': 'PASS', + 'case': 'text.from_bytes', + 'reason': NATIVE_REJECTION.decode()}) + for case in ('invalid-schema', 'invalid-signature'): + for route, flags in (('llvm-aot', []), ('llvm-jit', ['--jit'])): + target = work / ('snapshot-rejected-' + case + '-' + route) + run('snapshot-rejected-' + case + '-' + route, + [*base, '--native', *flags, '-o', target, + sources['rooted-file-snapshot-' + case + '.ny']], + rejection=SNAPSHOT_REJECTION) + if target.exists() or native_output_path(target).exists(): + raise AssertionError('invalid snapshot ABI published an executable') + report['native_rejections'].append({'route': route, 'case': case, + 'status': 'PASS', 'reason': SNAPSHOT_REJECTION.decode()}) + if len(report['snapshot_routes']) != (8 if args.llvm == 'ON' else 4): + raise AssertionError('snapshot route inventory incomplete') + if len(report['unbox_rejections']) != (4 if args.llvm == 'ON' else 2): + raise AssertionError('wrong-element extraction route inventory incomplete') + if len(report['conversion_routes']) != 2 or len(report['native_rejections']) != (6 if args.llvm == 'ON' else 0): + raise AssertionError('conversion route inventory incomplete') + for item in inputs.values(): + if support.file_record(Path(item['path'])) != {key: item[key] for key in ('bytes', 'sha256')}: + raise AssertionError('input changed during contract') + report['status'] = 'PASS' + except Exception as error: + report['error'] = str(error) + report['cleanup_verified'] = all(row.get('cleanup_verified') is True for row in report['commands']) + (output / 'report.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(report['status'] + ': ' + str(output / 'report.json')) + if report['status'] != 'PASS': + print(report.get('error', 'snapshot contract failed'), file=sys.stderr) + return 0 if report['status'] == 'PASS' else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) From 92dd8909da84c13304db7e0d8b0ed870ab871ea9 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 00:35:14 -0500 Subject: [PATCH 02/12] Qualify rooted file native contracts on both Mac architectures --- .../workflows/rooted-file-native-parity.yml | 326 ++++++++++++++++++ notebook/2026-11-12-web-server-boundaries.md | 90 +++++ notebook/README.md | 5 + 3 files changed, 421 insertions(+) create mode 100644 .github/workflows/rooted-file-native-parity.yml create mode 100644 notebook/2026-11-12-web-server-boundaries.md diff --git a/.github/workflows/rooted-file-native-parity.yml b/.github/workflows/rooted-file-native-parity.yml new file mode 100644 index 00000000..94c23989 --- /dev/null +++ b/.github/workflows/rooted-file-native-parity.yml @@ -0,0 +1,326 @@ +name: Rooted file native macOS checks + +"on": + workflow_dispatch: + +permissions: + contents: read + +# This campaign tests a fixed source revision. The workflow revision is recorded +# separately; neither an installed SDK nor any release artifact is replaced. +jobs: + native: + name: macOS ${{ matrix.host.arch }} rooted file native contract + runs-on: ${{ matrix.host.runner }} + timeout-minutes: 65 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + host: + - {runner: macos-15-intel, arch: x86_64} + - {runner: macos-15, arch: arm64} + defaults: + run: + shell: bash + env: + HOMEBREW_NO_AUTO_UPDATE: '1' + PYTHONDONTWRITEBYTECODE: '1' + PYTHONUTF8: '1' + NIL_TESTED_SOURCE: 93515c3d4c2b6e0e7c3b870eb283fce26392d613 + NIL_EXPECTED_ARCH: ${{ matrix.host.arch }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: 93515c3d4c2b6e0e7c3b870eb283fce26392d613 + path: source + persist-credentials: false + - name: Record identities and prepare bounded commands + timeout-minutes: 2 + run: | + mkdir -p build/rooted-native-evidence + cat > build/rooted_native_campaign.py <<'PY' + import hashlib, json, os, stat, subprocess, sys + from pathlib import Path + + ROOT = Path(os.environ['GITHUB_WORKSPACE']).resolve() + SOURCE = ROOT / 'source' + BUILD = ROOT / 'build/rooted-native' + EVIDENCE = ROOT / 'build/rooted-native-evidence' + sys.path.insert(0, str(SOURCE / 'tests/tools')) + import ordinary_io_parity_contract as support + + def save(name, value): + (EVIDENCE / name).write_text(json.dumps(value, indent=2) + '\n') + + def record(path): + return {**support.file_record(path), 'mode': stat.S_IMODE(path.stat().st_mode)} + + def source_identity(): + revision = subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], + text=True, timeout=15).strip() + dirty = subprocess.check_output(['git', '-C', str(SOURCE), 'status', '--porcelain', + '--untracked-files=no'], text=True, timeout=15) + assert revision == os.environ['NIL_TESTED_SOURCE'] and not dirty + return revision + + def run(name, command, timeout, environment=None): + # The canonical owner retains/reaps the process group and caps raw + # pipes. Keep full bounded logs, but print only their final 64 KiB. + support.MAX_OUTPUT = 8 * 1024**2 + row = support.run_process(support.load_ownership(SOURCE), command, ROOT, + environment or dict(os.environ), EVIDENCE, name, timeout) + save(name + '.json', row) + for key in ('stdout', 'stderr'): + path = EVIDENCE / row[key] + if path.is_file(): + data = path.read_bytes() + print(key + ': ' + str(len(data)) + ' bytes retained') + print(data[-65536:].decode('utf-8', errors='replace')) + assert row['status'] == 'PASS' and row['exit_code'] == 0 + assert row['cleanup_verified'] is True and not row.get('timed_out') and not row.get('error') + return row + + def seal(): + files = [BUILD / 'bin' / name for name in + ('nilcompiler', 'nilinterpreter', 'nil_rooted_file_snapshot_tests')] + files.append(BUILD / 'lib/libnil.a') + files.extend(path for path in (SOURCE / 'nilstd/src').rglob('*') if path.is_file()) + files.extend(SOURCE / name for name in ( + 'tests/tools/rooted_file_snapshot_contract.py', + 'tests/tools/ordinary_io_parity_contract.py', 'tests/native_output_staging.py', + 'validation/ny-3d/scripts/debugger_process.py', 'tests/rooted_file_snapshot_tests.cpp', + 'tests/language/rooted-file-snapshot.ny', 'tests/language/text-from-bytes.ny', + 'tests/language/text-from-bytes-native-reject.ny', + 'tests/language/rooted-file-snapshot-invalid-schema.ny', + 'tests/language/rooted-file-snapshot-invalid-signature.ny')) + return {str(path.relative_to(ROOT)): record(path) for path in sorted(files)} + PY + python3 - <<'PY' + import os, platform, re, subprocess, sys + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + assert platform.system() == 'Darwin' and platform.machine() == os.environ['NIL_EXPECTED_ARCH'] + workflow_revision = os.environ['GITHUB_WORKFLOW_SHA'] + assert re.fullmatch(r'[0-9a-f]{40}', workflow_revision) + campaign.save('identity.json', dict(tested_source_revision=campaign.source_identity(), + workflow_revision=workflow_revision, dispatch_revision=os.environ['GITHUB_SHA'], + workflow_ref=os.environ['GITHUB_WORKFLOW_REF'], run_id=os.environ['GITHUB_RUN_ID'], + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'], system=platform.system(), + machine=platform.machine(), python=sys.version, + macos=subprocess.check_output(['sw_vers'], text=True, timeout=15), + llvm=True, scope='Pinned source-library host, interpreter, LLVM AOT/JIT contracts; no installed SDK or release activation')) + PY + - name: Configure LLVM Release build + timeout-minutes: 8 + run: | + python3 - <<'PY' + import subprocess, sys + from pathlib import Path + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + prefix = Path(subprocess.check_output(['brew', '--prefix', 'llvm@18'], text=True, timeout=30).strip()) + version = subprocess.check_output([str(prefix/'bin/llvm-config'), '--version'], text=True, timeout=15).strip() + assert version.startswith('18.') + campaign.save('toolchain.json', dict(llvm_prefix=str(prefix), llvm_version=version, + sdk_path=subprocess.check_output(['xcrun', '--show-sdk-path'], text=True, timeout=15).strip(), + xcode=subprocess.check_output(['xcodebuild', '-version'], text=True, timeout=15))) + campaign.run('configure', ['cmake', '-S', str(campaign.SOURCE), '-B', str(campaign.BUILD), + '-DCMAKE_BUILD_TYPE=Release', '-DBUILD_TESTING=ON', '-DNIL_ENABLE_LLVM=ON', + '-DLLVM_DIR=' + str(prefix/'lib/cmake/llvm'), '-DPython3_EXECUTABLE=' + sys.executable, + '-DNILCOMPILER_SQLITE_INCLUDE_DIR=OFF', '-DNILCOMPILER_POSTGRESQL_INCLUDE_DIR=OFF'], 390) + PY + - name: Build compiler interpreter and host fixture + timeout-minutes: 40 + run: | + python3 - <<'PY' + import sys + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + campaign.run('build', ['cmake', '--build', str(campaign.BUILD), '--config', 'Release', + '--parallel', '2', '--target', 'nilcompiler', 'nilinterpreter', 'nil_rooted_file_snapshot_tests'], 2350) + campaign.save('seal-before.json', campaign.seal()) + PY + - name: Run exactly two rooted file contracts + timeout-minutes: 8 + run: | + python3 - <<'PY' + import json, os, subprocess, sys, xml.etree.ElementTree as ET + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + expected = {'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract'} + base = ['ctest', '--test-dir', str(campaign.BUILD), '-C', 'Release', '-R', + r'^nil[.]rooted-file-snapshot(-contract)?$'] + inventory = subprocess.check_output(base + ['--show-only=json-v1'], timeout=30) + (campaign.EVIDENCE/'inventory.json').write_bytes(inventory) + tests = json.loads(inventory)['tests'] + assert len(tests) == 2 and {test['name'] for test in tests} == expected + language = next(test for test in tests if test['name'].endswith('-contract'))['command'] + assert language[language.index('--llvm') + 1] == 'ON' + assert language[language.index('--compiler') + 1] == str(campaign.BUILD/'bin/nilcompiler') + assert language[language.index('--interpreter') + 1] == str(campaign.BUILD/'bin/nilinterpreter') + assert language[language.index('--nilstd') + 1] == str(campaign.SOURCE/'nilstd/src') + temporary = campaign.BUILD/'owned-test-temp' + temporary.mkdir(mode=0o700) + environment = dict(os.environ, TMPDIR=str(temporary) + os.sep) + campaign.run('ctest', base + ['--parallel', '1', '--no-tests=error', '--output-on-failure', + '--test-output-size-passed', '65536', '--test-output-size-failed', '65536', + '--output-junit', str(campaign.EVIDENCE/'results.xml')], 440, environment) + cases = list(ET.parse(campaign.EVIDENCE/'results.xml').getroot().iter('testcase')) + assert len(cases) == 2 and {case.get('name') for case in cases} == expected + assert all(case.get('status') == 'run' and all(case.find(kind) is None + for kind in ('failure', 'error', 'skipped')) for case in cases) + host = next(case for case in cases if case.get('name') == 'nil.rooted-file-snapshot') + assert host.findtext('system-out', '').strip() == '87 rooted snapshot checks passed' + assert not list(temporary.iterdir()), 'Owned host/compiler temporary files remain' + PY + - name: Independently verify route inventories raw output and seals + timeout-minutes: 3 + run: | + python3 - <<'PY' + import hashlib, json, sys + from pathlib import Path + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + roots = list((campaign.BUILD/'test-evidence').glob('rooted-file-snapshot-*')) + assert len(roots) == 1 and roots[0].is_dir() and not roots[0].is_symlink() + root = roots[0] + report = json.loads((root/'report.json').read_text()) + assert report['schema'] == 'nil.rooted-file-snapshot.v1' and report['status'] == 'PASS' + assert report['llvm_enabled'] is True and report['cleanup_verified'] is True + routes = {'text-interpreter', 'binary-interpreter', 'llvm-aot', 'llvm-jit'} + stages = {'initial', 'same-size-time-replacement'} + snapshot = report['snapshot_routes'] + assert len(snapshot) == 8 and {(row['stage'], row['route']) for row in snapshot} == { + (stage, route) for stage in stages for route in routes} + for row in snapshot: + seed = 19 if row['stage'] == 'initial' else 21 + digest = hashlib.sha256(bytes((index * 73 + seed) % 256 for index in range(256))).hexdigest() + assert row['status'] == 'PASS' and row['sha256'] == digest and row['bytes'] == 256 + assert row['exact_payload_bytes_checked'] == 256 and row['modified_ms'] == 1600000000123 + assert all(row[key] is True for key in ('empty_file_checked', + 'descendant_and_final_links_rejected', 'namespace_unchanged')) + unbox = report['unbox_rejections'] + assert len(unbox) == 4 and {row['route'] for row in unbox} == routes + conversions = report['conversion_routes'] + assert len(conversions) == 2 and {row['route'] for row in conversions} == {'text-interpreter', 'binary-interpreter'} + for row in conversions: + assert row['status'] == 'PASS' and all(row[key] is True for key in + ('all_256_byte_values', 'copy_independent_of_source_mutation', 'empty_and_1MiB_payloads')) + rejections = report['native_rejections'] + assert len(rejections) == 6 and {(row['case'], row['route']) for row in rejections} == { + (case, route) for case in ('text.from_bytes', 'invalid-schema', 'invalid-signature') + for route in ('llvm-aot', 'llvm-jit')} + expected_output = {'compile-snapshot-' + encoding: b'' for encoding in ('text', 'binary', 'native')} + for stage in stages: + for route in routes: + expected_output[stage + '-' + route] = b'snapshot-content-ok\nsnapshot-errors-ok\nsnapshot-links-ok\n' + expected_errors = {} + for row in unbox: + reason = ("anything type mismatch: expected 'array', got 'array'" + if row['route'].startswith('llvm-') else + "anything contains 'array', cannot extract 'array'") + assert row['status'] == 'PASS' and row['reason'] == reason + expected_errors['wrong-element-' + row['route']] = reason.encode() + for encoding in ('text', 'binary'): + expected_output['compile-conversion-' + encoding] = b'' + expected_output['conversion-' + encoding] = b'from-bytes-ok\n' + for row in rejections: + if row['case'] == 'text.from_bytes': + name = 'conversion-rejected-' + row['route'] + reason = "unsupported native annotation '@intrinsic(\"text.from_bytes\")'" + else: + name = 'snapshot-rejected-' + row['case'] + '-' + row['route'] + reason = 'file.read_rooted_snapshot requires (text,text,i64) -> Result' + assert row['status'] == 'PASS' and row['reason'] == reason + expected_errors[name] = reason.encode() + commands = report['commands'] + assert len(commands) == 25 and {row['name'] for row in commands} == set(expected_output) | set(expected_errors) + for row in commands: + assert row['cleanup_verified'] is True and not row.get('timed_out') and not row.get('error') + streams = {} + for stream in ('stdout', 'stderr'): + path = root/'logs'/row[stream] + assert path.parent == root/'logs' and path.name == row['name'] + '.' + stream + assert campaign.support.file_record(path, 1024**2) == row[stream + '_record'] + streams[stream] = path.read_bytes() + if row['name'] in expected_errors: + assert row['status'] == 'FAIL' and row['exit_code'] > 0 and not streams['stdout'] + assert expected_errors[row['name']] in streams['stderr'] + if '-rejected-' in row['name']: + target = Path(row['argv'][row['argv'].index('-o') + 1]) + assert not target.exists() and not target.is_symlink() + else: + assert row['status'] == 'PASS' and row['exit_code'] == 0 and not streams['stderr'] + assert streams['stdout'] == expected_output[row['name']] + paths = {name: campaign.SOURCE/'tests/language'/name for name in ( + 'rooted-file-snapshot.ny', 'text-from-bytes.ny', 'text-from-bytes-native-reject.ny', + 'rooted-file-snapshot-invalid-schema.ny', 'rooted-file-snapshot-invalid-signature.ny')} + paths.update(compiler=campaign.BUILD/'bin/nilcompiler', interpreter=campaign.BUILD/'bin/nilinterpreter', + file_library=campaign.SOURCE/'nilstd/src/io/file.ny', text_library=campaign.SOURCE/'nilstd/src/text/string_utils.ny', + contract=campaign.SOURCE/'tests/tools/rooted_file_snapshot_contract.py', + process_owner=campaign.SOURCE/'validation/ny-3d/scripts/debugger_process.py', + process_support=campaign.SOURCE/'tests/tools/ordinary_io_parity_contract.py') + assert set(report['inputs']) == set(paths) and report['nilstd'] == str(campaign.SOURCE/'nilstd/src') + for name, path in paths.items(): + item = report['inputs'][name] + assert item['path'] == str(path) and campaign.support.file_record(path) == { + key: item[key] for key in ('bytes', 'sha256')} + after = campaign.seal() + campaign.save('seal-after.json', after) + assert after == json.loads((campaign.EVIDENCE/'seal-before.json').read_text()) + assert not list((campaign.BUILD/'owned-test-temp').iterdir()) + campaign.save('replay.json', dict(status='PASS', tested_source_revision=campaign.source_identity(), + host_checks=87, snapshot_observations=8, wrong_element_rejections=4, + byte_conversions=2, native_compile_rejections=6, commands=25, raw_streams=50, + source_std_tools_unchanged=True, process_cleanup_verified=True, owned_temp_empty=True, + report=campaign.support.file_record(root/'report.json'))) + print('PASS: 87 host checks; 8 snapshot, 4 unbox, 2 conversion, 6 native rejection observations; 25 owned commands') + PY + - name: Preserve complete or partial evidence without product binaries + if: always() + timeout-minutes: 3 + run: | + python3 - <<'PY' + import hashlib, json, shutil, stat, sys + from pathlib import Path + sys.path.insert(0, 'build') + import rooted_native_campaign as campaign + destination = campaign.EVIDENCE/'contracts' + destination.mkdir(exist_ok=True) + for root in sorted((campaign.BUILD/'test-evidence').glob('rooted-file-snapshot-*')): + assert root.is_dir() and not root.is_symlink() + output = destination/root.name + output.mkdir() + paths = [root/'report.json', *sorted((root/'logs').glob('*'))] + for path in paths: + if not path.exists(): + continue + assert stat.S_ISREG(path.lstat().st_mode) and not path.is_symlink() + assert path.stat().st_size <= 8 * 1024**2 + target = output/path.relative_to(root) + target.parent.mkdir(exist_ok=True) + shutil.copyfile(path, target) + files = {} + total = 0 + for path in sorted(campaign.EVIDENCE.rglob('*')): + if path.is_dir(): + continue + assert stat.S_ISREG(path.lstat().st_mode) and not path.is_symlink() + assert path.stat().st_size <= 8 * 1024**2 + total += path.stat().st_size + assert total <= 64 * 1024**2 + files[str(path.relative_to(campaign.EVIDENCE))] = campaign.support.file_record(path, 8 * 1024**2) + campaign.save('evidence.json', dict(tested_source_revision=campaign.source_identity(), + workflow_revision=__import__('os').environ['GITHUB_WORKFLOW_SHA'], + scope='Evidence and bounded logs only; generated executables, objects, SDKs and fixture data excluded', + total_bytes=total, files=files)) + PY + - name: Upload complete or partial results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rooted-file-native-${{ matrix.host.arch }}-source-93515c3-${{ github.sha }}-${{ github.run_attempt }} + path: build/rooted-native-evidence/ + if-no-files-found: error + retention-days: 14 diff --git a/notebook/2026-11-12-web-server-boundaries.md b/notebook/2026-11-12-web-server-boundaries.md new file mode 100644 index 00000000..18280faa --- /dev/null +++ b/notebook/2026-11-12-web-server-boundaries.md @@ -0,0 +1,90 @@ +# Web server boundaries — 2026-11-12 + +## Status and question + +Implemented in framework `93515c3d4c2b6e0e7c3b870eb283fce26392d613` +([PR 39](https://github.com/exec-software/exec-nil-framework/pull/39)) and the +[web server PR](https://github.com/exec-software/exec-nil-framework-web/pull/1). +The application exposed gaps that isolated language examples did not: request +boundaries, file identity, filesystem containment and binary-value storage. + +The question was how to keep protocol policy in Matrix while giving it one +reliable filesystem observation on Linux, Windows and both macOS architectures. + +## Decision + +`read_rooted_snapshot(root, relative, maximum_bytes)` returns a typed +`FileSnapshot`: byte-array content, same-handle modification time and SHA-256 of +those exact bytes. It opens each descendant relative to an already-open +directory and rejects symlinks or Windows reparse points. Root spellings ending +in a separator or `/.` cannot disguise a final root link. Ancestors of the +trusted root, hard links and privileged mount changes remain outside this +boundary; the API is not a filesystem sandbox. + +Matrix retains HTTP framing, precondition ordering, range decisions and response +construction. A native content digest and one bulk byte-to-text copy avoid a +per-byte interpreter loop on every request. The pure Matrix SHA-256 code remains +a reference checked against independent vectors. The observed local 1 MiB +loopback response took about 10 ms; this is a measurement, not a service SLA. + +Raw `text.from_bytes` is explicitly interpreter-only until native text can +represent embedded NUL. The snapshot itself remains binary-safe on native +routes because its content is an array. Native compilation rejects unsupported +raw-text conversion instead of truncating data or silently changing its type. + +The work also corrected compact byte-array validation inside boxed records. +Exact `array` values from file operations are valid storage; raw pointer +allocations do not become arrays merely because their bytes look compatible. +Forced-GC and function-return controls verify that a boxed snapshot retains its +content and digest on the supported execution routes. + +## Evidence + +- Linux: 87 host filesystem checks; five focused CTest contracts, including + existing UTF-8, dynamic-value and native-runtime regressions. +- The rooted-file language contract checks textual and binary NIL, LLVM JIT + and AOT before and after an equal-size edit with its timestamp restored: + eight positive observations, four wrong-type extraction refusals, two raw + byte-text interpreter cases and six native declaration/provider refusals. +- [Four-host runtime run 43158309597](https://github.com/exec-software/exec-nil-framework/actions/runs/43158309597) + passed the exact eight-contract inventory on Linux, Windows, Intel macOS + and Apple Silicon macOS. These jobs use LLVM OFF; they are not native-route + or installed-SDK qualification. +- A fresh Windows LLVM build of `93515c3` passed 90 host checks and the + complete AOT/JIT snapshot contract. Existing checkouts and accepted SDKs + remained unchanged. Both macOS application/editor jobs also passed; the new + native snapshot contract is qualified separately on both Mac architectures. +- The web platform campaign requires 1,128 unit checks, 38 real-wire checks + including the actual first-byte deadline, and 15 static HTTP checks. Its + Windows run exposed CRLF checkout conversion of a byte-exact CSS fixture. + `.gitattributes` now requires LF text rather than weakening length or digest + expectations. The original failure remains in the CI record. A second run + found a test-oracle gap: Winsock reports deadline closure as error 10053. + The corrected oracle accepts that exact Windows closure, observes readiness + continuously, and rejects early disconnects outside the 19–25 second window. + Sixteen synthetic controls and 76 actual Windows wire checks passed across + two library locations, with closure observed at 20.004 and 20.027 seconds. +- Owned-process timeout controls passed on Linux and Windows, including a real + listening Matrix child, nested Windows Jobs and failed Job assignment before + command release. Test cleanup is part of the pass condition. + +The test implementations live in `tests/rooted_file_snapshot_tests.cpp` and +`tests/tools/rooted_file_snapshot_contract.py`; web consumer tests live in its +own repository. No accepted SDK or release deployment is replaced by this work. + +## Follow-up + +Complete native snapshot qualification on both macOS architectures, then record the corrected web campaign's four-host results. Keep the platform +checks required for public changes alongside the secret/path audit and review. + +The next project is a clean Matrix WebView package. The archived implementation +has no macOS provider and advertises some Linux-only functions through a shared +wrapper. Its replacement needs a checked provider ABI, bounded event queues and +observable asynchronous completion; generic legacy FFI and provider hot reload +do not establish those guarantees. Browser-origin provenance must come from +native evidence or be marked unavailable, never inferred from a page message. + +For future AI diagnostics, content identities can correlate a file observation +with an action without duplicating its bytes. This is a proposed use, not an +automatic logging or executable-context feature; fingerprints can themselves +be sensitive and need the same deliberate disclosure policy as other context. diff --git a/notebook/README.md b/notebook/README.md index ddf6fc5f..8c1b0d7a 100644 --- a/notebook/README.md +++ b/notebook/README.md @@ -11,6 +11,11 @@ and explicit about evidence, uncertainty, and implementation status. ## Contents +- [`2026-11-12-web-server-boundaries.md`](2026-11-12-web-server-boundaries.md) — + application-driven file snapshots, protocol boundaries, binary values and + the checked provider boundary needed by the planned WebView rewrite. +- [`2026-11-11-web-runtime-parity.md`](2026-11-11-web-runtime-parity.md) — + current-source and installed-SDK platform evidence for the web runtime. - [`2026-11-04-macos-parity-plan.md`](2026-11-04-macos-parity-plan.md) — initial Darwin/Apple Silicon gap audit, hosted and VM testing options, and the Windows-first sequence for establishing macOS parity. From aad707761460951c2e42401d3548faec226a8e79 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 00:35:24 -0500 Subject: [PATCH 03/12] Run the new native Mac campaign during workflow review --- .github/workflows/rooted-file-native-parity.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/rooted-file-native-parity.yml b/.github/workflows/rooted-file-native-parity.yml index 94c23989..978f7811 100644 --- a/.github/workflows/rooted-file-native-parity.yml +++ b/.github/workflows/rooted-file-native-parity.yml @@ -2,6 +2,12 @@ name: Rooted file native macOS checks "on": workflow_dispatch: + # A new workflow cannot be dispatched until it exists on the default branch. + # This scoped PR trigger qualifies it before review/merge without changing + # the fixed source revision under test. + pull_request: + paths: + - .github/workflows/rooted-file-native-parity.yml permissions: contents: read From 6923566732a38d7c2917466cc1befb6a5b4eee1f Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 00:56:06 -0500 Subject: [PATCH 04/12] Record successful server and native Mac qualification --- notebook/2026-11-12-web-server-boundaries.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/notebook/2026-11-12-web-server-boundaries.md b/notebook/2026-11-12-web-server-boundaries.md index 18280faa..51475ad7 100644 --- a/notebook/2026-11-12-web-server-boundaries.md +++ b/notebook/2026-11-12-web-server-boundaries.md @@ -74,8 +74,13 @@ own repository. No accepted SDK or release deployment is replaced by this work. ## Follow-up -Complete native snapshot qualification on both macOS architectures, then record the corrected web campaign's four-host results. Keep the platform -checks required for public changes alongside the secret/path audit and review. +Native snapshot qualification passed on both Mac architectures in +[run 43162404951](https://github.com/exec-software/exec-nil-framework/actions/runs/43162404951), +with the exact 87 host and 8/4/2/6 language-route inventories against fixed +source `93515c3`. The corrected [web run 43162279298](https://github.com/exec-software/exec-nil-framework-web/actions/runs/43162279298) +passed all four hosts at web head `4bcb43f`: 1,128 unit, 16 oracle, 38 wire +and 15 static checks each. The four `web-*` checks now join `secrets` as +required checks; review and the other existing branch protections are unchanged. The next project is a clean Matrix WebView package. The archived implementation has no macOS provider and advertises some Linux-only functions through a shared From 4ee601fa973439e092c080387dd1615334fb616f Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Thu, 12 Nov 2026 01:07:20 -0500 Subject: [PATCH 05/12] Add checked WebView provider operations across interpreter and LLVM routes --- CMakeLists.txt | 21 ++ common/webview_json_v1.hpp | 127 ++++++++ common/webview_provider_v1.h | 191 ++++++++++++ nilcompiler/CMakeLists.txt | 1 + nilcompiler/src/codegen.cpp | 3 +- nilcompiler/src/llvm_backend.cpp | 25 +- nilcompiler/src/native_runtime.cpp | 71 +++++ nilcompiler/src/native_runtime.hpp | 13 + nilcompiler/src/type_checker.cpp | 11 +- nilcompiler/src/vm.cpp | 53 ++++ nilcompiler/src/vm.hpp | 2 + nilcompiler/src/webview_intrinsic.hpp | 28 ++ nilcompiler/src/webview_runtime.cpp | 299 +++++++++++++++++++ nilcompiler/src/webview_runtime.hpp | 40 +++ notebook/2026-11-12-matrix-webview.md | 80 +++++ notebook/2026-11-12-web-server-boundaries.md | 9 +- notebook/README.md | 2 + tests/tools/webview_intrinsic_contract.py | 112 +++++++ tests/webview_provider_fixture.cpp | 89 ++++++ tests/webview_runtime_tests.cpp | 97 ++++++ 20 files changed, 1262 insertions(+), 12 deletions(-) create mode 100644 common/webview_json_v1.hpp create mode 100644 common/webview_provider_v1.h create mode 100644 nilcompiler/src/webview_intrinsic.hpp create mode 100644 nilcompiler/src/webview_runtime.cpp create mode 100644 nilcompiler/src/webview_runtime.hpp create mode 100644 notebook/2026-11-12-matrix-webview.md create mode 100644 tests/tools/webview_intrinsic_contract.py create mode 100644 tests/webview_provider_fixture.cpp create mode 100644 tests/webview_runtime_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bf26728e..e9b64881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,22 @@ if(BUILD_TESTING) LABELS "tools;windows;rc" TIMEOUT 30) endif() + # Synthetic checked provider peers exercise loading and ownership without + # a browser engine or signed-in desktop. Never packaged into the SDK. + add_library(nil_webview_fixture_good SHARED tests/webview_provider_fixture.cpp) + add_library(nil_webview_fixture_bad SHARED tests/webview_provider_fixture.cpp) + target_compile_definitions(nil_webview_fixture_bad PRIVATE NILWV_FIXTURE_BAD_ABI=1) + foreach(fixture IN ITEMS nil_webview_fixture_good nil_webview_fixture_bad) + target_compile_features(${fixture} PRIVATE cxx_std_17) + endforeach() + add_executable(nil_webview_runtime_tests tests/webview_runtime_tests.cpp) + target_link_libraries(nil_webview_runtime_tests PRIVATE nilruntime nil_webview_fixture_good Threads::Threads) + target_include_directories(nil_webview_runtime_tests PRIVATE nilcompiler/src) + add_dependencies(nil_webview_runtime_tests nil_webview_fixture_bad) + add_test(NAME nil.webview-runtime COMMAND $ + $ $) + set_tests_properties(nil.webview-runtime PROPERTIES TIMEOUT 20) + add_executable(nil_rooted_file_snapshot_tests tests/rooted_file_snapshot_tests.cpp) target_link_libraries(nil_rooted_file_snapshot_tests PRIVATE nilruntime) target_include_directories(nil_rooted_file_snapshot_tests PRIVATE nilcompiler/src) @@ -838,6 +854,11 @@ if(BUILD_TESTING) --lifecycle-test $ --llvm "${NIL_ENABLE_LLVM}" ${NIL_UI_DISPLAY_APPKIT_ARGS}) set_tests_properties(nil.ui-display PROPERTIES LABELS "ui;routes" SKIP_RETURN_CODE 77 TIMEOUT "${NIL_UI_DISPLAY_TIMEOUT}") + add_test(NAME nil.webview-intrinsic-contract + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/tools/webview_intrinsic_contract.py" + --compiler $ --interpreter $ + --llvm "${NIL_ENABLE_LLVM}") + set_tests_properties(nil.webview-intrinsic-contract PROPERTIES LABELS "webview;routes" TIMEOUT 180) add_test(NAME nil.ui-intrinsic-contract COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/tools/ui_intrinsic_contract.py" --compiler $ --interpreter $ diff --git a/common/webview_json_v1.hpp b/common/webview_json_v1.hpp new file mode 100644 index 00000000..b57a68cf --- /dev/null +++ b/common/webview_json_v1.hpp @@ -0,0 +1,127 @@ +// Copyright (C) 2026 Execute Software. SPDX-License-Identifier: MIT +#pragma once +#include +#include +#include + +// WebView v1 JSON text grammar. This validates without decoding or normalizing +// JSON.stringify output, including escaped lone UTF-16 surrogates. A configuration +// parser with duplicate-key/scalar restrictions is not interchangeable with it. +// The provider and framework carry identical copies of this versioned grammar. +namespace nilwebview::json_v1 { +namespace detail { +class Parser { +public: + explicit Parser(const std::string &source) : source_(source) {} + bool parse() noexcept { whitespace(); return value(0) && (whitespace(), offset_ == source_.size()); } +private: + void whitespace() noexcept { + while (offset_ < source_.size() && (source_[offset_] == ' ' || source_[offset_] == '\t' || + source_[offset_] == '\r' || source_[offset_] == '\n')) ++offset_; + } + bool take(char expected) noexcept { + if (offset_ == source_.size() || source_[offset_] != expected) return false; + ++offset_; return true; + } + bool literal(const char *text) noexcept { + const auto size = std::strlen(text); + if (source_.compare(offset_, size, text) != 0) return false; + offset_ += size; return true; + } + bool string() noexcept { + if (!take('"')) return false; + while (offset_ < source_.size()) { + const auto c = static_cast(source_[offset_++]); + if (c == '"') return true; + if (c < 0x20) return false; + if (c != '\\') continue; + if (offset_ == source_.size()) return false; + const auto escape = source_[offset_++]; + if (escape == 'u') { + for (int index = 0; index < 4; ++index) { + if (offset_ == source_.size()) return false; + const auto digit = source_[offset_++]; + if (!((digit >= '0' && digit <= '9') || (digit >= 'a' && digit <= 'f') || + (digit >= 'A' && digit <= 'F'))) return false; + } + } else if (!std::strchr("\"\\/bfnrt", escape)) return false; + } + return false; + } + bool number() noexcept { + take('-'); + if (!take('0')) { + if (offset_ == source_.size() || source_[offset_] < '1' || source_[offset_] > '9') return false; + do { ++offset_; } while (offset_ < source_.size() && source_[offset_] >= '0' && source_[offset_] <= '9'); + } + if (take('.')) { + const auto first = offset_; + while (offset_ < source_.size() && source_[offset_] >= '0' && source_[offset_] <= '9') ++offset_; + if (offset_ == first) return false; + } + if (take('e') || take('E')) { + if (!take('+')) take('-'); + const auto first = offset_; + while (offset_ < source_.size() && source_[offset_] >= '0' && source_[offset_] <= '9') ++offset_; + if (offset_ == first) return false; + } + return true; + } + bool value(unsigned depth) noexcept { + if (depth > 64 || offset_ == source_.size()) return false; + switch (source_[offset_]) { + case '"': return string(); + case 't': return literal("true"); + case 'f': return literal("false"); + case 'n': return literal("null"); + case '[': { + ++offset_; whitespace(); + if (take(']')) return true; + do { whitespace(); if (!value(depth + 1)) return false; whitespace(); } while (take(',')); + return take(']'); + } + case '{': { + ++offset_; whitespace(); + if (take('}')) return true; + do { + whitespace(); if (!string()) return false; + whitespace(); if (!take(':')) return false; + whitespace(); if (!value(depth + 1)) return false; + whitespace(); + } while (take(',')); + return take('}'); + } + default: return number(); + } + } + const std::string &source_; + std::size_t offset_ = 0; +}; +} // namespace detail + +inline bool valid_utf8(const std::string &text) noexcept { + for (std::size_t index = 0; index < text.size();) { + const auto first = static_cast(text[index++]); + if (!first) return false; + if (first < 0x80) continue; + unsigned count = 0; + std::uint32_t code = 0, minimum = 0; + if (first >= 0xc2 && first <= 0xdf) { count = 1; code = first & 0x1f; minimum = 0x80; } + else if (first >= 0xe0 && first <= 0xef) { count = 2; code = first & 0x0f; minimum = 0x800; } + else if (first >= 0xf0 && first <= 0xf4) { count = 3; code = first & 7; minimum = 0x10000; } + else return false; + if (text.size() - index < count) return false; + while (count--) { + const auto byte = static_cast(text[index++]); + if ((byte & 0xc0) != 0x80) return false; + code = (code << 6) | (byte & 0x3f); + } + if (code < minimum || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return false; + } + return true; +} + +inline bool valid_json(const std::string &text) noexcept { + return text.size() <= 1048576 && valid_utf8(text) && detail::Parser(text).parse(); +} +} // namespace nilwebview::json_v1 diff --git a/common/webview_provider_v1.h b/common/webview_provider_v1.h new file mode 100644 index 00000000..216964ee --- /dev/null +++ b/common/webview_provider_v1.h @@ -0,0 +1,191 @@ +/* Checked WebView provider ABI, version 1. + * Copyright (C) 2026 Execute Software. SPDX-License-Identifier: MIT + */ +#ifndef NILWEBVIEW_PROVIDER_V1_H +#define NILWEBVIEW_PROVIDER_V1_H +#include + +#if defined(_WIN32) +#define NILWV_CALL __cdecl +#else +#define NILWV_CALL +#endif +#if defined(_WIN32) && defined(NILWEBVIEW_PROVIDER_BUILD) +#define NILWV_EXPORT __declspec(dllexport) +#elif defined(__GNUC__) && defined(NILWEBVIEW_PROVIDER_BUILD) +#define NILWV_EXPORT __attribute__((visibility("default"))) +#else +#define NILWV_EXPORT +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define NILWV_ABI_V1 UINT32_C(1) +#define NILWV_MAX_TEXT_BYTES UINT64_C(1048576) +#define NILWV_MAX_QUEUED_BYTES UINT64_C(8388608) +#define NILWV_MAX_EVENTS UINT32_C(256) +#define NILWV_MAX_WINDOWS UINT32_C(16) +#define NILWV_OPERATION_TIMEOUT_MS UINT32_C(30000) + +/* Capability bits describe native sender metadata, never page assertions. + * A valid GTK6 provider may advertise zero. These bits do not prove that a + * display or browser runtime can initialize, or apply to every URL scheme. */ +#define NILWV_CAP_MESSAGE_URL UINT64_C(1) +#define NILWV_CAP_MESSAGE_ORIGIN UINT64_C(2) +#define NILWV_CAP_MESSAGE_FRAME UINT64_C(4) +#define NILWV_CAP_KNOWN_MASK UINT64_C(7) + +typedef struct nilwv_context_v1 nilwv_context_v1; +typedef struct nilwv_text_v1 { + const char *data; + uint64_t size; +} nilwv_text_v1; + +/* Fields are fixed-width; C enums below name values but never cross the ABI. + * UTF-8 is validated, with no embedded NUL. {NULL,0} is the empty string. + * Inputs are borrowed for the call; asynchronous operations copy them. + * Every returned text view is provider-owned until the next call using this + * context, or context destruction. The framework must copy it immediately. + * Error text uses the same rule; probe/create errors survive the next call + * on their owner thread. No allocator or C++ type crosses this boundary. */ +typedef struct nilwv_result_v1 { + uint32_t size; /* caller initializes to sizeof(result) */ + int32_t code; /* 0 success; negative error code */ + int64_t number; /* numeric result; zero when unused */ + nilwv_text_v1 text; /* text result, or diagnostic if code < 0 */ +} nilwv_result_v1; + +enum nilwv_error_code_v1 { + NILWV_E_UNAVAILABLE = -1, + NILWV_E_ARGUMENT = -2, + NILWV_E_HANDLE = -3, + NILWV_E_THREAD = -4, + NILWV_E_NOT_READY = -5, + NILWV_E_BUSY = -6, + NILWV_E_LIMIT = -7, + NILWV_E_NATIVE = -8, + NILWV_E_TIMEOUT = -9, + NILWV_E_PROTOCOL = -10, + NILWV_E_QUEUE_OVERFLOW = -11 +}; + +enum nilwv_event_kind_v1 { + NILWV_EVENT_READY = 1, + NILWV_EVENT_INITIALIZATION_FAILED = 2, + NILWV_EVENT_NAVIGATION_COMPLETED = 3, + NILWV_EVENT_NAVIGATION_FAILED = 4, + NILWV_EVENT_SCRIPT_COMPLETED = 5, + NILWV_EVENT_SCRIPT_FAILED = 6, + NILWV_EVENT_PAGE_MESSAGE = 7, + NILWV_EVENT_NAVIGATION_REQUESTED = 8, + NILWV_EVENT_CLOSE_REQUESTED = 9, + NILWV_EVENT_PROVIDER_FAILED = 10 +}; +enum nilwv_value_kind_v1 { + NILWV_VALUE_NONE = 0, NILWV_VALUE_JSON = 1, NILWV_VALUE_UNDEFINED = 2 +}; +enum nilwv_source_kind_v1 { + NILWV_SOURCE_UNKNOWN = 0, + NILWV_SOURCE_NATIVE_URL = 1, + NILWV_SOURCE_NATIVE_ORIGIN = 2 +}; +enum nilwv_frame_kind_v1 { + NILWV_FRAME_UNKNOWN = 0, NILWV_FRAME_MAIN = 1, NILWV_FRAME_CHILD = 2 +}; +enum nilwv_failure_kind_v1 { + NILWV_FAILURE_NONE = 0, + NILWV_FAILURE_NATIVE = 1, + NILWV_FAILURE_SCRIPT_EXCEPTION = 2, + NILWV_FAILURE_SERIALIZATION = 3, + NILWV_FAILURE_ASYNC_RESULT = 4, + NILWV_FAILURE_PAYLOAD_LIMIT = 5, + NILWV_FAILURE_TIMEOUT = 6, + NILWV_FAILURE_CANCELLED = 7 +}; + +/* One context per framework execution context, owned by its platform UI + * thread: the process main thread on macOS/Linux, the creating STA thread on + * Windows. All methods, including release and destruction, use that thread. + * Public numeric handles are positive, process-unique, never reused, <=INT64_MAX, + * and validated against context, resource kind and lifetime on every call. + * No callback from a provider enters Matrix or the VM. + * + * probe: no window/display initialization; result.number=1 on success. + * create: returns a context or NULL on error; profile state is provider-owned + * and isolated from personal browser profiles. Neither probe nor create starts + * a global event loop. window: returns a pending hidden window, then poll emits + * exactly one Ready/InitializationFailed unless explicitly closed first. + * + * show/close/release_event: number=1. navigate/load_html/evaluate_function: + * positive operation ID, then one matching completion/failure event. Reject + * overlapping navigation/script operations with BUSY; do not queue another. + * navigate authorizes one initial URL request. HTTP redirects and uninitiated + * navigations (including page actions/new windows) are cancelled and reported + * as NavigationRequested; a cancelled active navigation completes as failed. + * After that completion the application may explicitly follow a requested URL. + * load_html has no base URI; evaluate_function accepts a synchronous function + * body with explicit return, JSON.stringify result semantics, distinct undefined, + * and explicit failure for thenables or unsuccessful serialization. + * + * poll pumps a bounded batch (up to 64 native dispatch iterations, 2ms soft budget) and returns an + * owned immutable event snapshot ID, or 0 for empty. Retained snapshots count + * toward queue limits. The framework copies all fields then releases on every + * path. Overflow latches QUEUE_OVERFLOW; operations fail until context teardown. + * No messages or completion events are silently dropped to recover capacity. + * + * event_number names: window, kind, sequence, operation, value_kind, + * source_kind, frame_kind, failure_kind. operation=0 means no initiating call. + * event_text names: payload_json, source_url, source_origin, detail. + * source_url/origin are empty when unavailable; browser current URL and + * JavaScript-supplied URL/origin values must not be represented as sender proof. + * source_origin is only a native engine-reported origin, not parsed from a URL. + * Unknown event property names are ARGUMENT errors, not empty/zero results. + * + * close invalidates the window, its queued/retained events and pending operations. + * Closing again is HANDLE error. Teardown invalidates all context resources. + * Outstanding engine callbacks retain only independent cancelled state, never + * a raw freed context/window pointer. Late callbacks cannot publish new events. + * Browser processes may finish asynchronously after object closure; the provider + * remains responsible for its own isolated profile/cleanup state. No claim is + * made that cancelling a script stops executing JavaScript. + */ +typedef struct nilwv_api_v1 { + uint32_t abi_version; + uint32_t size; + uint64_t capabilities; + const char *provider_name; /* static NUL-terminated UTF-8 */ + const char *provider_build; /* static NUL-terminated UTF-8 */ + + void (NILWV_CALL *probe)(nilwv_result_v1 *); + void (NILWV_CALL *context_create)(nilwv_context_v1 **, nilwv_result_v1 *); + void (NILWV_CALL *context_destroy)(nilwv_context_v1 *); + void (NILWV_CALL *window)(nilwv_context_v1 *, nilwv_text_v1, int64_t, int64_t, nilwv_result_v1 *); + void (NILWV_CALL *show)(nilwv_context_v1 *, int64_t, nilwv_result_v1 *); + void (NILWV_CALL *navigate)(nilwv_context_v1 *, int64_t, nilwv_text_v1, nilwv_result_v1 *); + void (NILWV_CALL *load_html)(nilwv_context_v1 *, int64_t, nilwv_text_v1, nilwv_result_v1 *); + void (NILWV_CALL *evaluate_function)(nilwv_context_v1 *, int64_t, nilwv_text_v1, nilwv_result_v1 *); + void (NILWV_CALL *poll)(nilwv_context_v1 *, int64_t, nilwv_result_v1 *); + void (NILWV_CALL *event_number)(nilwv_context_v1 *, int64_t, nilwv_text_v1, nilwv_result_v1 *); + void (NILWV_CALL *event_text)(nilwv_context_v1 *, int64_t, nilwv_text_v1, nilwv_result_v1 *); + void (NILWV_CALL *release_event)(nilwv_context_v1 *, int64_t, nilwv_result_v1 *); + void (NILWV_CALL *close)(nilwv_context_v1 *, int64_t, nilwv_result_v1 *); +} nilwv_api_v1; + +typedef const nilwv_api_v1 *(NILWV_CALL *nilwv_get_api_v1_fn)(uint32_t, uint32_t); + +/* Exactly this exported C name; NULL rejects unsupported ABI/table size. + * The table and function pointers remain stable for process lifetime. + * Runtime loads only a caller-selected absolute provider path, validates every + * mandatory pointer and the exact ABI, calls probe, then creates its context. + * The runtime never unloads or hot-reloads a successfully loaded provider. + * No provider call may allow a C++ exception to cross this boundary. + */ +NILWV_EXPORT const nilwv_api_v1 *NILWV_CALL +nilwebview_provider_v1_get_api(uint32_t requested_version, uint32_t minimum_table_size); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/nilcompiler/CMakeLists.txt b/nilcompiler/CMakeLists.txt index b29a9cda..dec1beb2 100644 --- a/nilcompiler/CMakeLists.txt +++ b/nilcompiler/CMakeLists.txt @@ -69,6 +69,7 @@ add_library(nilruntime_objects OBJECT src/owned_runtime.cpp src/secure_file_io.cpp src/database_runtime.cpp + src/webview_runtime.cpp ${NIL_UI_PROVIDER_SOURCE} src/pattern_engine.cpp src/nildata.cpp diff --git a/nilcompiler/src/codegen.cpp b/nilcompiler/src/codegen.cpp index 219ef77b..660483af 100644 --- a/nilcompiler/src/codegen.cpp +++ b/nilcompiler/src/codegen.cpp @@ -6983,7 +6983,8 @@ codegen::assemble_module(const ast_program &program, std::string ret_type_str = "void"; if (intrinsic_name.rfind("regex.", 0) == 0 || intrinsic_name.rfind("nildata.", 0) == 0 || - intrinsic_name.rfind("ui.v1.", 0) == 0) { + intrinsic_name.rfind("ui.v1.", 0) == 0 || + intrinsic_name.rfind("webview.v1.", 0) == 0) { ret_type_str = canonical_nil_type_name(fn.return_type); } else if (is_declared_enum_type(*this, fn.return_type)) { ret_type_str = canonical_nil_type_name(fn.return_type); diff --git a/nilcompiler/src/llvm_backend.cpp b/nilcompiler/src/llvm_backend.cpp index 36d29a09..47a3eb2a 100644 --- a/nilcompiler/src/llvm_backend.cpp +++ b/nilcompiler/src/llvm_backend.cpp @@ -7,6 +7,7 @@ #include "diagnostics.hpp" #include "dynamic_type.hpp" #include "ui_intrinsic.hpp" +#include "webview_intrinsic.hpp" #include "executable_context.hpp" #include "native_runtime.hpp" #include "ordinary_io_intrinsic.hpp" @@ -2944,10 +2945,13 @@ bool NativeCodegen::generateModule(const nil_module &mod) { impl_->builder->CreateRet(value); } annotation_handled = true; - } else if (dispatch_name == "@intrinsic" && dispatch_provider.rfind("ui.v1.", 0) == 0) { - const auto contract = ui::intrinsic(dispatch_provider); + } else if (dispatch_name == "@intrinsic" && + (dispatch_provider.rfind("ui.v1.", 0) == 0 || dispatch_provider.rfind("webview.v1.", 0) == 0)) { + const bool is_webview = dispatch_provider.rfind("webview.v1.", 0) == 0; + const auto contract = is_webview ? webview::intrinsic(dispatch_provider) : ui::intrinsic(dispatch_provider); + const std::string family = is_webview ? "WebView" : "UI"; if (!contract) { - std::cerr << "[NativeCodegen] unknown UI intrinsic '" << dispatch_provider << "'\n"; + std::cerr << "[NativeCodegen] unknown " << family << " intrinsic '" << dispatch_provider << "'\n"; return false; } std::vector parameter_types; @@ -2969,7 +2973,7 @@ bool NativeCodegen::generateModule(const nil_module &mod) { valid = valid && schemas == 1; } if (!valid) { - std::cerr << "[NativeCodegen] " << dispatch_provider << " requires the exact UI ABI " + std::cerr << "[NativeCodegen] " << dispatch_provider << " requires the exact " << family << " ABI " << ui::signature(*contract) << " and the standard Result schema\n"; return false; } @@ -2989,7 +2993,7 @@ bool NativeCodegen::generateModule(const nil_module &mod) { std::vector types; for (auto *argument : arguments) types.push_back(argument->getType()); auto *runtime_type = llvm::FunctionType::get(capability ? word_type : llvm::Type::getInt32Ty(ctx), types, false); - auto runtime = impl_->module->getOrInsertFunction("nyrt_ui_v1_" + contract->operation, runtime_type); + auto runtime = impl_->module->getOrInsertFunction(std::string(is_webview ? "nyrt_webview_v1_" : "nyrt_ui_v1_") + contract->operation, runtime_type); auto *status = impl_->builder->CreateCall(runtime_type, runtime.getCallee(), arguments); if (capability) { impl_->builder->CreateRet(impl_->builder->CreateIntCast(status, impl_->current_function->getReturnType(), false)); @@ -9395,6 +9399,17 @@ int NativeCodegen::executeJIT(const std::vector &program_args) { llvm::orc::ExecutorSymbolDef(llvm::orc::ExecutorAddr::fromPtr(address), llvm::JITSymbolFlags::Exported); }; + expose_runtime_symbol("nyrt_webview_v1_configure", &nyrt_webview_v1_configure); + expose_runtime_symbol("nyrt_webview_v1_window", &nyrt_webview_v1_window); + expose_runtime_symbol("nyrt_webview_v1_show", &nyrt_webview_v1_show); + expose_runtime_symbol("nyrt_webview_v1_poll", &nyrt_webview_v1_poll); + expose_runtime_symbol("nyrt_webview_v1_release_event", &nyrt_webview_v1_release_event); + expose_runtime_symbol("nyrt_webview_v1_close", &nyrt_webview_v1_close); + expose_runtime_symbol("nyrt_webview_v1_navigate", &nyrt_webview_v1_navigate); + expose_runtime_symbol("nyrt_webview_v1_load_html", &nyrt_webview_v1_load_html); + expose_runtime_symbol("nyrt_webview_v1_evaluate_function", &nyrt_webview_v1_evaluate_function); + expose_runtime_symbol("nyrt_webview_v1_event_number", &nyrt_webview_v1_event_number); + expose_runtime_symbol("nyrt_webview_v1_event_text", &nyrt_webview_v1_event_text); expose_runtime_symbol("nyrt_ui_v1_available", &nyrt_ui_v1_available); expose_runtime_symbol("nyrt_ui_v1_window", &nyrt_ui_v1_window); expose_runtime_symbol("nyrt_ui_v1_widget", &nyrt_ui_v1_widget); diff --git a/nilcompiler/src/native_runtime.cpp b/nilcompiler/src/native_runtime.cpp index 7c20629b..4d2a7898 100644 --- a/nilcompiler/src/native_runtime.cpp +++ b/nilcompiler/src/native_runtime.cpp @@ -7,6 +7,7 @@ #include "dynamic_type.hpp" #include "database_runtime.hpp" #include "ui_runtime.hpp" +#include "webview_runtime.hpp" #include "integer_conversion.hpp" #include "memory_profile.hpp" #include "pattern_engine.hpp" @@ -97,6 +98,7 @@ struct runtime_context { ny::memory_profile::Session profile; ny::database::Context databases; ny::ui::Context ui; + ny::webview::Context webview; struct file { std::unique_ptr stream; bool readable = false; @@ -647,10 +649,77 @@ const char *ui_text_argument(const char *text) { if (!text) throw std::invalid_argument("ui: expected text argument"); return text; } +template +std::int32_t webview_number_result(std::int64_t *value, const char **error, + Invoke invoke) noexcept { + if (!value || !error) return 0; + *value = 0; *error = nullptr; + try { + auto result = invoke(); + if (!result) { set_error(error, result.error); return 0; } + *value = result.number; + return 1; + } catch (const std::invalid_argument &invalid) { set_error(error, invalid.what()); return 0; } + catch (...) { *error = "webview: host allocation failed"; return 0; } +} +template +std::int32_t webview_text_result(const char **value, const char **error, + Invoke invoke) noexcept { + if (!value || !error) return 0; + *value = nullptr; *error = nullptr; + try { + auto result = invoke(); + if (!result) { set_error(error, result.error); return 0; } + // The snapshot may be released immediately after this call. Never return + // provider storage, even when its lifetime presently spans the caller. + *value = own_string(result.text); + if (!*value) { *error = "webview: host allocation failed"; return 0; } + return 1; + } catch (const std::invalid_argument &invalid) { set_error(error, invalid.what()); return 0; } + catch (...) { *error = "webview: host allocation failed"; return 0; } +} +const char *webview_text_argument(const char *text) { + if (!text) throw std::invalid_argument("webview: expected text argument"); + return text; +} } // namespace extern "C" { +std::int32_t nyrt_webview_v1_configure(const char *path, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.configure(webview_text_argument(path)); }); +} +std::int32_t nyrt_webview_v1_window(const char *title, std::int64_t width, std::int64_t height, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.window(webview_text_argument(title), width, height); }); +} +std::int32_t nyrt_webview_v1_show(std::int64_t handle, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.show(handle); }); +} +std::int32_t nyrt_webview_v1_poll(std::int64_t handle, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.poll(handle); }); +} +std::int32_t nyrt_webview_v1_release_event(std::int64_t handle, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.release_event(handle); }); +} +std::int32_t nyrt_webview_v1_close(std::int64_t handle, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.close(handle); }); +} +std::int32_t nyrt_webview_v1_navigate(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.navigate(handle, webview_text_argument(text)); }); +} +std::int32_t nyrt_webview_v1_load_html(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.load_html(handle, webview_text_argument(text)); }); +} +std::int32_t nyrt_webview_v1_evaluate_function(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.evaluate_function(handle, webview_text_argument(text)); }); +} +std::int32_t nyrt_webview_v1_event_number(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept { + return webview_number_result(value, error, [&] { return context.webview.event_number(handle, webview_text_argument(text)); }); +} +std::int32_t nyrt_webview_v1_event_text(std::int64_t handle, const char *text, const char **value, const char **error) noexcept { + return webview_text_result(value, error, [&] { return context.webview.event_text(handle, webview_text_argument(text)); }); +} + std::int64_t nyrt_ui_v1_available() noexcept { return ny::ui::available() ? 1 : 0; } std::int32_t nyrt_ui_v1_window(const char *title, std::int64_t width, std::int64_t height, std::int64_t *value, const char **error) noexcept { @@ -869,6 +938,7 @@ void nyrt_context_end() noexcept { nyrt_fail("NIL native runtime error: memory capture failed\n"); context.databases.clear(); context.ui.clear(); + context.webview.clear(); context.files.clear(); nyrt_owned_v1_unwind(0); context.pending_fault.reset(); @@ -891,6 +961,7 @@ void nyrt_profile_site(const char *file, const char *function, [[noreturn]] void nyrt_fail(const char *message) noexcept { context.ui.clear(); + context.webview.clear(); (void)context.profile.finish("failed", false); if (message != nullptr) std::fputs(message, stderr); diff --git a/nilcompiler/src/native_runtime.hpp b/nilcompiler/src/native_runtime.hpp index 5435df50..306d5e0d 100644 --- a/nilcompiler/src/native_runtime.hpp +++ b/nilcompiler/src/native_runtime.hpp @@ -53,6 +53,19 @@ std::int32_t nyrt_ui_v1_destroy_widget(std::int64_t handle, std::int32_t nyrt_ui_v1_close(std::int64_t window, std::int64_t *value, const char **error) noexcept; +// Checked WebView provider bridge. All text is copied into managed storage. +std::int32_t nyrt_webview_v1_configure(const char *path, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_window(const char *title, std::int64_t width, std::int64_t height, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_show(std::int64_t handle, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_poll(std::int64_t handle, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_release_event(std::int64_t handle, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_close(std::int64_t handle, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_navigate(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_load_html(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_evaluate_function(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_event_number(std::int64_t handle, const char *text, std::int64_t *value, const char **error) noexcept; +std::int32_t nyrt_webview_v1_event_text(std::int64_t handle, const char *text, const char **value, const char **error) noexcept; + // Additive database ABI. Success is 1, failure 0; outputs are context-owned. std::int64_t nyrt_db2_available(const char *driver) noexcept; std::int32_t nyrt_db2_open(const char *driver, const char *address, diff --git a/nilcompiler/src/type_checker.cpp b/nilcompiler/src/type_checker.cpp index ce139964..9b053681 100644 --- a/nilcompiler/src/type_checker.cpp +++ b/nilcompiler/src/type_checker.cpp @@ -7,6 +7,7 @@ #include "diagnostics.hpp" #include "dynamic_type.hpp" #include "ui_intrinsic.hpp" +#include "webview_intrinsic.hpp" #include "nil_contract.hpp" #include "numeric_literal.hpp" #include "scalar_text.hpp" @@ -3309,10 +3310,12 @@ void type_checker::visit(ast_function_decl &p) { const auto &arguments = annotation.typed_arguments ? *annotation.typed_arguments : parsed.arguments; if (arguments.size() != 1 || arguments.front().name || arguments.front().kind != "text") continue; const auto &provider = arguments.front().value; - if (provider.rfind("ui.v1.", 0) != 0) continue; - const auto contract = ui::intrinsic(provider); + const bool is_webview = provider.rfind("webview.v1.", 0) == 0; + if (!is_webview && provider.rfind("ui.v1.", 0) != 0) continue; + const auto contract = is_webview ? webview::intrinsic(provider) : ui::intrinsic(provider); + const std::string family = is_webview ? "WebView" : "UI"; if (!contract) { - add_error(p.line, "Unknown UI intrinsic '" + provider + "'"); + add_error(p.line, "Unknown " + family + " intrinsic '" + provider + "'"); continue; } // Core Result may remain a short built-in name after ordinary import @@ -3334,7 +3337,7 @@ void type_checker::visit(ast_function_decl &p) { ordinary = ordinary && !parameter.is_ref && !parameter.is_variadic && !parameter.default_value; } if (!ordinary || !ui::signature_matches(*contract, parameters, p.return_type.value_or("void"))) - add_error(p.line, provider + " requires the exact UI ABI " + ui::signature(*contract) + + add_error(p.line, provider + " requires the exact " + family + " ABI " + ui::signature(*contract) + "; use a module-level, non-generic function without reference, variadic, or default parameters and import nil.std.core.result"); } diff --git a/nilcompiler/src/vm.cpp b/nilcompiler/src/vm.cpp index 50c7620b..22df6b69 100644 --- a/nilcompiler/src/vm.cpp +++ b/nilcompiler/src/vm.cpp @@ -10,6 +10,7 @@ #include "nil_contract.hpp" #include "dynamic_type.hpp" #include "ui_intrinsic.hpp" +#include "webview_intrinsic.hpp" #include "numeric_literal.hpp" #include "scalar_text.hpp" #include "integer_conversion.hpp" @@ -738,6 +739,7 @@ bool VM::finishMemoryProfile(const char *status, bool completed) noexcept { // This hook runs at every invocation terminal, even without profiling. // Paused debugger steps retain their UI until completion/reset. ui_.clear(); + webview_.clear(); closeOpenSockets(); if (!memory_profile_started_ || memory_profile_finished_) return true; @@ -5548,6 +5550,57 @@ void VM::executeInstruction(const Instruction &instr) { return; } + if (lib_name == "intrinsic" && fn_name.rfind("webview.v1.", 0) == 0) { + const auto contract = webview::intrinsic(fn_name); + if (!contract) throw std::runtime_error("webview: unknown intrinsic '" + fn_name + "'"); + const auto declared = call_stack_.empty() ? functions_.end() + : functions_.find(call_stack_.back().fn_name); + if (declared == functions_.end() || + !webview::signature_matches(*contract, declared->second.parameter_types, + declared->second.return_type) || + instr.args.size() != 4 || webview::abi_type(instr.args[3]) != contract->return_type() || + args.size() != contract->parameters.size()) + throw std::runtime_error(fn_name + " requires the exact WebView ABI " + webview::signature(*contract)); + if (contract->output != webview::Output::capability) { + std::size_t schemas = 0; + if (class_metadata_) + for (const auto &definition : class_metadata_->enums) + if (webview::result_schema_matches(*contract, definition)) ++schemas; + if (schemas != 1) + throw std::runtime_error(fn_name + " requires the standard Result schema with exact Ok and Error payloads"); + } + for (std::size_t index = 0; index < args.size(); ++index) { + const bool valid = contract->parameters[index] == "text" + ? std::holds_alternative(args[index]) + : std::holds_alternative(args[index]); + if (!valid) throw std::runtime_error(fn_name + " argument " + + std::to_string(index + 1) + " requires " + contract->parameters[index]); + } + auto text_arg = [&](std::size_t index) -> const std::string & { + return std::get(args[index]); + }; + auto word_arg = [&](std::size_t index) { return std::get(args[index]); }; + webview::Result result; + const auto &op = contract->operation; + if (op == "configure") result = webview_.configure(text_arg(0)); + else if (op == "window") result = webview_.window(text_arg(0), word_arg(1), word_arg(2)); + else if (op == "show") result = webview_.show(word_arg(0)); + else if (op == "navigate") result = webview_.navigate(word_arg(0), text_arg(1)); + else if (op == "load_html") result = webview_.load_html(word_arg(0), text_arg(1)); + else if (op == "evaluate_function") result = webview_.evaluate_function(word_arg(0), text_arg(1)); + else if (op == "poll") result = webview_.poll(word_arg(0)); + else if (op == "event_number") result = webview_.event_number(word_arg(0), text_arg(1)); + else if (op == "event_text") result = webview_.event_text(word_arg(0), text_arg(1)); + else if (op == "release_event") result = webview_.release_event(word_arg(0)); + else if (op == "close") result = webview_.close(word_arg(0)); + EnumValue value; + value.variant_name = result ? "Result::Ok" : "Result::Error"; + if (!result) value.payloads.push_back(std::move(result.error)); + else if (contract->output == webview::Output::text) value.payloads.push_back(std::move(result.text)); + else value.payloads.push_back(result.number); + pushStack(std::move(value)); + return; + } if (lib_name == "intrinsic" && fn_name.rfind("ui.v1.", 0) == 0) { const auto contract = ui::intrinsic(fn_name); if (!contract) throw std::runtime_error("ui: unknown intrinsic '" + fn_name + "'"); diff --git a/nilcompiler/src/vm.hpp b/nilcompiler/src/vm.hpp index 2f7db71c..e0419811 100644 --- a/nilcompiler/src/vm.hpp +++ b/nilcompiler/src/vm.hpp @@ -17,6 +17,7 @@ #include "source_debug.hpp" #include "database_runtime.hpp" #include "ui_runtime.hpp" +#include "webview_runtime.hpp" #include "memory_profile.hpp" namespace ny { @@ -561,6 +562,7 @@ class VM { // Heap management for manual memory (unsafe mode) and GC (safe mode) database::Context databases_; ui::Context ui_; + webview::Context webview_; struct HeapBlock { std::string type_name; // Logical element metadata for type_name == "array". Textual and binary diff --git a/nilcompiler/src/webview_intrinsic.hpp b/nilcompiler/src/webview_intrinsic.hpp new file mode 100644 index 00000000..744663d5 --- /dev/null +++ b/nilcompiler/src/webview_intrinsic.hpp @@ -0,0 +1,28 @@ +/* Copyright (C) 2026 Execute Software. SPDX-License-Identifier: MIT */ +#pragma once +#include "ui_intrinsic.hpp" + +namespace ny::webview { +using Intrinsic = ui::Intrinsic; +using Output = ui::Output; +using ui::abi_type; +using ui::signature; +using ui::signature_matches; +using ui::result_schema_matches; + +inline std::optional intrinsic(const std::string &name) { + if (name.rfind("webview.v1.", 0) != 0) return std::nullopt; + Intrinsic value; + value.operation = name.substr(11); + const auto &op = value.operation; + if (op == "configure") value.parameters = {"text"}; + else if (op == "window") value.parameters = {"text", "i64", "i64"}; + else if (op == "navigate" || op == "load_html" || op == "evaluate_function" || op == "event_number") + value.parameters = {"i64", "text"}; + else if (op == "event_text") { value.parameters = {"i64", "text"}; value.output = Output::text; } + else if (op == "show" || op == "poll" || op == "release_event" || op == "close") + value.parameters = {"i64"}; + else return std::nullopt; + return value; +} +} // namespace ny::webview diff --git a/nilcompiler/src/webview_runtime.cpp b/nilcompiler/src/webview_runtime.cpp new file mode 100644 index 00000000..5820cbe0 --- /dev/null +++ b/nilcompiler/src/webview_runtime.cpp @@ -0,0 +1,299 @@ +/* Copyright (C) 2026 Execute Software. SPDX-License-Identifier: MIT */ +#include "webview_runtime.hpp" +#include "../../common/webview_provider_v1.h" +#include "../../common/webview_json_v1.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +namespace ny::webview { +namespace { +Result failure(const std::string &message) { return {0, {}, "webview: " + message}; } +Result success(std::int64_t value = 1) { return {value, {}, {}}; } +bool valid_text(const std::string &value) { + if (value.size() > NILWV_MAX_TEXT_BYTES || value.find('\0') != std::string::npos) return false; + return nilwebview::json_v1::valid_utf8(value); +} +nilwv_text_v1 view(const std::string &value) { return {value.data(), value.size()}; } +std::atomic next_handle{1}; +std::int64_t fresh_handle() { + auto value = next_handle.load(std::memory_order_relaxed); + for (;;) { + if (value <= 0 || value == std::numeric_limits::max()) return 0; + if (next_handle.compare_exchange_weak(value, value + 1, std::memory_order_relaxed)) return value; + } +} +struct Library { + const nilwv_api_v1 *api; + void *module; // Intentionally never unloaded, even after a rejected handshake. +}; +// Process-lifetime registry: asynchronous engine callbacks can outlive a +// window/context. Do not let static destructor ordering unload their code. +struct Registry { std::mutex mutex; std::map libraries; }; +Registry ®istry() { static auto *value = new Registry; return *value; } +const nilwv_api_v1 *load_provider(const std::string &path) { + auto &cache = registry(); + std::lock_guard lock(cache.mutex); + auto found = cache.libraries.find(path); + if (found != cache.libraries.end()) return found->second.api; + if (cache.libraries.size() >= 16) return nullptr; + void *module = nullptr; + nilwv_get_api_v1_fn get_api = nullptr; +#ifdef _WIN32 + auto wide = std::filesystem::u8path(path).wstring(); + auto handle = LoadLibraryExW(wide.c_str(), nullptr, + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + module = handle; + if (handle) get_api = reinterpret_cast(GetProcAddress(handle, "nilwebview_provider_v1_get_api")); +#else + module = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (module) get_api = reinterpret_cast(dlsym(module, "nilwebview_provider_v1_get_api")); +#endif + if (!module) return nullptr; + // Insert before calling foreign code so allocation/validation failure cannot + // make a loaded library eligible for an accidental later unload. + auto &entry = cache.libraries.emplace(path, Library{nullptr, module}).first->second; + const nilwv_api_v1 *api = nullptr; + try { if (get_api) api = get_api(NILWV_ABI_V1, sizeof(nilwv_api_v1)); } catch (...) { return nullptr; } + if (!api || api->abi_version != NILWV_ABI_V1 || api->size < sizeof(nilwv_api_v1) || + (api->capabilities & ~NILWV_CAP_KNOWN_MASK) || + !api->probe || !api->context_create || !api->context_destroy || + !api->window || !api->show || !api->navigate || !api->load_html || + !api->evaluate_function || !api->poll || !api->event_number || + !api->event_text || !api->release_event || !api->close) return nullptr; + entry.api = api; + return api; +} +constexpr std::array number_fields{ + "window", "kind", "sequence", "operation", "value_kind", "source_kind", "frame_kind", "failure_kind"}; +constexpr std::array text_fields{"payload_json", "source_url", "source_origin", "detail"}; +template std::size_t field_index(const Array &fields, const std::string &key) { + return static_cast(std::find(fields.begin(), fields.end(), key) - fields.begin()); +} +} // namespace + +struct Context::Impl { + const std::thread::id thread = std::this_thread::get_id(); + const nilwv_api_v1 *api = nullptr; + nilwv_context_v1 *context = nullptr; + std::string path, fault; + std::map windows; + struct Event { std::array numbers{}; std::array texts; std::size_t bytes = 0; }; + std::map events; + std::size_t event_bytes = 0; + + Result check(bool configured = true) const { + if (thread != std::this_thread::get_id()) return failure("context belongs to another thread"); + if (!fault.empty()) return failure(fault); + if (configured && (!api || !context)) return failure("configure an explicit provider first"); + return success(); + } + Result protocol() { fault = "provider contract violation; close the execution context"; return failure(fault); } + template Result call(Invoke invoke) { + nilwv_result_v1 result{}; + result.size = sizeof(result); + try { invoke(&result); } catch (...) { return protocol(); } + if (result.size != sizeof(result) || result.code > 0 || + result.text.size > NILWV_MAX_TEXT_BYTES || (!result.text.data && result.text.size)) return protocol(); + std::string text; + if (result.text.size) text.assign(result.text.data, static_cast(result.text.size)); + if (!valid_text(text)) return protocol(); + if (result.code < 0) return failure(text.empty() ? "provider reported an unspecified failure" : text); + return {result.number, std::move(text), {}}; + } + std::int64_t native_window(std::int64_t window) const { + auto item = windows.find(window); return item == windows.end() ? 0 : item->second; + } + template Result window_call(std::int64_t window, Invoke invoke, bool operation = false) { + if (auto state = check(); !state) return state; + const auto native = native_window(window); + if (!native) return failure("invalid or closed window handle"); + auto result = call([&](auto *out) { invoke(native, out); }); + if (result && (operation ? result.number <= 0 : result.number != 1)) return protocol(); + return result; + } +}; + +Context::Context() : impl_(std::make_unique()) {} +Context::~Context() { clear(); } +void Context::clear() noexcept { + if (!impl_) return; + // Context ownership cannot be transferred. A foreign-thread teardown must + // not enter a toolkit on the wrong thread; retained provider state remains + // pinned, rather than risking callbacks into freed objects. + if (impl_->thread != std::this_thread::get_id()) return; + try { if (impl_->context && impl_->api) impl_->api->context_destroy(impl_->context); } catch (...) {} + impl_->context = nullptr; impl_->api = nullptr; + impl_->windows.clear(); impl_->events.clear(); impl_->event_bytes = 0; + impl_->fault.clear(); impl_->path.clear(); +} +Result Context::configure(const std::string &path) { + if (auto state = impl_->check(false); !state) return state; + if (!valid_text(path) || path.empty() || !std::filesystem::u8path(path).is_absolute()) + return failure("provider path must be absolute UTF-8 without NUL"); + if (impl_->context) + return path == impl_->path ? success(static_cast(impl_->api->capabilities)) + : failure("this execution already selected a provider"); + const auto *api = load_provider(path); + if (!api) return failure("provider could not be loaded or has an incompatible ABI"); + auto probe = impl_->call([&](auto *out) { api->probe(out); }); + if (!probe) return probe; + if (probe.number != 1) return impl_->protocol(); + nilwv_context_v1 *context = nullptr; + auto created = impl_->call([&](auto *out) { api->context_create(&context, out); }); + if (!created || !context || created.number != 1) { + try { if (context) api->context_destroy(context); } catch (...) {} + return !created ? created : impl_->protocol(); + } + impl_->api = api; impl_->context = context; impl_->path = path; + return success(static_cast(api->capabilities)); +} +Result Context::window(const std::string &title, std::int64_t width, std::int64_t height) { + if (auto state = impl_->check(); !state) return state; + if (!valid_text(title) || width < 1 || width > 16384 || height < 1 || height > 16384) + return failure("window requires valid text and dimensions from 1 to 16384"); + if (impl_->windows.size() >= NILWV_MAX_WINDOWS) return failure("window limit reached"); + auto result = impl_->call([&](auto *out) { impl_->api->window(impl_->context, view(title), width, height, out); }); + if (!result) return result; + if (result.number <= 0 || std::any_of(impl_->windows.begin(), impl_->windows.end(), + [&](const auto &entry) { return entry.second == result.number; })) return impl_->protocol(); + const auto handle = fresh_handle(); + if (!handle) { impl_->call([&](auto *out) { impl_->api->close(impl_->context, result.number, out); }); return failure("handle space exhausted"); } + try { impl_->windows.emplace(handle, result.number); } + catch (...) { impl_->call([&](auto *out) { impl_->api->close(impl_->context, result.number, out); }); throw; } + return success(handle); +} +Result Context::show(std::int64_t window) { + return impl_->window_call(window, [&](auto native, auto *out) { impl_->api->show(impl_->context, native, out); }); +} +#define NILWV_TEXT_OPERATION(name) \ +Result Context::name(std::int64_t window, const std::string &text) { \ + if (!valid_text(text)) return failure("text must be UTF-8 without NUL, at most 1048576 bytes"); \ + return impl_->window_call(window, [&](auto native, auto *out) { impl_->api->name(impl_->context, native, view(text), out); }, true); \ +} +NILWV_TEXT_OPERATION(navigate) +NILWV_TEXT_OPERATION(load_html) +NILWV_TEXT_OPERATION(evaluate_function) +#undef NILWV_TEXT_OPERATION + +Result Context::poll(std::int64_t window) { + if (auto state = impl_->check(); !state) return state; + const auto native = impl_->native_window(window); + if (!native) return failure("invalid or closed window handle"); + if (impl_->events.size() >= NILWV_MAX_EVENTS) return failure("release retained event snapshots before polling"); + auto polled = impl_->call([&](auto *out) { impl_->api->poll(impl_->context, native, out); }); + if (!polled || polled.number == 0) return polled; + if (polled.number < 0) return impl_->protocol(); + struct Release { + Impl *state; std::int64_t handle; bool active = true; + Result run() { active = false; return state->call([&](auto *out) { state->api->release_event(state->context, handle, out); }); } + ~Release() { if (active) { try { run(); } catch (...) {} } } + } release{impl_.get(), polled.number}; + Impl::Event event; + for (std::size_t i = 0; i < number_fields.size(); ++i) { + const std::string key = number_fields[i]; + auto field = impl_->call([&](auto *out) { impl_->api->event_number(impl_->context, polled.number, view(key), out); }); + if (!field) return field; + event.numbers[i] = field.number; + } + for (std::size_t i = 0; i < text_fields.size(); ++i) { + const std::string key = text_fields[i]; + auto field = impl_->call([&](auto *out) { impl_->api->event_text(impl_->context, polled.number, view(key), out); }); + if (!field) return field; + event.bytes += field.text.size(); event.texts[i] = std::move(field.text); + } + const auto &n = event.numbers; + if (n[0] != native || n[1] < NILWV_EVENT_READY || n[1] > NILWV_EVENT_PROVIDER_FAILED || + n[2] <= 0 || n[3] < 0 || n[4] < 0 || n[4] > NILWV_VALUE_UNDEFINED || + n[5] < 0 || n[5] > NILWV_SOURCE_NATIVE_ORIGIN || n[6] < 0 || n[6] > NILWV_FRAME_CHILD || + n[7] < 0 || n[7] > NILWV_FAILURE_CANCELLED) return impl_->protocol(); + const auto caps = impl_->api->capabilities; + if ((n[5] == NILWV_SOURCE_UNKNOWN && (!event.texts[1].empty() || !event.texts[2].empty())) || + (n[5] == NILWV_SOURCE_NATIVE_URL && (event.texts[1].empty() || !event.texts[2].empty() || !(caps & NILWV_CAP_MESSAGE_URL))) || + (n[5] == NILWV_SOURCE_NATIVE_ORIGIN && (event.texts[2].empty() || !(caps & NILWV_CAP_MESSAGE_ORIGIN))) || + (!event.texts[1].empty() && !(caps & NILWV_CAP_MESSAGE_URL)) || + (!event.texts[2].empty() && !(caps & NILWV_CAP_MESSAGE_ORIGIN)) || + (n[6] != NILWV_FRAME_UNKNOWN && !(caps & NILWV_CAP_MESSAGE_FRAME))) return impl_->protocol(); + if (n[4] == NILWV_VALUE_JSON) { + if (!nilwebview::json_v1::valid_json(event.texts[0])) return impl_->protocol(); + } else if (!event.texts[0].empty()) return impl_->protocol(); + const auto kind = n[1]; + const bool completion = kind >= NILWV_EVENT_NAVIGATION_COMPLETED && kind <= NILWV_EVENT_SCRIPT_FAILED; + const bool failed = kind == NILWV_EVENT_INITIALIZATION_FAILED || kind == NILWV_EVENT_NAVIGATION_FAILED || + kind == NILWV_EVENT_SCRIPT_FAILED || kind == NILWV_EVENT_PROVIDER_FAILED; + if ((completion ? n[3] <= 0 : n[3] != 0) || (failed ? n[7] == 0 : n[7] != 0) || + (kind != NILWV_EVENT_PAGE_MESSAGE && (n[5] != NILWV_SOURCE_UNKNOWN || n[6] != NILWV_FRAME_UNKNOWN))) + return impl_->protocol(); + if (kind == NILWV_EVENT_PAGE_MESSAGE || kind == NILWV_EVENT_NAVIGATION_COMPLETED || kind == NILWV_EVENT_NAVIGATION_REQUESTED) { + if (n[4] != NILWV_VALUE_JSON) return impl_->protocol(); + } else if (kind == NILWV_EVENT_SCRIPT_COMPLETED) { + if (n[4] != NILWV_VALUE_JSON && n[4] != NILWV_VALUE_UNDEFINED) return impl_->protocol(); + } else if (kind == NILWV_EVENT_NAVIGATION_FAILED) { + if (n[4] == NILWV_VALUE_UNDEFINED) return impl_->protocol(); + } else if (n[4] != NILWV_VALUE_NONE) return impl_->protocol(); + if ((kind == NILWV_EVENT_NAVIGATION_COMPLETED || kind == NILWV_EVENT_NAVIGATION_REQUESTED || + kind == NILWV_EVENT_NAVIGATION_FAILED) && n[4] == NILWV_VALUE_JSON) { + const auto first = event.texts[0].find_first_not_of(" \t\n\r"); + if (first == std::string::npos || event.texts[0][first] != '"') return impl_->protocol(); + } + auto released = release.run(); + if (!released || released.number != 1) return impl_->protocol(); + if (event.bytes > NILWV_MAX_QUEUED_BYTES - impl_->event_bytes) { + impl_->fault = "retained event byte limit exceeded; close the execution context"; + return failure(impl_->fault); + } + event.numbers[0] = window; + const auto handle = fresh_handle(); + if (!handle) return failure("handle space exhausted"); + const auto bytes = event.bytes; + impl_->events.emplace(handle, std::move(event)); impl_->event_bytes += bytes; + return success(handle); +} +Result Context::event_number(std::int64_t snapshot, const std::string &field) { + if (auto state = impl_->check(); !state) return state; + const auto event = impl_->events.find(snapshot); + const auto index = field_index(number_fields, field); + if (event == impl_->events.end()) return failure("invalid or released event handle"); + if (index == number_fields.size()) return failure("unknown numeric event field"); + return success(event->second.numbers[index]); +} +Result Context::event_text(std::int64_t snapshot, const std::string &field) { + if (auto state = impl_->check(); !state) return state; + const auto event = impl_->events.find(snapshot); + const auto index = field_index(text_fields, field); + if (event == impl_->events.end()) return failure("invalid or released event handle"); + if (index == text_fields.size()) return failure("unknown text event field"); + return {0, event->second.texts[index], {}}; +} +Result Context::release_event(std::int64_t snapshot) { + if (auto state = impl_->check(); !state) return state; + const auto event = impl_->events.find(snapshot); + if (event == impl_->events.end()) return failure("invalid or released event handle"); + impl_->event_bytes -= event->second.bytes; impl_->events.erase(event); + return success(); +} +Result Context::close(std::int64_t window) { + auto result = impl_->window_call(window, [&](auto native, auto *out) { impl_->api->close(impl_->context, native, out); }); + if (!result) return result; + impl_->windows.erase(window); + for (auto it = impl_->events.begin(); it != impl_->events.end();) { + if (it->second.numbers[0] == window) { impl_->event_bytes -= it->second.bytes; it = impl_->events.erase(it); } + else ++it; + } + return success(); +} +} // namespace ny::webview diff --git a/nilcompiler/src/webview_runtime.hpp b/nilcompiler/src/webview_runtime.hpp new file mode 100644 index 00000000..67d661a5 --- /dev/null +++ b/nilcompiler/src/webview_runtime.hpp @@ -0,0 +1,40 @@ +/* Copyright (C) 2026 Execute Software. SPDX-License-Identifier: MIT */ +#pragma once +#include +#include +#include + +namespace ny::webview { +struct Result { + std::int64_t number = 0; + std::string text, error; + explicit operator bool() const noexcept { return error.empty(); } +}; + +// Each execution owns a context on the UI thread. The explicit absolute path +// selects trusted native code; the loader is not a sandbox for untrusted DLLs. +// Loaded libraries stay pinned for process lifetime, including late browser +// callbacks. Context handles and copied events never expose native addresses. +class Context { +public: + Context(); + ~Context(); + Context(const Context &) = delete; + Context &operator=(const Context &) = delete; + void clear() noexcept; + Result configure(const std::string &absolute_provider_path); + Result window(const std::string &title, std::int64_t width, std::int64_t height); + Result show(std::int64_t window); + Result navigate(std::int64_t window, const std::string &url); + Result load_html(std::int64_t window, const std::string &html); + Result evaluate_function(std::int64_t window, const std::string &body); + Result poll(std::int64_t window); + Result event_number(std::int64_t snapshot, const std::string &field); + Result event_text(std::int64_t snapshot, const std::string &field); + Result release_event(std::int64_t snapshot); + Result close(std::int64_t window); +private: + struct Impl; + std::unique_ptr impl_; +}; +} // namespace ny::webview diff --git a/notebook/2026-11-12-matrix-webview.md b/notebook/2026-11-12-matrix-webview.md new file mode 100644 index 00000000..f8164301 --- /dev/null +++ b/notebook/2026-11-12-matrix-webview.md @@ -0,0 +1,80 @@ +# Matrix WebView provider boundary — 2026-11-12 + +## Status and question + +The archived Ny WebView project has been left untouched. Its modern replacement +is being built as `exec-nil-framework-webview`, with Matrix-facing library code +and separate WebView2, GTK4/WebKitGTK 6 and AppKit/WKWebView providers. Actual +browser qualification remains separate from the framework bridge tests below. + +The question is how a Matrix application can observe a browser's asynchronous +work consistently through interpreted NIL and LLVM AOT/JIT, without reviving the +old unchecked generic interop layer. + +## Decision + +The versioned `webview.v1.*` intrinsic family uses the standard nominal +`Result` and `Result` schemas. The frontend, loaded-NIL +interpreter and native backend each validate the declarations. A provider is +selected by an explicit absolute path to a trusted native library. The C ABI +contains fixed-width fields and borrowed text views; no toolkit object, callback +into Matrix, allocator or C++ container crosses it. Loaded modules remain pinned +for process lifetime because engine callbacks may outlive a closed window. + +Each execution owns its context on the UI thread. Runtime window and event +handles are process-unique and checked against that context. A poll copies all +event fields before releasing the provider's snapshot, including on malformed +output. Closing a window invalidates its retained runtime snapshots. Independent +contexts remain unaffected. Neither browser state nor these external handles is +part of the interpreter's pure same-process checkpoint profile. + +Creation yields a pending window and an eventual Ready or InitializationFailed +event. Navigation and synchronous function evaluation yield operation identities +with explicit completion or failure. Only one such operation is pending per +window; deadlines make a timed-out window unusable until closed and recreated. +Cancelling a request does not claim that executing JavaScript was forcibly stopped. + +The initial common navigation profile authorizes one URL request. Redirects and +page-initiated navigation are reported for explicit application approval instead +of being followed automatically. This avoids making different promises on macOS +15, whose public navigation-action API does not identify HTTP redirects the way +the other engines do. Popups, downloads and native host commands are outside this +profile. The migration document tracks legacy features separately rather than +advertising unimplemented cross-platform aliases. + +## Message and JSON identity + +A message's sender metadata must come from the native engine or remain Unknown. +WebKitGTK 6's public message callback does not expose authenticated sender frame +metadata. A JavaScript field or the browser's current URL cannot fill that gap. +Capability bits distinguish browser-reported URLs, security origins and frames. +An event cannot claim metadata that its provider did not advertise. + +Script results retain JSON.stringify's serialized text, with distinct Undefined, +script exception, serialization failure and unsupported Promise/thenable cases. +An independent review caught a real boundary mismatch: the framework's strict +configuration JSON parser rejected escaped lone surrogates and nesting levels +accepted by the provider. Both sides now use byte-identical WebView JSON grammar +headers: bounded UTF-8 JSON text, depth 64, no decoding of escaped UTF-16 units. +This preserves a legitimate `JSON.stringify` result without turning it into an +invalid UTF-8 scalar string. Depth 65 is rejected explicitly. + +For future AI tooling, operation identity and copied events provide an observable +boundary for browser actions. They do not grant page messages authority to invoke +host operations, and they do not automatically log page content or credentials. + +## Evidence and next qualification + +- 57 framework host checks cover provider ABI rejection, context/thread and + handle ownership, event release, provenance combinations and JSON boundaries. +- 28 intrinsic contract checks cover malformed declarations, disabled source + checking, typed failures and execution routes; the existing 40-check UI + intrinsic contract also passes after sharing native lowering code. +- The Matrix consumer contract passed text NIL, binary NIL, LLVM AOT and JIT + using a synthetic provider, including records retained after event/window + teardown. This is interface proof, not browser proof. +- Shared provider logic passed 196 native controls with sanitizers and 36 checks + of the generated JavaScript transport. Real browser builds and desktop tests + on Windows, Linux and both Mac architectures remain the next required gate. + +No accepted SDK, installer or release deployment is changed by this work. diff --git a/notebook/2026-11-12-web-server-boundaries.md b/notebook/2026-11-12-web-server-boundaries.md index 18280faa..51475ad7 100644 --- a/notebook/2026-11-12-web-server-boundaries.md +++ b/notebook/2026-11-12-web-server-boundaries.md @@ -74,8 +74,13 @@ own repository. No accepted SDK or release deployment is replaced by this work. ## Follow-up -Complete native snapshot qualification on both macOS architectures, then record the corrected web campaign's four-host results. Keep the platform -checks required for public changes alongside the secret/path audit and review. +Native snapshot qualification passed on both Mac architectures in +[run 43162404951](https://github.com/exec-software/exec-nil-framework/actions/runs/43162404951), +with the exact 87 host and 8/4/2/6 language-route inventories against fixed +source `93515c3`. The corrected [web run 43162279298](https://github.com/exec-software/exec-nil-framework-web/actions/runs/43162279298) +passed all four hosts at web head `4bcb43f`: 1,128 unit, 16 oracle, 38 wire +and 15 static checks each. The four `web-*` checks now join `secrets` as +required checks; review and the other existing branch protections are unchanged. The next project is a clean Matrix WebView package. The archived implementation has no macOS provider and advertises some Linux-only functions through a shared diff --git a/notebook/README.md b/notebook/README.md index 8c1b0d7a..8ea4f4eb 100644 --- a/notebook/README.md +++ b/notebook/README.md @@ -11,6 +11,8 @@ and explicit about evidence, uncertainty, and implementation status. ## Contents +- [`2026-11-12-matrix-webview.md`](2026-11-12-matrix-webview.md) — checked + browser providers, asynchronous operation identity and honest message provenance. - [`2026-11-12-web-server-boundaries.md`](2026-11-12-web-server-boundaries.md) — application-driven file snapshots, protocol boundaries, binary values and the checked provider boundary needed by the planned WebView rewrite. diff --git a/tests/tools/webview_intrinsic_contract.py b/tests/tools/webview_intrinsic_contract.py new file mode 100644 index 00000000..6134193b --- /dev/null +++ b/tests/tools/webview_intrinsic_contract.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Reject forged WebView declarations before loading any native provider.""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--compiler', required=True, type=Path) + parser.add_argument('--interpreter', required=True, type=Path) + parser.add_argument('--llvm', required=True, choices=('ON', 'OFF')) + args = parser.parse_args() + compiler, interpreter = args.compiler.resolve(), args.interpreter.resolve() + environment = {k: v for k, v in os.environ.items() if not k.startswith(('NIL_', 'NY_'))} + checks = 0 + with tempfile.TemporaryDirectory(prefix='nil-webview-abi-') as directory: + work = Path(directory) + base = [compiler, '--quiet', '--sdk-path', ROOT/'nilstd/src'] + + def run(command, status=0): + nonlocal checks + result = subprocess.run(list(map(str, command)), cwd=work, env=environment, + stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=60) + assert result.returncode == status, (result.returncode, result.stdout, result.stderr) + checks += 1 + return result + + imported = 'import nil.std.core.result\n' + entry = 'fn main() -> i32\n return 0\nend fn\n' + invalid = { + 'unknown': ('@intrinsic("webview.v1.unknown")\nfn provider() -> bool\n', 'Unknown WebView intrinsic'), + 'arity': (imported+'@intrinsic("webview.v1.configure")\nfn provider() -> Result\n', 'exact WebView ABI'), + 'width': (imported+'@intrinsic("webview.v1.show")\nfn provider(handle:i32) -> Result\n', 'exact WebView ABI'), + 'number-result': (imported+'@intrinsic("webview.v1.poll")\nfn provider(handle:i64) -> Result\n', 'exact WebView ABI'), + 'text-result': (imported+'@intrinsic("webview.v1.event_text")\nfn provider(handle:i64,field:text) -> Result\n', 'exact WebView ABI'), + 'error-payload': (imported+'@intrinsic("webview.v1.show")\nfn provider(handle:i64) -> Result\n', 'exact WebView ABI'), + 'reference': (imported+'@intrinsic("webview.v1.show")\nfn provider(ref handle:i64) -> Result\n', 'exact WebView ABI'), + 'default': (imported+'@intrinsic("webview.v1.show")\nfn provider(handle:i64=0) -> Result\n', 'exact WebView ABI'), + 'variadic': (imported+'@intrinsic("webview.v1.show")\nfn provider(..handles:i64) -> Result\n', 'exact WebView ABI'), + 'generic': (imported+'@intrinsic("webview.v1.show")\nfn provider(handle:i64) -> Result\n', 'exact WebView ABI'), + 'impostor': ('''module impostor +enum Result + Ok(T) + Error(E) +end enum +@intrinsic("webview.v1.show") +fn provider(handle:i64) -> Result +''', 'exact WebView ABI'), + } + for name, (declaration, diagnostic) in invalid.items(): + source, output = work/(name+'.ny'), work/(name+'.nil') + source.write_text(declaration+entry, newline='\n') + output.write_bytes(b'previous artifact') + result = run([*base, '--no-bundle', '-o', output, source], 1) + assert diagnostic in result.stderr + assert output.read_bytes() == b'previous artifact' + + for name, declaration, expression in ( + ('arity', imported+'@intrinsic("webview.v1.configure")\nfn provider() -> Result\n', 'provider()'), + ('return', '@intrinsic("webview.v1.poll")\nfn provider(handle:i64) -> i64\n', 'provider(0)'), + ): + source, output = work/('unchecked-'+name+'.ny'), work/('unchecked-'+name+'.nil') + source.write_text(declaration+'fn main() -> i32\n let value = '+expression+'\n return 0\nend fn\n', newline='\n') + run([*base, '--no-typecheck', '--no-bundle', '-o', output, source]) + result = run([interpreter, '--no-ai', output], 1) + assert 'WebView ABI' in result.stderr + if args.llvm == 'ON': + for flags in ([], ['--jit']): + output.write_bytes(b'previous artifact') + result = run([*base, '--no-typecheck', '--native', *flags, '-o', output, source], 1) + assert 'WebView ABI' in result.stderr + assert output.read_bytes() == b'previous artifact' + + source = work/'checked.ny' + source.write_text(imported+'''@intrinsic("webview.v1.configure") +fn configure(path:text) -> Result +@intrinsic("webview.v1.event_text") +fn event_text(snapshot:i64,field:text) -> Result +fn main() -> i32 + match configure("relative-provider") + Result.Ok(value) -> return 8 + Result.Error(message) -> println("checked WebView configuration failure") + end match + match event_text(0,"payload_json") + Result.Ok(value) -> return 9 + Result.Error(message) -> println("checked WebView event failure") + end match + return 0 +end fn +''', newline='\n') + for route in ['text', 'binary', 'bundle'] + (['native', 'jit'] if args.llvm == 'ON' else []): + output = work/('checked-'+route) + flags = (['--native', '--jit'] if route == 'jit' else ['--native'] if route == 'native' + else [] if route == 'bundle' else ['--no-bundle']) + if route == 'binary': flags += ['--emit-binary-nil'] + result = run([*base, *flags, '-o', output, source]) + if route != 'jit': + executable = output.with_suffix('.exe') if os.name == 'nt' and route in ('native', 'bundle') else output + command = [interpreter, '--no-ai', output] if route in ('text', 'binary') else [executable] + result = run(command) + assert result.stdout == 'checked WebView configuration failure\nchecked WebView event failure\n' + assert not result.stderr + print(f'WebView intrinsic ABI: {checks} checks passed') + + +if __name__ == '__main__': + main() diff --git a/tests/webview_provider_fixture.cpp b/tests/webview_provider_fixture.cpp new file mode 100644 index 00000000..7aadf460 --- /dev/null +++ b/tests/webview_provider_fixture.cpp @@ -0,0 +1,89 @@ +/* Synthetic ABI peer, never a browser provider. SPDX-License-Identifier: MIT */ +#define NILWEBVIEW_PROVIDER_BUILD +#include "../common/webview_provider_v1.h" +#include +#include +#include + +struct nilwv_context_v1 { + bool open = false, event = false; + std::int64_t kind = NILWV_EVENT_READY, operation = 0; + std::string payload, detail, control; +}; +namespace { +std::atomic contexts{0}, releases{0}; +void ok(nilwv_result_v1 *r, std::int64_t value = 1) { *r = {sizeof(*r), 0, value, {nullptr, 0}}; } +void bad(nilwv_result_v1 *r) { static const char text[] = "synthetic invalid handle"; *r = {sizeof(*r), NILWV_E_HANDLE, 0, {text, sizeof(text)-1}}; } +void probe(nilwv_result_v1 *r) { ok(r); } +void create(nilwv_context_v1 **c, nilwv_result_v1 *r) { *c = new nilwv_context_v1; ++contexts; ok(r); } +void destroy(nilwv_context_v1 *c) { delete c; --contexts; } +void window(nilwv_context_v1 *c, nilwv_text_v1, std::int64_t, std::int64_t, nilwv_result_v1 *r) { + if (c->open) { bad(r); return; } + c->open = c->event = true; c->kind = NILWV_EVENT_READY; ok(r, 17); +} +void show(nilwv_context_v1 *c, std::int64_t w, nilwv_result_v1 *r) { if (c->open && w == 17) ok(r); else bad(r); } +void navigate(nilwv_context_v1 *c, std::int64_t w, nilwv_text_v1, nilwv_result_v1 *r) { + if (!c->open || w != 17) { bad(r); return; } + c->event = true; c->kind = NILWV_EVENT_NAVIGATION_COMPLETED; c->operation = 31; c->payload = "\"about:blank\""; ok(r, 31); +} +void script(nilwv_context_v1 *c, std::int64_t w, nilwv_text_v1 body, nilwv_result_v1 *r) { + if (!c->open || w != 17) { bad(r); return; } + c->event = true; c->kind = NILWV_EVENT_SCRIPT_COMPLETED; c->operation = 32; + const std::string value(body.data, body.size); + c->control = value; + c->payload = value == "bad-json" ? "{invalid" : "{\"answer\":42}"; + if (value == "surrogate") c->payload = "\"\\ud800\""; + if (value == "deep33" || value == "deep64" || value == "deep65") { + const auto depth = value == "deep33" ? 33 : value == "deep64" ? 64 : 65; + c->payload = std::string(depth, '[') + "0" + std::string(depth, ']'); + } + if (value == "bad-message") { c->kind = NILWV_EVENT_PAGE_MESSAGE; c->operation = 0; c->payload.clear(); } + c->detail = value == "bad-text" ? std::string("\xc0\xaf", 2) : ""; + ok(r, 32); +} +void poll(nilwv_context_v1 *c, std::int64_t w, nilwv_result_v1 *r) { + if (!c->open || w != 17) { bad(r); return; } ok(r, c->event ? 19 : 0); +} +void number(nilwv_context_v1 *c, std::int64_t e, nilwv_text_v1 property, nilwv_result_v1 *r) { + if (!c->event || e != 19) { bad(r); return; } + const std::string field(property.data, property.size); + if (field == "window") ok(r,17); + else if (field == "kind") ok(r,c->kind); + else if (field == "sequence") ok(r,1); + else if (field == "operation") ok(r,c->operation); + else if (field == "value_kind") ok(r,c->payload.empty() ? NILWV_VALUE_NONE : NILWV_VALUE_JSON); + else if (field == "source_kind") ok(r,c->control == "bad-origin" ? NILWV_SOURCE_NATIVE_ORIGIN : c->control == "bad-url" ? NILWV_SOURCE_NATIVE_URL : 0); + else if (field == "frame_kind") ok(r,c->control == "bad-frame" ? NILWV_FRAME_MAIN : 0); + else if (field == "failure_kind") ok(r,0); + else bad(r); +} +void text(nilwv_context_v1 *c, std::int64_t e, nilwv_text_v1 property, nilwv_result_v1 *r) { + if (!c->event || e != 19) { bad(r); return; } + const std::string field(property.data, property.size); + ok(r,0); + if (field == "payload_json") r->text = {c->payload.data(), c->payload.size()}; + else if (field == "detail") r->text = {c->detail.data(), c->detail.size()}; + else if (field != "source_url" && field != "source_origin") bad(r); +} +void release(nilwv_context_v1 *c, std::int64_t e, nilwv_result_v1 *r) { + if (!c->event || e != 19) { bad(r); return; } + c->event = false; c->payload.assign("released storage"); c->detail.assign("released storage"); ++releases; ok(r); +} +void close(nilwv_context_v1 *c, std::int64_t w, nilwv_result_v1 *r) { + if (!c->open || w != 17) { bad(r); return; } c->open = c->event = false; ok(r); +} +const nilwv_api_v1 api{ +#ifdef NILWV_FIXTURE_BAD_ABI + 2, +#else + NILWV_ABI_V1, +#endif + sizeof(nilwv_api_v1), 0, "synthetic", "test-only", probe, create, destroy, + window, show, navigate, navigate, script, poll, number, text, release, close}; +} +extern "C" NILWV_EXPORT const nilwv_api_v1 *NILWV_CALL +nilwebview_provider_v1_get_api(std::uint32_t version, std::uint32_t size) { + return version == NILWV_ABI_V1 && size <= sizeof(api) ? &api : nullptr; +} +extern "C" NILWV_EXPORT std::int64_t nilwebview_fixture_live_contexts() { return contexts; } +extern "C" NILWV_EXPORT std::int64_t nilwebview_fixture_releases() { return releases; } diff --git a/tests/webview_runtime_tests.cpp b/tests/webview_runtime_tests.cpp new file mode 100644 index 00000000..73aef30e --- /dev/null +++ b/tests/webview_runtime_tests.cpp @@ -0,0 +1,97 @@ +/* Checked provider ownership/ABI controls, no desktop. SPDX-License-Identifier: MIT */ +#include "webview_runtime.hpp" +#include +#include +#include +#include +extern "C" std::int64_t nilwebview_fixture_live_contexts(); +extern "C" std::int64_t nilwebview_fixture_releases(); +namespace { +int checks = 0; +void require(bool condition, const char *label) { + ++checks; + if (!condition) { std::cerr << "FAIL: " << label << '\n'; std::exit(1); } +} +} +int main(int argc, char **argv) { + if (argc != 3) return 2; + using ny::webview::Context; + { + Context a, b; + require(!a.window("unset", 640, 480), "unconfigured context"); + require(!a.configure("relative-provider"), "relative provider refused"); + require(!a.configure(argv[2]), "incompatible ABI refused"); + auto configured = a.configure(argv[1]); + require(configured && configured.number == 0, "zero capabilities is successful configuration"); + require(bool(a.configure(argv[1])), "same provider configuration is idempotent"); + require(!a.configure(argv[2]), "cannot switch an active provider"); + require(bool(b.configure(argv[1])), "independent context"); + require(nilwebview_fixture_live_contexts() == 2, "exact live contexts"); + require(!a.window("bad",0,480), "zero dimension rejected"); + require(!a.window(std::string("x\0y",3),640,480), "NUL title rejected"); + require(!a.window(std::string("\xc0\xaf",2),640,480), "invalid UTF-8 title rejected"); + auto wa = a.window("first",640,480), wb = b.window("second",640,480); + require(wa && wb && wa.number != wb.number, "context handles distinct despite repeated native handles"); + require(!a.show(wb.number) && !b.show(wa.number), "cross-context window handles refused"); + bool wrong_thread = false; + std::thread other([&] { wrong_thread = !a.show(wa.number); }); other.join(); + require(wrong_thread, "foreign thread rejected"); + require(bool(a.show(wa.number)), "same-thread operation"); + auto ready = a.poll(wa.number); + require(ready && ready.number > 0, "ready copied"); + require(a.event_number(ready.number,"window").number == wa.number, "native window translated"); + require(a.event_number(ready.number,"kind").number == 1, "event kind preserved"); + require(!a.event_number(ready.number,"unknown"), "unknown number field refused"); + require(!a.event_text(ready.number,"unknown"), "unknown text field refused"); + require(!b.event_number(ready.number,"kind"), "cross-context event handle refused"); + require(nilwebview_fixture_releases() == 1, "provider event released after copy"); + require(a.poll(wa.number).number == 0, "empty event queue"); + require(bool(a.release_event(ready.number)), "copied event released"); + require(!a.release_event(ready.number), "event cannot be released twice"); + require(!a.navigate(wa.number,std::string("x\0y",3)), "NUL operation input rejected"); + auto script = a.evaluate_function(wa.number,"return 42;"); + require(script && script.number == 32, "accepted operation identity"); + auto completed = a.poll(wa.number); + require(completed && completed.number > 0, "script event copied"); + require(a.event_number(completed.number,"operation").number == script.number, "completion identity preserved"); + require(a.event_text(completed.number,"payload_json").text == "{\"answer\":42}", "snapshot survives provider storage mutation"); + require(bool(a.close(wa.number)), "window closed"); + require(!a.event_text(completed.number,"payload_json"), "close invalidates retained snapshots"); + require(!a.close(wa.number) && !a.show(wa.number), "stale window refused"); + require(bool(b.show(wb.number)), "unrelated context remains live"); + a.clear(); b.clear(); + require(nilwebview_fixture_live_contexts() == 0, "explicit context teardown"); + require(bool(a.configure(argv[1])), "context can be reused after terminal cleanup"); + auto w = a.window("invalid-result",640,480); + auto first = a.poll(w.number); a.release_event(first.number); + require(bool(a.evaluate_function(w.number,"bad-json")), "synthetic malformed result accepted asynchronously"); + const auto before = nilwebview_fixture_releases(); + require(!a.poll(w.number), "invalid provider JSON refused"); + require(nilwebview_fixture_releases() == before+1, "malformed event released on error"); + require(!a.show(w.number), "provider protocol fault remains latched"); + a.clear(); + require(bool(a.configure(argv[1])), "teardown resets protocol fault"); + w = a.window("bad-text",640,480); first = a.poll(w.number); a.release_event(first.number); + a.evaluate_function(w.number,"bad-text"); + require(!a.poll(w.number), "invalid provider UTF-8 refused"); + for (const auto *invalid : {"bad-origin", "bad-url", "bad-frame", "bad-message", "deep65"}) { + a.clear(); a.configure(argv[1]); w = a.window("invalid-event",640,480); + first = a.poll(w.number); a.release_event(first.number); + a.evaluate_function(w.number,invalid); + const auto released_before = nilwebview_fixture_releases(); + require(!a.poll(w.number), "inconsistent provenance, event kind or depth refused"); + require(nilwebview_fixture_releases() == released_before+1, "invalid snapshot always released"); + } + for (const auto *valid : {"surrogate", "deep33", "deep64"}) { + a.clear(); a.configure(argv[1]); w = a.window("JSON-stringify-result",640,480); + first = a.poll(w.number); a.release_event(first.number); + a.evaluate_function(w.number,valid); + auto found = a.poll(w.number); + require(found && found.number > 0, "bounded JSON.stringify text preserved"); + if (std::string(valid) == "surrogate") + require(a.event_text(found.number,"payload_json").text == "\"\\ud800\"", "escaped lone surrogate remains serialized text"); + } + } + require(nilwebview_fixture_live_contexts() == 0, "destructor cleanup"); + std::cout << checks << " WebView runtime checks passed\n"; +} From 891b7e652f2163ba4094ddccc509daab84f18f30 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 13:18:17 -0500 Subject: [PATCH 06/12] Record WebView handoff and separate editor dependency follow-up --- .../2026-11-13-formatted-markup-handoff.md | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/notebook/2026-11-13-formatted-markup-handoff.md b/notebook/2026-11-13-formatted-markup-handoff.md index 4c031418..cf530d03 100644 --- a/notebook/2026-11-13-formatted-markup-handoff.md +++ b/notebook/2026-11-13-formatted-markup-handoff.md @@ -108,8 +108,23 @@ No accepted SDK, installer or Lich release is changed. The [Matrix WebView repository](https://github.com/exec-software/exec-nil-framework-webview) is now public, with source secret/path audits, four required platform checks and an independent review requirement matching the web server repository. -Linux and both Mac architectures have passed real browser consumers through -text/binary NIL, AOT and JIT. Windows has passed compilation, shared/framework -contracts and synthetic routes; its real browser initialization correction is -being qualified. The repository's platform notes record exact source evidence. +Its [qualification PR](https://github.com/exec-software/exec-nil-framework-webview/pull/1) +tracks provider startup, navigation and teardown across textual/binary NIL, +AOT and JIT, with exact source evidence in the repository's platform notes. This follows the language handoff without activating an accepted SDK. + +## Separate editor dependency follow-up + +GitHub's existing [Dependabot alert 9](https://github.com/exec-software/exec-nil-framework/security/dependabot/9) +reports minimatch 10.2.6 in the editor lockfile. The +[upstream advisory](https://github.com/isaacs/minimatch/security/advisories/GHSA-g3hv-75wg-m249) +describes a moderate glob-pattern denial of service, fixed in 11.0.1. +Both `vscode-languageclient` and `@vscode/vsce` currently require the older +major. Minimatch 11 replaces the matcher and removes APIs; an override is a +compatibility change, not a routine lockfile refresh. + +Keep that update separate from this completed language qualification. Before +changing it, audit both consumers' API use, exercise adversarial and ordinary +glob patterns, and rerun extension compilation, packaging and actual editor +host checks on the supported platforms. No dependency override or alert +dismissal was made in this handoff. From 9c827a0b660c1b06f5a6ff60a84eae4e1932b697 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 13:29:42 -0500 Subject: [PATCH 07/12] Qualify integrated rooted file and WebView APIs on all platforms --- .github/workflows/macos-application-parity.yml | 14 +++++++++++--- .github/workflows/web-runtime-parity.yml | 8 ++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/macos-application-parity.yml b/.github/workflows/macos-application-parity.yml index 5fae6c5e..00c95ff6 100644 --- a/.github/workflows/macos-application-parity.yml +++ b/.github/workflows/macos-application-parity.yml @@ -25,6 +25,11 @@ name: macOS application platform checks - 'tests/tools/native_owned_lifetime.py' - 'tests/tools/native_symbol_surface.py' - 'tests/tools/file_modified_at_contract.py' + - 'tests/rooted_file_snapshot_tests.cpp' + - 'tests/tools/rooted_file_snapshot_contract.py' + - 'tests/language/rooted-file-snapshot*.ny' + - 'tests/webview_*' + - 'tests/tools/webview_intrinsic_contract.py' - 'tests/tools/formatted_text_contract.py' - 'tests/tools/markup_contract.py' - 'tests/language/formatted-text*' @@ -165,7 +170,8 @@ jobs: set -o pipefail cmake --build build/application --parallel 2 --target \ nilcompiler nilinterpreter nilassembler nillinker nillsp nildebugger nilbuild niltest aperture nilruntime_shared \ - nil_vscode_host_runner nil_file_time_support_tests nil_compiler_module_darwin_tests \ + nil_vscode_host_runner nil_file_time_support_tests nil_rooted_file_snapshot_tests nil_webview_runtime_tests \ + nil_compiler_module_darwin_tests \ nil_darwin_toolchain_inputs_tests nil_freestanding_hook_default nil_freestanding_hook_override \ nildebugger_core_tests nildebugger_source_metadata_tests nildebugger_dap_owner_tests \ 2>&1 | tee build/application-evidence/build.log @@ -234,7 +240,7 @@ jobs: run: >- python3 -B scripts/ci/macos-toolchain-census.py --build build/application --output build/application-evidence/toolchain-census - - name: Verify actual native lifetime and timestamp contracts + - name: Verify native lifetime, timestamps, rooted files and WebView contracts if: ${{ !cancelled() && steps.tools.outcome == 'success' && steps.heap.outcome == 'success' }} timeout-minutes: 16 run: | @@ -243,8 +249,10 @@ jobs: from pathlib import Path expected = {'compiler.darwin-module-fingerprint', 'ny.native-managed-lifetime', 'ny.native-owned-lifetime', 'nil.file-time-support', 'nil.file-modified-at-contract', + 'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract', + 'nil.webview-runtime', 'nil.webview-intrinsic-contract', 'nil.freestanding-hook-default', 'nil.freestanding-hook-override', 'nil.native-symbol-surface'} - pattern = r'^(compiler[.]darwin-module-fingerprint|ny[.]native-(managed|owned)-lifetime|nil[.]file-(time-support|modified-at-contract)|nil[.]freestanding-hook-(default|override)|nil[.]native-symbol-surface)$' + pattern = r'^(compiler[.]darwin-module-fingerprint|ny[.]native-(managed|owned)-lifetime|nil[.]file-(time-support|modified-at-contract)|nil[.]rooted-file-snapshot(-contract)?|nil[.]webview-(runtime|intrinsic-contract)|nil[.]freestanding-hook-(default|override)|nil[.]native-symbol-surface)$' base = ['ctest', '--test-dir', 'build/application', '-R', pattern] root = Path('build/application-evidence').resolve() inventory = subprocess.check_output(base+['--show-only=json-v1'], timeout=30) diff --git a/.github/workflows/web-runtime-parity.yml b/.github/workflows/web-runtime-parity.yml index 73f4e668..afd1d032 100644 --- a/.github/workflows/web-runtime-parity.yml +++ b/.github/workflows/web-runtime-parity.yml @@ -7,12 +7,15 @@ name: Web runtime platform checks - 'CMakeLists.txt' - 'nilcompiler/**' - 'nilstd/**' + - 'common/webview_*' - 'tests/net_*' - 'tests/file_time_support_tests.cpp' - 'tests/rooted_file_snapshot_tests.cpp' + - 'tests/webview_*' - 'tests/tools/net_socket_contract.py' - 'tests/tools/file_modified_at_contract.py' - 'tests/tools/rooted_file_snapshot_contract.py' + - 'tests/tools/webview_intrinsic_contract.py' - 'tests/language/rooted-file-snapshot.ny' - 'tests/language/text-from-bytes*.ny' - 'tests/tools/formatted_text_contract.py' @@ -91,7 +94,7 @@ jobs: cmake --build build/web-parity --config Release --parallel 2 \ --target nilcompiler nilinterpreter nil_net_support_tests \ nil_net_lifecycle_tests nil_file_time_support_tests \ - nil_rooted_file_snapshot_tests \ + nil_rooted_file_snapshot_tests nil_webview_runtime_tests \ 2>&1 | tee build/web-parity-evidence/build.log - name: Run real peer and language contracts timeout-minutes: 8 @@ -99,7 +102,7 @@ jobs: "$NIL_CI_PYTHON" - <<'PY' import json, subprocess from pathlib import Path - pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract|rooted-file-snapshot|rooted-file-snapshot-contract|formatted-text-contract|markup-contract)|matrix[.]execution)$' + pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract|rooted-file-snapshot|rooted-file-snapshot-contract|webview-runtime|webview-intrinsic-contract|formatted-text-contract|markup-contract)|matrix[.]execution)$' base = ['ctest', '--test-dir', 'build/web-parity', '-C', 'Release', '-R', pattern] root = Path('build/web-parity-evidence').resolve() inventory = subprocess.check_output(base+['--show-only=json-v1'], timeout=30) @@ -107,6 +110,7 @@ jobs: expected = {'net.support-contract', 'net.lifecycle-contract', 'net.socket-contract', 'nil.file-time-support', 'nil.file-modified-at-contract', 'matrix.execution', 'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract', + 'nil.webview-runtime', 'nil.webview-intrinsic-contract', 'nil.formatted-text-contract', 'nil.markup-contract'} assert {test['name'] for test in json.loads(inventory)['tests']} == expected with (root/'ctest.log').open('wb') as output: From b6a321e42ad17a5ede77373127a8820c17df1e75 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 14:01:26 -0500 Subject: [PATCH 08/12] Preserve Darwin symbol-reader inputs and keep historical replay manual --- .github/workflows/rooted-file-native-parity.yml | 11 ++++------- tests/tools/native_symbol_surface.py | 6 ++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/rooted-file-native-parity.yml b/.github/workflows/rooted-file-native-parity.yml index 978f7811..8445ddf5 100644 --- a/.github/workflows/rooted-file-native-parity.yml +++ b/.github/workflows/rooted-file-native-parity.yml @@ -1,13 +1,10 @@ -name: Rooted file native macOS checks +name: Historical rooted file native macOS replay "on": workflow_dispatch: - # A new workflow cannot be dispatched until it exists on the default branch. - # This scoped PR trigger qualifies it before review/merge without changing - # the fixed source revision under test. - pull_request: - paths: - - .github/workflows/rooted-file-native-parity.yml + # Replay the original fixed-source campaign only when explicitly dispatched. + # Current-source PR qualification is in macos-application-parity.yml, which + # includes the rooted-file and WebView contracts in its exact test inventory. permissions: contents: read diff --git a/tests/tools/native_symbol_surface.py b/tests/tools/native_symbol_surface.py index 750b32f8..ec5f184c 100644 --- a/tests/tools/native_symbol_surface.py +++ b/tests/tools/native_symbol_surface.py @@ -311,8 +311,7 @@ def main() -> int: ' let b: text = "right";\n' ' if a == b then\n return 1;\n end if\n' ' return 0;\n}\n') - compare, _ = emit(args.compiler, args.assembler, root, 'compare', - comparing) + compare, _ = emit_case('compare', comparing) assert compare_import in compare, ( 'a hosted program that compares text must order bytes through ' f'libc; imports: {sorted(compare)}') @@ -322,8 +321,7 @@ def main() -> int: f'unexpected: {sorted(compare - runtime_abi - {compare_import})}') checks += 1 - free_compare, _ = emit(args.compiler, args.assembler, root, - 'free_compare', comparing, freestanding=True) + free_compare, _ = emit_case('free_compare', comparing, freestanding=True) assert freestanding_compare_import in free_compare, ( 'a freestanding program that compares text must order bytes ' 'through the runtime; imports: ' From c1b1f5b69e21a937c54d47a2a500636b1551fb2e Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 15:07:10 -0500 Subject: [PATCH 09/12] Preserve bounded project timeout diagnostics and upload failures early --- .github/workflows/macos-application-parity.yml | 14 ++++++++++++++ tests/tools/project_build_contract.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.github/workflows/macos-application-parity.yml b/.github/workflows/macos-application-parity.yml index 00c95ff6..e7c85fe1 100644 --- a/.github/workflows/macos-application-parity.yml +++ b/.github/workflows/macos-application-parity.yml @@ -270,6 +270,7 @@ jobs: for kind in ['failure','error','skipped']) for case in cases) PYTHON - name: Verify project, discovery, assembler, context, archive and packaging contracts + id: sdk_contracts if: ${{ !cancelled() && steps.tools.outcome == 'success' }} timeout-minutes: 30 run: | @@ -296,6 +297,19 @@ jobs: assert all(case.get('status') == 'run' and all(case.find(kind) is None for kind in ['failure','error','skipped']) for case in cases) PYTHON + - name: Preserve early project and packaging failure evidence + if: ${{ failure() && steps.sdk_contracts.outcome == 'failure' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: macos-sdk-contract-failure-${{ matrix.host.arch }}-${{ github.sha }}-${{ github.run_attempt }} + path: | + build/application-evidence/sdk-contract-inventory.json + build/application-evidence/sdk-contracts.log + build/application-evidence/sdk-contract-results.xml + build/application-evidence/revision.txt + build/application-evidence/host.txt + if-no-files-found: error + retention-days: 14 - name: Verify default native toolchain and real incremental reuse if: ${{ !cancelled() && steps.tools.outcome == 'success' }} timeout-minutes: 20 diff --git a/tests/tools/project_build_contract.py b/tests/tools/project_build_contract.py index 4b3fede9..4cfae919 100644 --- a/tests/tools/project_build_contract.py +++ b/tests/tools/project_build_contract.py @@ -163,6 +163,22 @@ def invoke(command, *flags, status=0, cwd=root, explicit_tools=True, sdk=True, try: result = subprocess.run(call, cwd=cwd, text=True, capture_output=True, encoding='utf-8', timeout=timeout, env=environment) + except subprocess.TimeoutExpired as error: + for name, output in [('stdout', error.stdout), ('stderr', error.stderr)]: + limit = 32 * 1024 + if not output: + print(f'nilbuild contract: timeout {name}: ', flush=True) + continue + retained = output[:limit] + if isinstance(retained, bytes): + retained = retained.decode('utf-8', errors='replace') + encoded = retained.encode('utf-8', errors='replace') + truncated = len(output) > limit or len(encoded) > limit + retained = encoded[:limit].decode('utf-8', errors='ignore') + detail = f'truncated to {limit} UTF-8 bytes' if truncated else 'retained output' + print(f'nilbuild contract: timeout {name} ({detail}):', flush=True) + print(retained, flush=True) + raise finally: if measured_build: print(f'nilbuild contract: {command} elapsed {time.monotonic() - started:.3f}s', flush=True) From 3018021a03b1e989941290b03b16f29dd69d563e Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 15:46:03 -0500 Subject: [PATCH 10/12] Diagnose child setup failures and synchronize tuple editor checks --- README.md | 5 +++ nilbuild/process.hpp | 43 +++++++++++++++++-- notebook/2026-11-12-matrix-webview.md | 19 ++++++-- .../2026-11-13-formatted-markup-handoff.md | 32 ++++++++++++++ tests/tools/incremental_build_contract.py | 40 ++++++++++++++++- tests/tools/parallel_process_contract.py | 38 +++++++++++++--- vscode/test/language-values-checks.cjs | 26 ++++++++--- 7 files changed, 185 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7ee02c9e..9befdbd8 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,11 @@ File timestamps also pass textual/binary NIL, LLVM JIT and AOT checks everywhere The [platform support matrix](docs/platform-support.md) records exact SDK/source revisions, retained failures and remaining deployment boundaries. +The [current framework integration](https://github.com/exec-software/exec-nil-framework/pull/41) +combines rooted file snapshots, the Matrix WebView bridge and the completed +formatted-text/markup work. Its [qualification record](notebook/2026-11-13-formatted-markup-handoff.md#current-framework-integration) +tracks current-source checks separately from the installed SDK results above. + These are development results. Windows and macOS release qualification remains pending. The [release profile](docs/release/v1-profile.md) defines supported configurations diff --git a/nilbuild/process.hpp b/nilbuild/process.hpp index 47582b1e..3d7f63a7 100644 --- a/nilbuild/process.hpp +++ b/nilbuild/process.hpp @@ -201,6 +201,39 @@ class cancellation_scope { sigaction(SIGTERM, &old_term, nullptr); } }; + +namespace detail { +// Only fixed stack storage, integer operations and async-signal-safe syscalls +// are allowed here: this runs after fork and before exec. Use the original +// capture pipe because stderr redirection itself may be the failing step. +template +[[noreturn]] inline void child_setup_failed(int error_pipe, + const char (&stage)[N], + int saved_errno) noexcept { + static_assert(N <= 32, "child setup stage must remain bounded"); + char message[128]; + std::size_t length = 0; + constexpr char prefix[] = "nilbuild: child setup failed at "; + constexpr char middle[] = " (errno="; + for (std::size_t i = 0; i + 1 < sizeof(prefix); ++i) message[length++] = prefix[i]; + for (std::size_t i = 0; i + 1 < N; ++i) message[length++] = stage[i]; + for (std::size_t i = 0; i + 1 < sizeof(middle); ++i) message[length++] = middle[i]; + char digits[sizeof(unsigned) * 3]; + std::size_t count = 0; + auto number = static_cast(saved_errno); + do { digits[count++] = static_cast('0' + number % 10); number /= 10; } while (number); + while (count) message[length++] = digits[--count]; + message[length++] = ')'; + message[length++] = '\n'; + std::size_t written = 0; + for (unsigned attempt = 0; attempt != 4 && written < length; ++attempt) { + const auto result = write(error_pipe, message + written, length - written); + if (result > 0) written += static_cast(result); + else if (result != -1 || errno != EINTR) break; + } + _exit(126); +} +} // namespace detail #endif // The driver never invokes a shell. Compiler processes have bounded capture @@ -605,12 +638,14 @@ inline std::vector execute_many(const std::vector (1 << 63) - 1: + raise ValueError('fixture timestamp exceeds the diagnostic bound') + except (OSError, ValueError, AssertionError): + invalid += 1 + continue + counts[kind] += 1 + records.append({'kind': kind, 'monotonic_ns': timestamp}) + records.sort(key=lambda record: (record['monotonic_ns'], record['kind'])) + return {'counts': counts, 'latest': records[-16:], + 'invalid_records': invalid, 'truncated': truncated} + + def check_event_publication(root): path = root / 'publication-check.json' expected = ['start', str(root / 'source é.ny'), time.monotonic_ns()] @@ -215,8 +239,22 @@ def run(*arguments, cwd=root, status=0, tool=fake): # Keep fake/containment checks at 60 s and bound real Darwin builds # separately; this changes no assertion or cache admission rule. timeout = 300 if sys.platform == 'darwin' and tool == compiler else 60 - result = subprocess.run(command, cwd=cwd, text=True, capture_output=True, timeout=timeout) + before = event_summary(events) if arguments[0] == 'build' and tool == fake else None + + def report_fixture_events(): + if before is not None: + print('nilbuild fixture events: ' + json.dumps({ + 'before': before, 'after': event_summary(events)}, sort_keys=True), + file=sys.stderr, flush=True) + + try: + result = subprocess.run(command, cwd=cwd, text=True, capture_output=True, timeout=timeout) + except subprocess.TimeoutExpired: + report_fixture_events() + raise checks += 1 + if result.returncode != status: + report_fixture_events() assert result.returncode == status, (command, result.returncode, result.stdout, result.stderr) return result diff --git a/tests/tools/parallel_process_contract.py b/tests/tools/parallel_process_contract.py index b53c4ea8..454478f2 100644 --- a/tests/tools/parallel_process_contract.py +++ b/tests/tools/parallel_process_contract.py @@ -3,6 +3,7 @@ import argparse from contextlib import contextmanager +import errno import json import os from pathlib import Path @@ -27,7 +28,7 @@ std::string prefix(const std::string &text) { if (text.empty()) return "-"; std::ostringstream output; - for (unsigned char c : text.substr(0, 32)) + for (unsigned char c : text.substr(0, 128)) output << std::hex << std::setw(2) << std::setfill('0') << unsigned(c); return output.str(); } @@ -217,15 +218,17 @@ def process_diagnostic(returncode, output, error): f'stdout={output!r}\nstderr={error!r}') -def run_driver(call, **kwargs): +def run_driver(call, *, record_directory=None, **kwargs): completed = subprocess.run(call, text=True, capture_output=True, **console_options(), **kwargs) - record_driver(call, completed.returncode, completed.stdout, completed.stderr) + record_driver(call, completed.returncode, completed.stdout, completed.stderr, + directory=record_directory) return completed -def record_driver(call, returncode, output, error): - (Path(call[2]) / 'driver-result.json').write_text(json.dumps({ +def record_driver(call, returncode, output, error, *, directory=None): + output_directory = Path(call[2]) if directory is None else directory + (output_directory / 'driver-result.json').write_text(json.dumps({ 'command': call, 'returncode': returncode, 'exitHex': f'0x{returncode & 0xffffffff:08x}', 'stdout': output, 'stderr': error, @@ -265,6 +268,31 @@ def command(name, jobs, modes): return case, [str(driver), str(jobs), str(case), sys.executable, str(worker), *modes] + if os.name != 'nt': + # These exercise the real fork/exec path, preserving the setup + # failure distinction even when no child program has started. + case, call = command('missing-working-directory', 1, ['queued']) + call[2] = str(case / 'absent') + completed = run_driver(call, record_directory=case, timeout=10) + assert completed.returncode == 0, completed + record = results(completed.stdout)[0] + expected = f'nilbuild: child setup failed at chdir (errno={errno.ENOENT})\n' + assert record == {'status': 126, 'started': True, 'cancelled': False, + 'outsize': 0, 'errsize': len(expected), + 'out': '', 'err': expected}, record + assert not (case / 'started-0').exists() + + case, call = command('missing-executable', 1, ['queued']) + call[3] = str(case / 'absent-tool') + completed = run_driver(call, timeout=10) + assert completed.returncode == 0, completed + record = results(completed.stdout)[0] + expected = 'nilbuild: cannot execute child tool\n' + assert record == {'status': 127, 'started': True, 'cancelled': False, + 'outsize': 0, 'errsize': len(expected), + 'out': '', 'err': expected}, record + assert not (case / 'started-0').exists() + for jobs, count, delayed in ((1, 5, False), (2, 5, False), (2, 2, True)): case, call = command(f'success-{jobs}' + ('-delayed' if delayed else ''), jobs, ['success'] * count) diff --git a/vscode/test/language-values-checks.cjs b/vscode/test/language-values-checks.cjs index 28e3609f..d284b7ee 100644 --- a/vscode/test/language-values-checks.cjs +++ b/vscode/test/language-values-checks.cjs @@ -5,12 +5,12 @@ const vscode = require("vscode"); const pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); const label = item => typeof item.label === "string" ? item.label : item.label.label; -async function eventually(operation, description) { +async function eventually(operation, description, retryErrors = true) { const until = Date.now() + 15_000; let latest; do { try { const value = await operation(); if (value) return value; } - catch (error) { latest = error; } + catch (error) { if (!retryErrors) throw error; latest = error; } await pause(100); } while (Date.now() < until); throw new Error(`Timed out: ${description}${latest ? ` (${latest})` : ""}`); @@ -113,11 +113,27 @@ exports.run = async ({ check, workspace }) => { assert.deepEqual(await analyze(document), []); const occurrence = language === "ny" ? "println(height)" : "say height"; const selected = await complete(document, occurrence, "height", "hei", language === "ny" ? "f64" : "floating"); - const definitions = await vscode.commands.executeCommand("vscode.executeDefinitionProvider", document.uri, selected.start); + const version = document.version; + let definitionAttempts = 0; + const definitions = await eventually(async () => { + assert.equal(document.version, version, "Definition lookup must use the original tuple document version."); + definitionAttempts++; + const value = await vscode.commands.executeCommand("vscode.executeDefinitionProvider", document.uri, selected.start); + assert.equal(document.version, version, "Definition lookup must not change the tuple document version."); + assert.ok(value == null || Array.isArray(value), "Definition provider must return an array or no result."); + return value?.length && value; + }, "tuple definition after the editor generation settles", false); + // As with semantic-checks, only absence may be retried. A nonempty stale, + // duplicate, or guessed destination must fail immediately. + assert.equal(document.version, version, "Definition lookup must not change the tuple document version."); assert.equal(definitions.length, 1); const expected = range(document, "(width, height)", "height"); - assert.ok((definitions[0].targetSelectionRange || definitions[0].range).isEqual(expected)); - return { language, tupleArity: 2 }; + const [definition] = definitions; + assert.equal(definition.targetUri?.toString(), document.uri.toString()); + assert.ok(definition.originSelectionRange?.isEqual(selected)); + assert.ok(definition.targetSelectionRange?.isEqual(expected)); + assert.ok(definition.targetRange?.isEqual(expected)); + return { language, tupleArity: 2, documentVersion: version, definitionAttempts }; }); await check(`${language}: malformed interpolation reports an editor error and recovers after correction`, async () => { From 0745893c7eea4fd5fe44b51e7f5d776d6d4f8325 Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 15:47:22 -0500 Subject: [PATCH 11/12] Clarify where WebView engine evidence is recorded --- notebook/2026-11-12-matrix-webview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/2026-11-12-matrix-webview.md b/notebook/2026-11-12-matrix-webview.md index 38393c77..f57f0795 100644 --- a/notebook/2026-11-12-matrix-webview.md +++ b/notebook/2026-11-12-matrix-webview.md @@ -86,8 +86,8 @@ now passes on Linux, Windows, Intel macOS and Apple Silicon macOS at behavior groups through text NIL, binary NIL, AOT and JIT, alongside four synthetic-provider routes, 209 shared controls and 36 JavaScript checks. The [platform notes](https://github.com/exec-software/exec-nil-framework-webview/blob/63d3e49d953179c81231402cf07dc7d6dbe17939/docs/platform-support.md) -record the exact engine versions, request counts, cleanup checks, source seals -and limits of the implemented profile. This qualifies the initial API profile; +describe the engine-version evidence, request counts, cleanup checks, source +seals and limits of the implemented profile. This qualifies the initial API profile; it does not claim that every archived WebView feature has been ported. No accepted SDK, installer or release deployment is changed by this work. From 7419ac1a7d1d2b1458a3b6288c001b2bae1f4e3f Mon Sep 17 00:00:00 2001 From: Nathan Young Date: Fri, 13 Nov 2026 16:56:00 -0500 Subject: [PATCH 12/12] Refresh SDK library inventory and desktop provider documentation --- docs/platform-support.md | 9 +++++++-- docs/release/v1-profile.md | 6 +++--- docs/standard-library/index.md | 7 ++++--- nilstd/README.md | 15 ++++++++++----- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/platform-support.md b/docs/platform-support.md index 6e6e1078..446f72c1 100644 --- a/docs/platform-support.md +++ b/docs/platform-support.md @@ -14,8 +14,8 @@ boundary. A source update does not update an installed SDK. | macOS 15 Intel x86_64 | Relocated `1efbe66` SDK passes all 1,184 consumer routes, twelve PostgreSQL routes, 146 debugger checks, execution checkpoints, twenty installed AppKit desktop/input cases, four desktop DAP sessions and all 88 editor groups. Current `59ffa97` source fixtures pass all 22 application/tooling/packaging CTests. | The installed editor needed a diagnostic rerun after a project-build timeout; it passed unchanged deadlines, with the original cause unconfirmed. The original full source report retains four failures resolved by later fixtures and one ELF-only execution skip. Shared deployment and release qualification remain open. | | macOS 15 Apple Silicon arm64 | Relocated `1efbe66` SDK passes all 1,184 consumer routes, twelve PostgreSQL routes, 146 debugger checks, execution checkpoints, twenty installed AppKit desktop/input cases, four desktop DAP sessions and all 88 editor groups. Current `59ffa97` source fixtures also pass all 22 application/tooling/packaging CTests. | The original full source report retains four failures and one ELF-only execution skip; subsequent hosted checks resolve the four fixture failures. Shared deployment and release qualification remain open. | -The SDKs contain eleven root commands, eighteen compiled standard-library -modules, their source, native runtimes and HTML documentation. The installed +Those pinned baseline SDKs contain eleven root commands, eighteen compiled +standard-library modules, their source, native runtimes and HTML documentation. The installed consumer campaign covers both `nilstd` and `nillib`: 888 retained execution results are independently replayed from raw reports, while 296 object/link routes are checked by the pinned campaign driver. Each private candidate retains @@ -24,6 +24,11 @@ spaces and Unicode. Mac executables are checked as Mach-O for their expected architecture. These candidates have not replaced accepted SDKs or activated a release. +Current source SDKs additionally package `nil.std.markup`, bringing the logical +module inventory to nineteen. See the [current library status](standard-library/index.md); +the exact tools, source, binary modules and documentation are sealed in each +SDK's manifest. + The Linux editor result uses VS Code 1.144.1 in an isolated X11 session and empty profile. Both current Mac application campaigns use VS Code 1.146.0. Each passes 88 distinct groups, all nine test modules, the full-suite completion sentinel diff --git a/docs/release/v1-profile.md b/docs/release/v1-profile.md index ba8bf455..af552f4a 100644 --- a/docs/release/v1-profile.md +++ b/docs/release/v1-profile.md @@ -94,9 +94,9 @@ not extend the v1 language or debugger guarantees. The LSP has a deliberately limited capability set. `nilbuild` supports the [local-project profile](../tools/projects.md), with process execution qualified on Linux. `nypm` and `nilaix` are not part of this release. -Current development SDKs strictly compile and package eighteen logical -standard-library modules, including `prelude`. Public imports use `nil.std.*`, -with `ny.lang.*` retained as aliases. Both frontends share the flat module-family +Current development SDKs strictly compile and package nineteen logical +standard-library modules, including `prelude` and `nil.std.markup`. Public imports +use `nil.std.*`, with `ny.lang.*` retained as aliases. Both frontends share the flat module-family layout. Runtime support is API-specific, as described in the [library status](../standard-library/index.md). `nil.std.reflection` provides declaration/annotation snapshots, and `nil.std.db` diff --git a/docs/standard-library/index.md b/docs/standard-library/index.md index 28ed522e..bdd97174 100644 --- a/docs/standard-library/index.md +++ b/docs/standard-library/index.md @@ -1,7 +1,8 @@ # Standard-library status Ny and Matrix share `nilstd`, one standard library compiled to NIL. Its active -Ny and Matrix sources form 18 logical SDK modules, including `prelude`. +Ny and Matrix sources form 19 logical SDK modules, including `prelude` and +`nil.std.markup`. Support remains evidence-scoped. Focused fixtures exercise `nilstd` and `nillib` through textual/binary NIL, bundles, LLVM JIT and AOT, and standalone assembler/linker pipelines. Local SDK qualification records the exact source @@ -38,7 +39,7 @@ averaging, nonfinite compatibility behavior and numerical accuracy limits. ```text nilstd/ - src/ Ny and Matrix sources organized as 18 logical modules + src/ Ny and Matrix sources organized as 19 logical modules native/ 14 preserved C and C++ interoperability shims tests/ 8 preserved design-era library tests experimental/ 65 quarantined Ny sources excluded from the SDK @@ -49,7 +50,7 @@ The active tree uses neutral family folders: `nil.std.io` lives under remain aliases of the same module identities. The active logical modules are `prelude` and `nil.std.core`, `collections`, -`datetime`, `db`, `diagnostics`, `io`, `math`, `net`, `nildata`, `reflection`, `runtime`, +`datetime`, `db`, `diagnostics`, `io`, `markup`, `math`, `net`, `nildata`, `reflection`, `runtime`, `security`, `system`, `testing`, `text`, `threading`, and `ui`. Quarantine is broader than parser failure: an experimental source may compile yet still lack complete language semantics, an executable provider, or behavioral evidence. diff --git a/nilstd/README.md b/nilstd/README.md index 581173be..ee65ce1a 100644 --- a/nilstd/README.md +++ b/nilstd/README.md @@ -24,10 +24,10 @@ are first-party packages rather than standard-library modules and live in ## Current status -The active tree contains 80 sources (68 Ny and 12 Matrix) organized into 18 +The active tree contains 84 sources (72 Ny and 12 Matrix) organized into 19 logical SDK modules: `prelude` plus `nil.std.core`, `collections`, `datetime`, `db`, `diagnostics`, -`io`, `math`, `net`, `nildata`, `reflection`, `runtime`, `security`, `system`, `testing`, `text`, +`io`, `markup`, `math`, `net`, `nildata`, `reflection`, `runtime`, `security`, `system`, `testing`, `text`, `threading`, and `ui`. The SDK packaging gate compiles every logical module with normal type checking, emits a NIL library (v2, or v4 when carrying classes), and fails atomically rather than publishing a partial set. Matrix sources retain their @@ -46,7 +46,7 @@ below. | Area | Files | Status | | --- | ---: | --- | -| `src/` | 68 Ny files, 12 Matrix files | Eighteen logical SDK modules. Matrix contributes checked arrays/text spans, sorting, reflection values, vector helpers, random generators, testing helpers and the desktop UI library, consumed from both frontends. | +| `src/` | 72 Ny files, 12 Matrix files | Nineteen logical SDK modules. Matrix contributes checked arrays/text spans, sorting, reflection values, vector helpers, random generators, testing helpers and the desktop UI library, consumed from both frontends. | | `experimental/` | 65 Ny files | Design sources excluded from `nilstd`/`nillib` because language semantics, executable providers, or behavioral evidence are incomplete. | | `tests/` | 8 Ny files | Preserved design-era tests; these are still separate from the registered route fixtures under the repository-level `tests/` tree. | | `native/` | 14 C/C++ files | Preserved platform/provider shims; source presence alone is neither a build, shipment, nor native-support claim. | @@ -55,11 +55,16 @@ below. `nil.std.testing` supplies checks and checked value extraction with co-located niltest cases. `nil.std.ui` supplies typed windows, layouts, controls, events and -model synchronization through the optional GTK 4 provider on Linux and the -Windows development target. See the +model synchronization through GTK 4 on Linux, Win32 on Windows and AppKit on +macOS. See the [testing guide](../docs/tools/testing.md) and [UI reference](../docs/standard-library/ui.md) for execution evidence and current scope. +`nil.std.markup` provides serialized documents, composition and context-specific +escaping for checked markup blocks in Ny and Matrix. See the +[markup contract](../docs/standard-library/markup.md) for supported HTML contexts +and the explicit trusted `raw` bypass. + `nil.std.nildata` replaces NyData with versioned data blocks, one array kind, exact numeric tokens and checked JSON interchange. The compiler conversions and VM/native library share one engine. Legacy imports are explicit. See the