Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
332 changes: 332 additions & 0 deletions .github/workflows/rooted-file-native-parity.yml

Large diffs are not rendered by default.

8 changes: 7 additions & 1 deletion .github/workflows/web-runtime-parity.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,12 @@ name: Web runtime platform checks
- 'nilstd/**'
- 'tests/net_*'
- 'tests/file_time_support_tests.cpp'
- 'tests/rooted_file_snapshot_tests.cpp'
- 'tests/tools/net_socket_contract.py'
- 'tests/tools/file_modified_at_contract.py'
- 'tests/tools/rooted_file_snapshot_contract.py'
- 'tests/language/rooted-file-snapshot.ny'
- 'tests/language/text-from-bytes*.ny'
- 'tests/tools/formatted_text_contract.py'
- 'tests/tools/markup_contract.py'
- 'tests/language/formatted-text*'
Expand Down Expand Up @@ -87,20 +91,22 @@ jobs:
cmake --build build/web-parity --config Release --parallel 2 \
--target nilcompiler nilinterpreter nil_net_support_tests \
nil_net_lifecycle_tests nil_file_time_support_tests \
nil_rooted_file_snapshot_tests \
2>&1 | tee build/web-parity-evidence/build.log
- name: Run real peer and language contracts
timeout-minutes: 8
run: |
"$NIL_CI_PYTHON" - <<'PY'
import json, subprocess
from pathlib import Path
pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract|formatted-text-contract|markup-contract)|matrix[.]execution)$'
pattern = r'^(net[.](support-contract|lifecycle-contract|socket-contract)|nil[.](file-time-support|file-modified-at-contract|rooted-file-snapshot|rooted-file-snapshot-contract|formatted-text-contract|markup-contract)|matrix[.]execution)$'
base = ['ctest', '--test-dir', 'build/web-parity', '-C', 'Release', '-R', pattern]
root = Path('build/web-parity-evidence').resolve()
inventory = subprocess.check_output(base+['--show-only=json-v1'], timeout=30)
(root/'inventory.json').write_bytes(inventory)
expected = {'net.support-contract', 'net.lifecycle-contract', 'net.socket-contract',
'nil.file-time-support', 'nil.file-modified-at-contract', 'matrix.execution',
'nil.rooted-file-snapshot', 'nil.rooted-file-snapshot-contract',
'nil.formatted-text-contract', 'nil.markup-contract'}
assert {test['name'] for test in json.loads(inventory)['tests']} == expected
with (root/'ctest.log').open('wb') as output:
Expand Down
14 changes: 14 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,12 @@ if(BUILD_TESTING)
LABELS "tools;windows;rc" TIMEOUT 30)
endif()

add_executable(nil_rooted_file_snapshot_tests tests/rooted_file_snapshot_tests.cpp)
target_link_libraries(nil_rooted_file_snapshot_tests PRIVATE nilruntime)
target_include_directories(nil_rooted_file_snapshot_tests PRIVATE nilcompiler/src)
add_test(NAME nil.rooted-file-snapshot COMMAND $<TARGET_FILE:nil_rooted_file_snapshot_tests>)
set_tests_properties(nil.rooted-file-snapshot PROPERTIES LABELS "storage;runtime;rc" TIMEOUT 60)

add_executable(nil_utf8_file_paths_tests tests/utf8_file_paths_tests.cpp)
target_link_libraries(nil_utf8_file_paths_tests PRIVATE nilruntime)
target_include_directories(nil_utf8_file_paths_tests PRIVATE nilcompiler/src)
Expand Down Expand Up @@ -1216,6 +1222,14 @@ if(BUILD_TESTING)
--llvm "${NIL_ENABLE_LLVM}"
--output-parent "${CMAKE_BINARY_DIR}/test-evidence")
set_tests_properties(nil.file-modified-at-contract PROPERTIES LABELS "io;routes;rc" TIMEOUT 180)
add_test(NAME nil.rooted-file-snapshot-contract
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/tools/rooted_file_snapshot_contract.py"
--compiler $<TARGET_FILE:nilcompiler>
--interpreter $<TARGET_FILE:nilinterpreter>
--nilstd "${CMAKE_CURRENT_SOURCE_DIR}/nilstd/src"
--llvm "${NIL_ENABLE_LLVM}"
--output-parent "${CMAKE_BINARY_DIR}/test-evidence")
set_tests_properties(nil.rooted-file-snapshot-contract PROPERTIES LABELS "io;routes;rc" TIMEOUT 360)
add_test(NAME nillsp.semantic-protocol
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/lsp_semantic_protocol.py"
--lsp $<TARGET_FILE:nillsp>)
Expand Down
3 changes: 3 additions & 0 deletions docs/standard-library/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ the runtime and included in Aperture's text allocation measurements.
For untrusted file/container input, `nil.std.io.read_regular_bytes` accepts an
explicit byte ceiling and rejects the final-component symlink, special file,
short/growing read, and observed metadata change before returning bytes.
For paths below a trusted directory, [`read_rooted_snapshot`](rooted-file-snapshots.md)
also rejects links in descendant directories and returns the exact byte array,
descriptor-observed modification time, and SHA-256 together.
`write_bytes_atomic` stages and flushes bytes in the destination directory,
then atomically publishes them; `replace=false` cannot clobber an existing
entry (including a dangling symlink), while `replace=true` only replaces a
Expand Down
52 changes: 52 additions & 0 deletions docs/standard-library/rooted-file-snapshots.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Rooted file snapshots

`nil.std.io.read_rooted_snapshot(root, relative, maximum_bytes)` reads one
bounded regular file below a trusted directory. It returns
`Result<FileSnapshot, text>`:

| Field | Type | Meaning |
| --- | --- | --- |
| `content` | `array<u8>` | The exact bytes read, including NUL and arbitrary high bytes. |
| `modified_ms` | `i64` | Modification time observed on the final file handle, in Unix milliseconds. |
| `sha256` | `text` | The 64 lowercase hexadecimal digits of SHA-256 over `content`. |

The caller supplies a limit from zero through 512 MiB. A zero limit accepts
an empty file. Oversized input is rejected before allocating its contents.
SHA-256 is computed over the loaded bytes, so replacing content while keeping
its size and timestamp does not preserve its digest.

```ny
import nil.std.io.file

fn inspect_asset(root: text)
match nil::std::io::read_rooted_snapshot(root, "assets/site.css", 1048576)
Result.Ok(snapshot) ->
println(snapshot.content.len())
println(snapshot.modified_ms)
println(snapshot.sha256)
Result.Error(message) -> println(message)
end match
end fn
```

The `relative` path must be nonempty strict UTF-8, with `/` separating its
components. Empty components, `.` and `..`, absolute paths, backslashes,
colons, and NUL bytes are rejected. The root selects the trusted directory;
every descendant component is opened relative to an already-open directory.
Descendant symlinks and Windows reparse points are rejected, including the
final component. Directories and other special files cannot be returned as
content. Errors do not include caller-supplied paths.

The final handle supplies both bytes and metadata, and the runtime checks for
observed changes around the read. This does not make a file continuously
immutable or prevent privileged mount changes, hard links already placed in
the root, or every concurrent write. The returned digest describes the bytes
actually returned; it is not a separate path lookup or a metadata hash.

The snapshot record uses the same byte-array representation for interpreter,
LLVM JIT, and native executable routes. Code using the interpreter's raw-byte
text representation can convert `snapshot.content` with
[`nil.std.text.from_bytes`](text-spans.md#byte-values). That conversion remains
interpreter-only: native text has a NUL-terminated representation and must not
silently truncate binary content. Native consumers can inspect, hash-check, or
write the byte array without converting it to text.
14 changes: 10 additions & 4 deletions docs/standard-library/text-spans.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,20 @@ assembler/linker execution, and installed source/binary libraries.

## Byte values

`nil.std.text` measures and slices by byte — `length` of `"café"` is 5 — but
until these two existed there was no way to read a byte *as a number* or write
one as text.
`nil.std.text` measures and slices by byte — `length` of `"café"` is 5.
Use explicit byte operations to inspect or construct raw content.

| Function | Result |
| --- | --- |
| `byte_at(s: text, index: i64) -> i64` | The byte at an index, `0..255`, or minus one when the index is outside the text. |
| `from_byte(code: i64) -> text` | One byte as text; empty text for a code outside `0..255`. |
| `from_bytes(data: array<u8>) -> text` | An independent text copy of every array byte, including NUL and invalid UTF-8. |

These raw-byte operations are interpreter-only. Native text currently requires
NUL-free values; native compilation rejects the unsupported intrinsics rather
than silently truncating or replacing bytes. `from_bytes` copies the array in
one operation and does not change when the source array is later mutated.
Use a byte array directly for binary data shared with native routes.

The conversion that looks like it would do this is
`nil.std.core`'s `convert.char_to_i32`, and it will not: it takes a whole
Expand All @@ -85,7 +91,7 @@ isolated in order to inspect it whole, because isolating it needed the same
test. Percent-encoding anything outside ASCII was unreachable because of this,
and so was reading arbitrary bytes off a socket.

These two are about bytes and say so in their names. They know nothing about
These operations are about bytes and say so in their names. They know nothing about
UTF-8; assembling characters from bytes is the caller's job.

`from_byte(195)` is the single byte `0xC3`, where `convert.i32_to_char(195)` is
Expand Down
106 changes: 106 additions & 0 deletions nilcompiler/src/llvm_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3388,6 +3388,111 @@ bool NativeCodegen::generateModule(const nil_module &mod) {
return false;
impl_->builder->CreateRet(error_variant);
annotation_handled = true;
} else if (dispatch_name == "@intrinsic" &&
dispatch_provider == "file.read_rooted_snapshot") {
const auto result_arguments =
native_generic_arguments(fn.return_type, "result");
const auto *definition = result_arguments && result_arguments->size() == 2
? find_native_aggregate_definition(mod, (*result_arguments)[0])
: nullptr;
const auto layout_it = definition ? native_struct_layouts.find(definition)
: native_struct_layouts.end();
bool valid_schema = definition && definition->fields.size() == 3 &&
compact_nil_type(definition->name) == "nil::std::io::FileSnapshot" &&
layout_it != native_struct_layouts.end() &&
layout_it->second.hidden_fields == 0;
const std::pair<const char *, const char *> required_fields[] = {
{"content", "array<u8>"}, {"modified_ms", "i64"}, {"sha256", "text"}};
if (valid_schema) {
for (const auto &[name, type] : required_fields) {
const auto field = layout_it->second.field_indices.find(name);
if (field == layout_it->second.field_indices.end()) {
valid_schema = false;
break;
}
const auto &actual_type = layout_it->second.field_nil_types[field->second];
valid_schema = valid_schema &&
(std::string(type) == "text"
? native_handle_kind_for_nil_type(actual_type) == native_handle_kind::text
: normalize_nil_type(actual_type) == type);
}
}
if (!valid_schema || fn.params.size() != 3 ||
native_handle_kind_for_nil_type(nil_parameter_type(fn.params[0])) !=
native_handle_kind::text ||
native_handle_kind_for_nil_type(nil_parameter_type(fn.params[1])) !=
native_handle_kind::text ||
normalize_nil_type(nil_parameter_type(fn.params[2])) != "i64" ||
native_handle_kind_for_nil_type((*result_arguments)[1]) !=
native_handle_kind::text ||
impl_->current_function->arg_size() != 3 ||
!impl_->current_function->getReturnType()->isIntegerTy(64)) {
std::cerr << "[NativeCodegen] file.read_rooted_snapshot requires "
"(text,text,i64) -> Result<FileSnapshot,text> with the "
"exact content/modified_ms/sha256 field schema\n";
return false;
}
auto *pointer_type = llvm::PointerType::getUnqual(ctx);
auto *word_type = llvm::Type::getInt64Ty(ctx);
auto *bytes_out = impl_->builder->CreateAlloca(pointer_type);
auto *modified_out = impl_->builder->CreateAlloca(word_type);
auto *digest_out = impl_->builder->CreateAlloca(pointer_type);
auto *error_out = impl_->builder->CreateAlloca(pointer_type);
auto *runtime_type = llvm::FunctionType::get(
llvm::Type::getInt32Ty(ctx),
{pointer_type, pointer_type, word_type, pointer_type, pointer_type,
pointer_type, pointer_type}, false);
auto runtime = impl_->module->getOrInsertFunction(
"nyrt_read_rooted_snapshot", runtime_type);
auto *status = impl_->builder->CreateCall(runtime_type, runtime.getCallee(),
{impl_->current_function->getArg(0), impl_->current_function->getArg(1),
impl_->current_function->getArg(2), bytes_out, modified_out,
digest_out, error_out});
auto *success = llvm::BasicBlock::Create(ctx, "snapshot.success",
impl_->current_function);
auto *failure = llvm::BasicBlock::Create(ctx, "snapshot.failure",
impl_->current_function);
impl_->builder->CreateCondBr(impl_->builder->CreateICmpNE(
status, llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), 0)),
success, failure);
impl_->builder->SetInsertPoint(success);
const auto allocation_size = impl_->module->getDataLayout()
.getTypeAllocSize(layout_it->second.llvm_type).getFixedValue();
llvm::Value *record = nullptr;
if (impl_->memory_mode == MemoryMode::SAFE_GC) {
auto *allocate_type = llvm::FunctionType::get(pointer_type, {word_type}, false);
auto allocate = impl_->module->getOrInsertFunction("nyrt_alloc", allocate_type);
record = impl_->builder->CreateCall(allocate_type, allocate.getCallee(),
{llvm::ConstantInt::get(word_type, allocation_size)});
} else {
auto *allocate_type = llvm::FunctionType::get(pointer_type, {native_size_type}, false);
auto allocate = impl_->module->getOrInsertFunction("malloc", allocate_type);
record = impl_->builder->CreateCall(allocate_type, allocate.getCallee(),
{native_size_constant(allocation_size)});
}
emit_allocation_null_check(record, "FileSnapshot");
impl_->setNilType(record, definition->name);
llvm::Value *values[] = {
impl_->builder->CreateLoad(pointer_type, bytes_out),
impl_->builder->CreateLoad(word_type, modified_out),
impl_->builder->CreateLoad(pointer_type, digest_out)};
for (std::size_t index = 0; index < 3; ++index) {
const auto field_index = layout_it->second.field_indices.at(required_fields[index].first);
auto *field_pointer = impl_->builder->CreateStructGEP(
layout_it->second.llvm_type, record, field_index);
impl_->builder->CreateStore(values[index], field_pointer);
}
auto *ok = emit_single_payload_variant("Result::Ok", record,
fn.return_type, definition->name);
if (!ok) return false;
impl_->builder->CreateRet(ok);
impl_->builder->SetInsertPoint(failure);
auto *error = impl_->builder->CreateLoad(pointer_type, error_out);
auto *failed = emit_single_payload_variant("Result::Error", error,
fn.return_type, "text");
if (!failed) return false;
impl_->builder->CreateRet(failed);
annotation_handled = true;
} else if (ann.find("@intrinsic(\"file.read_regular_bytes\")") !=
std::string::npos ||
ann.find("@intrinsic(file.read_regular_bytes)") !=
Expand Down Expand Up @@ -9380,6 +9485,7 @@ int NativeCodegen::executeJIT(const std::vector<std::string> &program_args) {
expose_runtime_symbol("nyrt_buffer_get", &nyrt_buffer_get);
expose_runtime_symbol("nyrt_buffer_set", &nyrt_buffer_set);
expose_runtime_symbol("nyrt_read_regular_bytes", &nyrt_read_regular_bytes);
expose_runtime_symbol("nyrt_read_rooted_snapshot", &nyrt_read_rooted_snapshot);
expose_runtime_symbol("nyrt_write_bytes_atomic", &nyrt_write_bytes_atomic);
expose_runtime_symbol("nyrt_io_v1", &nyrt_io_v1);
expose_runtime_symbol("nyrt_try_i32", &nyrt_try_i32);
Expand Down
43 changes: 43 additions & 0 deletions nilcompiler/src/native_runtime.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include "native_runtime.hpp"
#include "../../common/standard_streams.hpp"
#include "../../common/strict_json.hpp"
#include "../../common/content_digest.hpp"
#include "../../common/windows_environment.hpp"
#include "ordinary_io_intrinsic.hpp"
#include "dynamic_type.hpp"
Expand Down Expand Up @@ -1955,6 +1956,48 @@ std::int32_t nyrt_io_v1(const char *operation, std::uint64_t first,
}
}

std::int32_t nyrt_read_rooted_snapshot(
const char *root, const char *relative, std::int64_t maximum_bytes,
void **bytes_out, std::int64_t *modified_ms_out, const char **sha256_out,
const char **error_out) noexcept {
if (bytes_out) *bytes_out = nullptr;
if (modified_ms_out) *modified_ms_out = 0;
if (sha256_out) *sha256_out = nullptr;
if (error_out) *error_out = nullptr;
if (!root || !relative || !bytes_out || !modified_ms_out || !sha256_out ||
maximum_bytes < 0 ||
static_cast<std::uint64_t>(maximum_bytes) > maximum_file_bytes) {
set_error(error_out, "file.read_rooted_snapshot: invalid arguments");
return 0;
}
try {
auto loaded = ny::detail::read_rooted_file_snapshot_utf8(
root, relative, static_cast<std::uint64_t>(maximum_bytes));
if (!loaded.ok) {
set_error(error_out, loaded.error);
return 0;
}
nil::detail::sha256 hash;
hash.update(loaded.bytes.data(), loaded.bytes.size());
const auto digest = hash.finish();
auto *array = make_array(NYRT_ARRAY_U8, loaded.bytes.size());
const auto *digest_text = own_string(digest);
if (!array || !digest_text) {
set_error(error_out, "file.read_rooted_snapshot: allocation failed");
return 0;
}
if (!loaded.bytes.empty())
std::memcpy(array->data, loaded.bytes.data(), loaded.bytes.size());
*bytes_out = array;
*modified_ms_out = loaded.modified_ms;
*sha256_out = digest_text;
return 1;
} catch (...) {
set_error(error_out, "file.read_rooted_snapshot: host operation failed");
return 0;
}
}

std::int32_t nyrt_read_regular_bytes(const char *path,
std::int64_t maximum_bytes,
void **bytes_out,
Expand Down
6 changes: 6 additions & 0 deletions nilcompiler/src/native_runtime.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,12 @@ std::int32_t nyrt_read_regular_bytes(const char *path,
std::int64_t maximum_bytes,
void **bytes_out,
const char **error_out) noexcept;
// All outputs describe the same bounded, rooted file read. The compiler owns
// the language record layout; this ABI only returns its independent fields.
std::int32_t nyrt_read_rooted_snapshot(
const char *root, const char *relative, std::int64_t maximum_bytes,
void **bytes_out, std::int64_t *modified_ms_out, const char **sha256_out,
const char **error_out) noexcept;
std::int32_t nyrt_write_bytes_atomic(const char *path, const void *bytes,
std::int32_t replace,
const char **error_out) noexcept;
Expand Down
Loading
Loading