From e4186b4f513c47bcc6504cd7046787cc48c03ca8 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 22 Oct 2026 10:47:20 +0200 Subject: [PATCH 01/38] add agave v4.3.2 --- hivezilla/direct-agave/Cargo.lock | 122 +++++++++++++------------- hivezilla/direct-agave/Cargo.toml | 4 +- hivezilla/direct-agave/README.md | 4 +- hivezilla/direct-agave/src/event.rs | 3 +- hivezilla/direct-agave/src/journal.rs | 4 +- 5 files changed, 70 insertions(+), 67 deletions(-) diff --git a/hivezilla/direct-agave/Cargo.lock b/hivezilla/direct-agave/Cargo.lock index 79bea19..6757628 100644 --- a/hivezilla/direct-agave/Cargo.lock +++ b/hivezilla/direct-agave/Cargo.lock @@ -20,8 +20,8 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "agave-feature-set" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "ahash", "solana-epoch-schedule", @@ -34,8 +34,8 @@ dependencies = [ [[package]] name = "agave-geyser-plugin-interface" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "log", "solana-clock", @@ -50,16 +50,16 @@ dependencies = [ [[package]] name = "agave-random" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "rand 0.9.4", ] [[package]] name = "agave-reserved-account-keys" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "solana-pubkey 4.3.0", @@ -86,8 +86,8 @@ dependencies = [ [[package]] name = "agave-votor-messages" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "bitvec", @@ -2868,8 +2868,8 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "Inflector", "base64", @@ -2912,8 +2912,8 @@ dependencies = [ [[package]] name = "solana-account-decoder-client-types" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "base64", "bs58", @@ -3044,8 +3044,8 @@ dependencies = [ [[package]] name = "solana-builtins-default-costs" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "ahash", @@ -3085,8 +3085,8 @@ checksum = "1517aa49dcfa9cb793ef90e7aac81346d62ca4a546bb1a754030a033e3972e1c" [[package]] name = "solana-compute-budget" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "solana-fee-structure", "solana-program-runtime", @@ -3094,8 +3094,8 @@ dependencies = [ [[package]] name = "solana-compute-budget-instruction" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "solana-borsh", @@ -3139,8 +3139,8 @@ dependencies = [ [[package]] name = "solana-cost-model" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "ahash", @@ -3227,8 +3227,8 @@ dependencies = [ [[package]] name = "solana-entry" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-votor-messages", "crossbeam-channel", @@ -3427,8 +3427,8 @@ dependencies = [ [[package]] name = "solana-leader-schedule" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-random", "itertools", @@ -3469,13 +3469,13 @@ dependencies = [ [[package]] name = "solana-measure" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" [[package]] name = "solana-merkle-tree" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "solana-hash", "solana-sha256-hasher", @@ -3502,8 +3502,8 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "crossbeam-channel", "gethostname", @@ -3565,8 +3565,8 @@ dependencies = [ [[package]] name = "solana-perf" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-transaction-view", "ahash", @@ -3651,8 +3651,8 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "base64", "bincode", @@ -3741,8 +3741,8 @@ dependencies = [ [[package]] name = "solana-runtime-transaction" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "agave-transaction-view", @@ -3969,8 +3969,8 @@ dependencies = [ [[package]] name = "solana-svm-callback" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "solana-account 4.5.0", "solana-clock", @@ -3980,26 +3980,26 @@ dependencies = [ [[package]] name = "solana-svm-feature-set" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" [[package]] name = "solana-svm-log-collector" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "log", ] [[package]] name = "solana-svm-measure" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" [[package]] name = "solana-svm-timings" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "eager", "enum-iterator", @@ -4022,8 +4022,8 @@ dependencies = [ [[package]] name = "solana-svm-type-overrides" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "rand 0.9.4", ] @@ -4133,8 +4133,8 @@ dependencies = [ [[package]] name = "solana-transaction-context" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "bincode", "serde", @@ -4145,6 +4145,7 @@ dependencies = [ "solana-rent", "solana-sbpf", "solana-sdk-ids", + "wincode", ] [[package]] @@ -4162,8 +4163,8 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "Inflector", "agave-reserved-account-keys", @@ -4206,8 +4207,8 @@ dependencies = [ [[package]] name = "solana-transaction-status-client-types" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "base64", "bincode", @@ -4229,8 +4230,8 @@ dependencies = [ [[package]] name = "solana-vote" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "log", "serde", @@ -4283,8 +4284,8 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "4.3.0" -source = "git+https://github.com/anza-xyz/agave?rev=825efd18292aff6ffcf9daa0f7612f21b3531a72#825efd18292aff6ffcf9daa0f7612f21b3531a72" +version = "4.3.2" +source = "git+https://github.com/anza-xyz/agave?rev=a203300045b405f86c9c84308b2eae76af69ac3b#a203300045b405f86c9c84308b2eae76af69ac3b" dependencies = [ "agave-feature-set", "bincode", @@ -4305,6 +4306,7 @@ dependencies = [ "solana-system-interface 3.3.0", "solana-transaction-context", "solana-vote-interface", + "wincode", ] [[package]] diff --git a/hivezilla/direct-agave/Cargo.toml b/hivezilla/direct-agave/Cargo.toml index d34e5cd..8edc808 100644 --- a/hivezilla/direct-agave/Cargo.toml +++ b/hivezilla/direct-agave/Cargo.toml @@ -34,8 +34,8 @@ wincode = { version = "=0.6.1", features = ["derive"] } blockzilla-archive-v2 = { path = "../../crates/compact-v2/blockzilla-archive-v2", default-features = false } blockzilla-compact = { path = "../../crates/compact-v2/blockzilla-compact", default-features = false } blockzilla-primitives = { path = "../../crates/blockzilla-primitives", default-features = false } -agave-geyser-plugin-interface = { git = "https://github.com/anza-xyz/agave", rev = "825efd18292aff6ffcf9daa0f7612f21b3531a72", features = ["agave-unstable-api"], optional = true } -solana-transaction-status = { git = "https://github.com/anza-xyz/agave", rev = "825efd18292aff6ffcf9daa0f7612f21b3531a72", features = ["agave-unstable-api"], optional = true } +agave-geyser-plugin-interface = { git = "https://github.com/anza-xyz/agave", rev = "a203300045b405f86c9c84308b2eae76af69ac3b", features = ["agave-unstable-api"], optional = true } +solana-transaction-status = { git = "https://github.com/anza-xyz/agave", rev = "a203300045b405f86c9c84308b2eae76af69ac3b", features = ["agave-unstable-api"], optional = true } solana-transaction = { version = "=4.2.0", features = ["wincode"], optional = true } solana-transaction-error = { version = "=3.4.0", features = ["wincode"], optional = true } solana-message = { version = "=4.5.0", features = ["wincode"], optional = true } diff --git a/hivezilla/direct-agave/README.md b/hivezilla/direct-agave/README.md index 2fc60f3..873ac34 100644 --- a/hivezilla/direct-agave/README.md +++ b/hivezilla/direct-agave/README.md @@ -16,7 +16,7 @@ It does not execute transactions a second time. archive catalog. Later root and footer evidence stays separate. Retirement requires a rooted custody proof that binds the payload and source bytes. - `libhivezilla_direct_agave.so`: Agave Geyser plugin. The interface and transaction - status crates use revision `825efd18292aff6ffcf9daa0f7612f21b3531a72` (4.3.0). + status crates use revision `a203300045b405f86c9c84308b2eae76af69ac3b` (v4.3.2). - `hivezilla-direct-agave build SESSION OUTPUT --blockzilla PATH --follow`: independent compact worker. The default batch is 32 blocks; the worker also limits source bytes per batch. It changes output directories at epoch boundaries. @@ -139,7 +139,7 @@ first bounded load and compare live rooted blocks before extending the run. ## Tested build and integration -The live Agave 4.3.0 executable uses Rust 1.97.1. This crate has its own workspace +The live Agave v4.3.2 executable uses Rust 1.97.1. This crate has its own workspace and lock file because the main CLI pins a different short-vec version. Build on the server with `cargo +1.97.1 build --release --features plugin --locked --ignore-rust-version`. The last flag permits shared internal crates that declare diff --git a/hivezilla/direct-agave/src/event.rs b/hivezilla/direct-agave/src/event.rs index 3703670..248c52e 100644 --- a/hivezilla/direct-agave/src/event.rs +++ b/hivezilla/direct-agave/src/event.rs @@ -5,8 +5,9 @@ use serde::{Deserialize, Serialize}; pub const REVISION: &str = match option_env!("HIVEZILLA_AGAVE_REVISION") { Some(revision) => revision, - None => "825efd18292aff6ffcf9daa0f7612f21b3531a72", + None => "a203300045b405f86c9c84308b2eae76af69ac3b", }; +pub const INTERFACE_VERSION: &str = "4.3.2"; pub const SCHEMA: &str = "hivezilla-agave-callback-event-v1"; pub const BANK_V2: &str = "hivezilla-agave-bank-observation-v2"; diff --git a/hivezilla/direct-agave/src/journal.rs b/hivezilla/direct-agave/src/journal.rs index 9be6bfc..8c0c842 100644 --- a/hivezilla/direct-agave/src/journal.rs +++ b/hivezilla/direct-agave/src/journal.rs @@ -1,4 +1,4 @@ -use crate::event::{Event, REVISION, SCHEMA, digest}; +use crate::event::{Event, INTERFACE_VERSION, REVISION, SCHEMA, digest}; use anyhow::{Context, Result, bail, ensure}; use crossbeam_channel::{Sender, bounded}; use rusqlite::{Connection, params}; @@ -143,7 +143,7 @@ impl Journal { reserved[4095] = b'\n'; failure_file.write_all(&reserved)?; failure_file.sync_all()?; - let session = serde_json::json!({"schema":SCHEMA,"payload_codec":"json-zstd-v1","retirement":"rooted-banks-v1","processed_custody":true,"bank_observations_v2":config.bank_v2,"agave_revision":REVISION,"interface_version":"4.3.0","session_id":session_id,"previous_unclean_sessions":previous_unclean,"config":config,"config_sha256":digest(&serde_json::to_vec(&config)?),"durability":"SQLite WAL synchronous=FULL; only committed rows are durable","unsealed_session_means_gap":true,"shreds":"separate receive hook; no completeness claim"}); + let session = serde_json::json!({"schema":SCHEMA,"payload_codec":"json-zstd-v1","retirement":"rooted-banks-v1","processed_custody":true,"bank_observations_v2":config.bank_v2,"agave_revision":REVISION,"interface_version":INTERFACE_VERSION,"session_id":session_id,"previous_unclean_sessions":previous_unclean,"config":config,"config_sha256":digest(&serde_json::to_vec(&config)?),"durability":"SQLite WAL synchronous=FULL; only committed rows are durable","unsealed_session_means_gap":true,"shreds":"separate receive hook; no completeness claim"}); atomic_json(&session_dir.join("SESSION.json"), &session)?; let mut db = open_writer(&session_dir.join("events.sqlite"))?; db.execute_batch("CREATE TABLE events(sequence INTEGER PRIMARY KEY, slot INTEGER NOT NULL, bank INTEGER, kind TEXT NOT NULL, sha256 TEXT NOT NULL, payload BLOB NOT NULL); CREATE INDEX bank_events ON events(slot,bank,sequence); CREATE INDEX roots ON events(kind,slot); CREATE TABLE status(id INTEGER PRIMARY KEY CHECK(id=1),state TEXT NOT NULL,durable_sequence INTEGER NOT NULL,error TEXT,latest_rooted_slot INTEGER NOT NULL,retired_events INTEGER NOT NULL); INSERT INTO status VALUES(1,'CAPTURING',0,NULL,0,0); CREATE TABLE retire_jobs(id TEXT PRIMARY KEY,bank_index INTEGER NOT NULL,deleted_events INTEGER NOT NULL,marker_sha TEXT NOT NULL);")?; From ccbbd6c91dc62ec551fac8e8a2dfcc4f61a37230 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 22 Oct 2026 11:53:11 +0200 Subject: [PATCH 02/38] add agave v4.3.2 for blockzilla --- blockzilla/cli/src/archive_v2/direct_agave.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/blockzilla/cli/src/archive_v2/direct_agave.rs b/blockzilla/cli/src/archive_v2/direct_agave.rs index 82c5316..8d15bc0 100644 --- a/blockzilla/cli/src/archive_v2/direct_agave.rs +++ b/blockzilla/cli/src/archive_v2/direct_agave.rs @@ -94,6 +94,7 @@ fn validate(r: &Receipt) -> Result<()> { ); ensure!( r.agave_revision == "825efd18292aff6ffcf9daa0f7612f21b3531a72" + || r.agave_revision == "a203300045b405f86c9c84308b2eae76af69ac3b" || option_env!("HIVEZILLA_AGAVE_REVISION") .is_some_and(|revision| r.agave_revision == revision), "unsupported Agave source revision" From c1227303dc464ac5a54d6a81ad9d5343793fca78 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 28 Oct 2026 11:09:40 +0100 Subject: [PATCH 03/38] add AG new info on the archive V2 --- AGENTS.md | 5 + blockzilla/archive-gateway/src/lib.rs | 5 + blockzilla/cli/src/archive_v2.rs | 55 ++- .../cli/src/archive_v2/registry_reprocess.rs | 52 +++ .../archive_v2/trusted_metadata_normalize.rs | 5 + blockzilla/cli/src/archive_verify.rs | 40 ++ .../src/bin/archive_v2_normalize_metadata.rs | 5 + .../cli/src/bin/archive_v2_pre_to_post.rs | 10 + blockzilla/cli/src/block_time_gaps.rs | 5 + blockzilla/cli/src/scheduler/mod.rs | 5 + .../src/convert/mod.rs | 5 + .../src/source_v2.rs | 4 + .../blockzilla-archive-v2/src/v2/mod.rs | 320 +++++++++++++++- .../src/archive_integrity.rs | 5 + .../src/archive_signatures.rs | 5 + .../src/compact_query.rs | 5 + .../src/reader.rs | 31 ++ .../src/reader/pipeline_tests.rs | 5 + .../src/bin/archive-v2-wire-profile-scan.rs | 5 + .../blockzilla-read-sdk-legacy/src/reader.rs | 25 ++ .../of-car-reader/src/car_block_group.rs | 91 +++++ crates/old-faithful/of-car-reader/src/node.rs | 348 +++++++++++++++++- .../wrangler.blockzilla-dev.jsonc | 20 + hivezilla/stream-relay/decoder/src/tests.rs | 5 + .../blockzilla-dump/tests/cases/scanner.rs | 4 + .../blockzilla-dump/tests/cases/verifier.rs | 5 + .../src/extract.rs | 15 + .../src/bin/archive-v2-account-projection.rs | 5 + ...r_program_index_wire_profile_transition.rs | 5 + runtime/blockzilla-replay/src/compact.rs | 5 + 30 files changed, 1066 insertions(+), 34 deletions(-) create mode 100644 edgezilla/foundation-watch/wrangler.blockzilla-dev.jsonc diff --git a/AGENTS.md b/AGENTS.md index 42c145f..f84a484 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,3 +8,8 @@ [the deployment record](hivezilla/deploy/direct-agave/README.md). Check live reports when current machine status is needed. Older handoff files are history. - Use ASD-STE100 Simplified Technical English when communicating with the user. +- Before you change code, write the plan: the files you will change, the + behavior you will keep, and the tests you will add. Wait for that plan to + be in the reply before you edit. +- After you change code, give a short record of the work: each file you + changed, what changed in it, and which tests passed. diff --git a/blockzilla/archive-gateway/src/lib.rs b/blockzilla/archive-gateway/src/lib.rs index 5e01887..8212279 100644 --- a/blockzilla/archive-gateway/src/lib.rs +++ b/blockzilla/archive-gateway/src/lib.rs @@ -1422,6 +1422,11 @@ mod tests { }], message_bytes: message, metadata_bytes: metadata, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); diff --git a/blockzilla/cli/src/archive_v2.rs b/blockzilla/cli/src/archive_v2.rs index 0bd4c63..6479609 100644 --- a/blockzilla/cli/src/archive_v2.rs +++ b/blockzilla/cli/src/archive_v2.rs @@ -21,12 +21,12 @@ use blockzilla_archive_v2::{ ArchiveV2BlockAccessBlob, ArchiveV2BlockAccessBlockhash, ArchiveV2BlockAccessIndexRow, ArchiveV2BlockAccessPubkey, ArchiveV2BlockAccessVoteHash, ArchiveV2ComputeBudgetInstructionData, ArchiveV2GetBlockIndexRow, ArchiveV2HotBlockBlob, - ArchiveV2HotBlockHeader, ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction, - ArchiveV2HotInstructionData, ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload, - ArchiveV2HotMetaRecord, ArchiveV2HotRewards, ArchiveV2HotTxRow, ArchiveV2HotV0Message, - ArchiveV2HotV1Message, ArchiveV2SystemInstructionData, ArchiveV2VoteHashRef, - ArchiveV2VoteLockoutOffset, ArchiveV2VoteStateUpdate, ArchiveV2VoteTowerSync, - BLOCK_TIME_GAP_FILE, WINCODE_ARCHIVE_V2_BLOCK_ACCESS_VERSION, + ArchiveV2HotBlockHeader, ArchiveV2HotBlockIndexRow, ArchiveV2HotConsensus, + ArchiveV2HotInstruction, ArchiveV2HotInstructionData, ArchiveV2HotLegacyMessage, + ArchiveV2HotMessagePayload, ArchiveV2HotMetaRecord, ArchiveV2HotRewards, ArchiveV2HotTxRow, + ArchiveV2HotV0Message, ArchiveV2HotV1Message, ArchiveV2SystemInstructionData, + ArchiveV2VoteHashRef, ArchiveV2VoteLockoutOffset, ArchiveV2VoteStateUpdate, + ArchiveV2VoteTowerSync, BLOCK_TIME_GAP_FILE, WINCODE_ARCHIVE_V2_BLOCK_ACCESS_VERSION, WINCODE_ARCHIVE_V2_FLAG_ALL_PUBKEY_REF_COUNTS, WINCODE_ARCHIVE_V2_FLAG_FIRST_SEEN_REGISTRY, WINCODE_ARCHIVE_V2_FLAG_LEB128, WINCODE_ARCHIVE_V2_FLAG_NO_REGISTRY, WINCODE_ARCHIVE_V2_HOT_BLOCK_VERSION, WINCODE_ARCHIVE_V2_VERSION, WincodeArchiveV2Block, @@ -687,7 +687,7 @@ pub(crate) fn build( .checked_sub(1) .and_then(|index| blockhashes.get(index)) .copied(); - let (record, tx_count, mut sidecar) = build_block_record( + let (record, tx_count, mut sidecar, _consensus) = build_block_record( &mut pending, block, &key_index, @@ -1000,7 +1000,7 @@ pub(crate) fn build_hot_blocks( .checked_sub(1) .and_then(|index| blockhashes.get(index)) .copied(); - let (mut record, tx_count, mut sidecar) = build_block_record( + let (mut record, tx_count, mut sidecar, consensus) = build_block_record( &mut pending, block, &key_index, @@ -1035,7 +1035,7 @@ pub(crate) fn build_hot_blocks( } else { None }; - let (hot_block, block_signature_count) = hot_block_from_archive_block( + let (mut hot_block, block_signature_count) = hot_block_from_archive_block( record, &known_program_ids, &slot_to_block_id, @@ -1046,6 +1046,7 @@ pub(crate) fn build_hot_blocks( &mut timings, ) .with_context(|| format!("slot {slot} hot block encode"))?; + apply_hot_consensus(&mut hot_block, consensus); patch_poh_entry_signature_counts(&mut sidecar.poh_entries, &hot_block.tx_rows) .with_context(|| format!("slot {slot} PoH entry signature counts"))?; block_bytes.clear(); @@ -1736,7 +1737,7 @@ pub(crate) fn build_hot_blocks_first_seen( timings.classify += classify_started.elapsed(); let previous_blockhash = last_blockhash.or_else(|| previous_tail.last().map(|previous| previous.hash)); - let (mut record, tx_count, mut sidecar) = build_block_record( + let (mut record, tx_count, mut sidecar, consensus) = build_block_record( &mut pending, block, &raw_key_index, @@ -1798,7 +1799,7 @@ pub(crate) fn build_hot_blocks_first_seen( known_program_ids.refresh_from_first_seen(®istry); vote_hashes.ensure_block(block_id); let block_shredding = std::mem::take(&mut record.header.compact.shredding); - let (hot_block, block_signature_count) = hot_block_from_first_seen_archive_block( + let (mut hot_block, block_signature_count) = hot_block_from_first_seen_archive_block( record, &first_seen_signatures, &known_program_ids, @@ -1809,6 +1810,7 @@ pub(crate) fn build_hot_blocks_first_seen( &mut timings, ) .with_context(|| format!("slot {slot} first-seen hot block encode"))?; + apply_hot_consensus(&mut hot_block, consensus); first_seen_signatures.record_block_write(); patch_poh_entry_signature_counts(&mut sidecar.poh_entries, &hot_block.tx_rows) .with_context(|| format!("slot {slot} PoH entry signature counts"))?; @@ -2692,7 +2694,7 @@ fn build_pre_hot_spool( let archive_block_id = block_id.saturating_add(blockhash_id_offset); let previous_blockhash = (archive_block_id > 0).then_some(last_blockhash).flatten(); rolling_blockhashes.prune_for_slot(block.slot)?; - let (record, tx_count, mut sidecar) = build_block_record( + let (record, tx_count, mut sidecar, _consensus) = build_block_record( &mut pending, block, &raw_key_index, @@ -6332,6 +6334,11 @@ fn hot_block_from_archive_block_with_signatures( tx_rows, message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, block_signature_count, )) @@ -6500,6 +6507,11 @@ fn hot_block_from_live_no_registry_block( tx_rows, message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, block_signature_count, )) @@ -6696,6 +6708,11 @@ fn hot_block_from_live_no_registry_block_signature_counts( tx_rows, message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, block_signature_count, )) @@ -17599,7 +17616,12 @@ fn build_block_record( zstd: &mut ZstdReusableDecoder, parallel_tx_decoder: Option<&mut HotTxDecodePool>, mut first_seen_signatures: Option<&mut FirstSeenBlockSignatures>, -) -> Result<(WincodeArchiveV2Block, u32, ArchiveV2BlockSidecar)> { +) -> Result<( + WincodeArchiveV2Block, + u32, + ArchiveV2BlockSidecar, + ArchiveV2HotConsensus, +)> { pending.last_slot = block.slot; let entries = ordered_entries(pending, &block)?; let sidecar = block_sidecar_from_entries( @@ -17765,9 +17787,16 @@ fn build_block_record( }, ordered_txs.len() as u32, sidecar, + ArchiveV2HotConsensus::from_slot_meta(&block.meta), )) } +fn apply_hot_consensus(block: &mut ArchiveV2HotBlockBlob, consensus: ArchiveV2HotConsensus) { + block.consensus_block_id = consensus.block_id; + block.parent_consensus_block_id = consensus.parent_block_id; + block.block_markers = consensus.markers; +} + fn build_no_registry_block_record( pending: &mut PendingBlock, block: RawBlockNode, diff --git a/blockzilla/cli/src/archive_v2/registry_reprocess.rs b/blockzilla/cli/src/archive_v2/registry_reprocess.rs index f39aee8..b8773b2 100644 --- a/blockzilla/cli/src/archive_v2/registry_reprocess.rs +++ b/blockzilla/cli/src/archive_v2/registry_reprocess.rs @@ -1784,6 +1784,10 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } } @@ -1808,6 +1812,10 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } } @@ -6841,6 +6849,11 @@ fn rewrite_source_block_with_scratch( tx_rows: std::mem::take(&mut scratch.rows), message_bytes: std::mem::take(&mut scratch.target_messages), metadata_bytes: std::mem::take(&mut scratch.target_metadata), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; (block, HotBlockOuterSchema::Current, access_references) } @@ -12643,6 +12656,11 @@ mod tests { }], message_bytes, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&encoded, 1).unwrap(); @@ -13466,6 +13484,11 @@ mod tests { }], message_bytes: Vec::new(), metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let analysis = analyze_source_exclusions(compressed_test_input(encoded, 1, 0)).unwrap(); @@ -13489,6 +13512,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let mut encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); encoded.push(0xa5); @@ -13674,6 +13702,10 @@ mod tests { ], message_bytes: [first_message, second_message, raw_message].concat(), metadata_bytes: [successful, failed, raw_metadata].concat(), + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } @@ -14219,6 +14251,11 @@ mod tests { ], message_bytes: raw_messages.to_vec(), metadata_bytes: [encoded_metadata, raw_metadata.to_vec()].concat(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let mut full: ArchiveV2HotBlockBlob = wincode::config::deserialize_exact( @@ -14345,6 +14382,11 @@ mod tests { ], message_bytes: [message_bytes, raw_message.to_vec()].concat(), metadata_bytes: [metadata_bytes, raw_metadata.to_vec()].concat(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let mut references = AccessReferenceSet::default(); super::super::collect_access_blockhash_id( @@ -14463,6 +14505,11 @@ mod tests { ], message_bytes: [first, second].concat(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let mut access = AccessReferenceSet::default(); @@ -14552,6 +14599,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let mut classes = Vec::new(); rewrite_block_pubkeys(&mut block, TEST_WIRE_PROFILE, |_key, class| { diff --git a/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs b/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs index fc5a794..1573db1 100644 --- a/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs +++ b/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs @@ -1985,6 +1985,11 @@ mod tests { )], message_bytes: message, metadata_bytes: metadata, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); diff --git a/blockzilla/cli/src/archive_verify.rs b/blockzilla/cli/src/archive_verify.rs index ee86ba4..bafba49 100644 --- a/blockzilla/cli/src/archive_verify.rs +++ b/blockzilla/cli/src/archive_verify.rs @@ -5023,6 +5023,11 @@ mod tests { tx_rows: vec![], message_bytes: vec![], metadata_bytes: vec![], + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let raw = wincode::config::serialize(&blob, blockzilla_primitives::wincode_leb128_config()) .unwrap(); @@ -5238,6 +5243,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, ArchiveV2HotBlockBlob { header: ArchiveV2HotBlockHeader { @@ -5262,6 +5272,11 @@ mod tests { }], message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, ]; let root = poh_migration_verify_fixture_root(label); @@ -5557,6 +5572,11 @@ mod tests { tx_rows: vec![row(1, 2), row(0, 1)], message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&hot_block, blockzilla_primitives::wincode_leb128_config()) @@ -5914,6 +5934,11 @@ mod tests { }], message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize( &hot_block, @@ -7511,6 +7536,11 @@ mod tests { ], message_bytes: vec![10, 11, 12, 13, 14], metadata_bytes: vec![20, 21, 22], + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&hot_block, blockzilla_primitives::wincode_leb128_config()) @@ -7530,6 +7560,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let empty_uncompressed = wincode::config::serialize( &empty_hot_block, @@ -7683,6 +7718,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&hot_block, blockzilla_primitives::wincode_leb128_config()) diff --git a/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs b/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs index fd98f51..a98a311 100644 --- a/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs +++ b/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs @@ -3542,6 +3542,11 @@ mod tests { tx_rows: vec![row], message_bytes: message.clone(), metadata_bytes: metadata, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); diff --git a/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs b/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs index b26d7f8..5848fdb 100644 --- a/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs +++ b/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs @@ -4623,6 +4623,11 @@ mod tests { }], message_bytes: message, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let owned = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let mut borrowed = Vec::new(); @@ -5438,6 +5443,11 @@ mod tests { tx_rows: vec![row], message_bytes: message.clone(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); diff --git a/blockzilla/cli/src/block_time_gaps.rs b/blockzilla/cli/src/block_time_gaps.rs index 4f02a51..e8ea8db 100644 --- a/blockzilla/cli/src/block_time_gaps.rs +++ b/blockzilla/cli/src/block_time_gaps.rs @@ -1266,6 +1266,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&encoded, 1).unwrap(); diff --git a/blockzilla/cli/src/scheduler/mod.rs b/blockzilla/cli/src/scheduler/mod.rs index 2d99b39..81ff9e0 100644 --- a/blockzilla/cli/src/scheduler/mod.rs +++ b/blockzilla/cli/src/scheduler/mod.rs @@ -30704,6 +30704,11 @@ mod tests { }], message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&encoded, 1).unwrap(); diff --git a/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs b/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs index 1363757..c9bd3b2 100644 --- a/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs +++ b/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs @@ -6385,6 +6385,11 @@ mod tests { tx_rows: vec![tx], message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, transactions: vec![ResolvedSourceTransaction { payload, diff --git a/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs b/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs index 2213584..55ae58e 100644 --- a/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs +++ b/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs @@ -2012,6 +2012,10 @@ mod tests { tx_rows: Vec::new(), message_bytes, metadata_bytes, + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } diff --git a/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs b/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs index b1df3dc..2a2c784 100644 --- a/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs +++ b/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs @@ -64,6 +64,15 @@ pub const WINCODE_ARCHIVE_V2_FLAG_FIRST_SEEN_REGISTRY: u32 = 1 << 2; /// Registry counts include every typed `CompactPubkey` reference, including rewards and logs. pub const WINCODE_ARCHIVE_V2_FLAG_ALL_PUBKEY_REF_COUNTS: u32 = 1 << 3; pub const ARCHIVE_V2_HOT_TX_ROW_LEN: usize = 28; +/// Marks the consensus tail appended after the historical hot-block payload. +/// +/// A block written before this tail has no bytes here. The value is not a +/// transaction count, so a historical payload cannot be read as a consensus tail. +pub const ARCHIVE_V2_HOT_CONSENSUS_MAGIC: u32 = 0xA7B1_0C4D; + +fn default_consensus_magic() -> u32 { + ARCHIVE_V2_HOT_CONSENSUS_MAGIC +} /// Header for the log-only wincode benchmark format. /// @@ -375,6 +384,63 @@ pub struct ArchiveV2HotBlockBlob { pub tx_rows: Vec, pub message_bytes: Vec, pub metadata_bytes: Vec, + /// Present on every newly written block. Historical bytes omit this tail. + #[serde(default = "default_consensus_magic")] + pub consensus_magic: u32, + /// Consensus block id of this block. Absent on a Tower block. + #[serde(default)] + pub consensus_block_id: Option<[u8; 32]>, + /// Consensus block id of the parent, taken from the last header or + /// update-parent marker. Absent when that marker is not present. + #[serde(default)] + pub parent_consensus_block_id: Option<[u8; 32]>, + /// Raw `VersionedBlockMarker` bytes, in source order. + #[serde(default)] + pub block_markers: Vec>, +} + +/// Consensus fields copied from a CAR `SlotMeta` onto a hot block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchiveV2HotConsensus { + pub block_id: Option<[u8; 32]>, + pub parent_block_id: Option<[u8; 32]>, + pub markers: Vec>, +} + +impl ArchiveV2HotConsensus { + pub fn from_slot_meta(meta: &SlotMeta) -> Self { + Self { + block_id: meta.block_id, + parent_block_id: meta.parent_consensus_block_id(), + markers: meta.block_markers.clone().unwrap_or_default(), + } + } +} + +/// Hot block payload written before the consensus tail existed. +#[derive(Debug, Serialize, Deserialize, SchemaRead, SchemaWrite)] +struct ArchiveV2HotBlockBlobLegacyPlain { + header: ArchiveV2HotBlockHeader, + tx_count: u32, + tx_rows: Vec, + message_bytes: Vec, + metadata_bytes: Vec, +} + +impl From for ArchiveV2HotBlockBlob { + fn from(value: ArchiveV2HotBlockBlobLegacyPlain) -> Self { + Self { + header: value.header, + tx_count: value.tx_count, + tx_rows: value.tx_rows, + message_bytes: value.message_bytes, + metadata_bytes: value.metadata_bytes, + consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), + } + } } /// A zero-copy view of the allocation-heavy fields in the current hot-block schema. @@ -392,6 +458,9 @@ pub struct BorrowedArchiveV2HotBlockBlob<'a> { tx_rows: &'a [[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], pub message_bytes: &'a [u8], pub metadata_bytes: &'a [u8], + pub consensus_block_id: Option<[u8; 32]>, + pub parent_consensus_block_id: Option<[u8; 32]>, + pub block_markers: Vec>, } /// Replay-oriented zero-copy view of a current hot block. @@ -408,6 +477,9 @@ pub struct BorrowedArchiveV2HotBlockBlobWithoutRewards<'a> { pub message_bytes: &'a [u8], pub metadata_bytes: &'a [u8], rewards_field_bytes: &'a [u8], + pub consensus_block_id: Option<[u8; 32]>, + pub parent_consensus_block_id: Option<[u8; 32]>, + pub block_markers: Vec>, } impl BorrowedArchiveV2HotBlockBlobWithoutRewards<'_> { @@ -525,6 +597,10 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, + consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } } @@ -569,6 +645,10 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, + consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } } @@ -602,16 +682,45 @@ pub fn deserialize_archive_v2_hot_block_slot_time(bytes: &[u8]) -> ReadResult<(u /// Read a historical hot-block source, preserving its schema fallback support. /// Integer spelling tolerance does not change the canonical writer grammar. +/// +/// A newly written block carries a consensus tail after the historical payload. +/// A block without that tail decodes as the previous schema and has an empty tail. pub fn deserialize_archive_v2_hot_block_blob(bytes: &[u8]) -> ReadResult { - match wincode::config::deserialize(bytes, historical_source_wincode_leb128_config()) { - Ok(block) => Ok(block), + if let Ok(block) = wincode::config::deserialize_exact::( + bytes, + historical_source_wincode_leb128_config(), + ) && block.consensus_magic == ARCHIVE_V2_HOT_CONSENSUS_MAGIC + { + return Ok(block); + } + let mut remaining = bytes; + if let Ok(block) = >::get(remaining.by_ref()) + { + if remaining.is_empty() { + return Ok(block.into()); + } + // The historical payload matched. Bytes after it are a consensus tail, including a + // truncated one. Do not try the older schemas: they can accept this prefix and hide + // the broken tail. + let (consensus_block_id, parent_consensus_block_id, block_markers) = + read_optional_consensus_tail::(&mut remaining)?; + if !remaining.is_empty() { + return Err(wincode::error::trailing_bytes()); + } + let mut block = ArchiveV2HotBlockBlob::from(block); + block.consensus_block_id = consensus_block_id; + block.parent_consensus_block_id = parent_consensus_block_id; + block.block_markers = block_markers; + return Ok(block); + } + match wincode::config::deserialize::( + bytes, + historical_source_wincode_leb128_config(), + ) { + Ok(block) => Ok(block.into()), Err(primary_error) => { - if let Ok(block) = wincode::config::deserialize::( - bytes, - historical_source_wincode_leb128_config(), - ) { - return Ok(block.into()); - } match wincode::config::deserialize::( bytes, historical_source_wincode_leb128_config(), @@ -634,19 +743,59 @@ pub fn deserialize_archive_v2_hot_block_blob(bytes: &[u8]) -> ReadResult ReadResult> { + if let Ok(( + header, + tx_count, + tx_rows, + message_bytes, + metadata_bytes, + consensus_magic, + consensus_block_id, + parent_consensus_block_id, + block_markers, + )) = wincode::config::deserialize_exact::< + ( + ArchiveV2HotBlockHeader, + u32, + &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], + &[u8], + &[u8], + u32, + Option<[u8; 32]>, + Option<[u8; 32]>, + Vec>, + ), + _, + >(bytes, historical_source_wincode_leb128_config()) + && consensus_magic == ARCHIVE_V2_HOT_CONSENSUS_MAGIC + { + return Ok(BorrowedArchiveV2HotBlockBlob { + header, + tx_count, + tx_rows, + message_bytes, + metadata_bytes, + consensus_block_id, + parent_consensus_block_id, + block_markers, + }); + } let (header, tx_count, tx_rows, message_bytes, metadata_bytes): ( ArchiveV2HotBlockHeader, u32, &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], &[u8], &[u8], - ) = wincode::config::deserialize(bytes, historical_source_wincode_leb128_config())?; + ) = wincode::config::deserialize_exact(bytes, historical_source_wincode_leb128_config())?; Ok(BorrowedArchiveV2HotBlockBlob { header, tx_count, tx_rows, message_bytes, metadata_bytes, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }) } @@ -660,22 +809,60 @@ pub fn deserialize_archive_v2_hot_block_blob_borrowed_current_with_preallocation >( bytes: &[u8], ) -> ReadResult> { + let config = bounded_historical_source_wincode_leb128_config::(); + if let Ok(( + header, + tx_count, + tx_rows, + message_bytes, + metadata_bytes, + consensus_magic, + consensus_block_id, + parent_consensus_block_id, + block_markers, + )) = wincode::config::deserialize_exact::< + ( + ArchiveV2HotBlockHeader, + u32, + &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], + &[u8], + &[u8], + u32, + Option<[u8; 32]>, + Option<[u8; 32]>, + Vec>, + ), + _, + >(bytes, config) + && consensus_magic == ARCHIVE_V2_HOT_CONSENSUS_MAGIC + { + return Ok(BorrowedArchiveV2HotBlockBlob { + header, + tx_count, + tx_rows, + message_bytes, + metadata_bytes, + consensus_block_id, + parent_consensus_block_id, + block_markers, + }); + } let (header, tx_count, tx_rows, message_bytes, metadata_bytes): ( ArchiveV2HotBlockHeader, u32, &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], &[u8], &[u8], - ) = wincode::config::deserialize_exact( - bytes, - bounded_historical_source_wincode_leb128_config::(), - )?; + ) = wincode::config::deserialize_exact(bytes, config)?; Ok(BorrowedArchiveV2HotBlockBlob { header, tx_count, tx_rows, message_bytes, metadata_bytes, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }) } @@ -744,7 +931,14 @@ pub fn deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards( &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], &[u8], &[u8], - ) = wincode::config::deserialize_exact(remaining, historical_source_wincode_leb128_config())?; + ) = <(u32, &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], &[u8], &[u8]) as SchemaRead< + HistoricalSourceWincodeLeb128Config, + >>::get(remaining.by_ref())?; + let (consensus_block_id, parent_consensus_block_id, block_markers) = + read_optional_consensus_tail::(&mut remaining)?; + if !remaining.is_empty() { + return Err(wincode::error::trailing_bytes()); + } Ok(BorrowedArchiveV2HotBlockBlobWithoutRewards { header: ArchiveV2HotBlockHeader { slot: header.slot, @@ -760,9 +954,28 @@ pub fn deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards( message_bytes, metadata_bytes, rewards_field_bytes, + consensus_block_id, + parent_consensus_block_id, + block_markers, }) } +fn read_optional_consensus_tail<'a, C: Config>( + bytes: &mut &'a [u8], +) -> ReadResult<(Option<[u8; 32]>, Option<[u8; 32]>, Vec>)> { + if bytes.is_empty() { + return Ok((None, None, Vec::new())); + } + let magic = >::get(bytes.by_ref())?; + if magic != ARCHIVE_V2_HOT_CONSENSUS_MAGIC { + return Err(wincode::error::invalid_tag_encoding(magic as usize)); + } + let consensus_block_id = as SchemaRead<'a, C>>::get(bytes.by_ref())?; + let parent_consensus_block_id = as SchemaRead<'a, C>>::get(bytes.by_ref())?; + let block_markers = > as SchemaRead<'a, C>>::get(bytes.by_ref())?; + Ok((consensus_block_id, parent_consensus_block_id, block_markers)) +} + #[cfg(test)] mod hot_block_slot_time_tests { use super::*; @@ -803,6 +1016,10 @@ mod hot_block_slot_time_tests { ], message_bytes: vec![10, 11, 12, 13, 14], metadata_bytes: vec![20, 21, 22], + consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } @@ -834,6 +1051,12 @@ mod hot_block_slot_time_tests { assert_eq!(replay.tx_rows().collect::>(), owned.tx_rows); assert_eq!(replay.message_bytes, owned.message_bytes); assert_eq!(replay.metadata_bytes, owned.metadata_bytes); + assert_eq!(replay.consensus_block_id, owned.consensus_block_id); + assert_eq!( + replay.parent_consensus_block_id, + owned.parent_consensus_block_id + ); + assert_eq!(replay.block_markers, owned.block_markers); } fn assert_exact_rewards_field(block: ArchiveV2HotBlockBlob) { @@ -863,6 +1086,65 @@ mod hot_block_slot_time_tests { ); } + #[test] + fn consensus_tail_round_trips_and_legacy_bytes_stay_empty() { + let mut block = current_hot_block_fixture(); + block.consensus_block_id = Some([0x21; 32]); + block.parent_consensus_block_id = Some([0x43; 32]); + block.block_markers = vec![vec![1, 0, 1, 4, 0], vec![9, 9]]; + let bytes = + wincode::config::serialize(&block, historical_source_wincode_leb128_config()).unwrap(); + let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); + assert_eq!(decoded.consensus_block_id, block.consensus_block_id); + assert_eq!( + decoded.parent_consensus_block_id, + block.parent_consensus_block_id + ); + assert_eq!(decoded.block_markers, block.block_markers); + assert_eq!(decoded.tx_count, block.tx_count); + let borrowed = deserialize_archive_v2_hot_block_blob_borrowed_current(&bytes).unwrap(); + assert_eq!(borrowed.consensus_block_id, Some([0x21; 32])); + assert_eq!(borrowed.parent_consensus_block_id, Some([0x43; 32])); + assert_eq!(borrowed.block_markers, block.block_markers); + let replay = + deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards(&bytes).unwrap(); + assert_eq!(replay.consensus_block_id, Some([0x21; 32])); + assert_eq!(replay.parent_consensus_block_id, Some([0x43; 32])); + assert_eq!(replay.block_markers, block.block_markers); + let exact = + deserialize_archive_v2_hot_block_blob_borrowed_current_with_preallocation_limit::< + { ARCHIVE_V2_DECODE_PREALLOCATION_LIMIT_BYTES }, + >(&bytes) + .unwrap(); + assert_eq!(exact.consensus_block_id, Some([0x21; 32])); + assert_eq!(exact.block_markers, block.block_markers); + + let legacy = ArchiveV2HotBlockBlobLegacyPlain { + header: block.header, + tx_count: 0, + tx_rows: Vec::new(), + message_bytes: Vec::new(), + metadata_bytes: Vec::new(), + }; + let legacy_bytes = + wincode::config::serialize(&legacy, historical_source_wincode_leb128_config()).unwrap(); + let decoded = deserialize_archive_v2_hot_block_blob(&legacy_bytes).unwrap(); + assert_eq!(decoded.consensus_block_id, None); + assert_eq!(decoded.parent_consensus_block_id, None); + assert!(decoded.block_markers.is_empty()); + assert_eq!(decoded.tx_count, 0); + let borrowed = + deserialize_archive_v2_hot_block_blob_borrowed_current(&legacy_bytes).unwrap(); + assert_eq!(borrowed.consensus_block_id, None); + assert!(borrowed.block_markers.is_empty()); + let replay = + deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards(&legacy_bytes) + .unwrap(); + assert_eq!(replay.consensus_block_id, None); + assert!(replay.block_markers.is_empty()); + assert_eq!(replay.tx_count, 0); + } + #[test] fn prefix_decoder_supports_current_hot_block_encoding() { let block = current_hot_block_fixture(); @@ -1280,6 +1562,16 @@ mod hot_block_slot_time_tests { assert!(deserialize_archive_v2_hot_block_blob_borrowed_current(&bytes).is_err()); assert!(deserialize_archive_v2_hot_block_blob(&bytes).is_err()); + + let mut empty = current_hot_block_fixture(); + empty.tx_count = 0; + empty.tx_rows.clear(); + empty.message_bytes.clear(); + empty.metadata_bytes.clear(); + let mut empty_bytes = wincode::config::serialize(&empty, wincode_leb128_config()).unwrap(); + empty_bytes.pop(); + assert!(deserialize_archive_v2_hot_block_blob_borrowed_current(&empty_bytes).is_err()); + assert!(deserialize_archive_v2_hot_block_blob(&empty_bytes).is_err()); } #[test] diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs index ae56014..dc482d2 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs @@ -3024,6 +3024,11 @@ mod tests { tx_rows, message_bytes, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 1).unwrap(); diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs index dc9a047..f1dfc40 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs @@ -714,6 +714,11 @@ mod tests { }], message_bytes, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 1).unwrap(); diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs index d04ab5a..a9d31f8 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs @@ -3823,6 +3823,11 @@ mod tests { tx_rows: rows, message_bytes: messages, metadata_bytes: metadata, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&blob, wincode_leb128_config()).unwrap(); diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs index 258a223..a4b169e 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs @@ -614,6 +614,10 @@ impl BorrowedDecodedBlock<'_> { tx_rows, message_bytes, metadata_bytes, + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: block.consensus_block_id, + parent_consensus_block_id: block.parent_consensus_block_id, + block_markers: block.block_markers, } } BorrowedDecodedBlockPayload::CurrentWithoutRewards(_) => { @@ -4838,6 +4842,10 @@ mod tests { ], message_bytes: first_messages, metadata_bytes: Vec::new(), + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: Some([0x21; 32]), + parent_consensus_block_id: Some([0x43; 32]), + block_markers: vec![vec![1, 2, 3]], }; let second_message = ArchiveV2HotMessagePayload::V0(ArchiveV2HotV0Message { @@ -4892,6 +4900,11 @@ mod tests { }], message_bytes: second_message, metadata_bytes: second_metadata, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let first_uncompressed = @@ -4993,6 +5006,11 @@ mod tests { tx_rows: Vec::new(), message_bytes: Vec::new(), metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -5139,6 +5157,11 @@ mod tests { }], message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, } } @@ -6517,6 +6540,9 @@ mod tests { let converted = decoded.into_owned().unwrap(); assert_eq!(converted.index_row.block_id, expected.index_row.block_id); assert_eq!(converted.index_row.slot, expected.index_row.slot); + assert_eq!(converted.block.consensus_block_id, Some([0x21; 32])); + assert_eq!(converted.block.parent_consensus_block_id, Some([0x43; 32])); + assert_eq!(converted.block.block_markers, vec![vec![1, 2, 3]]); assert_eq!( wincode::config::serialize(&converted.block, wincode_leb128_config()).unwrap(), expected_bytes @@ -6880,6 +6906,11 @@ mod tests { tx_rows: rows, message_bytes, metadata_bytes: vec![0xaa], + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, }; let scanned = archive.scan_decoded_block(&filter, decoded).unwrap(); diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs index 905a1c4..97e1a34 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs @@ -423,6 +423,11 @@ mod incremental_pipeline_regressions { .collect(), message_bytes: vec![0; transactions_per_block as usize], metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 1).unwrap(); diff --git a/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs b/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs index 70f0bf7..5b4a16b 100644 --- a/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs +++ b/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs @@ -1517,6 +1517,11 @@ mod tests { ], message_bytes, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); diff --git a/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs b/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs index c8fe20f..12c155a 100644 --- a/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs +++ b/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs @@ -4515,6 +4515,11 @@ mod tests { ], message_bytes: first_messages, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let second_message = ArchiveV2HotMessagePayload::V0(ArchiveV2HotV0Message { @@ -4569,6 +4574,11 @@ mod tests { }], message_bytes: second_message, metadata_bytes: second_metadata, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let first_uncompressed = @@ -5106,6 +5116,11 @@ mod tests { tx_rows: vec![row], message_bytes: message, metadata_bytes: vec![], + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, }; let filter = archive.compile_pubkey_filter([RAW_KEY]).unwrap(); @@ -6804,6 +6819,11 @@ mod tests { tx_rows: rows, message_bytes, metadata_bytes: vec![0xaa], + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, }; let scanned = archive.scan_decoded_block(&filter, decoded).unwrap(); @@ -7035,6 +7055,11 @@ mod tests { tx_rows: vec![row], message_bytes: Vec::new(), metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; assert!(matches!( diff --git a/crates/old-faithful/of-car-reader/src/car_block_group.rs b/crates/old-faithful/of-car-reader/src/car_block_group.rs index 650102c..065bbcb 100644 --- a/crates/old-faithful/of-car-reader/src/car_block_group.rs +++ b/crates/old-faithful/of-car-reader/src/car_block_group.rs @@ -89,6 +89,12 @@ pub struct CarBlockGroup { pub parent_slot: Option, pub block_time: Option, pub block_height: Option, + /// Consensus block id from a five-element SlotMeta. Empty on a Tower block. + pub block_id: Option<[u8; 32]>, + /// Raw Alpenglow block markers. Empty when the SlotMeta has no marker array. + pub block_markers: Vec>, + /// True when the SlotMeta array has the Alpenglow tail, including a null tail. + pub alpenglow: bool, // blockhash (from Entry.hash - last seen) pub blockhash: [u8; 32], @@ -162,6 +168,9 @@ impl CarBlockGroup { parent_slot: None, block_time: None, block_height: None, + block_id: None, + block_markers: Vec::new(), + alpenglow: false, blockhash: [0; 32], has_blockhash: false, @@ -213,6 +222,9 @@ impl CarBlockGroup { self.parent_slot = None; self.block_time = None; self.block_height = None; + self.block_id = None; + self.block_markers.clear(); + self.alpenglow = false; self.blockhash = [0; 32]; self.has_blockhash = false; @@ -584,6 +596,9 @@ impl CarBlockGroup { self.parent_slot = block.meta.parent_slot; self.block_time = block.meta.blocktime; self.block_height = block.meta.block_height; + self.block_id = block.meta.block_id; + self.block_markers = block.meta.block_markers.clone().unwrap_or_default(); + self.alpenglow = block.meta.alpenglow(); return Ok(true); } @@ -826,6 +841,9 @@ mod tests { assert_eq!(group.parent_slot, Some(41)); assert_eq!(group.block_time, Some(1_700_000_000)); assert_eq!(group.block_height, Some(99)); + assert!(!group.alpenglow); + assert_eq!(group.block_id, None); + assert!(group.block_markers.is_empty()); assert_eq!(group.get_len().0, 2); assert_eq!(group.entry_count(), 2); @@ -857,6 +875,43 @@ mod tests { assert_eq!(rewards.data, vec![0xaa, 0xbb, 0xcc]); } + #[test] + fn reads_alpenglow_slot_meta_onto_the_group() { + let marker = vec![9, 8, 7]; + let block_id = [0x44; 32]; + let mut car = Vec::new(); + push_uvarint(&mut car, 1); + car.push(0); + push_car_entry(&mut car, &[0x01; 36], &transaction_node(42, &[0x10])); + push_car_entry(&mut car, &[0x02; 36], &transaction_node(42, &[0x20])); + push_car_entry(&mut car, &[0x03; 36], &entry_node(3, [0x11; 32], 1)); + push_car_entry(&mut car, &[0x04; 36], &entry_node(4, [0x22; 32], 1)); + push_car_entry( + &mut car, + &[0x05; 36], + &rewards_node(42, Some(7), Some(0), Some(1), &[0xaa, 0xbb, 0xcc]), + ); + push_car_entry( + &mut car, + &[0x06; 36], + &block_node_alpenglow(42, 41, 1_700_000_000, 99, &marker, &block_id), + ); + + let mut reader = CarBlockReader::with_capacity(Cursor::new(car), 1024); + reader.skip_header().expect("skip header"); + let mut group = CarBlockGroup::new(); + assert!(reader.read_until_block_into(&mut group).expect("read")); + assert!(group.alpenglow); + assert_eq!(group.block_id, Some(block_id)); + assert_eq!(group.block_markers, vec![marker]); + assert_eq!(group.parent_slot, Some(41)); + + group.clear(); + assert!(!group.alpenglow); + assert_eq!(group.block_id, None); + assert!(group.block_markers.is_empty()); + } + #[test] fn reads_inline_identity_rewards_from_block_ref() { let mut car = Vec::new(); @@ -1473,6 +1528,42 @@ mod tests { e.into_writer() } + fn block_node_alpenglow( + slot: u64, + parent_slot: u64, + block_time: i64, + block_height: u64, + marker: &[u8], + block_id: &[u8; 32], + ) -> Vec { + let mut e = Encoder::new(Vec::new()); + e.array(6).expect("block array").u64(2).expect("kind"); + e.u64(slot).expect("slot"); + + e.array(2).expect("shredding"); + e.array(2).expect("shred 0"); + e.i64(10).expect("entry_end_idx 0"); + e.i64(11).expect("shred_end_idx 0"); + e.array(2).expect("shred 1"); + e.i64(20).expect("entry_end_idx 1"); + e.i64(21).expect("shred_end_idx 1"); + + e.array(2).expect("entries"); + encode_cid_ref(&mut e, 0x40); + encode_cid_ref(&mut e, 0x41); + + e.array(5).expect("meta"); + e.u64(parent_slot).expect("parent_slot"); + e.i64(block_time).expect("block_time"); + e.u64(block_height).expect("block_height"); + e.array(1).expect("markers"); + e.bytes(marker).expect("marker"); + e.bytes(block_id).expect("block id"); + + encode_cid_ref(&mut e, 0x50); + e.into_writer() + } + fn block_node_with_inline_rewards( slot: u64, parent_slot: u64, diff --git a/crates/old-faithful/of-car-reader/src/node.rs b/crates/old-faithful/of-car-reader/src/node.rs index 3f34f8a..b78edfc 100644 --- a/crates/old-faithful/of-car-reader/src/node.rs +++ b/crates/old-faithful/of-car-reader/src/node.rs @@ -190,15 +190,233 @@ impl<'a> From<&DataFrame<'a>> for OwnedDataFrame { } } -#[derive(Debug, Decode, Encode, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[cbor(array)] +/// Block metadata from an Old Faithful CAR block node. +/// +/// A Tower-era array has three elements. An Alpenglow array adds the raw block +/// markers and the 32-byte consensus block id. `meta_len` is the CBOR array +/// length, so a null tail still counts as Alpenglow. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SlotMeta { - #[n(0)] pub parent_slot: Option, - #[n(1)] pub blocktime: Option, - #[n(2)] pub block_height: Option, + /// `None` when the field is absent or CBOR null. `Some` is the marker array, + /// in source order. Each marker is the raw `VersionedBlockMarker` bytes. + #[serde(default)] + pub block_markers: Option>>, + /// Consensus block id. Absent or CBOR null on a Tower block. + #[serde(default)] + pub block_id: Option<[u8; 32]>, + /// CBOR array length. Three means Tower. More than three means Alpenglow. + #[serde(default = "tower_slot_meta_len")] + pub meta_len: u64, +} + +const fn tower_slot_meta_len() -> u64 { + 3 +} + +impl SlotMeta { + /// Parent consensus block id from the last header or update-parent marker. + /// + /// Other markers, including the footer and its certificates, stay raw. + pub fn parent_consensus_block_id(&self) -> Option<[u8; 32]> { + self.block_markers + .as_deref() + .and_then(parent_consensus_block_id_from_markers) + } + + pub fn alpenglow(&self) -> bool { + self.meta_len > 3 + } +} + +fn parent_consensus_block_id_from_markers(markers: &[Vec]) -> Option<[u8; 32]> { + markers.iter().rev().find_map(|marker| { + let bytes = marker.as_slice(); + if bytes.len() < 46 { + return None; + } + let version = u16::from_le_bytes([bytes[0], bytes[1]]); + let variant = bytes[2]; + let payload_len = u16::from_le_bytes([bytes[3], bytes[4]]); + if version != 1 || (variant != 1 && variant != 2) || payload_len != 41 { + return None; + } + if bytes.len() != 46 || bytes[5] != 1 { + return None; + } + let mut parent_block_id = [0u8; 32]; + parent_block_id.copy_from_slice(&bytes[14..46]); + Some(parent_block_id) + }) +} + +impl<'b, C> Decode<'b, C> for SlotMeta { + fn decode(d: &mut Decoder<'b>, _ctx: &mut C) -> core::result::Result { + let len = d + .array()? + .ok_or_else(|| CborError::message("indefinite slot meta array not supported"))?; + let parent_slot = if len >= 1 { + decode_optional_u64(d)? + } else { + None + }; + let blocktime = if len >= 2 { + decode_optional_i64(d)? + } else { + None + }; + let block_height = if len >= 3 { + decode_optional_u64(d)? + } else { + None + }; + let block_markers = if len >= 4 { + Some(decode_block_markers(d)?) + } else { + None + }; + let block_id = if len >= 5 { + Some(decode_consensus_block_id(d)?) + } else { + None + }; + for _ in 5..len { + d.skip()?; + } + Ok(Self { + parent_slot, + blocktime, + block_height, + block_markers: block_markers.flatten(), + block_id: block_id.flatten(), + meta_len: len, + }) + } +} + +fn decode_optional_i64(d: &mut Decoder<'_>) -> core::result::Result, CborError> { + match d.datatype()? { + Type::Null => { + d.null()?; + Ok(None) + } + Type::U8 | Type::U16 | Type::U32 | Type::U64 => { + let value = d.u64()?; + i64::try_from(value) + .map(Some) + .map_err(|_| CborError::message("slot meta integer does not fit in i64")) + } + Type::I8 | Type::I16 | Type::I32 | Type::I64 => Ok(Some(d.i64()?)), + other => Err(CborError::type_mismatch(other)), + } +} + +fn decode_block_markers( + d: &mut Decoder<'_>, +) -> core::result::Result>>, CborError> { + match d.datatype()? { + Type::Null => { + d.null()?; + Ok(None) + } + Type::Array => { + let count = d + .array()? + .ok_or_else(|| CborError::message("indefinite block marker array not supported"))?; + let mut markers = Vec::with_capacity(usize::try_from(count).unwrap_or(0)); + for _ in 0..count { + let bytes = d.bytes()?; + markers.push(bytes.to_vec()); + } + Ok(Some(markers)) + } + other => Err(CborError::type_mismatch(other)), + } +} + +fn decode_consensus_block_id( + d: &mut Decoder<'_>, +) -> core::result::Result, CborError> { + match d.datatype()? { + Type::Null => { + d.null()?; + Ok(None) + } + Type::Bytes => { + let bytes = d.bytes()?; + let id: [u8; 32] = bytes + .try_into() + .map_err(|_| CborError::message("consensus block id must be 32 bytes"))?; + Ok(Some(id)) + } + other => Err(CborError::type_mismatch(other)), + } +} + +impl Encode for SlotMeta { + fn encode( + &self, + e: &mut minicbor::Encoder, + _ctx: &mut C, + ) -> core::result::Result<(), minicbor::encode::Error> { + let len = self.meta_len.min(5); + e.array(len)?; + if len >= 1 { + encode_option_u64(e, self.parent_slot)?; + } + if len >= 2 { + encode_option_i64(e, self.blocktime)?; + } + if len >= 3 { + encode_option_u64(e, self.block_height)?; + } + if len >= 4 { + match &self.block_markers { + None => { + e.null()?; + } + Some(markers) => { + e.array(markers.len() as u64)?; + for marker in markers { + e.bytes(marker)?; + } + } + } + } + if len >= 5 { + match self.block_id { + None => { + e.null()?; + } + Some(id) => { + e.bytes(&id)?; + } + } + } + Ok(()) + } +} + +fn encode_option_u64( + e: &mut minicbor::Encoder, + value: Option, +) -> core::result::Result<(), minicbor::encode::Error> { + match value { + None => e.null().map(|_| ()), + Some(value) => e.u64(value).map(|_| ()), + } +} + +fn encode_option_i64( + e: &mut minicbor::Encoder, + value: Option, +) -> core::result::Result<(), minicbor::encode::Error> { + match value { + None => e.null().map(|_| ()), + Some(value) => e.i64(value).map(|_| ()), + } } #[derive(Debug, Decode, Encode, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -633,8 +851,8 @@ pub fn decode_entry_summary(data: &[u8]) -> crate::node::Result<(u64, &[u8], usi #[cfg(test)] mod tests { - use super::{DataFrame, Node, decode_node}; - use minicbor::{Decode, Decoder, Encoder}; + use super::{DataFrame, Node, SlotMeta, decode_node}; + use minicbor::{Decode, Decoder, Encode, Encoder}; #[test] fn dataframe_rejects_unsupported_next_array_shapes() { @@ -696,4 +914,120 @@ mod tests { assert_eq!(frame.hash, Some(42)); assert!(!frame.hash_was_negative); } + + fn header_marker(variant: u8, parent_slot: u64, parent_id: [u8; 32]) -> Vec { + let mut marker = Vec::new(); + marker.extend_from_slice(&1u16.to_le_bytes()); + marker.push(variant); + marker.extend_from_slice(&41u16.to_le_bytes()); + marker.push(1); + marker.extend_from_slice(&parent_slot.to_le_bytes()); + marker.extend_from_slice(&parent_id); + marker + } + + #[test] + fn slot_meta_keeps_three_fields_and_round_trips() { + let mut e = Encoder::new(Vec::new()); + e.array(3).unwrap(); + e.u64(41).unwrap(); + e.i64(1_700_000_000).unwrap(); + e.u64(99).unwrap(); + let bytes = e.into_writer(); + + let meta = SlotMeta::decode(&mut Decoder::new(&bytes), &mut ()).unwrap(); + assert!(!meta.alpenglow()); + assert_eq!(meta.parent_slot, Some(41)); + assert_eq!(meta.blocktime, Some(1_700_000_000)); + assert_eq!(meta.block_height, Some(99)); + assert_eq!(meta.block_markers, None); + assert_eq!(meta.block_id, None); + assert_eq!(meta.parent_consensus_block_id(), None); + + let mut encoded = Vec::new(); + meta.encode(&mut Encoder::new(&mut encoded), &mut ()) + .unwrap(); + assert_eq!(encoded, bytes); + } + + #[test] + fn slot_meta_keeps_a_short_parent_only_array() { + let mut e = Encoder::new(Vec::new()); + e.array(1).unwrap(); + e.u64(40).unwrap(); + let bytes = e.into_writer(); + + let meta = SlotMeta::decode(&mut Decoder::new(&bytes), &mut ()).unwrap(); + assert!(!meta.alpenglow()); + assert_eq!(meta.meta_len, 1); + assert_eq!(meta.parent_slot, Some(40)); + assert_eq!(meta.blocktime, None); + assert_eq!(meta.block_height, None); + + let mut encoded = Vec::new(); + meta.encode(&mut Encoder::new(&mut encoded), &mut ()) + .unwrap(); + assert_eq!(encoded, bytes); + } + + #[test] + fn slot_meta_keeps_raw_markers_and_block_id() { + let parent_id = [0x11; 32]; + let updated_parent = [0x22; 32]; + let block_id = [0x33; 32]; + let header = header_marker(1, 40, parent_id); + let update = header_marker(2, 39, updated_parent); + let footer = vec![1, 0, 0, 4, 0, 9, 9, 9]; + let mut e = Encoder::new(Vec::new()); + e.array(5).unwrap(); + e.u64(41).unwrap(); + e.null().unwrap(); + e.u64(99).unwrap(); + e.array(3).unwrap(); + e.bytes(&header).unwrap(); + e.bytes(&footer).unwrap(); + e.bytes(&update).unwrap(); + e.bytes(&block_id).unwrap(); + let bytes = e.into_writer(); + + let meta = SlotMeta::decode(&mut Decoder::new(&bytes), &mut ()).unwrap(); + assert!(meta.alpenglow()); + assert_eq!(meta.blocktime, None); + assert_eq!( + meta.block_markers.as_ref().unwrap(), + &vec![header, footer, update.clone()] + ); + assert_eq!(meta.block_id, Some(block_id)); + assert_eq!(meta.parent_consensus_block_id(), Some(updated_parent)); + + let mut encoded = Vec::new(); + meta.encode(&mut Encoder::new(&mut encoded), &mut ()) + .unwrap(); + let again = SlotMeta::decode(&mut Decoder::new(&encoded), &mut ()).unwrap(); + assert_eq!(again, meta); + } + + #[test] + fn slot_meta_rejects_a_short_block_id_and_keeps_null_tail() { + let mut e = Encoder::new(Vec::new()); + e.array(5).unwrap(); + e.u64(1).unwrap(); + e.u64(2).unwrap(); + e.u64(3).unwrap(); + e.null().unwrap(); + e.bytes(&[1, 2, 3]).unwrap(); + assert!(SlotMeta::decode(&mut Decoder::new(&e.into_writer()), &mut ()).is_err()); + + let mut e = Encoder::new(Vec::new()); + e.array(5).unwrap(); + e.null().unwrap(); + e.null().unwrap(); + e.null().unwrap(); + e.null().unwrap(); + e.null().unwrap(); + let meta = SlotMeta::decode(&mut Decoder::new(&e.into_writer()), &mut ()).unwrap(); + assert!(meta.alpenglow()); + assert_eq!(meta.block_markers, None); + assert_eq!(meta.block_id, None); + } } diff --git a/edgezilla/foundation-watch/wrangler.blockzilla-dev.jsonc b/edgezilla/foundation-watch/wrangler.blockzilla-dev.jsonc new file mode 100644 index 0000000..2cd56b8 --- /dev/null +++ b/edgezilla/foundation-watch/wrangler.blockzilla-dev.jsonc @@ -0,0 +1,20 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "blockzilla-dev-watch", + "account_id": "4d69e0b71fb882e3d955c871a84a53c3", + "main": "src/index.ts", + "compatibility_date": "2026-10-12", + "compatibility_flags": ["nodejs_compat"], + "workers_dev": true, + "vars": { + "MACHINE_LABEL": "blockzilla-dev", + "TELEGRAM_BOT_USERNAME": "REMPLACER_NOM_DU_BOT", + "STREAM_URL": "https://stream.blockzilla.dev/v1/blockhashes" + }, + "durable_objects": { + "bindings": [{ "name": "WATCH", "class_name": "FoundationWatch" }] + }, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["FoundationWatch"] }], + "triggers": { "crons": ["* * * * *"] }, + "observability": { "enabled": true, "head_sampling_rate": 1 } +} diff --git a/hivezilla/stream-relay/decoder/src/tests.rs b/hivezilla/stream-relay/decoder/src/tests.rs index 89b91ef..6fc87d0 100644 --- a/hivezilla/stream-relay/decoder/src/tests.rs +++ b/hivezilla/stream-relay/decoder/src/tests.rs @@ -127,6 +127,11 @@ fn fixture_with_tokens(loaded: bool, failed: bool, id: u32, tokens: bool) -> Vec }], message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let raw = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let mut index = b"BZV2HIX1".to_vec(); diff --git a/indexer/blockzilla-dump/tests/cases/scanner.rs b/indexer/blockzilla-dump/tests/cases/scanner.rs index f61ccd3..9985680 100644 --- a/indexer/blockzilla-dump/tests/cases/scanner.rs +++ b/indexer/blockzilla-dump/tests/cases/scanner.rs @@ -242,6 +242,10 @@ fn block(slot: u64, message: Vec, metadata: Vec, flags: u32) -> ArchiveV }], message_bytes: message, metadata_bytes: metadata, + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), } } diff --git a/indexer/blockzilla-dump/tests/cases/verifier.rs b/indexer/blockzilla-dump/tests/cases/verifier.rs index 71199e8..c4bbf4a 100644 --- a/indexer/blockzilla-dump/tests/cases/verifier.rs +++ b/indexer/blockzilla-dump/tests/cases/verifier.rs @@ -241,6 +241,11 @@ fn build_epoch_zero(root: &Path) { }], message_bytes: message, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 1).unwrap(); diff --git a/indexer/blockzilla-token-transaction-dump/src/extract.rs b/indexer/blockzilla-token-transaction-dump/src/extract.rs index 400a81d..ae6390e 100644 --- a/indexer/blockzilla-token-transaction-dump/src/extract.rs +++ b/indexer/blockzilla-token-transaction-dump/src/extract.rs @@ -8824,6 +8824,11 @@ mod tests { }], message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); @@ -9059,6 +9064,11 @@ mod tests { }], message_bytes, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); @@ -9217,6 +9227,11 @@ mod tests { tx_rows, message_bytes, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap(); diff --git a/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs b/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs index 85266b2..d665401 100644 --- a/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs +++ b/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs @@ -8118,6 +8118,11 @@ mod tests { tx_rows: vec![tx_row(message_bytes.len(), metadata_bytes.len(), flags, 1)], message_bytes, metadata_bytes, + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = serialize(&block); let compressed = zstd::bulk::compress(&uncompressed, 1).unwrap(); diff --git a/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs b/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs index 43073a5..1bbc1e7 100644 --- a/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs +++ b/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs @@ -1601,6 +1601,11 @@ mod tests { }], message_bytes: message, metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); let compressed = zstd::bulk::compress(&uncompressed, 1).unwrap(); diff --git a/runtime/blockzilla-replay/src/compact.rs b/runtime/blockzilla-replay/src/compact.rs index 1b43763..7f3e2cb 100644 --- a/runtime/blockzilla-replay/src/compact.rs +++ b/runtime/blockzilla-replay/src/compact.rs @@ -2165,6 +2165,11 @@ mod tests { tx_rows, message_bytes: vec![0; 3], metadata_bytes: Vec::new(), + + consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, + consensus_block_id: None, + parent_consensus_block_id: None, + block_markers: Vec::new(), }, }; From a35c8489ee01234540862c72934a142cb3725fdd Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 28 Oct 2026 15:16:49 +0100 Subject: [PATCH 04/38] reduce size of archive with AG --- blockzilla/archive-gateway/src/lib.rs | 2 - blockzilla/cli/src/archive_v2.rs | 7 - .../cli/src/archive_v2/registry_reprocess.rs | 22 -- .../archive_v2/trusted_metadata_normalize.rs | 2 - blockzilla/cli/src/archive_verify.rs | 16 -- .../src/bin/archive_v2_normalize_metadata.rs | 2 - .../cli/src/bin/archive_v2_pre_to_post.rs | 4 - blockzilla/cli/src/block_time_gaps.rs | 2 - blockzilla/cli/src/scheduler/mod.rs | 2 - .../src/convert/mod.rs | 2 - .../src/source_v2.rs | 2 - .../blockzilla-archive-v2/src/v2/mod.rs | 221 +++++------------- .../src/archive_integrity.rs | 2 - .../src/archive_signatures.rs | 2 - .../src/compact_query.rs | 2 - .../src/reader.rs | 13 -- .../src/reader/pipeline_tests.rs | 2 - .../src/bin/archive-v2-wire-profile-scan.rs | 2 - .../blockzilla-read-sdk-legacy/src/reader.rs | 10 - hivezilla/stream-relay/decoder/src/tests.rs | 2 - .../blockzilla-dump/tests/cases/scanner.rs | 2 - .../blockzilla-dump/tests/cases/verifier.rs | 2 - .../src/extract.rs | 6 - .../src/bin/archive-v2-account-projection.rs | 2 - ...r_program_index_wire_profile_transition.rs | 2 - runtime/blockzilla-replay/src/compact.rs | 2 - 26 files changed, 58 insertions(+), 277 deletions(-) diff --git a/blockzilla/archive-gateway/src/lib.rs b/blockzilla/archive-gateway/src/lib.rs index 8212279..e5d31b0 100644 --- a/blockzilla/archive-gateway/src/lib.rs +++ b/blockzilla/archive-gateway/src/lib.rs @@ -1423,9 +1423,7 @@ mod tests { message_bytes: message, metadata_bytes: metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/blockzilla/cli/src/archive_v2.rs b/blockzilla/cli/src/archive_v2.rs index 6479609..4c32662 100644 --- a/blockzilla/cli/src/archive_v2.rs +++ b/blockzilla/cli/src/archive_v2.rs @@ -6335,9 +6335,7 @@ fn hot_block_from_archive_block_with_signatures( message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, block_signature_count, @@ -6508,9 +6506,7 @@ fn hot_block_from_live_no_registry_block( message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, block_signature_count, @@ -6709,9 +6705,7 @@ fn hot_block_from_live_no_registry_block_signature_counts( message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, block_signature_count, @@ -17793,7 +17787,6 @@ fn build_block_record( fn apply_hot_consensus(block: &mut ArchiveV2HotBlockBlob, consensus: ArchiveV2HotConsensus) { block.consensus_block_id = consensus.block_id; - block.parent_consensus_block_id = consensus.parent_block_id; block.block_markers = consensus.markers; } diff --git a/blockzilla/cli/src/archive_v2/registry_reprocess.rs b/blockzilla/cli/src/archive_v2/registry_reprocess.rs index b8773b2..e420516 100644 --- a/blockzilla/cli/src/archive_v2/registry_reprocess.rs +++ b/blockzilla/cli/src/archive_v2/registry_reprocess.rs @@ -1784,9 +1784,7 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -1812,9 +1810,7 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -6850,9 +6846,7 @@ fn rewrite_source_block_with_scratch( message_bytes: std::mem::take(&mut scratch.target_messages), metadata_bytes: std::mem::take(&mut scratch.target_metadata), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; (block, HotBlockOuterSchema::Current, access_references) @@ -12657,9 +12651,7 @@ mod tests { message_bytes, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -13485,9 +13477,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -13513,9 +13503,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let mut encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -13702,9 +13690,7 @@ mod tests { ], message_bytes: [first_message, second_message, raw_message].concat(), metadata_bytes: [successful, failed, raw_metadata].concat(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -14252,9 +14238,7 @@ mod tests { message_bytes: raw_messages.to_vec(), metadata_bytes: [encoded_metadata, raw_metadata.to_vec()].concat(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -14383,9 +14367,7 @@ mod tests { message_bytes: [message_bytes, raw_message.to_vec()].concat(), metadata_bytes: [metadata_bytes, raw_metadata.to_vec()].concat(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let mut references = AccessReferenceSet::default(); @@ -14506,9 +14488,7 @@ mod tests { message_bytes: [first, second].concat(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let mut access = AccessReferenceSet::default(); @@ -14600,9 +14580,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let mut classes = Vec::new(); diff --git a/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs b/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs index 1573db1..5007383 100644 --- a/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs +++ b/blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs @@ -1986,9 +1986,7 @@ mod tests { message_bytes: message, metadata_bytes: metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/blockzilla/cli/src/archive_verify.rs b/blockzilla/cli/src/archive_verify.rs index bafba49..3cb8eea 100644 --- a/blockzilla/cli/src/archive_verify.rs +++ b/blockzilla/cli/src/archive_verify.rs @@ -5024,9 +5024,7 @@ mod tests { message_bytes: vec![], metadata_bytes: vec![], - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let raw = wincode::config::serialize(&blob, blockzilla_primitives::wincode_leb128_config()) @@ -5244,9 +5242,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, ArchiveV2HotBlockBlob { @@ -5273,9 +5269,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, ]; @@ -5573,9 +5567,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = @@ -5935,9 +5927,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize( @@ -7537,9 +7527,7 @@ mod tests { message_bytes: vec![10, 11, 12, 13, 14], metadata_bytes: vec![20, 21, 22], - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = @@ -7561,9 +7549,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let empty_uncompressed = wincode::config::serialize( @@ -7719,9 +7705,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = diff --git a/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs b/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs index a98a311..88e6f57 100644 --- a/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs +++ b/blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs @@ -3543,9 +3543,7 @@ mod tests { message_bytes: message.clone(), metadata_bytes: metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs b/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs index 5848fdb..517756a 100644 --- a/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs +++ b/blockzilla/cli/src/bin/archive_v2_pre_to_post.rs @@ -4624,9 +4624,7 @@ mod tests { message_bytes: message, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let owned = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -5444,9 +5442,7 @@ mod tests { message_bytes: message.clone(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/blockzilla/cli/src/block_time_gaps.rs b/blockzilla/cli/src/block_time_gaps.rs index e8ea8db..21118f4 100644 --- a/blockzilla/cli/src/block_time_gaps.rs +++ b/blockzilla/cli/src/block_time_gaps.rs @@ -1267,9 +1267,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/blockzilla/cli/src/scheduler/mod.rs b/blockzilla/cli/src/scheduler/mod.rs index 81ff9e0..a73d0d7 100644 --- a/blockzilla/cli/src/scheduler/mod.rs +++ b/blockzilla/cli/src/scheduler/mod.rs @@ -30705,9 +30705,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs b/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs index c9bd3b2..363afda 100644 --- a/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs +++ b/crates/archive-v3/blockzilla-archive-v3-convert/src/convert/mod.rs @@ -6386,9 +6386,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, transactions: vec![ResolvedSourceTransaction { diff --git a/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs b/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs index 55ae58e..094c3ea 100644 --- a/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs +++ b/crates/archive-v3/blockzilla-archive-v3-convert/src/source_v2.rs @@ -2012,9 +2012,7 @@ mod tests { tx_rows: Vec::new(), message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } diff --git a/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs b/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs index 2a2c784..e4b89fb 100644 --- a/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs +++ b/crates/compact-v2/blockzilla-archive-v2/src/v2/mod.rs @@ -23,9 +23,8 @@ use blockzilla_compact::{ SplitCompactIndexRecord, }; use blockzilla_primitives::{ - CompactPubkey, HistoricalSourceWincodeLeb128Config, - bounded_historical_source_wincode_leb128_config, historical_source_wincode_leb128_config, - wincode_leb128_config, + BoundedHistoricalSourceWincodeLeb128Config, CompactPubkey, HistoricalSourceWincodeLeb128Config, + historical_source_wincode_leb128_config, wincode_leb128_config, }; mod archive; @@ -64,15 +63,6 @@ pub const WINCODE_ARCHIVE_V2_FLAG_FIRST_SEEN_REGISTRY: u32 = 1 << 2; /// Registry counts include every typed `CompactPubkey` reference, including rewards and logs. pub const WINCODE_ARCHIVE_V2_FLAG_ALL_PUBKEY_REF_COUNTS: u32 = 1 << 3; pub const ARCHIVE_V2_HOT_TX_ROW_LEN: usize = 28; -/// Marks the consensus tail appended after the historical hot-block payload. -/// -/// A block written before this tail has no bytes here. The value is not a -/// transaction count, so a historical payload cannot be read as a consensus tail. -pub const ARCHIVE_V2_HOT_CONSENSUS_MAGIC: u32 = 0xA7B1_0C4D; - -fn default_consensus_magic() -> u32 { - ARCHIVE_V2_HOT_CONSENSUS_MAGIC -} /// Header for the log-only wincode benchmark format. /// @@ -384,17 +374,11 @@ pub struct ArchiveV2HotBlockBlob { pub tx_rows: Vec, pub message_bytes: Vec, pub metadata_bytes: Vec, - /// Present on every newly written block. Historical bytes omit this tail. - #[serde(default = "default_consensus_magic")] - pub consensus_magic: u32, /// Consensus block id of this block. Absent on a Tower block. #[serde(default)] pub consensus_block_id: Option<[u8; 32]>, - /// Consensus block id of the parent, taken from the last header or - /// update-parent marker. Absent when that marker is not present. - #[serde(default)] - pub parent_consensus_block_id: Option<[u8; 32]>, - /// Raw `VersionedBlockMarker` bytes, in source order. + /// Raw `VersionedBlockMarker` bytes, in source order. The parent consensus + /// block id stays inside the last header or update-parent marker. #[serde(default)] pub block_markers: Vec>, } @@ -403,7 +387,6 @@ pub struct ArchiveV2HotBlockBlob { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ArchiveV2HotConsensus { pub block_id: Option<[u8; 32]>, - pub parent_block_id: Option<[u8; 32]>, pub markers: Vec>, } @@ -411,7 +394,6 @@ impl ArchiveV2HotConsensus { pub fn from_slot_meta(meta: &SlotMeta) -> Self { Self { block_id: meta.block_id, - parent_block_id: meta.parent_consensus_block_id(), markers: meta.block_markers.clone().unwrap_or_default(), } } @@ -435,9 +417,7 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, - consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -459,7 +439,6 @@ pub struct BorrowedArchiveV2HotBlockBlob<'a> { pub message_bytes: &'a [u8], pub metadata_bytes: &'a [u8], pub consensus_block_id: Option<[u8; 32]>, - pub parent_consensus_block_id: Option<[u8; 32]>, pub block_markers: Vec>, } @@ -478,7 +457,6 @@ pub struct BorrowedArchiveV2HotBlockBlobWithoutRewards<'a> { pub metadata_bytes: &'a [u8], rewards_field_bytes: &'a [u8], pub consensus_block_id: Option<[u8; 32]>, - pub parent_consensus_block_id: Option<[u8; 32]>, pub block_markers: Vec>, } @@ -597,9 +575,7 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, - consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -645,9 +621,7 @@ impl From for ArchiveV2HotBlockBlob { tx_rows: value.tx_rows, message_bytes: value.message_bytes, metadata_bytes: value.metadata_bytes, - consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -686,32 +660,21 @@ pub fn deserialize_archive_v2_hot_block_slot_time(bytes: &[u8]) -> ReadResult<(u /// A newly written block carries a consensus tail after the historical payload. /// A block without that tail decodes as the previous schema and has an empty tail. pub fn deserialize_archive_v2_hot_block_blob(bytes: &[u8]) -> ReadResult { - if let Ok(block) = wincode::config::deserialize_exact::( - bytes, - historical_source_wincode_leb128_config(), - ) && block.consensus_magic == ARCHIVE_V2_HOT_CONSENSUS_MAGIC - { - return Ok(block); - } let mut remaining = bytes; if let Ok(block) = >::get(remaining.by_ref()) { - if remaining.is_empty() { - return Ok(block.into()); - } // The historical payload matched. Bytes after it are a consensus tail, including a // truncated one. Do not try the older schemas: they can accept this prefix and hide // the broken tail. - let (consensus_block_id, parent_consensus_block_id, block_markers) = + let (consensus_block_id, block_markers) = read_optional_consensus_tail::(&mut remaining)?; if !remaining.is_empty() { return Err(wincode::error::trailing_bytes()); } let mut block = ArchiveV2HotBlockBlob::from(block); block.consensus_block_id = consensus_block_id; - block.parent_consensus_block_id = parent_consensus_block_id; block.block_markers = block_markers; return Ok(block); } @@ -743,60 +706,7 @@ pub fn deserialize_archive_v2_hot_block_blob(bytes: &[u8]) -> ReadResult ReadResult> { - if let Ok(( - header, - tx_count, - tx_rows, - message_bytes, - metadata_bytes, - consensus_magic, - consensus_block_id, - parent_consensus_block_id, - block_markers, - )) = wincode::config::deserialize_exact::< - ( - ArchiveV2HotBlockHeader, - u32, - &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], - &[u8], - &[u8], - u32, - Option<[u8; 32]>, - Option<[u8; 32]>, - Vec>, - ), - _, - >(bytes, historical_source_wincode_leb128_config()) - && consensus_magic == ARCHIVE_V2_HOT_CONSENSUS_MAGIC - { - return Ok(BorrowedArchiveV2HotBlockBlob { - header, - tx_count, - tx_rows, - message_bytes, - metadata_bytes, - consensus_block_id, - parent_consensus_block_id, - block_markers, - }); - } - let (header, tx_count, tx_rows, message_bytes, metadata_bytes): ( - ArchiveV2HotBlockHeader, - u32, - &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], - &[u8], - &[u8], - ) = wincode::config::deserialize_exact(bytes, historical_source_wincode_leb128_config())?; - Ok(BorrowedArchiveV2HotBlockBlob { - header, - tx_count, - tx_rows, - message_bytes, - metadata_bytes, - consensus_block_id: None, - parent_consensus_block_id: None, - block_markers: Vec::new(), - }) + read_borrowed_hot_block::(bytes) } /// Decode exactly one current hot block with an explicit per-sequence allocation limit. @@ -809,60 +719,35 @@ pub fn deserialize_archive_v2_hot_block_blob_borrowed_current_with_preallocation >( bytes: &[u8], ) -> ReadResult> { - let config = bounded_historical_source_wincode_leb128_config::(); - if let Ok(( - header, - tx_count, - tx_rows, - message_bytes, - metadata_bytes, - consensus_magic, - consensus_block_id, - parent_consensus_block_id, - block_markers, - )) = wincode::config::deserialize_exact::< - ( + read_borrowed_hot_block::>( + bytes, + ) +} + +fn read_borrowed_hot_block<'a, C: Config>( + bytes: &'a [u8], +) -> ReadResult> { + let mut remaining = bytes; + let (header, tx_count, tx_rows, message_bytes, metadata_bytes) = + <( ArchiveV2HotBlockHeader, u32, &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], &[u8], &[u8], - u32, - Option<[u8; 32]>, - Option<[u8; 32]>, - Vec>, - ), - _, - >(bytes, config) - && consensus_magic == ARCHIVE_V2_HOT_CONSENSUS_MAGIC - { - return Ok(BorrowedArchiveV2HotBlockBlob { - header, - tx_count, - tx_rows, - message_bytes, - metadata_bytes, - consensus_block_id, - parent_consensus_block_id, - block_markers, - }); + ) as SchemaRead>::get(remaining.by_ref())?; + let (consensus_block_id, block_markers) = read_optional_consensus_tail::(&mut remaining)?; + if !remaining.is_empty() { + return Err(wincode::error::trailing_bytes()); } - let (header, tx_count, tx_rows, message_bytes, metadata_bytes): ( - ArchiveV2HotBlockHeader, - u32, - &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], - &[u8], - &[u8], - ) = wincode::config::deserialize_exact(bytes, config)?; Ok(BorrowedArchiveV2HotBlockBlob { header, tx_count, tx_rows, message_bytes, metadata_bytes, - consensus_block_id: None, - parent_consensus_block_id: None, - block_markers: Vec::new(), + consensus_block_id, + block_markers, }) } @@ -934,7 +819,7 @@ pub fn deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards( ) = <(u32, &[[u8; ARCHIVE_V2_HOT_TX_ROW_LEN]], &[u8], &[u8]) as SchemaRead< HistoricalSourceWincodeLeb128Config, >>::get(remaining.by_ref())?; - let (consensus_block_id, parent_consensus_block_id, block_markers) = + let (consensus_block_id, block_markers) = read_optional_consensus_tail::(&mut remaining)?; if !remaining.is_empty() { return Err(wincode::error::trailing_bytes()); @@ -955,25 +840,19 @@ pub fn deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards( metadata_bytes, rewards_field_bytes, consensus_block_id, - parent_consensus_block_id, block_markers, }) } fn read_optional_consensus_tail<'a, C: Config>( bytes: &mut &'a [u8], -) -> ReadResult<(Option<[u8; 32]>, Option<[u8; 32]>, Vec>)> { +) -> ReadResult<(Option<[u8; 32]>, Vec>)> { if bytes.is_empty() { - return Ok((None, None, Vec::new())); - } - let magic = >::get(bytes.by_ref())?; - if magic != ARCHIVE_V2_HOT_CONSENSUS_MAGIC { - return Err(wincode::error::invalid_tag_encoding(magic as usize)); + return Ok((None, Vec::new())); } let consensus_block_id = as SchemaRead<'a, C>>::get(bytes.by_ref())?; - let parent_consensus_block_id = as SchemaRead<'a, C>>::get(bytes.by_ref())?; let block_markers = > as SchemaRead<'a, C>>::get(bytes.by_ref())?; - Ok((consensus_block_id, parent_consensus_block_id, block_markers)) + Ok((consensus_block_id, block_markers)) } #[cfg(test)] @@ -1016,9 +895,7 @@ mod hot_block_slot_time_tests { ], message_bytes: vec![10, 11, 12, 13, 14], metadata_bytes: vec![20, 21, 22], - consensus_magic: ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } @@ -1052,10 +929,6 @@ mod hot_block_slot_time_tests { assert_eq!(replay.message_bytes, owned.message_bytes); assert_eq!(replay.metadata_bytes, owned.metadata_bytes); assert_eq!(replay.consensus_block_id, owned.consensus_block_id); - assert_eq!( - replay.parent_consensus_block_id, - owned.parent_consensus_block_id - ); assert_eq!(replay.block_markers, owned.block_markers); } @@ -1086,30 +959,47 @@ mod hot_block_slot_time_tests { ); } + fn header_marker(variant: u8, parent_block_id: [u8; 32]) -> Vec { + let mut marker = vec![0u8; 46]; + marker[0] = 1; + marker[2] = variant; + marker[3] = 41; + marker[5] = 1; + marker[14..46].copy_from_slice(&parent_block_id); + marker + } + #[test] fn consensus_tail_round_trips_and_legacy_bytes_stay_empty() { let mut block = current_hot_block_fixture(); + let header_parent = [0x43; 32]; + let updated_parent = [0x44; 32]; block.consensus_block_id = Some([0x21; 32]); - block.parent_consensus_block_id = Some([0x43; 32]); - block.block_markers = vec![vec![1, 0, 1, 4, 0], vec![9, 9]]; + block.block_markers = vec![ + header_marker(1, header_parent), + header_marker(2, updated_parent), + ]; let bytes = wincode::config::serialize(&block, historical_source_wincode_leb128_config()).unwrap(); let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); assert_eq!(decoded.consensus_block_id, block.consensus_block_id); - assert_eq!( - decoded.parent_consensus_block_id, - block.parent_consensus_block_id - ); assert_eq!(decoded.block_markers, block.block_markers); + let recovered = SlotMeta { + parent_slot: None, + blocktime: None, + block_height: None, + block_markers: Some(decoded.block_markers.clone()), + block_id: decoded.consensus_block_id, + meta_len: 5, + }; + assert_eq!(recovered.parent_consensus_block_id(), Some(updated_parent)); assert_eq!(decoded.tx_count, block.tx_count); let borrowed = deserialize_archive_v2_hot_block_blob_borrowed_current(&bytes).unwrap(); assert_eq!(borrowed.consensus_block_id, Some([0x21; 32])); - assert_eq!(borrowed.parent_consensus_block_id, Some([0x43; 32])); assert_eq!(borrowed.block_markers, block.block_markers); let replay = deserialize_archive_v2_hot_block_blob_borrowed_current_without_rewards(&bytes).unwrap(); assert_eq!(replay.consensus_block_id, Some([0x21; 32])); - assert_eq!(replay.parent_consensus_block_id, Some([0x43; 32])); assert_eq!(replay.block_markers, block.block_markers); let exact = deserialize_archive_v2_hot_block_blob_borrowed_current_with_preallocation_limit::< @@ -1118,6 +1008,12 @@ mod hot_block_slot_time_tests { .unwrap(); assert_eq!(exact.consensus_block_id, Some([0x21; 32])); assert_eq!(exact.block_markers, block.block_markers); + let tail = wincode::config::serialize( + &(block.consensus_block_id, block.block_markers.clone()), + historical_source_wincode_leb128_config(), + ) + .unwrap(); + assert!(bytes.ends_with(&tail)); let legacy = ArchiveV2HotBlockBlobLegacyPlain { header: block.header, @@ -1130,7 +1026,6 @@ mod hot_block_slot_time_tests { wincode::config::serialize(&legacy, historical_source_wincode_leb128_config()).unwrap(); let decoded = deserialize_archive_v2_hot_block_blob(&legacy_bytes).unwrap(); assert_eq!(decoded.consensus_block_id, None); - assert_eq!(decoded.parent_consensus_block_id, None); assert!(decoded.block_markers.is_empty()); assert_eq!(decoded.tx_count, 0); let borrowed = diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs index dc482d2..731ec78 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_integrity.rs @@ -3025,9 +3025,7 @@ mod tests { message_bytes, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs index f1dfc40..8202e95 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/archive_signatures.rs @@ -715,9 +715,7 @@ mod tests { message_bytes, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs index a9d31f8..4e5ac41 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/compact_query.rs @@ -3824,9 +3824,7 @@ mod tests { message_bytes: messages, metadata_bytes: metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs index a4b169e..7313000 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader.rs @@ -614,9 +614,7 @@ impl BorrowedDecodedBlock<'_> { tx_rows, message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: block.consensus_block_id, - parent_consensus_block_id: block.parent_consensus_block_id, block_markers: block.block_markers, } } @@ -4842,9 +4840,7 @@ mod tests { ], message_bytes: first_messages, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: Some([0x21; 32]), - parent_consensus_block_id: Some([0x43; 32]), block_markers: vec![vec![1, 2, 3]], }; @@ -4901,9 +4897,7 @@ mod tests { message_bytes: second_message, metadata_bytes: second_metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; @@ -5007,9 +5001,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = @@ -5158,9 +5150,7 @@ mod tests { message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, } @@ -6541,7 +6531,6 @@ mod tests { assert_eq!(converted.index_row.block_id, expected.index_row.block_id); assert_eq!(converted.index_row.slot, expected.index_row.slot); assert_eq!(converted.block.consensus_block_id, Some([0x21; 32])); - assert_eq!(converted.block.parent_consensus_block_id, Some([0x43; 32])); assert_eq!(converted.block.block_markers, vec![vec![1, 2, 3]]); assert_eq!( wincode::config::serialize(&converted.block, wincode_leb128_config()).unwrap(), @@ -6907,9 +6896,7 @@ mod tests { message_bytes, metadata_bytes: vec![0xaa], - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, }; diff --git a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs index 97e1a34..1f30ac9 100644 --- a/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs +++ b/crates/compact-v2/blockzilla-compact-v2-reader/src/reader/pipeline_tests.rs @@ -424,9 +424,7 @@ mod incremental_pipeline_regressions { message_bytes: vec![0; transactions_per_block as usize], metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs b/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs index 5b4a16b..537af6d 100644 --- a/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs +++ b/crates/compat/blockzilla-read-sdk-legacy/src/bin/archive-v2-wire-profile-scan.rs @@ -1518,9 +1518,7 @@ mod tests { message_bytes, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs b/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs index 12c155a..13551bb 100644 --- a/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs +++ b/crates/compat/blockzilla-read-sdk-legacy/src/reader.rs @@ -4516,9 +4516,7 @@ mod tests { message_bytes: first_messages, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; @@ -4575,9 +4573,7 @@ mod tests { message_bytes: second_message, metadata_bytes: second_metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; @@ -5117,9 +5113,7 @@ mod tests { message_bytes: message, metadata_bytes: vec![], - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, }; @@ -6820,9 +6814,7 @@ mod tests { message_bytes, metadata_bytes: vec![0xaa], - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, }; @@ -7056,9 +7048,7 @@ mod tests { message_bytes: Vec::new(), metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; diff --git a/hivezilla/stream-relay/decoder/src/tests.rs b/hivezilla/stream-relay/decoder/src/tests.rs index 6fc87d0..f76e16a 100644 --- a/hivezilla/stream-relay/decoder/src/tests.rs +++ b/hivezilla/stream-relay/decoder/src/tests.rs @@ -128,9 +128,7 @@ fn fixture_with_tokens(loaded: bool, failed: bool, id: u32, tokens: bool) -> Vec message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let raw = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/indexer/blockzilla-dump/tests/cases/scanner.rs b/indexer/blockzilla-dump/tests/cases/scanner.rs index 9985680..ea8e629 100644 --- a/indexer/blockzilla-dump/tests/cases/scanner.rs +++ b/indexer/blockzilla-dump/tests/cases/scanner.rs @@ -242,9 +242,7 @@ fn block(slot: u64, message: Vec, metadata: Vec, flags: u32) -> ArchiveV }], message_bytes: message, metadata_bytes: metadata, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), } } diff --git a/indexer/blockzilla-dump/tests/cases/verifier.rs b/indexer/blockzilla-dump/tests/cases/verifier.rs index c4bbf4a..8bdd55a 100644 --- a/indexer/blockzilla-dump/tests/cases/verifier.rs +++ b/indexer/blockzilla-dump/tests/cases/verifier.rs @@ -242,9 +242,7 @@ fn build_epoch_zero(root: &Path) { message_bytes: message, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/indexer/blockzilla-token-transaction-dump/src/extract.rs b/indexer/blockzilla-token-transaction-dump/src/extract.rs index ae6390e..c42c962 100644 --- a/indexer/blockzilla-token-transaction-dump/src/extract.rs +++ b/indexer/blockzilla-token-transaction-dump/src/extract.rs @@ -8825,9 +8825,7 @@ mod tests { message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -9065,9 +9063,7 @@ mod tests { message_bytes, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); @@ -9228,9 +9224,7 @@ mod tests { message_bytes, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs b/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs index d665401..bfc4aaf 100644 --- a/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs +++ b/indexer/blockzilla-user-program-index/src/bin/archive-v2-account-projection.rs @@ -8119,9 +8119,7 @@ mod tests { message_bytes, metadata_bytes, - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = serialize(&block); diff --git a/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs b/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs index 1bbc1e7..8aae308 100644 --- a/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs +++ b/indexer/blockzilla-user-program-index/src/user_program_index_wire_profile_transition.rs @@ -1602,9 +1602,7 @@ mod tests { message_bytes: message, metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }; let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap(); diff --git a/runtime/blockzilla-replay/src/compact.rs b/runtime/blockzilla-replay/src/compact.rs index 7f3e2cb..a3b087b 100644 --- a/runtime/blockzilla-replay/src/compact.rs +++ b/runtime/blockzilla-replay/src/compact.rs @@ -2166,9 +2166,7 @@ mod tests { message_bytes: vec![0; 3], metadata_bytes: Vec::new(), - consensus_magic: blockzilla_archive_v2::ARCHIVE_V2_HOT_CONSENSUS_MAGIC, consensus_block_id: None, - parent_consensus_block_id: None, block_markers: Vec::new(), }, }; From 403b96221a8b38c58a102dc5ab3ad5b819e74c8f Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 28 Oct 2026 15:36:46 +0100 Subject: [PATCH 05/38] add blockmarker to parse alpenglow markers --- .../of-car-reader/src/blockmarker.rs | 616 ++++++++++++++++++ crates/old-faithful/of-car-reader/src/lib.rs | 2 + crates/old-faithful/of-car-reader/src/node.rs | 26 +- 3 files changed, 621 insertions(+), 23 deletions(-) create mode 100644 crates/old-faithful/of-car-reader/src/blockmarker.rs diff --git a/crates/old-faithful/of-car-reader/src/blockmarker.rs b/crates/old-faithful/of-car-reader/src/blockmarker.rs new file mode 100644 index 0000000..54e3992 --- /dev/null +++ b/crates/old-faithful/of-car-reader/src/blockmarker.rs @@ -0,0 +1,616 @@ +//! Alpenglow `VersionedBlockMarker` bytes stored in a CAR `SlotMeta`. +//! +//! The wire layout is Agave's default Wincode spelling: fixed-width +//! little-endian integers and a one-byte `Option` tag. It is not the LEB128 +//! spelling of an Archive V2 hot block. BLS signatures are copied, not checked. +//! +//! [`parse`] splits one marker. [`decode_footer_certificates`] splits the +//! certificate bytes that remain after a footer's user agent. Callers that only +//! store markers do not need either function. + +use std::fmt; + +use crate::short_vec::decode_shortu16_len; + +const MARKER_VERSION: u16 = 1; +const PAYLOAD_VERSION: u8 = 1; +const ENVELOPE_LEN: usize = 5; +const HEADER_PAYLOAD_LEN: usize = 41; +const BLS_COMPRESSED_LEN: usize = 96; +const BLS_AFFINE_LEN: usize = 192; +const MAX_GENESIS_BITMAP_LEN: usize = 512; + +const VARIANT_FOOTER: u8 = 0; +const VARIANT_HEADER: u8 = 1; +const VARIANT_UPDATE_PARENT: u8 = 2; +const VARIANT_GENESIS: u8 = 3; + +/// One parsed Alpenglow block marker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Marker { + Header(HeaderV1), + UpdateParent(UpdateParentV1), + Footer(FooterV1), + Genesis(GenesisCertificate), +} + +/// Parent slot and parent consensus block id from a header marker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HeaderV1 { + pub parent_slot: u64, + pub parent_block_id: [u8; 32], +} + +/// Parent slot and parent consensus block id from an update-parent marker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateParentV1 { + pub new_parent_slot: u64, + pub new_parent_block_id: [u8; 32], +} + +/// Footer fields that precede the three optional certificates. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FooterV1 { + pub bank_hash: [u8; 32], + pub block_producer_time_nanos: u64, + pub block_user_agent: Vec, + /// Finalization, skip-reward, and notar-reward certificates, still encoded. + pub certificates: Vec, +} + +/// Genesis certificate. Its payload has no inner version byte. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GenesisCertificate { + pub slot: u64, + pub block_id: [u8; 32], + pub bls_signature: [u8; 192], + pub bitmap: Vec, +} + +/// The three certificates carried by a footer, in source order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FooterCertificates { + pub finalization: Option, + pub skip_reward: Option, + pub notar_reward: Option, +} + +/// Block finalization certificate from a footer. +/// +/// `notar_aggregate` is absent on a fast finalization of the parent block. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalizationCert { + pub slot: u64, + pub block_id: [u8; 32], + pub final_aggregate: VotesAggregate, + pub notar_aggregate: Option, +} + +/// Compressed BLS signature and the validator bitmap that signed it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VotesAggregate { + pub signature: [u8; BLS_COMPRESSED_LEN], + pub bitmap: Vec, +} + +/// Skip-reward certificate. The bitmap length is a Solana `ShortU16`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SkipRewardCert { + pub slot: u64, + pub signature: [u8; BLS_COMPRESSED_LEN], + pub bitmap: Vec, +} + +/// Notarization-reward certificate. The bitmap length is a Solana `ShortU16`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NotarRewardCert { + pub slot: u64, + pub block_id: [u8; 32], + pub signature: [u8; BLS_COMPRESSED_LEN], + pub bitmap: Vec, +} + +/// A marker or footer-certificate slice that does not match the Agave layout. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MarkerError { + Truncated, + TrailingBytes, + UnsupportedVersion(u16), + UnknownVariant(u8), + UnsupportedPayloadVersion(u8), + LengthMismatch { expected: usize, actual: usize }, + InvalidOption(u8), + InvalidBitmapLength, + BitmapTooLong(usize), +} + +impl fmt::Display for MarkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MarkerError::Truncated => write!(f, "truncated block marker"), + MarkerError::TrailingBytes => write!(f, "block marker has trailing bytes"), + MarkerError::UnsupportedVersion(version) => { + write!(f, "unsupported block marker version {version}") + } + MarkerError::UnknownVariant(variant) => { + write!(f, "unknown block marker variant {variant}") + } + MarkerError::UnsupportedPayloadVersion(version) => { + write!(f, "unsupported block marker payload version {version}") + } + MarkerError::LengthMismatch { expected, actual } => { + write!(f, "block marker length is {actual}, expected {expected}") + } + MarkerError::InvalidOption(tag) => { + write!(f, "block marker option tag {tag} is not 0 or 1") + } + MarkerError::InvalidBitmapLength => write!(f, "block marker bitmap length is invalid"), + MarkerError::BitmapTooLong(len) => { + write!(f, "block marker bitmap length {len} exceeds 512") + } + } + } +} + +impl std::error::Error for MarkerError {} + +/// Parent consensus block id from the last header or update-parent marker. +/// +/// A footer, a genesis marker, or a marker that does not parse is skipped. +pub fn parent_consensus_block_id(markers: &[Vec]) -> Option<[u8; 32]> { + markers + .iter() + .rev() + .find_map(|marker| match parse(marker).ok()? { + Marker::Header(header) => Some(header.parent_block_id), + Marker::UpdateParent(update) => Some(update.new_parent_block_id), + Marker::Footer(_) | Marker::Genesis(_) => None, + }) +} + +/// Parse one complete `VersionedBlockMarker`. +pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() < ENVELOPE_LEN { + return Err(MarkerError::Truncated); + } + let version = u16::from_le_bytes([bytes[0], bytes[1]]); + if version != MARKER_VERSION { + return Err(MarkerError::UnsupportedVersion(version)); + } + let variant = bytes[2]; + let payload_len = usize::from(u16::from_le_bytes([bytes[3], bytes[4]])); + let expected = ENVELOPE_LEN + payload_len; + if bytes.len() != expected { + return Err(MarkerError::LengthMismatch { + expected, + actual: bytes.len(), + }); + } + let payload = &bytes[ENVELOPE_LEN..]; + match variant { + VARIANT_FOOTER => Ok(Marker::Footer(decode_footer(payload)?)), + VARIANT_HEADER => { + let (parent_slot, parent_block_id) = decode_slot_and_block_id(payload)?; + Ok(Marker::Header(HeaderV1 { + parent_slot, + parent_block_id, + })) + } + VARIANT_UPDATE_PARENT => { + let (new_parent_slot, new_parent_block_id) = decode_slot_and_block_id(payload)?; + Ok(Marker::UpdateParent(UpdateParentV1 { + new_parent_slot, + new_parent_block_id, + })) + } + VARIANT_GENESIS => Ok(Marker::Genesis(decode_genesis(payload)?)), + other => Err(MarkerError::UnknownVariant(other)), + } +} + +/// Decode the three optional certificates that follow a footer's user agent. +/// +/// The slice must be consumed exactly. An empty slice is truncated: a footer +/// always carries the three presence tags. +pub fn decode_footer_certificates(bytes: &[u8]) -> Result { + let mut cursor = Cursor::new(bytes); + let finalization = read_optional(&mut cursor, read_finalization_cert)?; + let skip_reward = read_optional(&mut cursor, read_skip_reward)?; + let notar_reward = read_optional(&mut cursor, read_notar_reward)?; + cursor.finish()?; + Ok(FooterCertificates { + finalization, + skip_reward, + notar_reward, + }) +} + +fn decode_footer(payload: &[u8]) -> Result { + let mut cursor = Cursor::new(payload); + let version = cursor.u8()?; + if version != PAYLOAD_VERSION { + return Err(MarkerError::UnsupportedPayloadVersion(version)); + } + let bank_hash = cursor.array()?; + let block_producer_time_nanos = cursor.u64()?; + let user_agent_len = usize::from(cursor.u8()?); + let block_user_agent = cursor.take(user_agent_len)?.to_vec(); + let certificates = cursor.rest().to_vec(); + Ok(FooterV1 { + bank_hash, + block_producer_time_nanos, + block_user_agent, + certificates, + }) +} + +fn decode_slot_and_block_id(payload: &[u8]) -> Result<(u64, [u8; 32]), MarkerError> { + if payload.len() != HEADER_PAYLOAD_LEN { + return Err(MarkerError::LengthMismatch { + expected: HEADER_PAYLOAD_LEN, + actual: payload.len(), + }); + } + let mut cursor = Cursor::new(payload); + let version = cursor.u8()?; + if version != PAYLOAD_VERSION { + return Err(MarkerError::UnsupportedPayloadVersion(version)); + } + let slot = cursor.u64()?; + let block_id = cursor.array()?; + cursor.finish()?; + Ok((slot, block_id)) +} + +fn decode_genesis(payload: &[u8]) -> Result { + let mut cursor = Cursor::new(payload); + let slot = cursor.u64()?; + let block_id = cursor.array()?; + let bls_signature = cursor.array()?; + let bitmap_len = + usize::try_from(cursor.u64()?).map_err(|_| MarkerError::InvalidBitmapLength)?; + if bitmap_len > MAX_GENESIS_BITMAP_LEN { + return Err(MarkerError::BitmapTooLong(bitmap_len)); + } + if cursor.rest().len() != bitmap_len { + return Err(MarkerError::LengthMismatch { + expected: bitmap_len, + actual: cursor.rest().len(), + }); + } + let bitmap = cursor.take(bitmap_len)?.to_vec(); + cursor.finish()?; + Ok(GenesisCertificate { + slot, + block_id, + bls_signature, + bitmap, + }) +} + +fn read_finalization_cert(cursor: &mut Cursor<'_>) -> Result { + let slot = cursor.u64()?; + let block_id = cursor.array()?; + let final_aggregate = read_votes_aggregate(cursor)?; + let notar_aggregate = read_optional(cursor, read_votes_aggregate)?; + Ok(FinalizationCert { + slot, + block_id, + final_aggregate, + notar_aggregate, + }) +} + +fn read_votes_aggregate(cursor: &mut Cursor<'_>) -> Result { + let signature = cursor.array()?; + let bitmap_len = usize::from(cursor.u16()?); + let bitmap = cursor.take(bitmap_len)?.to_vec(); + Ok(VotesAggregate { signature, bitmap }) +} + +fn read_skip_reward(cursor: &mut Cursor<'_>) -> Result { + let slot = cursor.u64()?; + let signature = cursor.array()?; + let bitmap = read_short_bitmap(cursor)?; + Ok(SkipRewardCert { + slot, + signature, + bitmap, + }) +} + +fn read_notar_reward(cursor: &mut Cursor<'_>) -> Result { + let slot = cursor.u64()?; + let block_id = cursor.array()?; + let signature = cursor.array()?; + let bitmap = read_short_bitmap(cursor)?; + Ok(NotarRewardCert { + slot, + block_id, + signature, + bitmap, + }) +} + +fn read_short_bitmap(cursor: &mut Cursor<'_>) -> Result, MarkerError> { + let (bitmap_len, len_bytes) = + decode_shortu16_len(cursor.rest()).map_err(|_| MarkerError::InvalidBitmapLength)?; + cursor.take(len_bytes)?; + Ok(cursor.take(bitmap_len)?.to_vec()) +} + +fn read_optional( + cursor: &mut Cursor<'_>, + read: fn(&mut Cursor<'_>) -> Result, +) -> Result, MarkerError> { + match cursor.u8()? { + 0 => Ok(None), + 1 => Ok(Some(read(cursor)?)), + tag => Err(MarkerError::InvalidOption(tag)), + } +} + +struct Cursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } + + fn rest(&self) -> &'a [u8] { + &self.bytes[self.pos..] + } + + fn finish(self) -> Result<(), MarkerError> { + if self.pos == self.bytes.len() { + Ok(()) + } else { + Err(MarkerError::TrailingBytes) + } + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], MarkerError> { + let end = self.pos.checked_add(len).ok_or(MarkerError::Truncated)?; + if end > self.bytes.len() { + return Err(MarkerError::Truncated); + } + let slice = &self.bytes[self.pos..end]; + self.pos = end; + Ok(slice) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u16(&mut self) -> Result { + let bytes = self.take(2)?; + Ok(u16::from_le_bytes([bytes[0], bytes[1]])) + } + + fn u64(&mut self) -> Result { + let bytes = self.take(8)?; + Ok(u64::from_le_bytes( + bytes.try_into().expect("u64 is 8 bytes"), + )) + } + + fn array(&mut self) -> Result<[u8; N], MarkerError> { + let mut out = [0u8; N]; + out.copy_from_slice(self.take(N)?); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn marker(variant: u8, payload: &[u8]) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&MARKER_VERSION.to_le_bytes()); + out.push(variant); + out.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + out.extend_from_slice(payload); + out + } + + fn slot_and_id_payload(slot: u64, block_id: [u8; 32]) -> Vec { + let mut payload = vec![PAYLOAD_VERSION]; + payload.extend_from_slice(&slot.to_le_bytes()); + payload.extend_from_slice(&block_id); + payload + } + + fn votes(signature_byte: u8, bitmap: &[u8]) -> Vec { + let mut out = vec![signature_byte; BLS_COMPRESSED_LEN]; + out.extend_from_slice(&(bitmap.len() as u16).to_le_bytes()); + out.extend_from_slice(bitmap); + out + } + + fn short_u16(len: usize) -> Vec { + if len < 0x80 { + vec![len as u8] + } else { + vec![(len as u8 & 0x7f) | 0x80, (len >> 7) as u8] + } + } + + fn reward_body( + slot: u64, + block_id: Option<[u8; 32]>, + signature_byte: u8, + bitmap: &[u8], + ) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&slot.to_le_bytes()); + if let Some(block_id) = block_id { + out.extend_from_slice(&block_id); + } + out.extend(std::iter::repeat_n(signature_byte, BLS_COMPRESSED_LEN)); + out.extend_from_slice(&short_u16(bitmap.len())); + out.extend_from_slice(bitmap); + out + } + + fn footer_payload(user_agent: &str, certificates: &[u8]) -> Vec { + let mut payload = vec![PAYLOAD_VERSION]; + payload.extend_from_slice(&[0xab; 32]); + payload.extend_from_slice(&1_792_773_113_823_407_080u64.to_le_bytes()); + payload.push(user_agent.len() as u8); + payload.extend_from_slice(user_agent.as_bytes()); + payload.extend_from_slice(certificates); + payload + } + + #[test] + fn header_and_update_parent_expose_the_parent_block_id() { + let header_parent = [0x11; 32]; + let updated_parent = [0x22; 32]; + let header = marker(VARIANT_HEADER, &slot_and_id_payload(40, header_parent)); + let update = marker( + VARIANT_UPDATE_PARENT, + &slot_and_id_payload(39, updated_parent), + ); + assert_eq!(header.len(), 46); + assert_eq!(update.len(), 46); + + let Marker::Header(parsed_header) = parse(&header).unwrap() else { + panic!("header"); + }; + assert_eq!(parsed_header.parent_slot, 40); + assert_eq!(parsed_header.parent_block_id, header_parent); + let Marker::UpdateParent(parsed_update) = parse(&update).unwrap() else { + panic!("update parent"); + }; + assert_eq!(parsed_update.new_parent_slot, 39); + assert_eq!(parsed_update.new_parent_block_id, updated_parent); + + let footer = marker(VARIANT_FOOTER, &footer_payload("agave/4.3.2", &[0, 0, 0])); + let markers = vec![header, footer, vec![1, 2, 3], update]; + assert_eq!(parent_consensus_block_id(&markers), Some(updated_parent)); + } + + #[test] + fn fast_footer_keeps_certificates_raw_until_asked() { + let bitmap = [0xaa, 0xbb]; + let reward_bitmap = [0x10, 0x20, 0x30]; + let block_id = [0x44; 32]; + let mut certificates = vec![1]; + certificates.extend_from_slice(&460_512_003u64.to_le_bytes()); + certificates.extend_from_slice(&block_id); + certificates.extend_from_slice(&votes(0xa3, &bitmap)); + certificates.push(0); + certificates.push(0); + certificates.push(1); + certificates.extend_from_slice(&reward_body( + 460_511_996, + Some([0x55; 32]), + 0xa0, + &reward_bitmap, + )); + + let raw = marker( + VARIANT_FOOTER, + &footer_payload("agave/4.3.2", &certificates), + ); + let Marker::Footer(footer) = parse(&raw).unwrap() else { + panic!("footer"); + }; + assert_eq!(footer.bank_hash, [0xab; 32]); + assert_eq!(footer.block_producer_time_nanos, 1_792_773_113_823_407_080); + assert_eq!(footer.block_user_agent, b"agave/4.3.2"); + assert_eq!(footer.certificates, certificates); + + let decoded = decode_footer_certificates(&footer.certificates).unwrap(); + let finalization = decoded.finalization.unwrap(); + assert_eq!(finalization.slot, 460_512_003); + assert_eq!(finalization.block_id, block_id); + assert_eq!(finalization.final_aggregate.signature, [0xa3; 96]); + assert_eq!(finalization.final_aggregate.bitmap, bitmap); + assert!(finalization.notar_aggregate.is_none()); + assert!(decoded.skip_reward.is_none()); + let notar_reward = decoded.notar_reward.unwrap(); + assert_eq!(notar_reward.slot, 460_511_996); + assert_eq!(notar_reward.block_id, [0x55; 32]); + assert_eq!(notar_reward.signature, [0xa0; 96]); + assert_eq!(notar_reward.bitmap, reward_bitmap); + } + + #[test] + fn slow_footer_carries_both_aggregates_and_a_skip_reward() { + let skip_bitmap = vec![0x11; 128]; + let mut certificates = vec![1]; + certificates.extend_from_slice(&7u64.to_le_bytes()); + certificates.extend_from_slice(&[0x66; 32]); + certificates.extend_from_slice(&votes(0x01, &[0x01])); + certificates.push(1); + certificates.extend_from_slice(&votes(0x02, &[0x02, 0x03])); + certificates.push(1); + certificates.extend_from_slice(&reward_body(8, None, 0x03, &skip_bitmap)); + certificates.push(0); + + let decoded = decode_footer_certificates(&certificates).unwrap(); + let finalization = decoded.finalization.unwrap(); + assert_eq!(finalization.final_aggregate.bitmap, [0x01]); + let notar = finalization.notar_aggregate.unwrap(); + assert_eq!(notar.signature, [0x02; 96]); + assert_eq!(notar.bitmap, [0x02, 0x03]); + let skip = decoded.skip_reward.unwrap(); + assert_eq!(skip.slot, 8); + assert_eq!(skip.bitmap, skip_bitmap); + assert!(decoded.notar_reward.is_none()); + } + + #[test] + fn genesis_certificate_uses_the_affine_signature_and_u64_bitmap_length() { + let bitmap = vec![0x7e; 4]; + let mut payload = Vec::new(); + payload.extend_from_slice(&459_652_999u64.to_le_bytes()); + payload.extend_from_slice(&[0x91; 32]); + payload.extend(std::iter::repeat_n(0xbe, BLS_AFFINE_LEN)); + payload.extend_from_slice(&(bitmap.len() as u64).to_le_bytes()); + payload.extend_from_slice(&bitmap); + let raw = marker(VARIANT_GENESIS, &payload); + + let Marker::Genesis(cert) = parse(&raw).unwrap() else { + panic!("genesis"); + }; + assert_eq!(cert.slot, 459_652_999); + assert_eq!(cert.block_id, [0x91; 32]); + assert_eq!(cert.bls_signature, [0xbe; 192]); + assert_eq!(cert.bitmap, bitmap); + assert_eq!(parent_consensus_block_id(&[raw]), None); + } + + #[test] + fn truncated_unknown_and_trailing_markers_fail() { + assert_eq!(parse(&[1, 0, 1]).unwrap_err(), MarkerError::Truncated); + assert_eq!( + parse(&marker(9, &slot_and_id_payload(1, [2; 32]))).unwrap_err(), + MarkerError::UnknownVariant(9) + ); + let mut header = marker(VARIANT_HEADER, &slot_and_id_payload(1, [2; 32])); + header.push(0xff); + assert!(matches!( + parse(&header), + Err(MarkerError::LengthMismatch { .. }) + )); + + let short_footer = marker(VARIANT_FOOTER, &[PAYLOAD_VERSION]); + assert_eq!(parse(&short_footer).unwrap_err(), MarkerError::Truncated); + + assert_eq!( + decode_footer_certificates(&[]).unwrap_err(), + MarkerError::Truncated + ); + assert_eq!( + decode_footer_certificates(&[0, 0, 0, 0xff]).unwrap_err(), + MarkerError::TrailingBytes + ); + } +} diff --git a/crates/old-faithful/of-car-reader/src/lib.rs b/crates/old-faithful/of-car-reader/src/lib.rs index f41c7cc..22d2222 100644 --- a/crates/old-faithful/of-car-reader/src/lib.rs +++ b/crates/old-faithful/of-car-reader/src/lib.rs @@ -167,6 +167,7 @@ //! //! - `archive` opens a complete local or HTTP CAR epoch through the common //! ordered query interface when the `archive` feature is enabled. +//! - [`blockmarker`] parses Alpenglow block markers stored in a CAR `SlotMeta`. //! - [`node`] decodes Old Faithful CBOR nodes and peeks node kinds. //! - [`slot_ranges`] reads and writes worker slot range files. //! - [`genesis`] parses Solana genesis archives when the `genesis` feature is @@ -176,6 +177,7 @@ #[cfg(feature = "archive")] pub mod archive; +pub mod blockmarker; pub mod car_block_group; pub mod car_stream; #[cfg(feature = "compact-index")] diff --git a/crates/old-faithful/of-car-reader/src/node.rs b/crates/old-faithful/of-car-reader/src/node.rs index b78edfc..3d502d3 100644 --- a/crates/old-faithful/of-car-reader/src/node.rs +++ b/crates/old-faithful/of-car-reader/src/node.rs @@ -219,11 +219,12 @@ const fn tower_slot_meta_len() -> u64 { impl SlotMeta { /// Parent consensus block id from the last header or update-parent marker. /// - /// Other markers, including the footer and its certificates, stay raw. + /// A footer or genesis marker does not carry this id. A marker that does + /// not parse is skipped. pub fn parent_consensus_block_id(&self) -> Option<[u8; 32]> { self.block_markers .as_deref() - .and_then(parent_consensus_block_id_from_markers) + .and_then(crate::blockmarker::parent_consensus_block_id) } pub fn alpenglow(&self) -> bool { @@ -231,27 +232,6 @@ impl SlotMeta { } } -fn parent_consensus_block_id_from_markers(markers: &[Vec]) -> Option<[u8; 32]> { - markers.iter().rev().find_map(|marker| { - let bytes = marker.as_slice(); - if bytes.len() < 46 { - return None; - } - let version = u16::from_le_bytes([bytes[0], bytes[1]]); - let variant = bytes[2]; - let payload_len = u16::from_le_bytes([bytes[3], bytes[4]]); - if version != 1 || (variant != 1 && variant != 2) || payload_len != 41 { - return None; - } - if bytes.len() != 46 || bytes[5] != 1 { - return None; - } - let mut parent_block_id = [0u8; 32]; - parent_block_id.copy_from_slice(&bytes[14..46]); - Some(parent_block_id) - }) -} - impl<'b, C> Decode<'b, C> for SlotMeta { fn decode(d: &mut Decoder<'b>, _ctx: &mut C) -> core::result::Result { let len = d From fafdac59a55881ec68c75bf6e546c1dc8610c1c8 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 28 Oct 2026 16:13:26 +0100 Subject: [PATCH 06/38] add show markers CLI command --- blockzilla/cli/src/main.rs | 29 ++ .../cli/src/show_archive_v2_block_markers.rs | 465 ++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 blockzilla/cli/src/show_archive_v2_block_markers.rs diff --git a/blockzilla/cli/src/main.rs b/blockzilla/cli/src/main.rs index 41cfe32..baab486 100644 --- a/blockzilla/cli/src/main.rs +++ b/blockzilla/cli/src/main.rs @@ -28,6 +28,7 @@ mod index_tokens; mod pre_hot; mod predecessor_tail_seed; mod scheduler; +mod show_archive_v2_block_markers; mod split_compact; mod stream; mod token_events; @@ -1175,6 +1176,9 @@ enum Commands { #[arg(long)] entry: Option, }, + + /// Print consensus block ids and Alpenglow block markers from a hot-block Archive V2. + ShowArchiveV2BlockMarkers(show_archive_v2_block_markers::ShowArchiveV2BlockMarkersArgs), } /// Archive V3 candidate construction, taken from the former `ia-*` binaries. @@ -2147,6 +2151,7 @@ fn main() -> Result<()> { Commands::DumpCarPohEntry { input, slot, entry } => { car_debug::dump_car_poh_entry(&input, slot, entry) } + Commands::ShowArchiveV2BlockMarkers(args) => show_archive_v2_block_markers::run(args), Commands::VerifyArchiveV2Poh { archive_dir, threads, @@ -3574,4 +3579,28 @@ mod cli_tests { _ => panic!("unexpected command"), } } + + #[test] + fn show_archive_v2_block_markers_command_parses() { + let cli = Cli::try_parse_from([ + "blockzilla", + "show-archive-v2-block-markers", + "epoch-out", + "--slot", + "460512004", + "--max-blocks", + "8", + "--check-parent-chain", + ]) + .unwrap(); + match cli.command { + Commands::ShowArchiveV2BlockMarkers(args) => { + assert_eq!(args.archive_dir, Path::new("epoch-out")); + assert_eq!(args.slot, Some(460512004)); + assert_eq!(args.max_blocks, Some(8)); + assert!(args.check_parent_chain); + } + _ => panic!("unexpected command"), + } + } } diff --git a/blockzilla/cli/src/show_archive_v2_block_markers.rs b/blockzilla/cli/src/show_archive_v2_block_markers.rs new file mode 100644 index 0000000..cba897a --- /dev/null +++ b/blockzilla/cli/src/show_archive_v2_block_markers.rs @@ -0,0 +1,465 @@ +//! Print the consensus id and Alpenglow markers stored on hot-block Archive V2. + +use std::{ + fs::File, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result, anyhow, ensure}; +use blockzilla_archive_v2::{ + ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKS_FILE, ARCHIVE_V2_HOT_INDEX_FLAG_DICTIONARY, + ARCHIVE_V2_HOT_INDEX_FLAG_RAW_BLOCKS, ArchiveV2HotBlockBlob, ArchiveV2HotBlockIndexRow, + archive_v2_hot_index_path, deserialize_archive_v2_hot_block_blob, + read_archive_v2_hot_block_index, +}; +use clap::Args; +use of_car_reader::blockmarker::{self, Marker}; + +#[derive(Args, Debug)] +pub(crate) struct ShowArchiveV2BlockMarkersArgs { + /// Directory that contains archive-v2-blocks.zstd and archive-v2-blocks.index. + pub archive_dir: PathBuf, + /// Print this slot only. + #[arg(long)] + pub slot: Option, + /// Stop after N blocks, in index order. + #[arg(long)] + pub max_blocks: Option, + /// Require each parent consensus id to equal the previous block's consensus id. + /// + /// The first shown block has no previous id. A previous block with no consensus id + /// starts the chain again. + #[arg(long)] + pub check_parent_chain: bool, +} + +pub(crate) fn run(args: ShowArchiveV2BlockMarkersArgs) -> Result<()> { + let mut stdout = std::io::stdout().lock(); + write!(stdout, "{}", render(&args)?)?; + Ok(()) +} + +fn render(args: &ShowArchiveV2BlockMarkersArgs) -> Result { + if let Some(0) = args.max_blocks { + return Err(anyhow!("--max-blocks must be greater than zero")); + } + let blocks_path = args.archive_dir.join(ARCHIVE_V2_BLOCKS_FILE); + let index = read_archive_v2_hot_block_index(&archive_v2_hot_index_path(&blocks_path)) + .with_context(|| { + format!( + "read {}", + args.archive_dir.join(ARCHIVE_V2_BLOCK_INDEX_FILE).display() + ) + })?; + let raw_blocks = index.flags & ARCHIVE_V2_HOT_INDEX_FLAG_RAW_BLOCKS != 0; + let rows = selected_rows(&index.rows, args)?; + if !raw_blocks && index.flags & ARCHIVE_V2_HOT_INDEX_FLAG_DICTIONARY != 0 && !rows.is_empty() { + return Err(anyhow!( + "archive index uses a zstd dictionary; this command reads plain zstd frames" + )); + } + + let mut file = + File::open(&blocks_path).with_context(|| format!("open {}", blocks_path.display()))?; + let mut out = format!("blocks {}\n", rows.len()); + let mut previous_consensus: Option<[u8; 32]> = None; + for row in rows { + let plain = read_block_bytes(&mut file, raw_blocks, row)?; + let block = deserialize_archive_v2_hot_block_blob(&plain) + .map_err(|err| anyhow!("slot {} hot block: {err}", row.slot))?; + ensure!( + block.header.slot == row.slot, + "slot {} index row decodes as slot {}", + row.slot, + block.header.slot + ); + ensure!( + block.tx_count == row.tx_count, + "slot {} index tx_count {} decodes as {}", + row.slot, + row.tx_count, + block.tx_count + ); + let parent = blockmarker::parent_consensus_block_id(&block.block_markers); + if args.check_parent_chain { + if let Some(previous) = previous_consensus { + ensure!( + parent == Some(previous), + "slot {} parent {} does not match previous consensus {}", + block.header.slot, + hex(parent), + hex(Some(previous)) + ); + } + previous_consensus = block.consensus_block_id; + } + append_block(&mut out, &block, plain.len(), parent)?; + } + Ok(out) +} + +fn selected_rows<'a>( + rows: &'a [ArchiveV2HotBlockIndexRow], + args: &ShowArchiveV2BlockMarkersArgs, +) -> Result> { + let mut selected: Vec<_> = rows + .iter() + .filter(|row| args.slot.is_none_or(|slot| row.slot == slot)) + .collect(); + if let Some(slot) = args.slot { + ensure!(!selected.is_empty(), "slot {slot} not found"); + } + if let Some(max_blocks) = args.max_blocks { + selected.truncate(usize::try_from(max_blocks).unwrap_or(usize::MAX)); + } + Ok(selected) +} + +fn read_block_bytes( + file: &mut File, + raw_blocks: bool, + row: &ArchiveV2HotBlockIndexRow, +) -> Result> { + file.seek(SeekFrom::Start(row.compressed_offset)) + .with_context(|| format!("seek slot {}", row.slot))?; + if raw_blocks { + let mut bytes = vec![0; row.uncompressed_len as usize]; + file.read_exact(&mut bytes) + .with_context(|| format!("read raw slot {}", row.slot))?; + return Ok(bytes); + } + let mut input = vec![0; row.compressed_len as usize]; + file.read_exact(&mut input) + .with_context(|| format!("read zstd slot {}", row.slot))?; + let plain = zstd::bulk::decompress(&input, row.uncompressed_len as usize) + .with_context(|| format!("decompress slot {}", row.slot))?; + ensure!( + plain.len() == row.uncompressed_len as usize, + "slot {} decompressed to {} bytes, index says {}", + row.slot, + plain.len(), + row.uncompressed_len + ); + Ok(plain) +} + +fn append_block( + out: &mut String, + block: &ArchiveV2HotBlockBlob, + uncompressed_len: usize, + parent: Option<[u8; 32]>, +) -> Result<()> { + out.push_str(&format!( + "slot {} txs {} uncompressed {uncompressed_len} consensus {} parent {} markers {}\n", + block.header.slot, + block.tx_count, + hex(block.consensus_block_id), + hex(parent), + block.block_markers.len() + )); + for (index, marker) in block.block_markers.iter().enumerate() { + let parsed = blockmarker::parse(marker) + .map_err(|err| anyhow!("slot {} marker {index}: {err}", block.header.slot))?; + match parsed { + Marker::Header(header) => { + out.push_str(&format!( + " marker {index} header parent_slot {} parent {}\n", + header.parent_slot, + hex(Some(header.parent_block_id)) + )); + } + Marker::UpdateParent(update) => { + out.push_str(&format!( + " marker {index} update-parent slot {} parent {}\n", + update.new_parent_slot, + hex(Some(update.new_parent_block_id)) + )); + } + Marker::Footer(footer) => { + let certs = blockmarker::decode_footer_certificates(&footer.certificates).map_err( + |err| anyhow!("slot {} marker {index} footer: {err}", block.header.slot), + )?; + let kind = match certs.finalization.as_ref() { + Some(cert) if cert.notar_aggregate.is_some() => "slow", + Some(_) => "fast", + None => "no-finalization", + }; + out.push_str(&format!( + " marker {index} footer bytes {} bank {} agent {:?} finalization {kind} skip {} notar-reward {}\n", + marker.len(), + hex(Some(footer.bank_hash)), + String::from_utf8_lossy(&footer.block_user_agent), + certs.skip_reward.is_some(), + certs.notar_reward.is_some() + )); + } + Marker::Genesis(cert) => { + out.push_str(&format!( + " marker {index} genesis slot {} block {}\n", + cert.slot, + hex(Some(cert.block_id)) + )); + } + } + } + Ok(()) +} + +fn hex(bytes: Option<[u8; 32]>) -> String { + match bytes { + None => "none".to_string(), + Some(bytes) => bytes.iter().map(|byte| format!("{byte:02x}")).collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use blockzilla_archive_v2::{ArchiveV2HotBlockHeader, write_archive_v2_hot_block_index}; + use blockzilla_primitives::historical_source_wincode_leb128_config; + + fn header_marker(variant: u8, parent_slot: u64, parent: [u8; 32]) -> Vec { + let mut marker = vec![0u8; 46]; + marker[0] = 1; + marker[2] = variant; + marker[3] = 41; + marker[5] = 1; + marker[6..14].copy_from_slice(&parent_slot.to_le_bytes()); + marker[14..46].copy_from_slice(&parent); + marker + } + + fn footer_marker() -> Vec { + let mut payload = Vec::new(); + payload.push(1); + payload.extend_from_slice(&[0xab; 32]); + payload.extend_from_slice(&7u64.to_le_bytes()); + let agent = b"agave/4.3.2"; + payload.push(u8::try_from(agent.len()).unwrap()); + payload.extend_from_slice(agent); + payload.extend_from_slice(&[0, 0, 0]); + let mut marker = Vec::new(); + marker.extend_from_slice(&1u16.to_le_bytes()); + marker.push(0); + marker.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_le_bytes()); + marker.extend(payload); + marker + } + + fn hot_block( + slot: u64, + consensus: Option<[u8; 32]>, + markers: Vec>, + ) -> ArchiveV2HotBlockBlob { + ArchiveV2HotBlockBlob { + header: ArchiveV2HotBlockHeader { + slot, + parent_slot: slot.saturating_sub(1), + blockhash_id: 1, + previous_blockhash_id: 0, + block_time: None, + block_height: None, + rewards: None, + }, + tx_count: 0, + tx_rows: Vec::new(), + message_bytes: Vec::new(), + metadata_bytes: Vec::new(), + consensus_block_id: consensus, + block_markers: markers, + } + } + + fn write_archive(dir: &Path, blocks: &[ArchiveV2HotBlockBlob], flags: u32) { + let raw = flags & ARCHIVE_V2_HOT_INDEX_FLAG_RAW_BLOCKS != 0; + let mut blob = Vec::new(); + let mut rows = Vec::new(); + for (index, block) in blocks.iter().enumerate() { + let plain = + wincode::config::serialize(block, historical_source_wincode_leb128_config()) + .unwrap(); + let stored = if raw { + plain.clone() + } else { + zstd::bulk::compress(&plain, 1).unwrap() + }; + rows.push(ArchiveV2HotBlockIndexRow { + block_id: u32::try_from(index).unwrap(), + slot: block.header.slot, + compressed_offset: u64::try_from(blob.len()).unwrap(), + compressed_len: u32::try_from(stored.len()).unwrap(), + uncompressed_len: u32::try_from(plain.len()).unwrap(), + tx_count: block.tx_count, + first_tx_ordinal: 0, + first_signature_ordinal: 0, + signature_count: 0, + }); + blob.extend(stored); + } + std::fs::write(dir.join(ARCHIVE_V2_BLOCKS_FILE), &blob).unwrap(); + write_archive_v2_hot_block_index( + &dir.join(ARCHIVE_V2_BLOCK_INDEX_FILE), + u64::try_from(blob.len()).unwrap(), + 1, + flags, + &rows, + ) + .unwrap(); + } + + fn args(dir: &Path) -> ShowArchiveV2BlockMarkersArgs { + ShowArchiveV2BlockMarkersArgs { + archive_dir: dir.to_path_buf(), + slot: None, + max_blocks: None, + check_parent_chain: false, + } + } + + #[test] + fn prints_markers_and_accepts_a_parent_chain() { + let dir = tempfile::tempdir().unwrap(); + let first = [0x11; 32]; + let second = [0x22; 32]; + write_archive( + dir.path(), + &[ + hot_block( + 10, + Some(first), + vec![header_marker(1, 9, [0x10; 32]), footer_marker()], + ), + hot_block( + 11, + Some(second), + vec![ + header_marker(1, 10, [0x99; 32]), + header_marker(2, 10, first), + ], + ), + ], + 0, + ); + let mut request = args(dir.path()); + request.check_parent_chain = true; + let text = render(&request).unwrap(); + assert!(text.starts_with("blocks 2\n")); + let slot10 = text + .lines() + .find(|line| line.starts_with("slot 10 ")) + .unwrap(); + assert!(slot10.contains("txs 0 ")); + assert!(slot10.contains(&format!( + "consensus {} parent {} markers 2", + hex(Some(first)), + hex(Some([0x10; 32])) + ))); + assert!(text.contains(&format!( + "marker 0 header parent_slot 9 parent {}", + hex(Some([0x10; 32])) + ))); + assert!(text.contains("finalization no-finalization skip false notar-reward false")); + assert!(text.contains("agent \"agave/4.3.2\"")); + let slot11 = text + .lines() + .find(|line| line.starts_with("slot 11 ")) + .unwrap(); + assert!(slot11.contains(&format!( + "consensus {} parent {} markers 2", + hex(Some(second)), + hex(Some(first)) + ))); + assert!(text.contains(&format!( + "marker 0 header parent_slot 10 parent {}", + hex(Some([0x99; 32])) + ))); + assert!(text.contains(&format!( + "marker 1 update-parent slot 10 parent {}", + hex(Some(first)) + ))); + } + + #[test] + fn parent_chain_failure_names_the_slot() { + let dir = tempfile::tempdir().unwrap(); + write_archive( + dir.path(), + &[ + hot_block(10, Some([0x11; 32]), vec![header_marker(1, 9, [0x10; 32])]), + hot_block(11, Some([0x22; 32]), vec![header_marker(1, 10, [0x33; 32])]), + ], + 0, + ); + let mut request = args(dir.path()); + request.check_parent_chain = true; + let error = render(&request).unwrap_err().to_string(); + assert!(error.contains("slot 11 parent")); + assert!(error.contains("does not match previous consensus")); + } + + #[test] + fn slot_and_max_blocks_select_rows() { + let dir = tempfile::tempdir().unwrap(); + write_archive( + dir.path(), + &[ + hot_block(10, Some([0x11; 32]), Vec::new()), + hot_block(11, Some([0x22; 32]), Vec::new()), + hot_block(12, None, Vec::new()), + ], + 0, + ); + let mut request = args(dir.path()); + request.max_blocks = Some(1); + let text = render(&request).unwrap(); + assert!(text.starts_with("blocks 1\n")); + assert!(text.contains("slot 10 ")); + assert!(!text.contains("slot 11 ")); + + request.max_blocks = None; + request.slot = Some(12); + let text = render(&request).unwrap(); + assert!(text.contains("slot 12 txs 0")); + assert!(text.contains("consensus none parent none markers 0\n")); + assert!(!text.contains("slot 10 ")); + + request.slot = Some(99); + let error = render(&request).unwrap_err().to_string(); + assert!(error.contains("slot 99 not found")); + + request.slot = None; + request.max_blocks = Some(0); + let error = render(&request).unwrap_err().to_string(); + assert!(error.contains("--max-blocks must be greater than zero")); + } + + #[test] + fn reads_raw_block_bytes() { + let dir = tempfile::tempdir().unwrap(); + write_archive( + dir.path(), + &[hot_block( + 10, + Some([0x44; 32]), + vec![header_marker(1, 9, [0x10; 32])], + )], + ARCHIVE_V2_HOT_INDEX_FLAG_RAW_BLOCKS, + ); + let text = render(&args(dir.path())).unwrap(); + assert!(text.contains(&format!("consensus {}", hex(Some([0x44; 32]))))); + assert!(text.contains("marker 0 header parent_slot 9")); + } + + #[test] + fn dictionary_frames_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + write_archive( + dir.path(), + &[hot_block(10, None, Vec::new())], + ARCHIVE_V2_HOT_INDEX_FLAG_DICTIONARY, + ); + let error = render(&args(dir.path())).unwrap_err().to_string(); + assert!(error.contains("zstd dictionary")); + } +} From cc58fa46a1fe00b71d5440d25589ac0ea7a057bb Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 28 Oct 2026 16:20:07 +0100 Subject: [PATCH 07/38] correct warning --- blockzilla/cli/src/show_archive_v2_block_markers.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blockzilla/cli/src/show_archive_v2_block_markers.rs b/blockzilla/cli/src/show_archive_v2_block_markers.rs index cba897a..75c1b85 100644 --- a/blockzilla/cli/src/show_archive_v2_block_markers.rs +++ b/blockzilla/cli/src/show_archive_v2_block_markers.rs @@ -3,7 +3,7 @@ use std::{ fs::File, io::{Read, Seek, SeekFrom, Write}, - path::{Path, PathBuf}, + path::PathBuf, }; use anyhow::{Context, Result, anyhow, ensure}; @@ -216,6 +216,8 @@ fn hex(bytes: Option<[u8; 32]>) -> String { #[cfg(test)] mod tests { use super::*; + use std::path::Path; + use blockzilla_archive_v2::{ArchiveV2HotBlockHeader, write_archive_v2_hot_block_index}; use blockzilla_primitives::historical_source_wincode_leb128_config; From d1012330f31f55fec1ad819bf77f77227ceb311a Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Tue, 3 Nov 2026 16:24:20 +0100 Subject: [PATCH 08/38] test build archive from ledger on server --- Cargo.toml | 5 +- hivezilla/ledger-archive/Cargo.lock | 9289 ++++++++++++++++++ hivezilla/ledger-archive/Cargo.toml | 39 + hivezilla/ledger-archive/rust-toolchain.toml | 2 + hivezilla/ledger-archive/src/check.rs | 279 + hivezilla/ledger-archive/src/convert.rs | 675 ++ hivezilla/ledger-archive/src/ledger.rs | 763 ++ hivezilla/ledger-archive/src/main.rs | 102 + hivezilla/ledger-archive/src/markers.rs | 43 + hivezilla/ledger-archive/src/source.rs | 304 + 10 files changed, 11499 insertions(+), 2 deletions(-) create mode 100644 hivezilla/ledger-archive/Cargo.lock create mode 100644 hivezilla/ledger-archive/Cargo.toml create mode 100644 hivezilla/ledger-archive/rust-toolchain.toml create mode 100644 hivezilla/ledger-archive/src/check.rs create mode 100644 hivezilla/ledger-archive/src/convert.rs create mode 100644 hivezilla/ledger-archive/src/ledger.rs create mode 100644 hivezilla/ledger-archive/src/main.rs create mode 100644 hivezilla/ledger-archive/src/markers.rs create mode 100644 hivezilla/ledger-archive/src/source.rs diff --git a/Cargo.toml b/Cargo.toml index 2e2abd4..aa74682 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [workspace] resolver = "2" -# The validator plugin has an independent lock file for its exact Agave ABI. -exclude = ["hivezilla/direct-agave", "hivezilla/stream-relay", "hivezilla/stream-relay/decoder"] +# The validator plugin and the ledger archive tool have independent lock files +# for their exact Agave ABI. +exclude = ["hivezilla/direct-agave", "hivezilla/stream-relay", "hivezilla/stream-relay/decoder", "hivezilla/ledger-archive"] members = [ # Shared models, primitives, and byte sources. "crates/blockzilla-model", diff --git a/hivezilla/ledger-archive/Cargo.lock b/hivezilla/ledger-archive/Cargo.lock new file mode 100644 index 0000000..f7008b0 --- /dev/null +++ b/hivezilla/ledger-archive/Cargo.lock @@ -0,0 +1,9289 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "aes" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.1", +] + +[[package]] +name = "aes-gcm-siv" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c32325128cb24de2a2d14d92da7569dae7073f9e47334521d494747fa94d902" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ctutils", + "polyval", + "zeroize", +] + +[[package]] +name = "agave-bls-cert-verify" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-votor-messages", + "bitvec", + "qualifier_attr", + "rayon", + "solana-bls-signatures", + "solana-signer-store", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "agave-cpu-utils" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "libc", +] + +[[package]] +name = "agave-feature-set" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "ahash 0.8.12", + "solana-epoch-schedule", + "solana-hash", + "solana-keypair", + "solana-pubkey 4.4.0", + "solana-sha256-hasher", + "solana-svm-feature-set", +] + +[[package]] +name = "agave-fs" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-io-uring", + "io-uring", + "libc", + "log", + "slab", + "smallvec", + "tempfile", + "thiserror 2.0.21", +] + +[[package]] +name = "agave-io-uring" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "io-uring", + "libc", + "log", + "slab", + "smallvec", +] + +[[package]] +name = "agave-precompiles" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "bincode", + "ed25519-dalek", + "libsecp256k1", + "openssl", + "solana-ed25519-program", + "solana-keccak-hasher", + "solana-message", + "solana-precompile-error", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-secp256k1-program", + "solana-secp256r1-program", +] + +[[package]] +name = "agave-random" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "rand 0.9.5", +] + +[[package]] +name = "agave-reserved-account-keys" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "solana-pubkey 4.4.0", + "solana-sdk-ids", +] + +[[package]] +name = "agave-snapshots" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-fs", + "bincode", + "bzip2", + "crossbeam-channel", + "log", + "lz4", + "rand 0.9.5", + "regex", + "semver", + "solana-accounts-db", + "solana-clock", + "solana-genesis-config", + "solana-hash", + "solana-lattice-hash", + "solana-measure", + "solana-metrics", + "strum", + "symlink", + "tar", + "tempfile", + "thiserror 2.0.21", + "wincode", + "zstd 0.14.3", +] + +[[package]] +name = "agave-transaction-view" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8c52171e8afbe9c1f32996b71fa12fed24cf4c061651198f2d861886405a76" +dependencies = [ + "bytes", + "solana-hash", + "solana-message", + "solana-packet", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-short-vec", + "solana-signature", + "solana-svm-transaction", + "solana-transaction", +] + +[[package]] +name = "agave-votor-messages" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "bitvec", + "crossbeam-channel", + "log", + "serde", + "smallvec", + "solana-address 2.9.0", + "solana-bls-signatures", + "solana-clock", + "solana-epoch-schedule", + "solana-hash", + "solana-leader-schedule", + "solana-pubkey 4.4.0", + "solana-short-vec", + "solana-signer-store", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "agave-xdp" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-cpu-utils", + "agave-xdp-ebpf", + "arc-swap", + "arrayvec", + "aya", + "bytes", + "caps", + "crossbeam-channel", + "crossbeam-queue", + "libc", + "log", + "thiserror 2.0.21", +] + +[[package]] +name = "agave-xdp-ebpf" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "aya-ebpf", +] + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" + +[[package]] +name = "alloc-stdlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "anza-quinn" +version = "0.11.12-issue2860.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd524576ab56786d8241905b12498e8ec6b9a09a58bfe8ef9e9dfcf511aeecf4" +dependencies = [ + "anza-quinn-proto", + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "anza-quinn-proto" +version = "0.11.18-issue2860.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9dfece0ce5ffc483085c0112b8499dabab137e2eb028f1ff3a28d5b24d6d477" +dependencies = [ + "bytes", + "fastbloom", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "archery" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca55ee147b1926dbea904f50fe4902494e97bc742205abbbf10c709e43815f" + +[[package]] +name = "ark-bn254" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69eab57e8d2663efa5c63135b2af4f396d66424f88954c21104125ab6b3e6bc" +dependencies = [ + "ark-ec", + "ark-ff", + "ark-std", +] + +[[package]] +name = "ark-ec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d68f2d516162846c1238e755a7c4d131b892b70cc70c471a8e3ca3ed818fce" +dependencies = [ + "ahash 0.8.12", + "ark-ff", + "ark-poly", + "ark-serialize", + "ark-std", + "educe", + "fnv", + "hashbrown 0.15.5", + "itertools 0.13.0", + "num-bigint", + "num-integer", + "num-traits", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" +dependencies = [ + "ark-ff-asm", + "ark-ff-macros", + "ark-serialize", + "ark-std", + "arrayvec", + "digest 0.10.7", + "educe", + "itertools 0.13.0", + "num-bigint", + "num-traits", + "paste", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-ff-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-poly" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579305839da207f02b89cd1679e50e67b4331e2f9294a57693e5051b7703fe27" +dependencies = [ + "ahash 0.8.12", + "ark-ff", + "ark-serialize", + "ark-std", + "educe", + "fnv", + "hashbrown 0.15.5", +] + +[[package]] +name = "ark-serialize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" +dependencies = [ + "ark-serialize-derive", + "ark-std", + "arrayvec", + "digest 0.10.7", + "num-bigint", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-std" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" +dependencies = [ + "num-traits", + "rand 0.8.8", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + +[[package]] +name = "async-compression" +version = "0.4.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "atomic-wait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55b94919229f2c42292fd71ffa4b75e83193bffdd77b1e858cd55fd2d0b0ea8" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64 0.22.1", + "flate2", + "http", + "log", + "native-tls", + "serde", + "serde_json", + "serde_urlencoded", + "url", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + +[[package]] +name = "aya" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e644424fada9fff4fdc63848db1732fb69b626e8328202ef55c03df1f4d939" +dependencies = [ + "assert_matches", + "aya-obj", + "bitflags 2.13.2", + "hashbrown 0.17.1", + "libc", + "log", + "object", + "once_cell", + "scopeguard", + "thiserror 2.0.21", +] + +[[package]] +name = "aya-build" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a356366c974ffb737700f9691962d7aabe52b3843702e381773afef5a0f51c9f" +dependencies = [ + "anyhow", + "cargo_metadata", + "rustc_version", + "which", +] + +[[package]] +name = "aya-ebpf" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eace5970943931fea46a8cbdca208a8f6a9e0413d85a5a6c4e168f0b909570bc" +dependencies = [ + "aya-build", + "aya-ebpf-bindings", + "aya-ebpf-cty", + "aya-ebpf-macros", + "rustversion", +] + +[[package]] +name = "aya-ebpf-bindings" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1f19286ebbda6673099b68d049034838f08e4939cef66c998a8d3e62825d76" +dependencies = [ + "aya-build", + "aya-ebpf-cty", +] + +[[package]] +name = "aya-ebpf-cty" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0884264bc8a51a49fed03df8feada3a21d703cf501dae33c28bdba5d2313ef33" +dependencies = [ + "aya-build", +] + +[[package]] +name = "aya-ebpf-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e3a7f86e914df106dd00f9d757bc93f7d5c93380c51eead23d8d2ccd15a87a" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aya-obj" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c76b9c75d9cdc155ff8f6a06d61e873f67bf47be8cfa92a3b5aaea43f4b4077" +dependencies = [ + "bytes", + "log", + "object", + "thiserror 2.0.21", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "binout" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff5fa7be63dbf62cca511b0e64e4b869ebc68f2b99d65fb1aa127771c5b7cd0d" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitm" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31468ea4a856000d83cb61960dfdc2980ecd96b15b61321c8c76cc96aea6e688" +dependencies = [ + "dyn_size_of", +] + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "serde", + "tap", + "wyz", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blockzilla-archive-v2" +version = "0.2.0" +dependencies = [ + "anyhow", + "blockzilla-compact", + "blockzilla-primitives", + "blockzilla-program-logs", + "of-car-reader", + "serde", + "sha2 0.11.1", + "wincode", +] + +[[package]] +name = "blockzilla-compact" +version = "0.2.0" +dependencies = [ + "anyhow", + "blockzilla-log-parser", + "blockzilla-primitives", + "blockzilla-program-logs", + "blockzilla-registry", + "data-encoding", + "heapless", + "of-car-reader", + "prost", + "serde", + "solana-pubkey 4.4.0", + "wincode", +] + +[[package]] +name = "blockzilla-log-parser" +version = "0.2.0" + +[[package]] +name = "blockzilla-primitives" +version = "0.2.0" +dependencies = [ + "anyhow", + "serde", + "solana-pubkey 4.4.0", + "wincode", +] + +[[package]] +name = "blockzilla-program-logs" +version = "0.2.0" +dependencies = [ + "blockzilla-primitives", + "data-encoding", + "serde", + "solana-pubkey 4.4.0", + "wincode", +] + +[[package]] +name = "blockzilla-registry" +version = "0.2.0" +dependencies = [ + "anyhow", + "blockzilla-primitives", + "five8", + "gxhash", + "memmap2 0.9.12", + "ph", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "blst" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20659f9bbee16cbbd2f7393e40ab6309f5a98f76a2eb57a995ec508b72387fe" +dependencies = [ + "cc", + "glob", + "threadpool", + "zeroize", +] + +[[package]] +name = "blstrs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a8a8ed6fefbeef4a8c7b460e4110e12c5e22a5b7cf32621aae6ad650c4dcf29" +dependencies = [ + "blst", + "byte-slice-cast", + "ff 0.13.1", + "group 0.13.0", + "pairing", + "rand_core 0.6.4", + "serde", + "subtle", +] + +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "brotli" +version = "9.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34c6783f42c21c952ad7112b354f0ba2f10760e9decea5b1b60523d6e270faaf" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bv" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8834bb1d8ee5dc048ee3124f2c7c1afcc6bc9aed03f11e9dfd8c69470a5db340" +dependencies = [ + "feature-probe", + "serde", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +dependencies = [ + "serde", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" +dependencies = [ + "serde_core", +] + +[[package]] +name = "caps" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1ddba47aba30b6a889298ad0109c3b8dcb0e8fc993b459daa7067d46f865e0" +dependencies = [ + "libc", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "cc" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "cfg_eval" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45565fc9416b9896014f5732ac776f810ee53a66730c17e4020c3ec064a8f88f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-humanize" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799627e6b4d27827a814e837b9d8a504832086081806d45b1afa34dc982b023b" +dependencies = [ + "chrono", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap", + "unicode-width 0.1.14", + "vec_map", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width 0.2.3", + "windows-sys 0.59.0", +] + +[[package]] +name = "console" +version = "0.16.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width 0.2.3", + "windows-sys 0.61.2", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", + "subtle", +] + +[[package]] +name = "curve25519-dalek" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rand_core 0.6.4", + "rustc_version", + "serde", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.12", + "rayon", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + +[[package]] +name = "derivation-path" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e5c37193a1db1d8ed868c03ec7b152175f26160a5b740e5e484143877e0adf0" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "dyn_size_of" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a742b95783b1f45b900129082cbc47717b6a77ee8d17eea70a8ea62462f5de3" + +[[package]] +name = "eager" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe71d579d1812060163dff96056261deb5bf6729b100fa2e36a68b9649ba3d3" + +[[package]] +name = "ecdsa" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" +dependencies = [ + "der", + "digest 0.11.3", + "elliptic-curve", + "rfc6979", + "signature 3.0.0", + "spki", + "zeroize", +] + +[[package]] +name = "ed25519" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +dependencies = [ + "signature 1.6.4", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "signature 2.2.0", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +dependencies = [ + "curve25519-dalek 3.2.0", + "ed25519 1.5.3", + "rand 0.7.3", + "serde", + "sha2 0.9.9", + "zeroize", +] + +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "elliptic-curve" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" +dependencies = [ + "base16ct", + "crypto-bigint", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "pkcs8", + "rand_core 0.10.1", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "enum-iterator" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4549325971814bda7a44061bf3fe7e487d447cba01e4220a4b454d630d7a016" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enum-ordinalize" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89dd01549b09589510cf0647475075d12071456586d70f5c75c98ae2a5537677" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher 1.0.4", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "feature-probe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "bitvec", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "five8" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f76610e969fa1784327ded240f1e28a3fd9520c9cec93b636fcf62dd37f772" +dependencies = [ + "five8_core", +] + +[[package]] +name = "five8_const" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a0f1728185f277989ca573a402716ae0beaaea3f76a8ff87ef9dd8fb19436c5" +dependencies = [ + "five8_core", +] + +[[package]] +name = "five8_core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059c31d7d36c43fe39d89e55711858b4da8be7eb6dabac23c7289b1a19489406" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "goauth" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6a66933ee399eb5dcb9fa142d0d0eaf3faca037a06add0e8de4fdec25f11d21" +dependencies = [ + "arc-swap", + "attohttpc", + "log", + "serde", + "serde_derive", + "serde_json", + "simpl", + "smpl_jwt", + "time", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff 0.13.1", + "rand 0.8.8", + "rand_core 0.6.4", + "rand_xorshift", + "subtle", +] + +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "gxhash" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ce1bab7aa741d4e7042b2aae415b78741f267a98a7271ea226cd5ba6c43d7d" +dependencies = [ + "rustversion", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32", + "serde_core", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hivezilla-ledger-archive" +version = "0.2.0" +dependencies = [ + "anyhow", + "blockzilla-archive-v2", + "blockzilla-compact", + "blockzilla-primitives", + "blockzilla-registry", + "bs58", + "clap 4.6.7", + "of-car-reader", + "solana-entry", + "solana-hash", + "solana-instruction-error 3.0.0", + "solana-ledger", + "solana-message", + "solana-reward-info", + "solana-transaction-error", + "solana-transaction-status", + "tempfile", + "wincode", + "zstd 0.13.4", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "hmac-drbg" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" +dependencies = [ + "digest 0.9.0", + "generic-array", + "hmac 0.8.1", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "subtle", + "typenum", + "zeroize", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "imbl" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" +dependencies = [ + "archery", + "equivalent", + "imbl-sized-chunks", + "rand_core 0.9.5", + "rand_xoshiro", + "rayon", + "version_check", + "wide", +] + +[[package]] +name = "imbl-sized-chunks" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console 0.16.6", + "portable-atomic", + "unicode-width 0.2.3", + "unit-prefix", + "web-time", +] + +[[package]] +name = "inout" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine 4.6.9", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "k256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f50113171a713f4a4231ef82eb26703607139b35dcb56241f0ceab2ae1f7d8" +dependencies = [ + "cpubits", + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.11.1", + "signature 3.0.0", + "wnaf", +] + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + +[[package]] +name = "lazy-lru" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35523c6dfa972e1fd19132ef647eff4360a6546c6271807e1327ca6e8797f96" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f454fd6831a29c0e827ed075bca2a15be201bd57f4168f0fd1041124eab17e" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "librocksdb-sys" +version = "0.17.3+10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", +] + +[[package]] +name = "libsecp256k1" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79019718125edc905a079a70cfa5f3820bc76139fc91d6f9abc27ea2a887139" +dependencies = [ + "arrayref", + "base64 0.22.1", + "digest 0.9.0", + "hmac-drbg", + "libsecp256k1-core", + "libsecp256k1-gen-ecmult", + "libsecp256k1-gen-genmult", + "rand 0.8.8", + "serde", + "sha2 0.9.9", + "typenum", +] + +[[package]] +name = "libsecp256k1-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be9b9bb642d8522a44d533eab56c16c738301965504753b03ad1de3425d5451" +dependencies = [ + "crunchy", + "digest 0.9.0", + "subtle", +] + +[[package]] +name = "libsecp256k1-gen-ecmult" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3038c808c55c87e8a172643a7d87187fc6c4174468159cb3090659d55bcb4809" +dependencies = [ + "libsecp256k1-core", +] + +[[package]] +name = "libsecp256k1-gen-genmult" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db8d6ba2cec9eacc40e6e8ccc98931840301f1006e95647ceb2dd5c3aa06f7c" +dependencies = [ + "libsecp256k1-core", +] + +[[package]] +name = "libz-sys" +version = "1.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "light-poseidon" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29d3992daf44efce5f36b8e61dc5373ee5ff7485cc59d02f5d8afac8a57067eb" +dependencies = [ + "ark-bn254", + "ark-ff", + "num-bigint", + "thiserror 1.0.69", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memmap2" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "merlin" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" +dependencies = [ + "byteorder", + "keccak 0.1.6", + "rand_core 0.6.4", + "zeroize", +] + +[[package]] +name = "minicbor" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7244eafd1e09e7c4e33043881997aa6ab6dffff3ed2cee3395a278980481d6e" +dependencies = [ + "half", + "minicbor-derive", +] + +[[package]] +name = "minicbor-derive" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f5ad8dfe10176465fe33f19ecfcf08783ba66630fdb40b2e4542547c1046f0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "mockall" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a6ceddfe3ce334925e96bf420fdb2dcee5bed6c632a168ece622676dadeaf8a" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfe16fbe8a314aeec0b861ac24e60b1e123e97634bab045475b9d6a18416fd8" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "modular-bitfield" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2956e537fc68236d2aa048f55704f231cc93f1c4de42fe1ecb5bd7938061fc4a" +dependencies = [ + "modular-bitfield-impl", + "static_assertions", +] + +[[package]] +name = "modular-bitfield-impl" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b43b4fd69e3437618106f7754f34021b831a514f9e1a98ae863cabcd8d8dad" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e98dc3b890f6c23a0f9d3d491a2823d0dea0fa656302a13dd225fa924112a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi 0.5.3", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "crc32fast", + "hashbrown 0.17.1", + "indexmap", + "memchr", +] + +[[package]] +name = "of-car-reader" +version = "0.2.1" +dependencies = [ + "minicbor", + "pb-rs", + "prost", + "prost-build", + "protoc-bin-vendored", + "quick-protobuf", + "serde", + "sha2 0.11.1", + "wincode", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl" +version = "0.10.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de8111e94a56adca7ebfe6c365b6d42f99deb69b74b6becacd4204e2f193aa2" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c413d6ec9ebaf5e1c829d355cb3e9f783778d700f135be0921d52fce3c60ae" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "pairing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" +dependencies = [ + "group 0.13.0", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pb-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "354a34df9c65b596152598001c0fe3393379ec2db03ae30b9985659422e2607e" +dependencies = [ + "clap 2.34.0", + "env_logger", + "log", + "nom", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "ph" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a447f203c7254ebb06aa4f111480e821debb361e80fb4d760554a8460f236550" +dependencies = [ + "arrayvec", + "binout", + "bitm", + "dyn_size_of", + "rayon", + "seedable_hash", + "voracious_radix_sort", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.1", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common 0.2.2", + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8760a25b6ff9c620324822737e468478fa092234190d2e449760344354896ed9" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fa2624782ca04cd44f51554566717377acd240e4c0016d757dd74fccc9324f" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2417e9817fa237dab803ad4dda7357a111656e242959cc6b8f9a1a583367d42" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d189c34636356a46a7ed3188233dc8a88c431278cc54d4a19b096a2d270e985" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171e39f1e846e5f322ced1ac3b8d4cd3a3833ca24b6e5d58b3632574fe6204fa" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873cdcc097593432086661aa432b8078f1cd87bfb02847c332e98ae2c119e966" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb72df001783b8297847fe8f5f874ee400fd742c843d60583e8c23d96977c7f" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04652167eca899dda05f32f5481adeaf25c623a98ce2fc146a001cc59a2add7" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263a3f48f01e7309e857138bd47f785585b4a005e8e56c6d2824ce91195999c3" + +[[package]] +name = "pulldown-cmark" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "qstring" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "qualifier_attr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "reed-solomon-erasure" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" +dependencies = [ + "cc", + "libc", + "libm", + "lru 0.7.8", + "parking_lot 0.11.2", + "smallvec", + "spin", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "reqwest-middleware" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest 0.12.28", + "serde", + "thiserror 1.0.69", + "tower-service", +] + +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint", + "hmac 0.13.0", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rocksdb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct", + "ctutils", + "der", + "hybrid-array", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "seedable_hash" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba2a159211665e29bbf5a2fbb42da50dd6eadff23eef7a6a7ae4a9b0a7cd0152" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "seqlock" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c67b6f14ecc5b86c66fa63d76b5092352678545a8a3cdae80aef5128371910" +dependencies = [ + "parking_lot 0.12.5", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak 0.1.6", +] + +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.2", + "sponge-cursor", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "simpl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "smpl_jwt" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45432dc6b645c982d4ef68966b4507fb57d98ca67289df780cd7ca4a4369f5e" +dependencies = [ + "base64 0.22.1", + "log", + "openssl", + "serde", + "serde_derive", + "serde_json", + "simpl", + "time", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "solana-account" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f544a28b0ef4177dee1d0299357a39fab0e6ecc394304c80955aa7990ac077a" +dependencies = [ + "serde", + "serde_bytes", + "serde_derive", + "solana-account-info", + "solana-clock", + "solana-instruction-error 3.0.0", + "solana-pubkey 4.4.0", + "wincode", +] + +[[package]] +name = "solana-account-decoder" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "Inflector", + "base64 0.23.1", + "bincode", + "bs58", + "bv", + "serde", + "serde_json", + "solana-account", + "solana-account-decoder-client-types", + "solana-address-lookup-table-interface", + "solana-clock", + "solana-config-interface", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-loader-v3-interface", + "solana-nonce", + "solana-program-option", + "solana-program-pack", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sdk-ids", + "solana-slot-hashes", + "solana-slot-history", + "solana-stake-history", + "solana-stake-interface", + "solana-sysvar", + "solana-vote-interface", + "solana-zk-sdk-pod", + "spl-generic-token", + "spl-token-2022-interface", + "spl-token-group-interface", + "spl-token-interface", + "spl-token-metadata-interface", + "thiserror 2.0.21", + "wincode", + "zstd 0.14.3", +] + +[[package]] +name = "solana-account-decoder-client-types" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "base64 0.23.1", + "bs58", + "serde", + "serde_json", + "solana-account", + "solana-pubkey 4.4.0", + "zstd 0.14.3", +] + +[[package]] +name = "solana-account-info" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9cf16495d9eb53e3d04e72366a33bb1c20c24e78c171d8b8f5978357b63ae95" +dependencies = [ + "solana-address 2.9.0", + "solana-program-error", + "solana-program-memory", +] + +[[package]] +name = "solana-accounts-db" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-fs", + "ahash 0.8.12", + "blake3", + "bv", + "dashmap", + "itertools 0.15.0", + "log", + "modular-bitfield", + "num_cpus", + "rand 0.9.5", + "rayon", + "semver", + "seqlock", + "serde", + "smallvec", + "solana-account", + "solana-address-lookup-table-interface", + "solana-bucket-map", + "solana-clock", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-hash", + "solana-lattice-hash", + "solana-measure", + "solana-message", + "solana-metrics", + "solana-nohash-hasher", + "solana-nonce", + "solana-pubkey 4.4.0", + "solana-rayon-threadlimit", + "solana-reward-info", + "solana-slot-hashes", + "solana-svm-transaction", + "solana-system-interface", + "solana-sysvar", + "solana-time-utils", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", + "spl-generic-token", + "tempfile", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-address" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ecac8e1b7f74c2baa9e774c42817e3e75b20787134b76cc4d45e8a604488f5" +dependencies = [ + "solana-address 2.9.0", +] + +[[package]] +name = "solana-address" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451294fdd4dde59ae27213695e2333eaaa39035e1f099f68644d2aa3ca0e2edd" +dependencies = [ + "borsh", + "bytemuck", + "bytemuck_derive", + "curve25519-dalek 4.1.3", + "five8", + "five8_const", + "rand 0.9.5", + "serde", + "serde_derive", + "sha2-const-stable", + "solana-atomic-u64", + "solana-define-syscall 5.4.0", + "solana-nullable", + "solana-program-error", + "solana-sanitize", + "solana-sha256-hasher", + "wincode", +] + +[[package]] +name = "solana-address-lookup-table-interface" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c09d0da218d985406ed11c1c84169fc9e02e61a5a0012398026b329b9af608" +dependencies = [ + "bincode", + "bytemuck", + "serde", + "serde_derive", + "solana-clock", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-slot-hashes", + "wincode", +] + +[[package]] +name = "solana-atomic-u64" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "085db4906d89324cef2a30840d59eaecf3d4231c560ec7c9f6614a93c652f501" +dependencies = [ + "parking_lot 0.12.5", +] + +[[package]] +name = "solana-big-mod-exp" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05789c4afc39164132db9dcc117218d412ca1f1271d2c629fc6f554c19e5a44" +dependencies = [ + "num-bigint", + "solana-define-syscall 5.4.0", +] + +[[package]] +name = "solana-bincode" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6e56bdda53ef11a97350c776266f81550e76a7fe4675a542ef63354439c333" +dependencies = [ + "bincode", + "serde_core", + "solana-instruction-error 3.0.0", +] + +[[package]] +name = "solana-blake3-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7116e1d942a2432ca3f514625104757ab8a56233787e95144c93950029e31176" +dependencies = [ + "blake3", + "solana-define-syscall 4.0.1", + "solana-hash", +] + +[[package]] +name = "solana-bls-signatures" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7228c72fd3c1a4821c719e3b8caa60c0db155694a2eee84fc80007423542bf7" +dependencies = [ + "base64 0.22.1", + "blst", + "blstrs", + "bytemuck", + "cfg_eval", + "ff 0.13.1", + "group 0.13.0", + "pairing", + "rand 0.8.8", + "rayon", + "serde", + "serde_json", + "serde_with", + "solana-signature", + "solana-signer", + "subtle", + "thiserror 2.0.21", + "wincode", + "zeroize", +] + +[[package]] +name = "solana-bls12-381-syscall" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e8256fda0542709fd1320a96d81c3c66e10ac5f680cca9b991d7d3b0e3fe86" +dependencies = [ + "blst", + "blstrs", + "bytemuck", + "bytemuck_derive", + "pairing", +] + +[[package]] +name = "solana-bn254" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ff13a8867fcc7b0f1114764e1bf6191b4551dcaf93729ddc676cd4ec6abc9f" +dependencies = [ + "ark-bn254", + "ark-ec", + "ark-ff", + "ark-serialize", + "bytemuck", + "solana-define-syscall 5.4.0", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-borsh" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c04abbae16f57178a163125805637b8a076175bb5c0002fb04f4792bea901cf7" +dependencies = [ + "borsh", +] + +[[package]] +name = "solana-bpf-loader-program" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "bincode", + "qualifier_attr", + "solana-account", + "solana-bincode", + "solana-clock", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-loader-v3-interface", + "solana-packet", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-sbpf", + "solana-sdk-ids", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-measure", + "solana-svm-type-overrides", + "solana-syscalls", + "solana-system-interface", + "solana-transaction-context", +] + +[[package]] +name = "solana-bucket-map" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "ahash 0.8.12", + "bv", + "bytemuck", + "bytemuck_derive", + "memmap2 0.9.12", + "modular-bitfield", + "num_enum", + "rand 0.9.5", + "solana-clock", + "solana-measure", + "solana-pubkey 4.4.0", + "tempfile", +] + +[[package]] +name = "solana-builtins" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "solana-bpf-loader-program", + "solana-compute-budget-program", + "solana-hash", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-system-program", + "solana-vote-program", + "solana-zk-elgamal-proof-program", + "solana-zk-token-proof-program", +] + +[[package]] +name = "solana-builtins-default-costs" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "ahash 0.8.12", + "solana-pubkey 4.4.0", + "solana-sdk-ids", +] + +[[package]] +name = "solana-client-traits" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0398e621a2d56c8641a583739411077916c044762e77528e218dd426ee29d74" +dependencies = [ + "solana-account", + "solana-commitment-config", + "solana-epoch-info", + "solana-hash", + "solana-instruction 4.0.0", + "solana-keypair", + "solana-message", + "solana-pubkey 4.4.0", + "solana-signature", + "solana-signer", + "solana-system-interface", + "solana-transaction", + "solana-transaction-error", +] + +[[package]] +name = "solana-clock" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "178fc0c31655aa16190cc112919dcdf279437e14b2f2f9790fe1ec29157ff20c" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-cluster-type" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0cb69036cd1ac8b8a2c644115535396794b057976de2a1033635c25aba333d5" +dependencies = [ + "serde", + "serde_derive", + "solana-hash", +] + +[[package]] +name = "solana-commitment-config" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1517aa49dcfa9cb793ef90e7aac81346d62ca4a546bb1a754030a033e3972e1c" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "solana-compute-budget" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "solana-fee-structure", + "solana-program-runtime", +] + +[[package]] +name = "solana-compute-budget-instruction" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "solana-borsh", + "solana-builtins-default-costs", + "solana-compute-budget", + "solana-compute-budget-interface", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-svm-transaction", + "solana-transaction-error", +] + +[[package]] +name = "solana-compute-budget-interface" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0139c131989ae133c23b31c04ecf03949c198a335e5dc028ddd54bb5c9fe6ee8" +dependencies = [ + "borsh", + "solana-instruction 3.5.1", + "solana-sdk-ids", +] + +[[package]] +name = "solana-compute-budget-program" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "solana-program-runtime", +] + +[[package]] +name = "solana-config-interface" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b7646765c887c471b9cd2cba0d88d5c0729100ffb7c2b3ee240dfa9d5944f9" +dependencies = [ + "serde", + "serde_derive", + "solana-instruction 3.5.1", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-short-vec", + "solana-system-interface", + "wincode", +] + +[[package]] +name = "solana-cost-model" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "ahash 0.8.12", + "log", + "solana-bincode", + "solana-compute-budget", + "solana-packet", + "solana-pubkey 4.4.0", + "solana-runtime-transaction", + "solana-sdk-ids", + "solana-svm-transaction", + "solana-system-interface", + "solana-transaction-error", + "solana-vote-program", +] + +[[package]] +name = "solana-cpi" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dea26709d867aada85d0d3617db0944215c8bb28d3745b912de7db13a23280c" +dependencies = [ + "solana-account-info", + "solana-define-syscall 4.0.1", + "solana-instruction 3.5.1", + "solana-program-error", + "solana-pubkey 4.4.0", + "solana-stable-layout 3.0.1", +] + +[[package]] +name = "solana-curve25519" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b4d2a4bf0d0b0a86c22111917e86e8bd39a7b31420fb2c7d73eb83761fc7af" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "curve25519-dalek 4.1.3", + "solana-define-syscall 5.4.0", + "subtle", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-define-syscall" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5b1c0bc1d4a4d10c88a4100499d954c09d3fecfae4912c1a074dff68b1738" + +[[package]] +name = "solana-define-syscall" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8506988b0997022c4f2801985d9c4dccfb18c16e6a38f30390a47d54efeca805" + +[[package]] +name = "solana-derivation-path" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff71743072690fdbdfcdc37700ae1cb77485aaad49019473a81aee099b1e0b8c" +dependencies = [ + "derivation-path", + "qstring", + "uriparse", +] + +[[package]] +name = "solana-download-utils" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-snapshots", + "log", + "solana-clock", + "solana-file-download", + "solana-genesis-config", + "solana-runtime", +] + +[[package]] +name = "solana-ed25519" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4c7caf3fe3d70815869a5838ee45b4d2b009eaacea1790ef3cd9b3c89edbbd" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "curve25519-dalek-derive", + "digest 0.11.3", + "ed25519 2.2.3", + "hashbrown 0.15.5", + "rand 0.10.3", + "rand_core 0.10.1", + "rustc_version", + "sha2 0.11.1", + "subtle", + "zeroize", +] + +[[package]] +name = "solana-ed25519-program" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1419197f1c06abf760043f6d64ba9d79a03ad5a43f18c7586471937122094da" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "solana-instruction 3.5.1", + "solana-sdk-ids", +] + +[[package]] +name = "solana-entry" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-transaction-view", + "agave-votor-messages", + "bytes", + "crossbeam-channel", + "log", + "num_cpus", + "rayon", + "smallvec", + "solana-address 2.9.0", + "solana-bls-signatures", + "solana-clock", + "solana-cost-model", + "solana-hash", + "solana-measure", + "solana-merkle-tree", + "solana-packet", + "solana-perf", + "solana-runtime-transaction", + "solana-sha256-hasher", + "solana-signature", + "solana-transaction", + "solana-transaction-error", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-epoch-info" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e093c84f6ece620a6b10cd036574b0cd51944231ab32d81f80f76d54aba833e6" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "solana-epoch-rewards" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0788d74ee15778deecaa15ed1a1e37727ba954f86cbc35225450a1f2b5012969" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-hash", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-epoch-rewards-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee8beac9bff4db9225e57d532d169b0be5e447f1e6601a2f50f27a01bf5518f" +dependencies = [ + "siphasher 0.3.11", + "solana-address 2.9.0", + "solana-hash", +] + +[[package]] +name = "solana-epoch-schedule" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae73018043cff5d3990b3998d846fab2e823c07e3b2f49be602ea584be3d4f" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-program-error", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-feature-gate-interface" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e429812a07f338b834f5fb14ecacc94750e1c3efcc6af8ad8d6c22ffc526af0" +dependencies = [ + "serde", + "serde_derive", + "solana-account", + "solana-account-info", + "solana-instruction 4.0.0", + "solana-program-error", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-system-interface", + "wincode", +] + +[[package]] +name = "solana-fee" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "solana-fee-structure", + "solana-svm-transaction", +] + +[[package]] +name = "solana-fee-calculator" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55add16a1c8f8953379a2b91ff3e94ee7096feac787ca33fad0292c6ca86dbe5" +dependencies = [ + "log", + "serde", + "serde_derive", + "wincode", +] + +[[package]] +name = "solana-fee-structure" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3591f4827715ebb49c290b829f707ec1747a152973436f27699897f752a31cd6" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "solana-file-download" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b72d3025bed3ca0a12d4cf62b4a5a766e817657bc1c1ea1af6e628dc92b176c" +dependencies = [ + "console 0.15.11", + "indicatif", + "log", + "reqwest 0.13.5", +] + +[[package]] +name = "solana-genesis-config" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8f376bc0e0363e3ada8f1409375670903eac588936073723e11ddc61ae120d6" +dependencies = [ + "bincode", + "chrono", + "memmap2 0.5.10", + "serde", + "serde_derive", + "solana-account", + "solana-clock", + "solana-cluster-type", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-hash", + "solana-inflation", + "solana-keypair", + "solana-poh-config", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sdk-ids", + "solana-sha256-hasher", + "solana-shred-version", + "solana-signer", + "solana-time-utils", +] + +[[package]] +name = "solana-genesis-utils" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-snapshots", + "log", + "solana-download-utils", + "solana-genesis-config", + "solana-hash", + "solana-rpc-client", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-get-sysvar" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef3bc859fc036ed490146793557386cbfae614ebba4adc704c37d94350824ed4" +dependencies = [ + "solana-address 2.9.0", + "solana-define-syscall 5.4.0", + "solana-program-error", +] + +[[package]] +name = "solana-hard-forks" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12163308c71e7fb6c392fc4b3e5719597539a6dd6ecfee00d418974af36edbac" +dependencies = [ + "serde", + "serde_derive", + "wincode", +] + +[[package]] +name = "solana-hash" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0df9b01495ed31100aca97a7f5862d5e19ab1636d60d1a9f02391408dd9dec84" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "five8", + "rand 0.9.5", + "serde", + "serde_derive", + "solana-atomic-u64", + "solana-sanitize", + "wincode", +] + +[[package]] +name = "solana-hash-512" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee9c987913768643be3f4fc5f8ec667e8ec19e59e6d131688ade8d1430dafe9" + +[[package]] +name = "solana-inflation" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b59421998ca91673b914fbc0b39ad49afe6ea7df3c455b067d6ad57ff2c592a1" +dependencies = [ + "serde", + "serde_derive", + "wincode", +] + +[[package]] +name = "solana-instruction" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d8f113d1312870e2d8698033e9d153d3329c520e1065f05145a55a889e21c91" +dependencies = [ + "solana-instruction 4.0.0", + "solana-instruction-error 2.5.0", +] + +[[package]] +name = "solana-instruction" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea8442b67a58dd54d5497bfc6cd3106b00f9fe6f6877d33c271c3ec4672eedc" +dependencies = [ + "bincode", + "serde", + "serde_derive", + "solana-define-syscall 5.4.0", + "solana-pubkey 4.4.0", + "wincode", +] + +[[package]] +name = "solana-instruction-error" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e7db78c122d02189619490b1fef04a2a042c389662920617c0e6381fdf8fdd" +dependencies = [ + "num-traits", + "solana-program-error", +] + +[[package]] +name = "solana-instruction-error" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d45854131ca87ba37ca0d537d06b460f8e0e99f00a8cd5f15492be961c85338" +dependencies = [ + "num-traits", + "serde", + "serde_derive", + "solana-program-error", + "wincode", +] + +[[package]] +name = "solana-instructions-sysvar" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e0732294560e88ecdb2bbc656e67383e9f88c78ec09469cef172f0d28cd1bcd" +dependencies = [ + "bitflags 2.13.2", + "solana-account-info", + "solana-instruction 3.5.1", + "solana-instruction-error 2.5.0", + "solana-program-error", + "solana-sanitize", + "solana-sdk-ids", + "solana-serialize-utils 3.1.2", + "solana-sysvar-id", +] + +[[package]] +name = "solana-instructions-sysvar" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b74c3b078450525dcb84536c51299d5b0dc052f33699b07276d6870bd1ebed6" +dependencies = [ + "bitflags 2.13.2", + "solana-account-info", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-program-error", + "solana-sanitize", + "solana-sdk-ids", + "solana-serialize-utils 4.0.0", + "solana-sysvar-id", +] + +[[package]] +name = "solana-keccak-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed1c0d16d6fdeba12291a1f068cdf0d479d9bff1141bf44afd7aa9d485f65ef8" +dependencies = [ + "sha3", + "solana-define-syscall 4.0.1", + "solana-hash", +] + +[[package]] +name = "solana-keypair" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240dc6ce700f5b34df364ed433208079617f7f68cbfc005bf487d1c78ce4c970" +dependencies = [ + "five8", + "five8_core", + "rand 0.9.5", + "solana-address 2.9.0", + "solana-derivation-path", + "solana-ed25519", + "solana-seed-derivable", + "solana-seed-phrase", + "solana-signature", + "solana-signer", + "zeroize", +] + +[[package]] +name = "solana-last-restart-slot" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5099e736c3c06c451b307a60637d69eb65b3144143ebeed899e2fd1134f4fc35" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-lattice-hash" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "base64 0.23.1", + "blake3", + "bs58", + "bytemuck", +] + +[[package]] +name = "solana-leader-schedule" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-random", + "itertools 0.15.0", + "rand_chacha 0.9.0", + "solana-clock", + "solana-pubkey 4.4.0", + "solana-vote", +] + +[[package]] +name = "solana-ledger" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "agave-reserved-account-keys", + "agave-snapshots", + "agave-transaction-view", + "agave-votor-messages", + "anyhow", + "assert_matches", + "atomic-wait", + "bitflags 2.13.2", + "bytes", + "bzip2", + "chrono", + "chrono-humanize", + "crossbeam-channel", + "crossbeam-utils", + "dashmap", + "eager", + "fs_extra", + "futures", + "itertools 0.15.0", + "lazy-lru", + "log", + "lru 0.18.5", + "mockall", + "num_cpus", + "num_enum", + "parking_lot 0.12.5", + "prost", + "qualifier_attr", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rayon", + "reed-solomon-erasure", + "rocksdb", + "scopeguard", + "serde", + "smallvec", + "solana-account", + "solana-accounts-db", + "solana-address-lookup-table-interface", + "solana-clock", + "solana-cost-model", + "solana-entry", + "solana-epoch-schedule", + "solana-genesis-config", + "solana-genesis-utils", + "solana-hash", + "solana-instruction 4.0.0", + "solana-keypair", + "solana-leader-schedule", + "solana-measure", + "solana-message", + "solana-metrics", + "solana-native-token", + "solana-net-utils", + "solana-nohash-hasher", + "solana-packet", + "solana-perf", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-runtime", + "solana-runtime-transaction", + "solana-sha256-hasher", + "solana-shred-version", + "solana-signature", + "solana-signer", + "solana-stake-interface", + "solana-storage-bigtable", + "solana-storage-proto", + "solana-streamer", + "solana-svm", + "solana-svm-timings", + "solana-svm-transaction", + "solana-system-interface", + "solana-system-transaction", + "solana-time-utils", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", + "solana-transaction-status", + "solana-vote", + "solana-vote-program", + "static_assertions", + "strum", + "tar", + "tempfile", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "trees", + "wincode", +] + +[[package]] +name = "solana-loader-v2-interface" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88abae245ad0aedbbb1498836c1ee4ac79903d7d632c2492d5a376554ac6fe74" +dependencies = [ + "serde", + "serde_bytes", + "serde_derive", + "solana-instruction 4.0.0", + "solana-pubkey 4.4.0", + "solana-sdk-ids", +] + +[[package]] +name = "solana-loader-v3-interface" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1be12013e6364447d9ac7142a90ebff74c676b99bd4a5dca0986126eae6b9e7" +dependencies = [ + "serde", + "serde_bytes", + "serde_derive", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-system-interface", + "wincode", +] + +[[package]] +name = "solana-loader-v4-interface" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c948b33ff81fa89699911b207059e493defdba9647eaf18f23abdf3674e0fb" +dependencies = [ + "solana-instruction 3.5.1", + "solana-pubkey 3.0.0", + "solana-sdk-ids", +] + +[[package]] +name = "solana-measure" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" + +[[package]] +name = "solana-merkle-tree" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "solana-hash", + "solana-sha256-hasher", +] + +[[package]] +name = "solana-message" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904166464d72cad48c4767e0317463a1662a54f9a113071554b083dc2ae12ebe" +dependencies = [ + "blake3", + "serde", + "serde_derive", + "solana-address 2.9.0", + "solana-hash", + "solana-instruction 4.0.0", + "solana-sanitize", + "solana-sdk-ids", + "solana-short-vec", + "solana-transaction-error", + "wincode", +] + +[[package]] +name = "solana-metrics" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "crossbeam-channel", + "gethostname", + "log", + "reqwest 0.12.28", + "solana-cluster-type", + "solana-sha256-hasher", + "solana-time-utils", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-msg" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726b7cbbc6be6f1c6f29146ac824343b9415133eee8cce156452ad1db93f8008" +dependencies = [ + "solana-define-syscall 5.4.0", +] + +[[package]] +name = "solana-native-token" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8dd4c280dca9d046139eb5b7a5ac9ad10403fbd64964c7d7571214950d758f" + +[[package]] +name = "solana-net-utils" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-xdp", + "anza-quinn", + "bincode", + "bytes", + "cfg-if", + "dashmap", + "itertools 0.15.0", + "log", + "nix", + "rand 0.9.5", + "serde", + "socket2", + "solana-serde", + "solana-svm-type-overrides", + "thiserror 2.0.21", + "tokio", + "url", +] + +[[package]] +name = "solana-nohash-hasher" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8a731ed60e89177c8a7ab05fe0f1511cedd3e70e773f288f9de33a9cfdc21e" + +[[package]] +name = "solana-nonce" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f13b7ec92de6dd4ef713ed763d22585efa0042cd244bce81997e52077885c47" +dependencies = [ + "serde", + "serde_derive", + "solana-fee-calculator", + "solana-hash", + "solana-pubkey 4.4.0", + "solana-sha256-hasher", + "wincode", +] + +[[package]] +name = "solana-nonce-account" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1520dab045b453567cc5bad802e75bd55d5e85da5bde687bc96aa49d76b06c8d" +dependencies = [ + "solana-account", + "solana-hash", + "solana-nonce", + "solana-sdk-ids", +] + +[[package]] +name = "solana-nullable" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "889194d8c5faec648f2f6fadddb60566249921ebb074e2707e7095458d5864e2" +dependencies = [ + "borsh", + "bytemuck", +] + +[[package]] +name = "solana-packet" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f9d2969eaffcaaa017d5de6d282d584ad3def7a0718185a3660a4cca3c2c6" +dependencies = [ + "bincode", + "bitflags 2.13.2", + "cfg_eval", + "serde", + "serde_derive", + "serde_with", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "solana-perf" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-transaction-view", + "ahash 0.8.12", + "arc-swap", + "bincode", + "bytes", + "caps", + "libc", + "log", + "nix", + "num_cpus", + "rand 0.9.5", + "rayon", + "serde", + "solana-hash", + "solana-message", + "solana-metrics", + "solana-packet", + "solana-pubkey 4.4.0", + "solana-runtime-transaction", + "solana-sdk-ids", + "solana-short-vec", + "solana-signature", + "solana-time-utils", + "solana-transaction-context", + "wincode", +] + +[[package]] +name = "solana-poh-config" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f2b4747ef01194ea4eb679101476c2769fb4ee46d939d77e482f44394e5f02" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "solana-poseidon" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a066cf8c003a052ee6cab5e639174b4ddb395d38816dc6fb33d1b86fd2cd13a" +dependencies = [ + "ark-bn254", + "light-poseidon", + "solana-define-syscall 5.4.0", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-precompile-error" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cafcd950de74c6c39d55dc8ca108bbb007799842ab370ef26cf45a34453c31e1" +dependencies = [ + "num-traits", +] + +[[package]] +name = "solana-program-entrypoint" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c9b0a1ff494e05f503a08b3d51150b73aa639544631e510279d6375f290997" +dependencies = [ + "solana-account-info", + "solana-define-syscall 4.0.1", + "solana-program-error", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "solana-program-error" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f04fa578707b3612b095f0c8e19b66a1233f7c42ca8082fcb3b745afcc0add6" +dependencies = [ + "borsh", +] + +[[package]] +name = "solana-program-memory" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4068648649653c2c50546e9a7fb761791b5ab0cda054c771bb5808d3a4b9eb52" +dependencies = [ + "solana-define-syscall 4.0.1", +] + +[[package]] +name = "solana-program-option" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a88006a9b8594088cec9027ab77caaaa258a2aaa2083d3f086c44b42e50aeab" + +[[package]] +name = "solana-program-pack" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7701cb15b90667ae1c89ef4ac35a59c61e66ce58ddee13d729472af7f41d59" +dependencies = [ + "solana-program-error", +] + +[[package]] +name = "solana-program-runtime" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "base64 0.23.1", + "bincode", + "cfg-if", + "itertools 0.15.0", + "log", + "rand 0.9.5", + "serde", + "solana-account", + "solana-account-info", + "solana-clock", + "solana-epoch-rewards", + "solana-epoch-schedule", + "solana-fee-structure", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-last-restart-slot", + "solana-loader-v3-interface", + "solana-program-entrypoint", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sbpf", + "solana-sdk-ids", + "solana-slot-hashes", + "solana-stable-layout 4.0.0", + "solana-stake-history", + "solana-svm-callback", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-measure", + "solana-svm-timings", + "solana-svm-transaction", + "solana-svm-type-overrides", + "solana-system-interface", + "solana-sysvar", + "solana-sysvar-id", + "solana-transaction-context", + "static_assertions", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-pubkey" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8909d399deb0851aa524420beeb5646b115fd253ef446e35fe4504c904da3941" +dependencies = [ + "solana-address 1.1.0", +] + +[[package]] +name = "solana-pubkey" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b8ba5d677d77a76d557b59ce7ecf9a8578743089cb25c73990752c5f005885" +dependencies = [ + "rand 0.9.5", + "solana-address 2.9.0", +] + +[[package]] +name = "solana-rayon-threadlimit" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "log", + "num_cpus", +] + +[[package]] +name = "solana-rent" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7c7590d4e53446a00c363783d98199d8c8f5099ca60eaf2a0f1f50d0e7a31e" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-reward-info" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d20f0760fe5f41208593d5111a13dff725a0b1af9986924c4a4ce5461520d0d" +dependencies = [ + "serde", + "serde_derive", + "wincode", +] + +[[package]] +name = "solana-rpc-client" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-votor-messages", + "async-trait", + "base64 0.23.1", + "bincode", + "bs58", + "futures", + "log", + "reqwest 0.12.28", + "reqwest-middleware", + "semver", + "serde", + "serde_json", + "solana-account", + "solana-account-decoder", + "solana-account-decoder-client-types", + "solana-bls-signatures", + "solana-clock", + "solana-commitment-config", + "solana-epoch-info", + "solana-epoch-schedule", + "solana-feature-gate-interface", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-message", + "solana-pubkey 4.4.0", + "solana-rpc-client-api", + "solana-signature", + "solana-transaction", + "solana-transaction-error", + "solana-transaction-status-client-types", + "solana-version", + "solana-vote-interface", + "tokio", + "wincode", +] + +[[package]] +name = "solana-rpc-client-api" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "anyhow", + "jsonrpc-core", + "reqwest 0.12.28", + "reqwest-middleware", + "serde", + "serde_json", + "solana-clock", + "solana-rpc-client-types", + "solana-signer", + "solana-transaction-error", + "solana-transaction-status-client-types", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-rpc-client-types" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "base64 0.23.1", + "bs58", + "semver", + "serde", + "serde_json", + "solana-account-decoder-client-types", + "solana-address 2.9.0", + "solana-clock", + "solana-commitment-config", + "solana-fee-calculator", + "solana-inflation", + "solana-reward-info", + "solana-transaction", + "solana-transaction-error", + "solana-transaction-status-client-types", + "solana-version", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-runtime" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-bls-cert-verify", + "agave-feature-set", + "agave-fs", + "agave-precompiles", + "agave-reserved-account-keys", + "agave-snapshots", + "agave-transaction-view", + "agave-votor-messages", + "ahash 0.8.12", + "arc-swap", + "arrayref", + "assert_matches", + "base64 0.23.1", + "bitvec", + "bytemuck", + "bytes", + "crossbeam-channel", + "crossbeam-queue", + "crossbeam-utils", + "dashmap", + "imbl", + "itertools 0.15.0", + "libc", + "log", + "mockall", + "num-derive 0.5.1", + "num-traits", + "qualifier_attr", + "rand 0.9.5", + "rayon", + "regex", + "semver", + "serde", + "serde_json", + "smallvec", + "solana-account", + "solana-account-decoder", + "solana-account-info", + "solana-accounts-db", + "solana-address-lookup-table-interface", + "solana-bls-signatures", + "solana-builtins", + "solana-client-traits", + "solana-clock", + "solana-cluster-type", + "solana-commitment-config", + "solana-compute-budget", + "solana-compute-budget-instruction", + "solana-compute-budget-interface", + "solana-config-interface", + "solana-cost-model", + "solana-cpi", + "solana-ed25519-program", + "solana-entry", + "solana-epoch-info", + "solana-epoch-rewards-hasher", + "solana-epoch-schedule", + "solana-feature-gate-interface", + "solana-fee", + "solana-fee-calculator", + "solana-fee-structure", + "solana-genesis-config", + "solana-hard-forks", + "solana-hash", + "solana-inflation", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-keypair", + "solana-lattice-hash", + "solana-leader-schedule", + "solana-loader-v3-interface", + "solana-measure", + "solana-message", + "solana-metrics", + "solana-native-token", + "solana-nonce", + "solana-nonce-account", + "solana-packet", + "solana-poh-config", + "solana-precompile-error", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-rayon-threadlimit", + "solana-rent", + "solana-reward-info", + "solana-runtime-transaction", + "solana-sdk-ids", + "solana-secp256k1-program", + "solana-seed-derivable", + "solana-serde", + "solana-sha256-hasher", + "solana-signature", + "solana-signer", + "solana-signer-store", + "solana-slot-hashes", + "solana-slot-history", + "solana-stake-history", + "solana-stake-interface", + "solana-svm", + "solana-svm-callback", + "solana-svm-timings", + "solana-svm-transaction", + "solana-syscalls", + "solana-system-interface", + "solana-system-transaction", + "solana-sysvar", + "solana-sysvar-id", + "solana-time-utils", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", + "solana-transaction-status", + "solana-transaction-status-client-types", + "solana-unified-scheduler-logic", + "solana-version", + "solana-vote", + "solana-vote-interface", + "solana-vote-program", + "spl-generic-token", + "static_assertions", + "strum", + "strum_macros", + "tempfile", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-runtime-transaction" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "agave-transaction-view", + "bytes", + "solana-compute-budget", + "solana-compute-budget-instruction", + "solana-hash", + "solana-message", + "solana-program-entrypoint", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-signature", + "solana-svm-transaction", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", + "wincode", +] + +[[package]] +name = "solana-sanitize" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf09694a0fc14e5ffb18f9b7b7c0f15ecb6eac5b5610bf76a1853459d19daf9" + +[[package]] +name = "solana-sbpf" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59635731a7d919ef964155beede88bb02b82aecb6667b3e8c93594424180d238" +dependencies = [ + "byteorder", + "combine 3.8.1", + "hash32", + "libc", + "log", + "rand 0.8.8", + "rustc-demangle", + "thiserror 2.0.21", + "winapi", +] + +[[package]] +name = "solana-sdk-ids" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def234c1956ff616d46c9dd953f251fa7096ddbaa6d52b165218de97882b7280" +dependencies = [ + "solana-address 2.9.0", +] + +[[package]] +name = "solana-sdk-macro" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765316242300c48242d84a41614cb3388229ec353ba464f6fe62a733e41806f" +dependencies = [ + "bs58", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "solana-secp256k1-program" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46dbeb15416393115ae664250c118abda41a742ac3f9067f7322591e661aa186" +dependencies = [ + "k256", + "serde", + "serde_derive", + "solana-keccak-hasher", + "solana-signature", +] + +[[package]] +name = "solana-secp256k1-recover" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f00f973301968be9f05f4af951974a28848b6513fb16edf77c8af20e681914" +dependencies = [ + "k256", + "solana-define-syscall 5.4.0", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-secp256r1-program" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445d8e12592631d76fc4dc57858bae66c9fd7cc838c306c62a472547fc9d0ce6" +dependencies = [ + "bytemuck", + "openssl", + "solana-instruction 3.5.1", + "solana-sdk-ids", +] + +[[package]] +name = "solana-seed-derivable" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7bdb72758e3bec33ed0e2658a920f1f35dfb9ed576b951d20d63cb61ecd95c" +dependencies = [ + "solana-derivation-path", +] + +[[package]] +name = "solana-seed-phrase" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc905b200a95f2ea9146e43f2a7181e3aeb55de6bc12afb36462d00a3c7310de" +dependencies = [ + "hmac 0.12.1", + "pbkdf2", + "sha2 0.10.9", +] + +[[package]] +name = "solana-serde" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709a93cab694c70f40b279d497639788fc2ccbcf9b4aa32273d4b361322c02dd" +dependencies = [ + "serde", +] + +[[package]] +name = "solana-serde-varint" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950e5b83e839dc0f92c66afc124bb8f40e89bc90f0579e8ec5499296d27f54e3" +dependencies = [ + "serde", +] + +[[package]] +name = "solana-serialize-utils" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761357b0853c9623bf12c1d2314b3d6160a85b087b84c45224fb85766d22616b" +dependencies = [ + "solana-instruction-error 2.5.0", + "solana-pubkey 4.4.0", + "solana-sanitize", +] + +[[package]] +name = "solana-serialize-utils" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f844dab1b29cc8c1a3413b96bf0ef9fba44f47be53ec498ce01507ab4b5520" +dependencies = [ + "solana-instruction-error 3.0.0", + "solana-pubkey 4.4.0", + "solana-sanitize", +] + +[[package]] +name = "solana-sha256-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7dc3011ea4c0334aaaa7e7128cb390ecf546b28d412e9bf2064680f57f588f" +dependencies = [ + "sha2 0.10.9", + "solana-define-syscall 4.0.1", + "solana-hash", +] + +[[package]] +name = "solana-sha512-hasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11e10e103ab5bd52af341e7bc2f4123a2f4e66bb7ba4e6ca40f1e00cd6314e02" +dependencies = [ + "sha2 0.10.9", + "solana-define-syscall 5.4.0", + "solana-hash-512", +] + +[[package]] +name = "solana-short-vec" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75782529cc4521114c1ecd71c7e2cdebd41a6fe9844b5d088fd36ef08c57c36" +dependencies = [ + "serde_core", + "wincode", +] + +[[package]] +name = "solana-shred-version" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c79722e299d957958bf33695f7cd1ef6724ff55563c60fd9e3e24487cccde2" +dependencies = [ + "solana-hard-forks", + "solana-hash", + "solana-sha256-hasher", +] + +[[package]] +name = "solana-signature" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea8b3e375e6191ceafca9ca916c395d445caa9c42f13d83d8ee0419944896b0" +dependencies = [ + "five8", + "serde", + "serde-big-array", + "serde_derive", + "solana-ed25519", + "solana-sanitize", + "wincode", +] + +[[package]] +name = "solana-signer" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a11c290ae825ea4cd26553873ba220f965c8894c5a4ace5f41ce7106fa6801bc" +dependencies = [ + "solana-pubkey 4.4.0", + "solana-signature", + "solana-transaction-error", +] + +[[package]] +name = "solana-signer-store" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36329bba208f0e41954389ae4ad5d973fe15952672cfd71a9b49deb7d2ecbc2f" +dependencies = [ + "bitvec", + "num-derive 0.4.2", + "num-traits", +] + +[[package]] +name = "solana-slot-hashes" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3da667c6de47dd04fa03cd3a29f2ce046627407f0de0dc56c5d78ed1ba23785" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-hash", + "solana-sdk-ids", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-slot-history" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4560fde841a6f2001aedc33b74fe99840ee3d56a71fe9b98ee332303b8597900" +dependencies = [ + "bv", + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-stable-layout" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9f6a291ba063a37780af29e7db14bdd3dc447584d8ba5b3fc4b88e2bbc982fa" +dependencies = [ + "solana-instruction 3.5.1", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "solana-stable-layout" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31bb3fe4ad6c73690848997e71d3f59b7a9dccafaae63fc129936223e5cba682" +dependencies = [ + "solana-instruction 4.0.0", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "solana-stake-history" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edab56849103bda42aed2f34ccdd65cd764af9e598117ca54ebc5ec7211292b4" +dependencies = [ + "serde", + "serde_derive", + "solana-clock", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-stake-interface" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36bba7bab030c8e7eb0150abd8db29a959924cea44b55f89a587694706ccda9b" +dependencies = [ + "num-traits", + "serde", + "serde_derive", + "solana-clock", + "solana-cpi", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-program-error", + "solana-pubkey 4.4.0", + "solana-stake-history", + "solana-system-interface", + "wincode", +] + +[[package]] +name = "solana-storage-bigtable" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-reserved-account-keys", + "base64 0.23.1", + "bincode", + "bzip2", + "flate2", + "futures", + "goauth", + "http", + "hyper-util", + "log", + "prost", + "prost-types", + "serde", + "smpl_jwt", + "solana-clock", + "solana-entry", + "solana-message", + "solana-metrics", + "solana-pubkey 4.4.0", + "solana-serde", + "solana-signature", + "solana-storage-proto", + "solana-time-utils", + "solana-transaction", + "solana-transaction-error", + "solana-transaction-status", + "thiserror 2.0.21", + "tokio", + "tonic", + "tonic-prost", + "wincode", + "zstd 0.14.3", +] + +[[package]] +name = "solana-storage-proto" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "bs58", + "prost", + "protobuf-src", + "serde", + "solana-account-decoder", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-message", + "solana-pubkey 4.4.0", + "solana-serde", + "solana-signature", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", + "solana-transaction-status", + "thiserror 2.0.21", + "tonic-prost-build", + "wincode", +] + +[[package]] +name = "solana-streamer" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "anza-quinn", + "bytes", + "crossbeam-channel", + "futures", + "indexmap", + "itertools 0.15.0", + "libc", + "log", + "nix", + "num_cpus", + "pem", + "rand 0.9.5", + "rustls", + "smallvec", + "solana-keypair", + "solana-measure", + "solana-metrics", + "solana-net-utils", + "solana-packet", + "solana-perf", + "solana-pubkey 4.4.0", + "solana-signer", + "solana-time-utils", + "solana-tls-utils", + "solana-transaction-error", + "thiserror 2.0.21", + "tokio", + "tokio-util", +] + +[[package]] +name = "solana-svm" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "ahash 0.8.12", + "bincode", + "log", + "serde", + "solana-account", + "solana-bpf-loader-program", + "solana-clock", + "solana-fee-structure", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-instructions-sysvar 5.0.0", + "solana-loader-v3-interface", + "solana-loader-v4-interface", + "solana-message", + "solana-nonce", + "solana-nonce-account", + "solana-program-entrypoint", + "solana-program-pack", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sbpf", + "solana-sdk-ids", + "solana-svm-callback", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-measure", + "solana-svm-timings", + "solana-svm-transaction", + "solana-svm-type-overrides", + "solana-system-interface", + "solana-sysvar-id", + "solana-transaction-context", + "solana-transaction-error", + "spl-generic-token", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-svm-callback" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "solana-account", + "solana-clock", + "solana-precompile-error", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "solana-svm-feature-set" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" + +[[package]] +name = "solana-svm-log-collector" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "log", +] + +[[package]] +name = "solana-svm-measure" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" + +[[package]] +name = "solana-svm-timings" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "eager", + "enum-iterator", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "solana-svm-transaction" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e1d4d6932de8290b7835f9926281c210291718e14732f5a736ae31ca54e2e47" +dependencies = [ + "solana-hash", + "solana-message", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-signature", + "solana-transaction", +] + +[[package]] +name = "solana-svm-type-overrides" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "rand 0.9.5", +] + +[[package]] +name = "solana-syscalls" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "bincode", + "libsecp256k1", + "num-traits", + "solana-account", + "solana-account-info", + "solana-big-mod-exp", + "solana-blake3-hasher", + "solana-bls12-381-syscall", + "solana-bn254", + "solana-clock", + "solana-cpi", + "solana-curve25519", + "solana-define-syscall 5.4.0", + "solana-hash", + "solana-hash-512", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-keccak-hasher", + "solana-poseidon", + "solana-program-entrypoint", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-sbpf", + "solana-sdk-ids", + "solana-secp256k1-recover", + "solana-sha256-hasher", + "solana-sha512-hasher", + "solana-stable-layout 4.0.0", + "solana-stake-history", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-type-overrides", + "solana-sysvar-id", + "solana-transaction-context", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-system-interface" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e2d5311d0584af231c6c2b0503ba1c3aa50118ead4f503ab215ccbdd1ffac57" +dependencies = [ + "num-traits", + "serde", + "serde_derive", + "solana-address 2.9.0", + "solana-instruction 3.5.1", + "solana-msg", + "solana-program-error", + "wincode", +] + +[[package]] +name = "solana-system-program" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "bincode", + "log", + "solana-account", + "solana-bincode", + "solana-fee-calculator", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-nonce", + "solana-nonce-account", + "solana-packet", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-svm-log-collector", + "solana-system-interface", + "solana-sysvar", + "solana-transaction-context", +] + +[[package]] +name = "solana-system-transaction" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf5f8c21796c8e92a08423deab8c8d237024ffac0732eec58758b6dc6971ea2f" +dependencies = [ + "solana-hash", + "solana-keypair", + "solana-message", + "solana-pubkey 4.4.0", + "solana-signer", + "solana-system-interface", + "solana-transaction", +] + +[[package]] +name = "solana-sysvar" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcfc1dead2b328974e09500feb2e413fc2be82bb3ffebc87132804cdce87d31b" +dependencies = [ + "base64 0.22.1", + "lazy_static", + "serde", + "serde_derive", + "solana-account-info", + "solana-clock", + "solana-define-syscall 5.4.0", + "solana-epoch-rewards", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-get-sysvar", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-last-restart-slot", + "solana-program-entrypoint", + "solana-program-error", + "solana-program-memory", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-slot-hashes", + "solana-slot-history", + "solana-stake-history", + "solana-sysvar-id", + "wincode", +] + +[[package]] +name = "solana-sysvar-id" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17358d1e9a13e5b9c2264d301102126cf11a47fd394cdf3dec174fe7bc96e1de" +dependencies = [ + "solana-address 2.9.0", + "solana-sdk-ids", +] + +[[package]] +name = "solana-time-utils" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced92c60aa76ec4780a9d93f3bd64dfa916e1b998eacc6f1c110f3f444f02c9" + +[[package]] +name = "solana-tls-utils" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "anza-quinn", + "rustls", + "rustls-webpki", + "solana-keypair", + "solana-pubkey 4.4.0", + "solana-signer", +] + +[[package]] +name = "solana-transaction" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29242fe2ba04e84aa298c431f5aa687e89e42d27e85aea4a5c39c467f0f47799" +dependencies = [ + "serde", + "serde_derive", + "solana-address 2.9.0", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-message", + "solana-sanitize", + "solana-sdk-ids", + "solana-short-vec", + "solana-signature", + "solana-signer", + "solana-transaction-error", + "wincode", +] + +[[package]] +name = "solana-transaction-context" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "bincode", + "serde", + "solana-account", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-instructions-sysvar 5.0.0", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sbpf", + "solana-sdk-ids", + "wincode", +] + +[[package]] +name = "solana-transaction-error" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85286bfd93440b49a90dbb0b62c2514bcb95ef6b3d34a36dc7ca0c3e4f4a9f45" +dependencies = [ + "serde", + "serde_derive", + "solana-instruction-error 3.0.0", + "solana-sanitize", + "wincode", +] + +[[package]] +name = "solana-transaction-status" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "Inflector", + "agave-reserved-account-keys", + "base64 0.23.1", + "bincode", + "borsh", + "bs58", + "serde", + "serde_json", + "solana-account-decoder", + "solana-address-lookup-table-interface", + "solana-clock", + "solana-entry", + "solana-hash", + "solana-instruction 4.0.0", + "solana-loader-v2-interface", + "solana-loader-v3-interface", + "solana-message", + "solana-program-option", + "solana-pubkey 4.4.0", + "solana-reward-info", + "solana-sdk-ids", + "solana-signature", + "solana-stake-interface", + "solana-system-interface", + "solana-transaction", + "solana-transaction-error", + "solana-transaction-status-client-types", + "solana-vote-interface", + "solana-zk-sdk-pod", + "spl-associated-token-account-interface", + "spl-memo-interface", + "spl-token-2022-interface", + "spl-token-group-interface", + "spl-token-interface", + "spl-token-metadata-interface", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-transaction-status-client-types" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "base64 0.23.1", + "bincode", + "bs58", + "serde", + "serde_json", + "solana-account-decoder-client-types", + "solana-commitment-config", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-message", + "solana-reward-info", + "solana-signature", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-unified-scheduler-logic" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "assert_matches", + "solana-clock", + "solana-hash", + "solana-pubkey 4.4.0", + "solana-runtime-transaction", + "solana-svm-transaction", + "static_assertions", +] + +[[package]] +name = "solana-version" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "rand 0.9.5", + "semver", + "serde", + "solana-sanitize", + "solana-serde-varint", + "solana-wincode-varint", + "wincode", +] + +[[package]] +name = "solana-vote" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "log", + "serde", + "solana-account", + "solana-bincode", + "solana-clock", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-keypair", + "solana-packet", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-serialize-utils 4.0.0", + "solana-signature", + "solana-signer", + "solana-svm-transaction", + "solana-transaction", + "solana-vote-interface", + "thiserror 2.0.21", + "wincode", +] + +[[package]] +name = "solana-vote-interface" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b3909c886ea5e6dbbda3fe5046244ad05566209e3d2f96a278f75e8956d9d51" +dependencies = [ + "bincode", + "cfg_eval", + "num-derive 0.4.2", + "num-traits", + "serde", + "serde_derive", + "serde_with", + "solana-clock", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-pubkey 4.4.0", + "solana-sdk-ids", + "solana-serde-varint", + "solana-serialize-utils 4.0.0", + "solana-short-vec", + "solana-system-interface", + "solana-wincode-varint", + "wincode", +] + +[[package]] +name = "solana-vote-program" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "agave-feature-set", + "bincode", + "log", + "solana-account", + "solana-bincode", + "solana-bls-signatures", + "solana-clock", + "solana-epoch-schedule", + "solana-hash", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-packet", + "solana-program-runtime", + "solana-pubkey 4.4.0", + "solana-rent", + "solana-sdk-ids", + "solana-slot-hashes", + "solana-system-interface", + "solana-transaction-context", + "solana-vote-interface", + "wincode", +] + +[[package]] +name = "solana-wincode-varint" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15737770cec70afc784649526db191289e2b7ca988a878ed2d171f2cdd60b66" +dependencies = [ + "wincode", +] + +[[package]] +name = "solana-zero-copy" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "369c96063ef15bc310508155ad57bc527c64d814e46ac02d9ebb751a41eb8ea1" +dependencies = [ + "bytemuck", + "bytemuck_derive", +] + +[[package]] +name = "solana-zk-elgamal-proof-interface" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8da7f01db2148a1dc16261ff1dc6f3930a1e255a33cece4f1b56658694f27f7" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "num-derive 0.4.2", + "num-traits", + "solana-address 2.9.0", + "solana-instruction 3.5.1", + "solana-sdk-ids", + "solana-zk-sdk-pod", +] + +[[package]] +name = "solana-zk-elgamal-proof-interface" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eecaadf2ceaa3b4c2219d299b17e3fe079dbe47e155bea52ee9dce636ac2681a" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "num-derive 0.5.1", + "num-traits", + "solana-address 2.9.0", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-sdk-ids", + "solana-zk-sdk-pod", +] + +[[package]] +name = "solana-zk-elgamal-proof-program" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "bytemuck", + "solana-instruction 4.0.0", + "solana-instruction-error 3.0.0", + "solana-program-runtime", + "solana-sdk-ids", + "solana-svm-log-collector", + "solana-zk-elgamal-proof-interface 1.0.0", + "solana-zk-sdk", +] + +[[package]] +name = "solana-zk-sdk" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78a601443971629485248f27272ca2456c94e8aca559ce6e44b8335f11e80ed4" +dependencies = [ + "aes-gcm-siv", + "base64 0.23.1", + "bincode", + "bytemuck", + "curve25519-dalek 4.1.3", + "getrandom 0.2.17", + "hkdf", + "itertools 0.15.0", + "merlin", + "serde", + "serde_derive", + "serde_json", + "sha2 0.11.1", + "sha3", + "shake", + "solana-address 2.9.0", + "solana-derivation-path", + "solana-seed-derivable", + "solana-seed-phrase", + "solana-signature", + "solana-signer", + "solana-zk-elgamal-proof-interface 1.0.0", + "solana-zk-sdk-pod", + "subtle", + "thiserror 2.0.21", + "zeroize", +] + +[[package]] +name = "solana-zk-sdk-pod" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a800583b7a4cea3e851686af162cc6e4712eef97fa91dfa9aca2459b95c84777" +dependencies = [ + "base64 0.22.1", + "bytemuck", + "bytemuck_derive", + "solana-nullable", + "thiserror 2.0.21", +] + +[[package]] +name = "solana-zk-token-proof-program" +version = "4.4.0-rc.1" +source = "git+https://github.com/anza-xyz/agave?tag=v4.4.0-rc.1#b6149a719f8fa84123ac3ce1e5f201af64bca8ee" +dependencies = [ + "solana-program-runtime", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "spl-associated-token-account-interface" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6433917b60441d68d99a17e121d9db0ea15a9a69c0e5afa34649cf5ba12612f" +dependencies = [ + "borsh", + "solana-instruction 3.5.1", + "solana-pubkey 3.0.0", +] + +[[package]] +name = "spl-discriminator" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e597c5ff9ed7c74a54dbc47bae2f06e4db8c98f4356ad280200dc11878266db1" +dependencies = [ + "bytemuck", + "solana-program-error", + "solana-sha256-hasher", + "spl-discriminator-derive", +] + +[[package]] +name = "spl-discriminator-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e8418ea6269dcfb01c712f0444d2c75542c04448b480e87de59d2865edc750" +dependencies = [ + "quote", + "spl-discriminator-syn", + "syn 2.0.119", +] + +[[package]] +name = "spl-discriminator-syn" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1dbc82ab91422345b6df40a79e2b78c7bce1ebb366da323572dd60b7076b67" +dependencies = [ + "proc-macro2", + "quote", + "sha2 0.10.9", + "syn 2.0.119", + "thiserror 1.0.69", +] + +[[package]] +name = "spl-generic-token" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233df81b75ab99b42f002b5cdd6e65a7505ffa930624f7096a7580a56765e9cf" +dependencies = [ + "bytemuck", + "solana-pubkey 3.0.0", +] + +[[package]] +name = "spl-memo-interface" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3745d384b0afee980d43d62b66c27bdcbbd03507732b8d3626d3413cb72084f2" +dependencies = [ + "solana-instruction 3.5.1", + "solana-pubkey 4.4.0", +] + +[[package]] +name = "spl-token-2022-interface" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8b75ca59053d3a79bb8cbaaef12c63ec08f6f1f0b8292352a39b56c455a248" +dependencies = [ + "arrayref", + "bytemuck", + "getrandom 0.2.17", + "num-derive 0.5.1", + "num-traits", + "num_enum", + "solana-account-info", + "solana-address 2.9.0", + "solana-instruction 3.5.1", + "solana-nullable", + "solana-program-error", + "solana-program-option", + "solana-program-pack", + "solana-sdk-ids", + "solana-zero-copy", + "solana-zk-elgamal-proof-interface 0.1.3", + "solana-zk-sdk-pod", + "spl-token-confidential-transfer-proof-extraction", + "spl-token-group-interface", + "spl-token-metadata-interface", + "spl-type-length-value", + "thiserror 2.0.21", +] + +[[package]] +name = "spl-token-confidential-transfer-proof-extraction" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a845ec724e807643a04f1d43439ee3571dd913848112ac76aa90b850eaf7c" +dependencies = [ + "bytemuck", + "solana-account-info", + "solana-address 2.9.0", + "solana-curve25519", + "solana-instruction 3.5.1", + "solana-instructions-sysvar 3.0.1", + "solana-msg", + "solana-program-error", + "solana-sdk-ids", + "solana-zk-elgamal-proof-interface 0.1.3", + "solana-zk-sdk-pod", + "thiserror 2.0.21", +] + +[[package]] +name = "spl-token-group-interface" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841cbd6f2322d02719be4da1affedbe6495b1048b7b985ec9796032564026e22" +dependencies = [ + "bytemuck", + "num-derive 0.4.2", + "num-traits", + "num_enum", + "solana-address 2.9.0", + "solana-instruction 3.5.1", + "solana-nullable", + "solana-program-error", + "solana-zero-copy", + "spl-discriminator", + "thiserror 2.0.21", +] + +[[package]] +name = "spl-token-interface" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7143c4e676984047a400e2fbd7ac29a1659f2b1b52b897cfde19316b562e4589" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive 0.4.2", + "num-traits", + "num_enum", + "solana-instruction 3.5.1", + "solana-program-error", + "solana-program-option", + "solana-program-pack", + "solana-pubkey 3.0.0", + "solana-sdk-ids", + "thiserror 2.0.21", +] + +[[package]] +name = "spl-token-metadata-interface" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3d96f175e7022ff200464dfa75a3708a4e9b70c83c4ecd04fe52ee479f4fef" +dependencies = [ + "borsh", + "num-derive 0.4.2", + "num-traits", + "num_enum", + "solana-address 2.9.0", + "solana-borsh", + "solana-instruction 3.5.1", + "solana-nullable", + "solana-program-error", + "spl-discriminator", + "spl-type-length-value", + "thiserror 2.0.21", +] + +[[package]] +name = "spl-type-length-value" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2504631748c48d2a937414d64a12dcac4588d34bd07d355d648619c189d29435" +dependencies = [ + "bytemuck", + "num-derive 0.4.2", + "num-traits", + "num_enum", + "solana-account-info", + "solana-program-error", + "solana-zero-copy", + "spl-discriminator", + "thiserror 2.0.21", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.14", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot 0.12.5", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + +[[package]] +name = "tonic" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea216ddc448e08e7839ab09e20e60b06b6a1c32745ccb212c0ebd161c199f1b" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags 2.13.2", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "trees" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de5f738ceab88e2491a94ddc33c3feeadfa95fedc60363ef110845df12f3878" + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "uriparse" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" +dependencies = [ + "fnv", + "lazy_static", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" +dependencies = [ + "libc", +] + +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "wincode" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfc6339f1ba427bf7ad7c42403b28e524832ba2ddb5eef1bb2cc3b85db6b7b75" +dependencies = [ + "bv", + "bytes", + "pastey", + "proc-macro2", + "quote", + "smallvec", + "thiserror 2.0.21", + "wincode-derive", +] + +[[package]] +name = "wincode-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d582d9fc9d813264407a9e16bfa16846ccf15ef032f9bbcd700741bdc00255" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "primefield", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb02c3132d251ab1606e9fc913a7da6c6ed08cc263df15edf082e1dc497d157" +dependencies = [ + "zstd-safe 7.4.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", +] + +[[package]] +name = "zstd-safe" +version = "7.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8ea04668f5de8d8d51051714ba560c9536519cadcdb8829bba4fe2ad159b03" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/hivezilla/ledger-archive/Cargo.toml b/hivezilla/ledger-archive/Cargo.toml new file mode 100644 index 0000000..2733001 --- /dev/null +++ b/hivezilla/ledger-archive/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "hivezilla-ledger-archive" +version = "0.2.0" +edition = "2024" +rust-version = "1.98.1" +license = "MIT" +publish = false +description = "Build Compact V2 hot archives from an Agave validator ledger" + +[[bin]] +name = "hivezilla-ledger-archive" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.104" +blockzilla-archive-v2 = { path = "../../crates/compact-v2/blockzilla-archive-v2", default-features = false } +blockzilla-compact = { path = "../../crates/compact-v2/blockzilla-compact", default-features = false } +blockzilla-primitives = { path = "../../crates/blockzilla-primitives", default-features = false } +blockzilla-registry = { path = "../../crates/compact-v2/blockzilla-registry" } +bs58 = "0.5.1" +clap = { version = "4.6.6", features = ["derive"] } +of-car-reader = { path = "../../crates/old-faithful/of-car-reader", default-features = false } +# Agave 4.4.0-rc.1, commit b6149a719f8fa84123ac3ce1e5f201af64bca8ee. +# The local reference checkout is branch v4.4.0-rc.1 at that tag. +solana-entry = { git = "https://github.com/anza-xyz/agave", tag = "v4.4.0-rc.1" } +solana-hash = "=4.6.0" +solana-instruction-error = "=3.0.0" +solana-ledger = { git = "https://github.com/anza-xyz/agave", tag = "v4.4.0-rc.1", features = ["agave-unstable-api"] } +solana-message = "=5.0.0" +solana-reward-info = "=7.0.0" +solana-transaction-error = "=4.0.0" +solana-transaction-status = { git = "https://github.com/anza-xyz/agave", tag = "v4.4.0-rc.1", features = ["agave-unstable-api"] } +wincode = { version = "=0.6.1", features = ["derive"] } +zstd = "0.13.3" + +[dev-dependencies] +tempfile = "3.27.0" + +[workspace] diff --git a/hivezilla/ledger-archive/rust-toolchain.toml b/hivezilla/ledger-archive/rust-toolchain.toml new file mode 100644 index 0000000..6422209 --- /dev/null +++ b/hivezilla/ledger-archive/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "1.98.1" diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs new file mode 100644 index 0000000..cb51262 --- /dev/null +++ b/hivezilla/ledger-archive/src/check.rs @@ -0,0 +1,279 @@ +use anyhow::{bail, Context, Result}; + +use crate::markers::reward_names; +use crate::source::{LedgerView, REWARD_SLOT_DISTANCE}; + +/// Blocks and proven skips for one inclusive slot window. +/// +/// `check-ledger-range` and `build-archive-v2-from-ledger` both call this +/// function. A check that passes on a snapshot will not later stop the build +/// of that same snapshot for a hole, a missing status, or a missing parent. +#[derive(Debug, Clone)] +pub(crate) struct Inspected { + pub block_slots: Vec, + pub skips: Vec, + pub boundary_hash: [u8; 32], +} + +pub(crate) fn inspect(view: &LedgerView) -> Result { + if view.start_slot > view.end_slot { + bail!( + "start slot {} is after end slot {}", + view.start_slot, + view.end_slot + ); + } + + let mut block_slots = Vec::new(); + let mut skips = Vec::new(); + for slot in view.start_slot..=view.end_slot { + if view.slots.contains_key(&slot) { + accept_block(view, slot, &block_slots)?; + block_slots.push(slot); + } else { + classify_hole(view, slot, &mut skips)?; + } + } + + let Some(first_slot) = block_slots.first().copied() else { + bail!( + "slots {} to {} have no block", + view.start_slot, + view.end_slot + ); + }; + let boundary_hash = view.slots[&first_slot] + .previous_blockhash + .context("first block is missing parent entries")?; + + Ok(Inspected { + block_slots, + skips, + boundary_hash, + }) +} + +fn accept_block(view: &LedgerView, slot: u64, produced: &[u64]) -> Result<()> { + let data = &view.slots[&slot]; + if !data.is_root { + bail!("slot {slot} has data and is not rooted"); + } + if !data.is_full { + bail!("slot {slot} is rooted and is not full"); + } + let Some(parent_slot) = data.parent_slot else { + bail!("slot {slot} has no parent slot"); + }; + if parent_slot >= slot { + bail!("slot {slot} parent {parent_slot} is not before the slot"); + } + let Some(previous_hash) = data.previous_blockhash else { + bail!("slot {slot} is missing parent entries"); + }; + if !data.markers.is_empty() && data.consensus_block_id.is_none() { + bail!("slot {slot} has block markers and no consensus block id"); + } + + if let Some(previous_slot) = produced.last().copied() { + let previous = &view.slots[&previous_slot]; + if parent_slot != previous.slot { + bail!( + "slot {slot} parent {parent_slot} is not the previous block {}", + previous.slot + ); + } + if previous_hash != previous.blockhash { + bail!( + "slot {slot} previous blockhash does not match slot {}", + previous.slot + ); + } + } else if (view.start_slot..=view.end_slot).contains(&parent_slot) { + bail!("slot {slot} parent {parent_slot} is inside the range and is not a block"); + } + + for (index, tx) in data.transactions.iter().enumerate() { + if tx.status.is_none() { + bail!("slot {slot} transaction {index} is missing status"); + } + } + Ok(()) +} + +fn classify_hole(view: &LedgerView, slot: u64, skips: &mut Vec) -> Result<()> { + let carrier = slot + .checked_add(REWARD_SLOT_DISTANCE) + .context("slot + 8 overflows")?; + let Some(data) = view.slots.get(&carrier) else { + bail!("slot {slot} has no SlotMeta and slot {carrier} has no footer proof"); + }; + let names = reward_names(&data.markers) + .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; + if names.notars.contains(&slot) { + bail!("slot {slot} is missing and slot {carrier} has a notarization reward for it"); + } + if names.skips.contains(&slot) { + skips.push(slot); + return Ok(()); + } + bail!("slot {slot} has no SlotMeta and slot {carrier} does not prove a skip"); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::source::{test_block, SlotData}; + + fn view(start: u64, end: u64, slots: Vec) -> LedgerView { + LedgerView { + start_slot: start, + end_slot: end, + slots: slots.into_iter().map(|slot| (slot.slot, slot)).collect(), + } + } + + fn header_marker(parent_slot: u64, parent_id: [u8; 32]) -> Vec { + let mut payload = vec![1u8]; + payload.extend_from_slice(&parent_slot.to_le_bytes()); + payload.extend_from_slice(&parent_id); + marker(1, &payload) + } + + fn skip_footer(skipped_slot: u64) -> Vec { + let mut certificates = vec![0, 1]; + certificates.extend_from_slice(&skipped_slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(7u8, 96)); + certificates.push(0); + certificates.push(0); + footer_marker(&certificates) + } + + fn notar_footer(notarized_slot: u64) -> Vec { + let mut certificates = vec![0, 0, 1]; + certificates.extend_from_slice(¬arized_slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(9u8, 32)); + certificates.extend(std::iter::repeat_n(8u8, 96)); + certificates.push(0); + footer_marker(&certificates) + } + + fn footer_marker(certificates: &[u8]) -> Vec { + let mut payload = vec![1u8]; + payload.extend(std::iter::repeat_n(0xabu8, 32)); + payload.extend_from_slice(&1u64.to_le_bytes()); + payload.push(0); + payload.extend_from_slice(certificates); + marker(0, &payload) + } + + fn marker(variant: u8, payload: &[u8]) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&1u16.to_le_bytes()); + out.push(variant); + out.extend_from_slice(&(u16::try_from(payload.len()).unwrap()).to_le_bytes()); + out.extend_from_slice(payload); + out + } + + #[test] + fn missing_status_is_an_error() { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.transactions.push(crate::source::SourceTx { + signatures: vec![[4; 64]], + message: crate::source::SourceMessage::Legacy(crate::source::SourceMessageBody { + header: crate::source::SourceHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: Vec::new(), + recent_blockhash: [1; 32], + instructions: Vec::new(), + lookups: Vec::new(), + }), + status: None, + }); + block.entries[0].tx_count = 1; + + let error = inspect(&view(10, 10, vec![block])).unwrap_err(); + assert!( + error.to_string().contains("missing status"), + "{error:#}" + ); + } + + #[test] + fn missing_parent_entries_are_an_error() { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.previous_blockhash = None; + let error = inspect(&view(10, 10, vec![block])).unwrap_err(); + assert!( + error.to_string().contains("missing parent entries"), + "{error:#}" + ); + } + + #[test] + fn a_proven_skip_is_omitted() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut second = test_block(12, 10, [5; 32], [4; 32]); + second.entries[0].hash = [5; 32]; + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.markers = vec![skip_footer(11)]; + carrier.is_root = false; + + let inspected = inspect(&view(10, 12, vec![first, second, carrier])).unwrap(); + assert_eq!(inspected.block_slots, vec![10, 12]); + assert_eq!(inspected.skips, vec![11]); + assert_eq!(inspected.boundary_hash, [3; 32]); + } + + #[test] + fn a_notarized_missing_slot_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let second = test_block(12, 10, [5; 32], [4; 32]); + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.markers = vec![notar_footer(11)]; + + let error = inspect(&view(10, 12, vec![first, second, carrier])).unwrap_err(); + assert!( + error.to_string().contains("notarization reward"), + "{error:#}" + ); + } + + #[test] + fn an_unproven_hole_is_an_error() { + let block = test_block(10, 9, [4; 32], [3; 32]); + let error = inspect(&view(10, 11, vec![block])).unwrap_err(); + assert!(error.to_string().contains("does not prove a skip") || error.to_string().contains("no footer proof"), "{error:#}"); + } + + #[test] + fn a_non_root_slot_is_an_error() { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.is_root = false; + let error = inspect(&view(10, 10, vec![block])).unwrap_err(); + assert!(error.to_string().contains("not rooted"), "{error:#}"); + } + + #[test] + fn a_rooted_partial_slot_is_an_error() { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.is_full = false; + let error = inspect(&view(10, 10, vec![block])).unwrap_err(); + assert!(error.to_string().contains("not full"), "{error:#}"); + } + + #[test] + fn markers_without_a_consensus_id_are_an_error() { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.markers = vec![header_marker(9, [7; 32])]; + let error = inspect(&view(10, 10, vec![block])).unwrap_err(); + assert!( + error.to_string().contains("no consensus block id"), + "{error:#}" + ); + } + +} diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs new file mode 100644 index 0000000..de6bef1 --- /dev/null +++ b/hivezilla/ledger-archive/src/convert.rs @@ -0,0 +1,675 @@ +use std::fs::{self, File}; +use std::io::{BufWriter, Write}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use blockzilla_archive_v2::{ + write_archive_v2_hot_block_index, ArchiveV2HotBlockBlob, ArchiveV2HotBlockHeader, + ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction, ArchiveV2HotInstructionData, + ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload, ArchiveV2HotRewards, + ArchiveV2HotTxRow, ArchiveV2HotV0Message, ArchiveV2HotV1Message, WincodeArchiveV2PohRecord, + WincodeArchiveV2ShreddingRecord, ARCHIVE_V2_BLOCKS_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, + ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_POH_FILE, ARCHIVE_V2_SHREDDING_FILE, + ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_TX_FLAG_HAS_ERROR, ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, + ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES, ARCHIVE_V2_TX_FLAG_HAS_LOGS, + ARCHIVE_V2_TX_FLAG_HAS_METADATA, ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA, + ARCHIVE_V2_TX_FLAG_HAS_TOKEN_BALANCES, ARCHIVE_V2_TX_FLAG_MESSAGE_V0, + ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, +}; +use blockzilla_compact::{ + CompactInnerInstruction, CompactInnerInstructions, CompactMessageHeader, CompactMetaV1, + CompactPohEntry, CompactPubkey, CompactReturnData, CompactReward, CompactShredding, + CompactTokenBalance, CompactTransactionConfig, OwnedCompactAddressTableLookup, + OwnedCompactRecentBlockhash, +}; +use blockzilla_primitives::{wincode_leb128_config, WincodeLeb128FramedWriter}; +use blockzilla_registry::RawPubkeyCompactor; +use blockzilla_compact::parse_logs_with_compactor; + +use crate::check::Inspected; +use crate::markers::footer_bank_hash; +use crate::source::{ + LedgerView, SlotData, SourceInnerGroup, SourceMessage, SourceMessageBody, SourceReward, + SourceStatus, SourceTokenBalance, SourceTx, SourceV1Message, +}; + +pub(crate) const ZSTD_LEVEL: i32 = 3; +const INDEX_FLAGS: u32 = 0; +const VOTE_HASH_ROW_LEN: usize = 65; + +pub(crate) fn write_archive(output: &Path, view: &LedgerView, inspected: &Inspected) -> Result<()> { + fs::create_dir_all(output) + .with_context(|| format!("create {}", output.display()))?; + + let blocks_tmp = output.join(format!("{ARCHIVE_V2_BLOCKS_FILE}.tmp")); + let signatures_tmp = output.join(format!("{ARCHIVE_V2_SIGNATURES_FILE}.tmp")); + let blockhash_tmp = output.join(format!("{ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE}.tmp")); + let vote_tmp = output.join(format!("{ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE}.tmp")); + let poh_tmp = output.join(format!("{ARCHIVE_V2_POH_FILE}.tmp")); + let shredding_tmp = output.join(format!("{ARCHIVE_V2_SHREDDING_FILE}.tmp")); + let temps = [ + &blocks_tmp, + &signatures_tmp, + &blockhash_tmp, + &vote_tmp, + &poh_tmp, + &shredding_tmp, + ]; + + let write_result = write_archive_files( + output, + view, + inspected, + &blocks_tmp, + &signatures_tmp, + &blockhash_tmp, + &vote_tmp, + &poh_tmp, + &shredding_tmp, + ); + if write_result.is_err() { + for path in temps { + let _ = fs::remove_file(path); + } + } + write_result +} + +fn write_archive_files( + output: &Path, + view: &LedgerView, + inspected: &Inspected, + blocks_tmp: &Path, + signatures_tmp: &Path, + blockhash_tmp: &Path, + vote_tmp: &Path, + poh_tmp: &Path, + shredding_tmp: &Path, +) -> Result<()> { + let mut blocks = BufWriter::new( + File::create(blocks_tmp).with_context(|| format!("create {}", blocks_tmp.display()))?, + ); + let mut signatures = Vec::new(); + let mut blockhashes = Vec::with_capacity(32 * (inspected.block_slots.len() + 1)); + blockhashes.extend(inspected.boundary_hash); + let mut vote_hashes = Vec::with_capacity(VOTE_HASH_ROW_LEN * inspected.block_slots.len()); + let mut poh = WincodeLeb128FramedWriter::new(BufWriter::new( + File::create(poh_tmp).with_context(|| format!("create {}", poh_tmp.display()))?, + )); + let mut shredding = WincodeLeb128FramedWriter::new(BufWriter::new( + File::create(shredding_tmp) + .with_context(|| format!("create {}", shredding_tmp.display()))?, + )); + + let mut rows = Vec::with_capacity(inspected.block_slots.len()); + let mut compressed_offset = 0u64; + let mut next_tx_ordinal = 0u64; + let mut next_signature_ordinal = 0u64; + + for (ordinal, slot) in inspected.block_slots.iter().copied().enumerate() { + let block = &view.slots[&slot]; + let block_id = u32::try_from(ordinal).context("archive block id exceeds u32")?; + let uncompressed = encode_hot_block(block, block_id)?; + let compressed = zstd::bulk::compress(&uncompressed, ZSTD_LEVEL) + .with_context(|| format!("compress slot {slot}"))?; + let compressed_len = u32::try_from(compressed.len()).context("compressed block exceeds u32")?; + let uncompressed_len = + u32::try_from(uncompressed.len()).context("block bytes exceed u32")?; + blocks + .write_all(&compressed) + .with_context(|| format!("write slot {slot}"))?; + + let signature_count = append_signatures(&mut signatures, block)?; + blockhashes.extend(block.blockhash); + vote_hashes.extend(vote_hash_row(block)?); + poh.write(&poh_record(block, block_id)) + .with_context(|| format!("write PoH for slot {slot}"))?; + shredding + .write(&shredding_record(block, block_id)) + .with_context(|| format!("write shredding for slot {slot}"))?; + + let tx_count = u32::try_from(block.transactions.len()).context("transaction count exceeds u32")?; + rows.push(ArchiveV2HotBlockIndexRow { + block_id, + slot, + compressed_offset, + compressed_len, + uncompressed_len, + tx_count, + first_tx_ordinal: next_tx_ordinal, + first_signature_ordinal: next_signature_ordinal, + signature_count, + }); + compressed_offset = compressed_offset + .checked_add(u64::from(compressed_len)) + .context("compressed archive exceeds u64")?; + next_tx_ordinal = next_tx_ordinal + .checked_add(u64::from(tx_count)) + .context("transaction ordinal exceeds u64")?; + next_signature_ordinal = next_signature_ordinal + .checked_add(u64::from(signature_count)) + .context("signature ordinal exceeds u64")?; + } + + blocks + .flush() + .with_context(|| format!("flush {}", blocks_tmp.display()))?; + drop(blocks); + poh.flush().context("flush PoH sidecar")?; + let poh_writer = poh.into_inner(); + poh_writer + .into_inner() + .map_err(|err| anyhow::anyhow!(err)) + .context("flush PoH sidecar")?; + shredding.flush().context("flush shredding sidecar")?; + let shredding_writer = shredding.into_inner(); + shredding_writer + .into_inner() + .map_err(|err| anyhow::anyhow!(err)) + .context("flush shredding sidecar")?; + + fs::write(signatures_tmp, &signatures) + .with_context(|| format!("write {}", signatures_tmp.display()))?; + fs::write(blockhash_tmp, &blockhashes) + .with_context(|| format!("write {}", blockhash_tmp.display()))?; + fs::write(vote_tmp, &vote_hashes) + .with_context(|| format!("write {}", vote_tmp.display()))?; + + rename_finished(blocks_tmp, &output.join(ARCHIVE_V2_BLOCKS_FILE))?; + rename_finished(signatures_tmp, &output.join(ARCHIVE_V2_SIGNATURES_FILE))?; + rename_finished( + blockhash_tmp, + &output.join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE), + )?; + rename_finished(vote_tmp, &output.join(ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE))?; + rename_finished(poh_tmp, &output.join(ARCHIVE_V2_POH_FILE))?; + rename_finished(shredding_tmp, &output.join(ARCHIVE_V2_SHREDDING_FILE))?; + write_archive_v2_hot_block_index( + &output.join(ARCHIVE_V2_BLOCK_INDEX_FILE), + compressed_offset, + ZSTD_LEVEL, + INDEX_FLAGS, + &rows, + )?; + Ok(()) +} + +fn rename_finished(from: &Path, to: &Path) -> Result<()> { + fs::rename(from, to) + .with_context(|| format!("rename {} to {}", from.display(), to.display())) +} + +pub(crate) fn encode_hot_block(block: &SlotData, block_id: u32) -> Result> { + let entry_txs = block.entries.iter().try_fold(0u32, |total, entry| { + total + .checked_add(entry.tx_count) + .context("entry transaction count overflows") + })?; + if usize::from(entry_txs) != block.transactions.len() { + bail!( + "slot {} entry transaction count {entry_txs} does not match {}", + block.slot, + block.transactions.len() + ); + } + + let mut message_bytes = Vec::new(); + let mut metadata_bytes = Vec::new(); + let mut rows = Vec::with_capacity(block.transactions.len()); + for (tx_index, tx) in block.transactions.iter().enumerate() { + let status = tx.status.as_ref().with_context(|| { + format!("slot {} transaction {tx_index} is missing status", block.slot) + })?; + let row = encode_tx_row( + tx, + status, + u32::try_from(tx_index).context("transaction index exceeds u32")?, + &mut message_bytes, + &mut metadata_bytes, + )?; + rows.push(row); + } + + let rewards = if block.rewards.is_empty() && block.num_partitions.is_none() { + None + } else { + Some(ArchiveV2HotRewards { + num_partitions: block.num_partitions, + decoded: block.rewards.iter().map(compact_reward).collect(), + }) + }; + let blob = ArchiveV2HotBlockBlob { + header: ArchiveV2HotBlockHeader { + slot: block.slot, + parent_slot: block + .parent_slot + .with_context(|| format!("slot {} has no parent slot", block.slot))?, + blockhash_id: block_id + .checked_add(1) + .context("blockhash id exceeds u32")?, + previous_blockhash_id: block_id, + block_time: block.block_time, + block_height: block.block_height, + rewards, + }, + tx_count: u32::try_from(block.transactions.len()).context("transaction count exceeds u32")?, + tx_rows: rows, + message_bytes, + metadata_bytes, + consensus_block_id: block.consensus_block_id, + block_markers: block.markers.clone(), + }; + wincode::config::serialize(&blob, wincode_leb128_config()).context("serialize hot block") +} + +fn encode_tx_row( + tx: &SourceTx, + status: &SourceStatus, + tx_index: u32, + message_bytes: &mut Vec, + metadata_bytes: &mut Vec, +) -> Result { + let (payload, mut flags) = encode_message(&tx.message)?; + flags |= ARCHIVE_V2_TX_FLAG_HAS_METADATA; + if status.err.is_some() { + flags |= ARCHIVE_V2_TX_FLAG_HAS_ERROR; + } + if status.return_data.is_some() { + flags |= ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA; + } + if status.logs.is_some() { + flags |= ARCHIVE_V2_TX_FLAG_HAS_LOGS; + } + if status.inner_instructions.is_some() { + flags |= ARCHIVE_V2_TX_FLAG_HAS_INNER_IX; + } + if !status.pre_token_balances.is_empty() || !status.post_token_balances.is_empty() { + flags |= ARCHIVE_V2_TX_FLAG_HAS_TOKEN_BALANCES; + } + if !status.loaded_writable.is_empty() || !status.loaded_readonly.is_empty() { + flags |= ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES; + } + + let message = wincode::config::serialize(&payload, wincode_leb128_config()) + .context("serialize transaction message")?; + let metadata = wincode::config::serialize(&compact_meta(status), wincode_leb128_config()) + .context("serialize transaction metadata")?; + let message_offset = u32::try_from(message_bytes.len()).context("message region exceeds u32")?; + let message_len = u32::try_from(message.len()).context("message exceeds u32")?; + let metadata_offset = + u32::try_from(metadata_bytes.len()).context("metadata region exceeds u32")?; + let metadata_len = u32::try_from(metadata.len()).context("metadata exceeds u32")?; + message_bytes.extend(message); + metadata_bytes.extend(metadata); + let signature_count = u8::try_from(tx.signatures.len()) + .context("transaction has more than 255 signatures")?; + + Ok(ArchiveV2HotTxRow { + tx_index, + flags, + message_offset, + message_len, + metadata_offset, + metadata_len, + signature_count, + reserved: [0; 3], + }) +} + +fn encode_message(message: &SourceMessage) -> Result<(ArchiveV2HotMessagePayload, u32)> { + match message { + SourceMessage::Legacy(body) => Ok(( + ArchiveV2HotMessagePayload::Legacy(ArchiveV2HotLegacyMessage { + header: compact_header(body), + account_keys: raw_keys(&body.account_keys), + recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), + instructions: raw_instructions(&body.instructions), + }), + 0, + )), + SourceMessage::V0(body) => Ok(( + ArchiveV2HotMessagePayload::V0(ArchiveV2HotV0Message { + header: compact_header(body), + account_keys: raw_keys(&body.account_keys), + recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), + instructions: raw_instructions(&body.instructions), + address_table_lookups: body + .lookups + .iter() + .map(|lookup| OwnedCompactAddressTableLookup { + account_key: CompactPubkey::raw(lookup.account_key), + writable_indexes: lookup.writable_indexes.clone(), + readonly_indexes: lookup.readonly_indexes.clone(), + }) + .collect(), + }), + ARCHIVE_V2_TX_FLAG_MESSAGE_V0, + )), + SourceMessage::V1(body) => Ok(( + ArchiveV2HotMessagePayload::V1(ArchiveV2HotV1Message { + header: compact_header_v1(body), + config: CompactTransactionConfig { + priority_fee: body.config.priority_fee, + compute_unit_limit: body.config.compute_unit_limit, + loaded_accounts_data_size_limit: body.config.loaded_accounts_data_size_limit, + heap_size: body.config.heap_size, + }, + account_keys: raw_keys(&body.account_keys), + recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), + instructions: raw_instructions(&body.instructions), + }), + 0, + )), + } +} + +fn compact_header(body: &SourceMessageBody) -> CompactMessageHeader { + CompactMessageHeader { + num_required_signatures: body.header.num_required_signatures, + num_readonly_signed_accounts: body.header.num_readonly_signed_accounts, + num_readonly_unsigned_accounts: body.header.num_readonly_unsigned_accounts, + } +} + +fn compact_header_v1(body: &SourceV1Message) -> CompactMessageHeader { + CompactMessageHeader { + num_required_signatures: body.header.num_required_signatures, + num_readonly_signed_accounts: body.header.num_readonly_signed_accounts, + num_readonly_unsigned_accounts: body.header.num_readonly_unsigned_accounts, + } +} + +fn raw_keys(keys: &[[u8; 32]]) -> Vec { + keys.iter().copied().map(CompactPubkey::raw).collect() +} + +fn raw_instructions(instructions: &[crate::source::SourceInstruction]) -> Vec { + instructions + .iter() + .map(|instruction| ArchiveV2HotInstruction { + program_id_index: instruction.program_id_index, + accounts: instruction.accounts.clone(), + data: ArchiveV2HotInstructionData::Raw(instruction.data.clone()), + }) + .collect() +} + +fn compact_meta(status: &SourceStatus) -> CompactMetaV1 { + CompactMetaV1 { + err: status.err.clone(), + fee: status.fee, + pre_balances: status.pre_balances.clone(), + post_balances: status.post_balances.clone(), + inner_instructions: status + .inner_instructions + .as_ref() + .map(|groups| groups.iter().map(compact_inner_group).collect()), + logs: status + .logs + .as_ref() + .map(|lines| parse_logs_with_compactor(lines, &RawPubkeyCompactor)), + pre_token_balances: status + .pre_token_balances + .iter() + .map(compact_token_balance) + .collect(), + post_token_balances: status + .post_token_balances + .iter() + .map(compact_token_balance) + .collect(), + rewards: status.rewards.iter().map(compact_reward).collect(), + loaded_writable_addresses: raw_keys(&status.loaded_writable), + loaded_readonly_addresses: raw_keys(&status.loaded_readonly), + return_data: status.return_data.as_ref().map(|data| CompactReturnData { + program_id: CompactPubkey::raw(data.program_id), + data: data.data.clone(), + }), + compute_units_consumed: status.compute_units_consumed, + cost_units: status.cost_units, + } +} + +fn compact_inner_group(group: &SourceInnerGroup) -> CompactInnerInstructions { + CompactInnerInstructions { + index: group.index, + instructions: group + .instructions + .iter() + .map(|instruction| CompactInnerInstruction { + program_id_index: instruction.program_id_index, + accounts: instruction.accounts.clone(), + data: instruction.data.clone(), + stack_height: instruction.stack_height, + }) + .collect(), + } +} + +fn compact_token_balance(balance: &SourceTokenBalance) -> CompactTokenBalance { + CompactTokenBalance { + account_index: balance.account_index, + mint: Some(CompactPubkey::raw(balance.mint)), + owner: balance.owner.map(CompactPubkey::raw), + program_id: balance.program_id.map(CompactPubkey::raw), + amount: balance.amount, + decimals: balance.decimals, + } +} + +fn compact_reward(reward: &SourceReward) -> CompactReward { + CompactReward { + pubkey: CompactPubkey::raw(reward.pubkey), + lamports: reward.lamports, + post_balance: reward.post_balance, + reward_type: reward.reward_type, + commission: reward.commission, + } +} + +fn append_signatures(out: &mut Vec, block: &SlotData) -> Result { + let mut count = 0u32; + for tx in &block.transactions { + for signature in &tx.signatures { + out.extend(signature); + count = count.checked_add(1).context("signature count exceeds u32")?; + } + } + Ok(count) +} + +fn poh_record(block: &SlotData, block_id: u32) -> WincodeArchiveV2PohRecord { + WincodeArchiveV2PohRecord { + block_id, + slot: block.slot, + entries: block + .entries + .iter() + .map(|entry| CompactPohEntry { + num_hashes: entry.num_hashes, + hash: entry.hash, + tx_count: entry.tx_count, + signature_count: entry.signature_count, + }) + .collect(), + } +} + +pub(crate) fn shredding_record(block: &SlotData, block_id: u32) -> WincodeArchiveV2ShreddingRecord { + WincodeArchiveV2ShreddingRecord { + block_id, + slot: block.slot, + shredding: block + .shredding + .iter() + .map(|group| CompactShredding { + entry_end_idx: group.entry_end_idx, + shred_end_idx: group.shred_end_idx, + }) + .collect(), + } +} + +fn vote_hash_row(block: &SlotData) -> Result<[u8; VOTE_HASH_ROW_LEN]> { + let bank_hash = footer_bank_hash(&block.markers) + .with_context(|| format!("slot {} footer does not parse", block.slot))?; + let mut row = [0u8; VOTE_HASH_ROW_LEN]; + row[0] = u8::from(bank_hash.is_some()) | (u8::from(block.consensus_block_id.is_some()) << 1); + if let Some(hash) = bank_hash { + row[1..33].copy_from_slice(&hash); + } + if let Some(hash) = block.consensus_block_id { + row[33..65].copy_from_slice(&hash); + } + Ok(row) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::check::inspect; + use crate::source::{test_block, LedgerView, SourceShredGroup, REWARD_SLOT_DISTANCE}; + use blockzilla_archive_v2::deserialize_archive_v2_hot_block_blob; + use blockzilla_primitives::WincodeLeb128FramedReader; + use of_car_reader::blockmarker::parent_consensus_block_id; + use std::io::Cursor; + + fn header_marker(parent_slot: u64, parent_id: [u8; 32]) -> Vec { + let mut payload = vec![1u8]; + payload.extend_from_slice(&parent_slot.to_le_bytes()); + payload.extend_from_slice(&parent_id); + let mut out = Vec::new(); + out.extend_from_slice(&1u16.to_le_bytes()); + out.push(1); + out.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_le_bytes()); + out.extend(payload); + out + } + + #[test] + fn empty_tower_tail_is_two_zero_bytes_and_has_no_consensus_id() { + let block = test_block(10, 9, [2; 32], [1; 32]); + let bytes = encode_hot_block(&block, 0).unwrap(); + assert_eq!(&bytes[bytes.len() - 2..], &[0, 0]); + let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); + assert_eq!(decoded.consensus_block_id, None); + assert!(decoded.block_markers.is_empty()); + assert_eq!(decoded.header.previous_blockhash_id, 0); + assert_eq!(decoded.header.blockhash_id, 1); + } + + #[test] + fn header_marker_fills_the_consensus_id_and_parent_id() { + let parent_id = [7; 32]; + let consensus_id = [9; 32]; + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.consensus_block_id = Some(consensus_id); + block.markers = vec![header_marker(9, parent_id)]; + + let bytes = encode_hot_block(&block, 0).unwrap(); + let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); + assert_eq!(decoded.consensus_block_id, Some(consensus_id)); + assert_eq!( + parent_consensus_block_id(&decoded.block_markers), + Some(parent_id) + ); + } + + #[test] + fn two_shredding_groups_become_two_rows() { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.shredding = vec![ + SourceShredGroup { + entry_end_idx: 4, + shred_end_idx: 31, + }, + SourceShredGroup { + entry_end_idx: 11, + shred_end_idx: 95, + }, + ]; + let record = shredding_record(&block, 0); + assert_eq!(record.shredding.len(), 2); + assert_eq!(record.shredding[0].entry_end_idx, 4); + assert_eq!(record.shredding[0].shred_end_idx, 31); + assert_eq!(record.shredding[1].entry_end_idx, 11); + assert_eq!(record.shredding[1].shred_end_idx, 95); + } + + #[test] + fn a_proven_skip_writes_no_blob() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let second = test_block(12, 10, [5; 32], [4; 32]); + let mut carrier = test_block(11 + REWARD_SLOT_DISTANCE, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_footer(11)]; + let view = LedgerView { + start_slot: 10, + end_slot: 12, + slots: [first, second, carrier] + .into_iter() + .map(|slot| (slot.slot, slot)) + .collect(), + }; + let inspected = inspect(&view).unwrap(); + let dir = tempfile::tempdir().unwrap(); + write_archive(dir.path(), &view, &inspected).unwrap(); + + let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); + assert_eq!(&index[..8], b"BZV2HIX1"); + assert_eq!(u64::from_le_bytes(index[12..20].try_into().unwrap()), 2); + assert_eq!(i32::from_le_bytes(index[28..32].try_into().unwrap()), ZSTD_LEVEL); + assert_eq!(u32::from_le_bytes(index[32..36].try_into().unwrap()), 0); + let row_len = 52; + let slots: Vec = index[36..] + .chunks_exact(row_len) + .map(|row| u64::from_le_bytes(row[4..12].try_into().unwrap())) + .collect(); + assert_eq!(slots, vec![10, 12]); + + let hashes = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE)).unwrap(); + assert_eq!(hashes.len(), 96); + assert_eq!(&hashes[..32], &[3; 32]); + assert_eq!(&hashes[32..64], &[4; 32]); + assert_eq!(&hashes[64..], &[5; 32]); + + let mut reader = WincodeLeb128FramedReader::new(Cursor::new( + std::fs::read(dir.path().join(ARCHIVE_V2_SHREDDING_FILE)).unwrap(), + )); + let first_record = reader + .read::() + .unwrap() + .unwrap() + .1; + let second_record = reader + .read::() + .unwrap() + .unwrap() + .1; + assert!(reader + .read::() + .unwrap() + .is_none()); + assert_eq!(first_record.slot, 10); + assert_eq!(second_record.slot, 12); + } + + fn skip_footer(skipped_slot: u64) -> Vec { + let mut certificates = vec![0, 1]; + certificates.extend_from_slice(&skipped_slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(7u8, 96)); + certificates.push(0); + certificates.push(0); + let mut payload = vec![1u8]; + payload.extend(std::iter::repeat_n(0xabu8, 32)); + payload.extend_from_slice(&1u64.to_le_bytes()); + payload.push(0); + payload.extend(certificates); + let mut out = Vec::new(); + out.extend_from_slice(&1u16.to_le_bytes()); + out.push(0); + out.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_le_bytes()); + out.extend(payload); + out + } +} diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs new file mode 100644 index 0000000..e2366b2 --- /dev/null +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -0,0 +1,763 @@ +use std::path::Path; + +use anyhow::{anyhow, bail, Context, Result}; +use blockzilla_compact::{CompactInstructionError, CompactTransactionError}; +use solana_entry::block_component::BlockComponent; +use solana_instruction_error::InstructionError; +use solana_ledger::blockstore::error::BlockstoreError; +use solana_ledger::blockstore::Blockstore; +use solana_ledger::blockstore_meta::BlockLocation; +use solana_ledger::blockstore_options::{ + AccessType, BlockstoreOptions, BLOCKSTORE_DIRECTORY_ROCKS_LEVEL, +}; +use solana_message::VersionedMessage; +use solana_reward_info::RewardType; +use solana_transaction_error::TransactionError; +use solana_transaction_status::{Reward, TransactionStatusMeta}; + +use crate::source::{ + LedgerView, RawComponent, SlotData, SourceEntry, SourceHeader, SourceInnerGroup, + SourceInnerInstruction, SourceInstruction, SourceLookup, SourceMessage, SourceMessageBody, + SourceReturnData, SourceReward, SourceStatus, SourceTokenBalance, SourceTx, SourceTxConfig, + SourceV1Message, REWARD_SLOT_DISTANCE, +}; + +/// Open a ledger only in `AccessType::ReadOnly`. +/// +/// `Blockstore::open` creates a missing ledger directory. This function +/// returns before that call when `rocksdb` is not already a directory. +pub(crate) fn read_only_options(ledger: &Path) -> Result { + let rocks = ledger.join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL); + if !rocks.is_dir() { + bail!("rocksdb directory {} does not exist", rocks.display()); + } + let mut options = BlockstoreOptions::default(); + options.access_type = AccessType::ReadOnly; + reject_non_readonly(&options.access_type)?; + Ok(options) +} + +pub(crate) fn reject_non_readonly(access: &AccessType) -> Result<()> { + if *access == AccessType::ReadOnly { + Ok(()) + } else { + bail!("ledger archive opens AccessType::ReadOnly only"); + } +} + +pub(crate) fn load_range(ledger: &Path, start_slot: u64, end_slot: u64) -> Result { + if start_slot > end_slot { + bail!("start slot {start_slot} is after end slot {end_slot}"); + } + let options = read_only_options(ledger)?; + let blockstore = Blockstore::open_with_options(ledger, options) + .with_context(|| format!("open ledger {}", ledger.display()))?; + load_blockstore(&blockstore, start_slot, end_slot) +} + +fn load_blockstore(blockstore: &Blockstore, start_slot: u64, end_slot: u64) -> Result { + let last_carrier = end_slot + .checked_add(REWARD_SLOT_DISTANCE) + .context("end slot + 8 overflows")?; + let mut slots = std::collections::BTreeMap::new(); + for slot in start_slot..=last_carrier { + let Some(meta) = blockstore + .meta(slot) + .with_context(|| format!("read SlotMeta for slot {slot}"))? + else { + continue; + }; + let is_root = blockstore.is_root(slot); + let data = if slot <= end_slot && is_root && meta.is_full() { + load_rooted_block(blockstore, slot)? + } else if slot > end_slot { + let mut data = empty_slot(slot, meta.parent_slot, is_root, meta.is_full()); + data.markers = read_markers(blockstore, slot)?; + data + } else { + empty_slot(slot, meta.parent_slot, is_root, meta.is_full()) + }; + slots.insert(slot, data); + } + Ok(LedgerView { + start_slot, + end_slot, + slots, + }) +} + +fn load_rooted_block(blockstore: &Blockstore, slot: u64) -> Result { + let (components, ranges, is_full) = blockstore + .get_slot_components_with_shred_info(slot, 0, false) + .with_context(|| format!("read shreds for slot {slot}"))?; + if !is_full { + bail!("slot {slot} is not full"); + } + if components.len() != ranges.len() { + bail!("slot {slot} component count does not match its shred ranges"); + } + + let mut markers = Vec::new(); + let mut raw_entries = Vec::new(); + let mut raw_components = Vec::with_capacity(components.len()); + for (component, range) in components.into_iter().zip(ranges) { + let entry_count = match component { + BlockComponent::EntryBatch(entries) => { + let count = u64::try_from(entries.len()).context("entry batch exceeds u64")?; + raw_entries.extend(entries); + count + } + BlockComponent::BlockMarker(marker) => { + let bytes = wincode::serialize(&marker) + .with_context(|| format!("serialize a block marker in slot {slot}"))?; + markers.push(bytes); + 0 + } + }; + raw_components.push(RawComponent { + entry_count, + shred_range_end: range.end, + }); + } + let shredding = crate::source::shred_groups(&raw_components) + .with_context(|| format!("shred ranges for slot {slot}"))?; + + let block = match blockstore.get_rooted_block(slot, true) { + Ok(block) => block, + Err(BlockstoreError::ParentEntriesUnavailable) => { + bail!("slot {slot} is missing parent entries"); + } + Err(BlockstoreError::MissingTransactionMetadata) => { + bail!("slot {slot} is missing transaction status"); + } + Err(err) => return Err(anyhow!(err).context(format!("read rooted block {slot}"))), + }; + + let has_markers = !markers.is_empty(); + let consensus_block_id = consensus_block_id(blockstore, slot, has_markers)?; + let mut tx_cursor = 0usize; + let mut entries = Vec::with_capacity(raw_entries.len()); + for entry in &raw_entries { + let tx_count = entry.transactions.len(); + let rooted = block.transactions.get(tx_cursor..tx_cursor + tx_count).with_context(|| { + format!("slot {slot} entry transactions do not match the rooted block") + })?; + let mut signature_count = 0u32; + for (entry_tx, rooted_tx) in entry.transactions.iter().zip(rooted) { + if entry_tx.signatures != rooted_tx.transaction.signatures { + bail!("slot {slot} entry transactions do not match the rooted block"); + } + let count = u32::try_from(entry_tx.signatures.len()) + .context("entry signature count exceeds u32")?; + signature_count = signature_count + .checked_add(count) + .context("entry signature count overflows")?; + } + entries.push(SourceEntry { + num_hashes: entry.num_hashes, + hash: copy32(entry.hash.as_ref())?, + tx_count: u32::try_from(tx_count).context("entry has more than u32 transactions")?, + signature_count, + }); + tx_cursor += tx_count; + } + if tx_cursor != block.transactions.len() { + bail!("slot {slot} rooted block has transactions that are not in its entries"); + } + + let transactions = block + .transactions + .iter() + .enumerate() + .map(|(index, tx)| map_transaction(slot, index, tx)) + .collect::>>()?; + + Ok(SlotData { + slot, + is_root: true, + is_full: true, + parent_slot: Some(block.parent_slot), + previous_blockhash: Some(decode_hash( + &block.previous_blockhash, + &format!("slot {slot} previous blockhash"), + )?), + blockhash: decode_hash(&block.blockhash, &format!("slot {slot} blockhash"))?, + block_time: block.block_time, + block_height: block.block_height, + num_partitions: block.num_partitions, + rewards: block + .rewards + .iter() + .map(|reward| map_reward(slot, reward)) + .collect::>>()?, + consensus_block_id: Some(consensus_block_id), + markers, + entries, + shredding, + transactions, + }) +} + +fn consensus_block_id(blockstore: &Blockstore, slot: u64, has_markers: bool) -> Result<[u8; 32]> { + let hash = if has_markers { + blockstore + .get_double_merkle_root(slot, BlockLocation::Original) + .with_context(|| format!("read double merkle root for slot {slot}"))? + } else { + blockstore + .get_last_shred_merkle_root(slot) + .with_context(|| format!("read last shred merkle root for slot {slot}"))? + }; + let Some(hash) = hash else { + bail!("slot {slot} has no consensus block id"); + }; + copy32(hash.as_ref()) +} + +fn read_markers(blockstore: &Blockstore, slot: u64) -> Result>> { + let (components, _, _) = blockstore + .get_slot_components_with_shred_info(slot, 0, false) + .with_context(|| format!("read markers for slot {slot}"))?; + let mut markers = Vec::new(); + for component in components { + if let BlockComponent::BlockMarker(marker) = component { + markers.push( + wincode::serialize(&marker) + .with_context(|| format!("serialize a block marker in slot {slot}"))?, + ); + } + } + Ok(markers) +} + +fn empty_slot(slot: u64, parent_slot: Option, is_root: bool, is_full: bool) -> SlotData { + SlotData { + slot, + is_root, + is_full, + parent_slot, + previous_blockhash: None, + blockhash: [0; 32], + block_time: None, + block_height: None, + num_partitions: None, + rewards: Vec::new(), + consensus_block_id: None, + markers: Vec::new(), + entries: Vec::new(), + shredding: Vec::new(), + transactions: Vec::new(), + } +} + +fn map_transaction( + slot: u64, + index: usize, + tx: &solana_transaction_status::VersionedTransactionWithStatusMeta, +) -> Result { + if tx.transaction.signatures.len() > usize::from(u8::MAX) { + bail!("slot {slot} transaction {index} has more than 255 signatures"); + } + let signatures = tx + .transaction + .signatures + .iter() + .map(|signature| copy64(signature.as_ref())) + .collect::>>()?; + Ok(SourceTx { + signatures, + message: map_message(&tx.transaction.message)?, + status: Some(map_status(slot, index, &tx.meta)?), + }) +} + +fn map_message(message: &VersionedMessage) -> Result { + match message { + VersionedMessage::Legacy(message) => Ok(SourceMessage::Legacy(SourceMessageBody { + header: map_header(&message.header), + account_keys: message + .account_keys + .iter() + .map(|key| copy32(key.as_ref())) + .collect::>>()?, + recent_blockhash: copy32(message.recent_blockhash.as_ref())?, + instructions: message + .instructions + .iter() + .map(map_instruction) + .collect(), + lookups: Vec::new(), + })), + VersionedMessage::V0(message) => Ok(SourceMessage::V0(SourceMessageBody { + header: map_header(&message.header), + account_keys: message + .account_keys + .iter() + .map(|key| copy32(key.as_ref())) + .collect::>>()?, + recent_blockhash: copy32(message.recent_blockhash.as_ref())?, + instructions: message + .instructions + .iter() + .map(map_instruction) + .collect(), + lookups: message + .address_table_lookups + .iter() + .map(|lookup| { + Ok(SourceLookup { + account_key: copy32(lookup.account_key.as_ref())?, + writable_indexes: lookup.writable_indexes.clone(), + readonly_indexes: lookup.readonly_indexes.clone(), + }) + }) + .collect::>>()?, + })), + VersionedMessage::V1(message) => Ok(SourceMessage::V1(SourceV1Message { + header: map_header(&message.header), + config: SourceTxConfig { + priority_fee: message.config.priority_fee, + compute_unit_limit: message.config.compute_unit_limit, + loaded_accounts_data_size_limit: message.config.loaded_accounts_data_size_limit, + heap_size: message.config.heap_size, + }, + account_keys: message + .account_keys + .iter() + .map(|key| copy32(key.as_ref())) + .collect::>>()?, + recent_blockhash: copy32(message.lifetime_specifier.as_ref())?, + instructions: message + .instructions + .iter() + .map(map_instruction) + .collect(), + })), + } +} + +fn map_header(header: &solana_message::MessageHeader) -> SourceHeader { + SourceHeader { + num_required_signatures: header.num_required_signatures, + num_readonly_signed_accounts: header.num_readonly_signed_accounts, + num_readonly_unsigned_accounts: header.num_readonly_unsigned_accounts, + } +} + +fn map_instruction(instruction: &solana_message::compiled_instruction::CompiledInstruction) -> SourceInstruction { + SourceInstruction { + program_id_index: instruction.program_id_index, + accounts: instruction.accounts.clone(), + data: instruction.data.clone(), + } +} + +fn map_status(slot: u64, index: usize, meta: &TransactionStatusMeta) -> Result { + let err = match &meta.status { + Ok(()) => None, + Err(err) => Some( + map_transaction_error(err) + .with_context(|| format!("slot {slot} transaction {index} error"))?, + ), + }; + Ok(SourceStatus { + err, + fee: meta.fee, + pre_balances: meta.pre_balances.clone(), + post_balances: meta.post_balances.clone(), + inner_instructions: meta + .inner_instructions + .as_ref() + .map(|groups| { + groups + .iter() + .map(|group| SourceInnerGroup { + index: u32::from(group.index), + instructions: group + .instructions + .iter() + .map(|instruction| SourceInnerInstruction { + program_id_index: u32::from(instruction.instruction.program_id_index), + accounts: instruction.instruction.accounts.clone(), + data: instruction.instruction.data.clone(), + stack_height: instruction.stack_height, + }) + .collect(), + }) + .collect() + }), + logs: meta.log_messages.clone(), + pre_token_balances: map_token_balances(slot, index, meta.pre_token_balances.as_deref())?, + post_token_balances: map_token_balances(slot, index, meta.post_token_balances.as_deref())?, + rewards: meta + .rewards + .as_deref() + .unwrap_or(&[]) + .iter() + .map(|reward| map_reward(slot, reward)) + .collect::>>()?, + loaded_writable: meta + .loaded_addresses + .writable + .iter() + .map(|key| copy32(key.as_ref())) + .collect::>>()?, + loaded_readonly: meta + .loaded_addresses + .readonly + .iter() + .map(|key| copy32(key.as_ref())) + .collect::>>()?, + return_data: meta + .return_data + .as_ref() + .map(|data| { + Ok(SourceReturnData { + program_id: copy32(data.program_id.as_ref())?, + data: data.data.clone(), + }) + }) + .transpose()?, + compute_units_consumed: meta.compute_units_consumed, + cost_units: meta.cost_units, + }) +} + +fn map_token_balances( + slot: u64, + index: usize, + balances: Option<&[solana_transaction_status::TransactionTokenBalance]>, +) -> Result> { + let Some(balances) = balances else { + return Ok(Vec::new()); + }; + balances + .iter() + .map(|balance| { + let amount = balance + .ui_token_amount + .amount + .parse::() + .with_context(|| { + format!( + "slot {slot} transaction {index} token amount {}", + balance.ui_token_amount.amount + ) + })?; + Ok(SourceTokenBalance { + account_index: u32::from(balance.account_index), + mint: decode_key(&balance.mint, "token mint")?, + owner: decode_optional_key(&balance.owner, "token owner")?, + program_id: decode_optional_key(&balance.program_id, "token program")?, + amount, + decimals: balance.ui_token_amount.decimals, + }) + }) + .collect() +} + +fn map_reward(slot: u64, reward: &Reward) -> Result { + if reward.commission.is_none() && reward.commission_bps.is_some() { + bail!("slot {slot} reward stores commission_bps without a u8 commission"); + } + if let (Some(commission), Some(bps)) = (reward.commission, reward.commission_bps) { + if u16::from(commission).checked_mul(100) != Some(bps) { + bail!( + "slot {slot} reward commission {commission} does not match commission_bps {bps}" + ); + } + } + Ok(SourceReward { + pubkey: decode_key(&reward.pubkey, "reward pubkey")?, + lamports: reward.lamports, + post_balance: reward.post_balance, + reward_type: reward_type_code(reward.reward_type), + commission: reward.commission, + }) +} + +pub(crate) fn reward_type_code(kind: Option) -> i32 { + match kind { + None => 0, + Some(RewardType::Fee) => 1, + Some(RewardType::Rent) => 2, + Some(RewardType::Staking) => 3, + Some(RewardType::Voting) => 4, + Some(RewardType::DeactivatedStake) => 5, + Some(RewardType::VATDebit) => 6, + } +} + +pub(crate) fn map_transaction_error(err: &TransactionError) -> Result { + Ok(match err { + TransactionError::AccountInUse => CompactTransactionError::AccountInUse, + TransactionError::AccountLoadedTwice => CompactTransactionError::AccountLoadedTwice, + TransactionError::AccountNotFound => CompactTransactionError::AccountNotFound, + TransactionError::ProgramAccountNotFound => CompactTransactionError::ProgramAccountNotFound, + TransactionError::InsufficientFundsForFee => CompactTransactionError::InsufficientFundsForFee, + TransactionError::InvalidAccountForFee => CompactTransactionError::InvalidAccountForFee, + TransactionError::AlreadyProcessed => CompactTransactionError::AlreadyProcessed, + TransactionError::BlockhashNotFound => CompactTransactionError::BlockhashNotFound, + TransactionError::InstructionError(index, err) => CompactTransactionError::InstructionError( + *index, + map_instruction_error(err)?, + ), + TransactionError::CallChainTooDeep => CompactTransactionError::CallChainTooDeep, + TransactionError::MissingSignatureForFee => CompactTransactionError::MissingSignatureForFee, + TransactionError::InvalidAccountIndex => CompactTransactionError::InvalidAccountIndex, + TransactionError::SignatureFailure => CompactTransactionError::SignatureFailure, + TransactionError::InvalidProgramForExecution => { + CompactTransactionError::InvalidProgramForExecution + } + TransactionError::SanitizeFailure => CompactTransactionError::SanitizeFailure, + TransactionError::ClusterMaintenance => CompactTransactionError::ClusterMaintenance, + TransactionError::AccountBorrowOutstanding => { + CompactTransactionError::AccountBorrowOutstanding + } + TransactionError::WouldExceedMaxBlockCostLimit => { + CompactTransactionError::WouldExceedMaxBlockCostLimit + } + TransactionError::UnsupportedVersion => CompactTransactionError::UnsupportedVersion, + TransactionError::InvalidWritableAccount => CompactTransactionError::InvalidWritableAccount, + TransactionError::WouldExceedMaxAccountCostLimit => { + CompactTransactionError::WouldExceedMaxAccountCostLimit + } + TransactionError::WouldExceedAccountDataBlockLimit => { + CompactTransactionError::WouldExceedAccountDataBlockLimit + } + TransactionError::TooManyAccountLocks => CompactTransactionError::TooManyAccountLocks, + TransactionError::AddressLookupTableNotFound => { + CompactTransactionError::AddressLookupTableNotFound + } + TransactionError::InvalidAddressLookupTableOwner => { + CompactTransactionError::InvalidAddressLookupTableOwner + } + TransactionError::InvalidAddressLookupTableData => { + CompactTransactionError::InvalidAddressLookupTableData + } + TransactionError::InvalidAddressLookupTableIndex => { + CompactTransactionError::InvalidAddressLookupTableIndex + } + TransactionError::InvalidRentPayingAccount => { + CompactTransactionError::InvalidRentPayingAccount + } + TransactionError::WouldExceedMaxVoteCostLimit => { + CompactTransactionError::WouldExceedMaxVoteCostLimit + } + TransactionError::WouldExceedAccountDataTotalLimit => { + CompactTransactionError::WouldExceedAccountDataTotalLimit + } + TransactionError::DuplicateInstruction(index) => { + CompactTransactionError::DuplicateInstruction(*index) + } + TransactionError::InsufficientFundsForRent { account_index } => { + CompactTransactionError::InsufficientFundsForRent { + account_index: *account_index, + } + } + TransactionError::MaxLoadedAccountsDataSizeExceeded => { + CompactTransactionError::MaxLoadedAccountsDataSizeExceeded + } + TransactionError::InvalidLoadedAccountsDataSizeLimit => { + CompactTransactionError::InvalidLoadedAccountsDataSizeLimit + } + TransactionError::ResanitizationNeeded => CompactTransactionError::ResanitizationNeeded, + TransactionError::ProgramExecutionTemporarilyRestricted { account_index } => { + CompactTransactionError::ProgramExecutionTemporarilyRestricted { + account_index: *account_index, + } + } + TransactionError::UnbalancedTransaction => CompactTransactionError::UnbalancedTransaction, + TransactionError::ProgramCacheHitMaxLimit => CompactTransactionError::ProgramCacheHitMaxLimit, + TransactionError::CommitCancelled => CompactTransactionError::CommitCancelled, + TransactionError::BailOut => { + bail!("transaction error BailOut has no compact encoding"); + } + other => bail!("unsupported transaction error {other:?}"), + }) +} + +#[allow(deprecated)] +pub(crate) fn map_instruction_error(err: &InstructionError) -> Result { + Ok(match err { + InstructionError::GenericError => CompactInstructionError::GenericError, + InstructionError::InvalidArgument => CompactInstructionError::InvalidArgument, + InstructionError::InvalidInstructionData => CompactInstructionError::InvalidInstructionData, + InstructionError::InvalidAccountData => CompactInstructionError::InvalidAccountData, + InstructionError::AccountDataTooSmall => CompactInstructionError::AccountDataTooSmall, + InstructionError::InsufficientFunds => CompactInstructionError::InsufficientFunds, + InstructionError::IncorrectProgramId => CompactInstructionError::IncorrectProgramId, + InstructionError::MissingRequiredSignature => { + CompactInstructionError::MissingRequiredSignature + } + InstructionError::AccountAlreadyInitialized => { + CompactInstructionError::AccountAlreadyInitialized + } + InstructionError::UninitializedAccount => CompactInstructionError::UninitializedAccount, + InstructionError::UnbalancedInstruction => CompactInstructionError::UnbalancedInstruction, + InstructionError::ModifiedProgramId => CompactInstructionError::ModifiedProgramId, + InstructionError::ExternalAccountLamportSpend => { + CompactInstructionError::ExternalAccountLamportSpend + } + InstructionError::ExternalAccountDataModified => { + CompactInstructionError::ExternalAccountDataModified + } + InstructionError::ReadonlyLamportChange => CompactInstructionError::ReadonlyLamportChange, + InstructionError::ReadonlyDataModified => CompactInstructionError::ReadonlyDataModified, + InstructionError::DuplicateAccountIndex => CompactInstructionError::DuplicateAccountIndex, + InstructionError::ExecutableModified => CompactInstructionError::ExecutableModified, + InstructionError::RentEpochModified => CompactInstructionError::RentEpochModified, + InstructionError::NotEnoughAccountKeys => CompactInstructionError::NotEnoughAccountKeys, + InstructionError::AccountDataSizeChanged => CompactInstructionError::AccountDataSizeChanged, + InstructionError::AccountNotExecutable => CompactInstructionError::AccountNotExecutable, + InstructionError::AccountBorrowFailed => CompactInstructionError::AccountBorrowFailed, + InstructionError::AccountBorrowOutstanding => { + CompactInstructionError::AccountBorrowOutstanding + } + InstructionError::DuplicateAccountOutOfSync => { + CompactInstructionError::DuplicateAccountOutOfSync + } + InstructionError::Custom(code) => CompactInstructionError::Custom(*code), + InstructionError::InvalidError => CompactInstructionError::InvalidError, + InstructionError::ExecutableDataModified => CompactInstructionError::ExecutableDataModified, + InstructionError::ExecutableLamportChange => { + CompactInstructionError::ExecutableLamportChange + } + InstructionError::ExecutableAccountNotRentExempt => { + CompactInstructionError::ExecutableAccountNotRentExempt + } + InstructionError::UnsupportedProgramId => CompactInstructionError::UnsupportedProgramId, + InstructionError::CallDepth => CompactInstructionError::CallDepth, + InstructionError::MissingAccount => CompactInstructionError::MissingAccount, + InstructionError::ReentrancyNotAllowed => CompactInstructionError::ReentrancyNotAllowed, + InstructionError::MaxSeedLengthExceeded => CompactInstructionError::MaxSeedLengthExceeded, + InstructionError::InvalidSeeds => CompactInstructionError::InvalidSeeds, + InstructionError::InvalidRealloc => CompactInstructionError::InvalidRealloc, + InstructionError::ComputationalBudgetExceeded => { + CompactInstructionError::ComputationalBudgetExceeded + } + InstructionError::PrivilegeEscalation => CompactInstructionError::PrivilegeEscalation, + InstructionError::ProgramEnvironmentSetupFailure => { + CompactInstructionError::ProgramEnvironmentSetupFailure + } + InstructionError::ProgramFailedToComplete => CompactInstructionError::ProgramFailedToComplete, + InstructionError::ProgramFailedToCompile => CompactInstructionError::ProgramFailedToCompile, + InstructionError::Immutable => CompactInstructionError::Immutable, + InstructionError::IncorrectAuthority => CompactInstructionError::IncorrectAuthority, + InstructionError::BorshIoError => CompactInstructionError::BorshIoError(String::new()), + InstructionError::AccountNotRentExempt => CompactInstructionError::AccountNotRentExempt, + InstructionError::InvalidAccountOwner => CompactInstructionError::InvalidAccountOwner, + InstructionError::ArithmeticOverflow => CompactInstructionError::ArithmeticOverflow, + InstructionError::UnsupportedSysvar => CompactInstructionError::UnsupportedSysvar, + InstructionError::IllegalOwner => CompactInstructionError::IllegalOwner, + InstructionError::MaxAccountsDataAllocationsExceeded => { + CompactInstructionError::MaxAccountsDataAllocationsExceeded + } + InstructionError::MaxAccountsExceeded => CompactInstructionError::MaxAccountsExceeded, + InstructionError::MaxInstructionTraceLengthExceeded => { + CompactInstructionError::MaxInstructionTraceLengthExceeded + } + InstructionError::BuiltinProgramsMustConsumeComputeUnits => { + CompactInstructionError::BuiltinProgramsMustConsumeComputeUnits + } + InstructionError::BailOut => { + bail!("instruction error BailOut has no compact encoding"); + } + other => bail!("unsupported instruction error {other:?}"), + }) +} + +fn decode_hash(text: &str, what: &str) -> Result<[u8; 32]> { + decode_key(text, what) +} + +fn decode_key(text: &str, what: &str) -> Result<[u8; 32]> { + if text.is_empty() { + bail!("{what} is empty"); + } + let bytes = bs58::decode(text) + .into_vec() + .with_context(|| format!("{what} is not base58"))?; + <[u8; 32]>::try_from(bytes).map_err(|_| anyhow!("{what} is not 32 bytes")) +} + +fn decode_optional_key(text: &str, what: &str) -> Result> { + if text.is_empty() { + Ok(None) + } else { + decode_key(text, what).map(Some) + } +} + +fn copy32(bytes: &[u8]) -> Result<[u8; 32]> { + <[u8; 32]>::try_from(bytes).map_err(|_| anyhow!("key is not 32 bytes")) +} + +fn copy64(bytes: &[u8]) -> Result<[u8; 64]> { + <[u8; 64]>::try_from(bytes).map_err(|_| anyhow!("signature is not 64 bytes")) +} + +#[cfg(test)] +mod tests { + use super::*; + use solana_entry::block_component::{BlockHeaderV1, VersionedBlockMarker}; + use solana_hash::Hash; + + #[test] + fn read_only_options_do_not_create_a_ledger() { + let dir = tempfile::tempdir().unwrap(); + let error = read_only_options(dir.path()).unwrap_err(); + assert!(error.to_string().contains("does not exist"), "{error:#}"); + assert!(!dir.path().join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL).exists()); + } + + #[test] + fn read_only_options_require_readonly_access() { + let dir = tempfile::tempdir().unwrap(); + let rocks = dir.path().join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL); + std::fs::create_dir(&rocks).unwrap(); + let options = read_only_options(dir.path()).unwrap(); + assert_eq!(options.access_type, AccessType::ReadOnly); + assert!(reject_non_readonly(&AccessType::Primary).is_err()); + assert!(reject_non_readonly(&AccessType::PrimaryForMaintenance).is_err()); + assert!(reject_non_readonly(&AccessType::ReadOnly).is_ok()); + } + + #[test] + fn agave_header_marker_roundtrips_the_parent_block_id() { + let parent = [0x3c; 32]; + let marker = VersionedBlockMarker::from_block_header(BlockHeaderV1 { + parent_slot: 40, + parent_block_id: Hash::new_from_array(parent), + }); + let bytes = wincode::serialize(&marker).unwrap(); + assert_eq!( + of_car_reader::blockmarker::parent_consensus_block_id(&[bytes]), + Some(parent) + ); + } + + #[test] + fn known_transaction_error_maps_and_bailout_fails() { + assert_eq!( + map_transaction_error(&TransactionError::AccountInUse).unwrap(), + CompactTransactionError::AccountInUse + ); + assert!(map_transaction_error(&TransactionError::BailOut).is_err()); + assert!(map_instruction_error(&InstructionError::BailOut).is_err()); + assert_eq!( + map_instruction_error(&InstructionError::BorshIoError).unwrap(), + CompactInstructionError::BorshIoError(String::new()) + ); + } + + #[test] + fn reward_type_codes_keep_the_archive_numbers() { + assert_eq!(reward_type_code(Some(RewardType::Fee)), 1); + assert_eq!(reward_type_code(Some(RewardType::Rent)), 2); + assert_eq!(reward_type_code(Some(RewardType::Staking)), 3); + assert_eq!(reward_type_code(Some(RewardType::Voting)), 4); + assert_eq!(reward_type_code(Some(RewardType::DeactivatedStake)), 5); + assert_eq!(reward_type_code(Some(RewardType::VATDebit)), 6); + } +} diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs new file mode 100644 index 0000000..3d022c5 --- /dev/null +++ b/hivezilla/ledger-archive/src/main.rs @@ -0,0 +1,102 @@ +mod check; +mod convert; +mod ledger; +mod markers; +mod source; + +use std::path::PathBuf; + +use anyhow::{bail, Result}; +use clap::{Parser, Subcommand}; + +use crate::check::inspect; +use crate::convert::write_archive; +use crate::ledger::load_range; + +#[derive(Parser)] +#[command(name = "hivezilla-ledger-archive")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Check one inclusive slot range in a snapshot or stopped ledger. + CheckLedgerRange { + ledger: PathBuf, + #[arg(long)] + start_slot: u64, + #[arg(long)] + end_slot: u64, + }, + /// Write a Compact V2 hot archive for one inclusive slot range. + BuildArchiveV2FromLedger { + ledger: PathBuf, + output: PathBuf, + #[arg(long)] + start_slot: u64, + #[arg(long)] + end_slot: u64, + }, +} + +fn main() { + if let Err(err) = run() { + eprintln!("error: {err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + match Cli::parse().command { + Command::CheckLedgerRange { + ledger, + start_slot, + end_slot, + } => { + let inspected = inspect_ledger(&ledger, start_slot, end_slot)?; + print_report(&inspected); + Ok(()) + } + Command::BuildArchiveV2FromLedger { + ledger, + output, + start_slot, + end_slot, + } => { + let (view, inspected) = load_and_inspect(&ledger, start_slot, end_slot)?; + write_archive(&output, &view, &inspected)?; + print_report(&inspected); + println!("Sortie : {}", output.display()); + Ok(()) + } + } +} + +fn inspect_ledger(ledger: &std::path::Path, start_slot: u64, end_slot: u64) -> Result { + Ok(load_and_inspect(ledger, start_slot, end_slot)?.1) +} + +fn load_and_inspect( + ledger: &std::path::Path, + start_slot: u64, + end_slot: u64, +) -> Result<(source::LedgerView, check::Inspected)> { + if start_slot > end_slot { + bail!("start slot {start_slot} is after end slot {end_slot}"); + } + let view = load_range(ledger, start_slot, end_slot)?; + let inspected = inspect(&view)?; + Ok((view, inspected)) +} + +fn print_report(inspected: &check::Inspected) { + let first = inspected.block_slots.first().copied().unwrap_or(0); + let last = inspected.block_slots.last().copied().unwrap_or(0); + println!("Blocs : {}", inspected.block_slots.len()); + println!("Sauts prouvés : {}", inspected.skips.len()); + println!("Premier bloc : {first}"); + println!("Dernier bloc : {last}"); + println!("Hash de frontière : {}", bs58::encode(inspected.boundary_hash).into_string()); +} diff --git a/hivezilla/ledger-archive/src/markers.rs b/hivezilla/ledger-archive/src/markers.rs new file mode 100644 index 0000000..01a028a --- /dev/null +++ b/hivezilla/ledger-archive/src/markers.rs @@ -0,0 +1,43 @@ +use anyhow::{Context, Result}; +use of_car_reader::blockmarker::{self, Marker}; + +#[derive(Debug, Default)] +pub(crate) struct RewardNames { + pub skips: Vec, + pub notars: Vec, +} + +/// Skip and notarization reward slots named by the footers of one carrier slot. +pub(crate) fn reward_names(markers: &[Vec]) -> Result { + let mut names = RewardNames::default(); + for marker in markers { + let parsed = blockmarker::parse(marker).context("block marker does not parse")?; + let Marker::Footer(footer) = parsed else { + continue; + }; + let certificates = blockmarker::decode_footer_certificates(&footer.certificates) + .context("footer certificates do not parse")?; + if let Some(skip) = certificates.skip_reward { + names.skips.push(skip.slot); + } + if let Some(notar) = certificates.notar_reward { + names.notars.push(notar.slot); + } + } + Ok(names) +} + +/// Bank hash from the last footer. A block with no footer returns `None`. +pub(crate) fn footer_bank_hash(markers: &[Vec]) -> Result> { + let mut bank_hash = None; + for marker in markers { + let parsed = blockmarker::parse(marker).context("block marker does not parse")?; + let Marker::Footer(footer) = parsed else { + continue; + }; + blockmarker::decode_footer_certificates(&footer.certificates) + .context("footer certificates do not parse")?; + bank_hash = Some(footer.bank_hash); + } + Ok(bank_hash) +} diff --git a/hivezilla/ledger-archive/src/source.rs b/hivezilla/ledger-archive/src/source.rs new file mode 100644 index 0000000..0e7227a --- /dev/null +++ b/hivezilla/ledger-archive/src/source.rs @@ -0,0 +1,304 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, Result}; +use blockzilla_compact::CompactTransactionError; + +/// Agave `NUM_SLOTS_FOR_REWARD`. A skip or notarization reward for slot N is +/// carried by the footer of slot N+8. +pub(crate) const REWARD_SLOT_DISTANCE: u64 = 8; + +/// One inclusive ledger window and every slot the checker may read. +/// +/// A missing key means the slot has no `SlotMeta`. Slots past `end_slot` are +/// certificate carriers for holes near the end of the window. +#[derive(Debug, Clone)] +pub(crate) struct LedgerView { + pub start_slot: u64, + pub end_slot: u64, + pub slots: BTreeMap, +} + +#[derive(Debug, Clone)] +pub(crate) struct SlotData { + pub slot: u64, + pub is_root: bool, + pub is_full: bool, + pub parent_slot: Option, + pub previous_blockhash: Option<[u8; 32]>, + pub blockhash: [u8; 32], + pub block_time: Option, + pub block_height: Option, + pub num_partitions: Option, + pub rewards: Vec, + pub consensus_block_id: Option<[u8; 32]>, + pub markers: Vec>, + pub entries: Vec, + pub shredding: Vec, + pub transactions: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceEntry { + pub num_hashes: u64, + pub hash: [u8; 32], + pub tx_count: u32, + pub signature_count: u32, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct SourceShredGroup { + pub entry_end_idx: i64, + pub shred_end_idx: i64, +} + +/// One completed shred range before the cumulative entry index is applied. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RawComponent { + pub entry_count: u64, + /// Exclusive end of the Agave shred range. + pub shred_range_end: u32, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceTx { + pub signatures: Vec<[u8; 64]>, + pub message: SourceMessage, + pub status: Option, +} + +#[derive(Debug, Clone)] +pub(crate) enum SourceMessage { + Legacy(SourceMessageBody), + V0(SourceMessageBody), + V1(SourceV1Message), +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceMessageBody { + pub header: SourceHeader, + pub account_keys: Vec<[u8; 32]>, + pub recent_blockhash: [u8; 32], + pub instructions: Vec, + pub lookups: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceV1Message { + pub header: SourceHeader, + pub config: SourceTxConfig, + pub account_keys: Vec<[u8; 32]>, + pub recent_blockhash: [u8; 32], + pub instructions: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct SourceHeader { + pub num_required_signatures: u8, + pub num_readonly_signed_accounts: u8, + pub num_readonly_unsigned_accounts: u8, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct SourceTxConfig { + pub priority_fee: Option, + pub compute_unit_limit: Option, + pub loaded_accounts_data_size_limit: Option, + pub heap_size: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceInstruction { + pub program_id_index: u8, + pub accounts: Vec, + pub data: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceLookup { + pub account_key: [u8; 32], + pub writable_indexes: Vec, + pub readonly_indexes: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceStatus { + pub err: Option, + pub fee: u64, + pub pre_balances: Vec, + pub post_balances: Vec, + pub inner_instructions: Option>, + pub logs: Option>, + pub pre_token_balances: Vec, + pub post_token_balances: Vec, + pub rewards: Vec, + pub loaded_writable: Vec<[u8; 32]>, + pub loaded_readonly: Vec<[u8; 32]>, + pub return_data: Option, + pub compute_units_consumed: Option, + pub cost_units: Option, +} + +impl SourceStatus { + pub(crate) fn success() -> Self { + Self { + err: None, + fee: 0, + pre_balances: Vec::new(), + post_balances: Vec::new(), + inner_instructions: None, + logs: None, + pre_token_balances: Vec::new(), + post_token_balances: Vec::new(), + rewards: Vec::new(), + loaded_writable: Vec::new(), + loaded_readonly: Vec::new(), + return_data: None, + compute_units_consumed: None, + cost_units: None, + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceInnerGroup { + pub index: u32, + pub instructions: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceInnerInstruction { + pub program_id_index: u32, + pub accounts: Vec, + pub data: Vec, + pub stack_height: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceTokenBalance { + pub account_index: u32, + pub mint: [u8; 32], + pub owner: Option<[u8; 32]>, + pub program_id: Option<[u8; 32]>, + pub amount: u64, + pub decimals: u8, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceReturnData { + pub program_id: [u8; 32], + pub data: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct SourceReward { + pub pubkey: [u8; 32], + pub lamports: i64, + pub post_balance: u64, + pub reward_type: i32, + pub commission: Option, +} + +/// `entry_end_idx` is the cumulative entry count after the component. +/// `shred_end_idx` is the last shred in the range, so it is `range.end - 1`. +pub(crate) fn shred_groups(components: &[RawComponent]) -> Result> { + let mut entry_end = 0u64; + let mut groups = Vec::with_capacity(components.len()); + for component in components { + entry_end = entry_end + .checked_add(component.entry_count) + .context("entry index overflow")?; + let shred_end = component + .shred_range_end + .checked_sub(1) + .context("shred range is empty")?; + groups.push(SourceShredGroup { + entry_end_idx: i64::try_from(entry_end).context("entry index exceeds i64")?, + shred_end_idx: i64::from(shred_end), + }); + } + Ok(groups) +} + +#[cfg(test)] +pub(crate) fn test_block( + slot: u64, + parent_slot: u64, + blockhash: [u8; 32], + previous_blockhash: [u8; 32], +) -> SlotData { + SlotData { + slot, + is_root: true, + is_full: true, + parent_slot: Some(parent_slot), + previous_blockhash: Some(previous_blockhash), + blockhash, + block_time: None, + block_height: None, + num_partitions: None, + rewards: Vec::new(), + consensus_block_id: None, + markers: Vec::new(), + entries: vec![SourceEntry { + num_hashes: 1, + hash: blockhash, + tx_count: 0, + signature_count: 0, + }], + shredding: vec![SourceShredGroup { + entry_end_idx: 1, + shred_end_idx: 31, + }], + transactions: Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn two_entry_batches_produce_two_shredding_rows() { + let groups = shred_groups(&[ + RawComponent { + entry_count: 4, + shred_range_end: 32, + }, + RawComponent { + entry_count: 7, + shred_range_end: 96, + }, + ]) + .unwrap(); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].entry_end_idx, 4); + assert_eq!(groups[0].shred_end_idx, 31); + assert_eq!(groups[1].entry_end_idx, 11); + assert_eq!(groups[1].shred_end_idx, 95); + } + + #[test] + fn a_marker_keeps_the_entry_cursor_and_still_adds_a_row() { + let groups = shred_groups(&[ + RawComponent { + entry_count: 4, + shred_range_end: 32, + }, + RawComponent { + entry_count: 0, + shred_range_end: 64, + }, + RawComponent { + entry_count: 7, + shred_range_end: 96, + }, + ]) + .unwrap(); + + assert_eq!(groups.len(), 3); + assert_eq!(groups[1].entry_end_idx, 4); + assert_eq!(groups[1].shred_end_idx, 63); + assert_eq!(groups[2].entry_end_idx, 11); + } +} From fdb220c266f8389a8e76637f4625955eb1f4b886 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Tue, 3 Nov 2026 16:53:19 +0100 Subject: [PATCH 09/38] use the Agave RocksDB fork so ledger-archive tests build --- hivezilla/ledger-archive/Cargo.lock | 16 ++++++---------- hivezilla/ledger-archive/Cargo.toml | 9 +++++++++ hivezilla/ledger-archive/src/convert.rs | 6 +++--- hivezilla/ledger-archive/src/ledger.rs | 12 ++++++------ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/hivezilla/ledger-archive/Cargo.lock b/hivezilla/ledger-archive/Cargo.lock index f7008b0..26f5b1b 100644 --- a/hivezilla/ledger-archive/Cargo.lock +++ b/hivezilla/ledger-archive/Cargo.lock @@ -984,8 +984,7 @@ dependencies = [ [[package]] name = "blstrs" version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8a8ed6fefbeef4a8c7b460e4110e12c5e22a5b7cf32621aae6ad650c4dcf29" +source = "git+https://github.com/anza-xyz/blstrs?rev=a1faffa3420bdaf4102425c43f00cdd815d4f145#a1faffa3420bdaf4102425c43f00cdd815d4f145" dependencies = [ "blst", "byte-slice-cast", @@ -1482,9 +1481,8 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +version = "0.9.20" +source = "git+https://github.com/anza-xyz/crossbeam?rev=5ee1c6ec671d94db14ed9238eb99e59a84555aad#5ee1c6ec671d94db14ed9238eb99e59a84555aad" dependencies = [ "crossbeam-utils", ] @@ -3071,8 +3069,7 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "librocksdb-sys" version = "0.17.3+10.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +source = "git+https://github.com/anza-xyz/rust-rocksdb?rev=4dba80dfe9434e595a971f5b2985ff9ba4fb31e7#4dba80dfe9434e595a971f5b2985ff9ba4fb31e7" dependencies = [ "bindgen", "bzip2-sys", @@ -4512,8 +4509,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +source = "git+https://github.com/anza-xyz/rust-rocksdb?rev=4dba80dfe9434e595a971f5b2985ff9ba4fb31e7#4dba80dfe9434e595a971f5b2985ff9ba4fb31e7" dependencies = [ "libc", "librocksdb-sys", @@ -8122,7 +8118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/hivezilla/ledger-archive/Cargo.toml b/hivezilla/ledger-archive/Cargo.toml index 2733001..d05c2e2 100644 --- a/hivezilla/ledger-archive/Cargo.toml +++ b/hivezilla/ledger-archive/Cargo.toml @@ -37,3 +37,12 @@ zstd = "0.13.3" tempfile = "3.27.0" [workspace] + +# Same crates.io replacements as Agave v4.4.0-rc.1. Cargo applies +# [patch] only from this workspace, not from the Agave git dependency. +[patch.crates-io] +blstrs = { git = "https://github.com/anza-xyz/blstrs", rev = "a1faffa3420bdaf4102425c43f00cdd815d4f145" } +# for details, see https://github.com/anza-xyz/crossbeam/commit/5ee1c6ec671d94db14ed9238eb99e59a84555aad +crossbeam-epoch = { git = "https://github.com/anza-xyz/crossbeam", rev = "5ee1c6ec671d94db14ed9238eb99e59a84555aad" } +librocksdb-sys = { git = "https://github.com/anza-xyz/rust-rocksdb", rev = "4dba80dfe9434e595a971f5b2985ff9ba4fb31e7" } +rocksdb = { git = "https://github.com/anza-xyz/rust-rocksdb", rev = "4dba80dfe9434e595a971f5b2985ff9ba4fb31e7" } diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index de6bef1..6fe515d 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -18,11 +18,11 @@ use blockzilla_archive_v2::{ }; use blockzilla_compact::{ CompactInnerInstruction, CompactInnerInstructions, CompactMessageHeader, CompactMetaV1, - CompactPohEntry, CompactPubkey, CompactReturnData, CompactReward, CompactShredding, + CompactPohEntry, CompactReturnData, CompactReward, CompactShredding, CompactTokenBalance, CompactTransactionConfig, OwnedCompactAddressTableLookup, OwnedCompactRecentBlockhash, }; -use blockzilla_primitives::{wincode_leb128_config, WincodeLeb128FramedWriter}; +use blockzilla_primitives::{wincode_leb128_config, CompactPubkey, WincodeLeb128FramedWriter}; use blockzilla_registry::RawPubkeyCompactor; use blockzilla_compact::parse_logs_with_compactor; @@ -205,7 +205,7 @@ pub(crate) fn encode_hot_block(block: &SlotData, block_id: u32) -> Result Result Result { Ok(SourceReturnData { program_id: copy32(data.program_id.as_ref())?, data: data.data.clone(), @@ -739,16 +739,16 @@ mod tests { #[test] fn known_transaction_error_maps_and_bailout_fails() { - assert_eq!( + assert!(matches!( map_transaction_error(&TransactionError::AccountInUse).unwrap(), CompactTransactionError::AccountInUse - ); + )); assert!(map_transaction_error(&TransactionError::BailOut).is_err()); assert!(map_instruction_error(&InstructionError::BailOut).is_err()); - assert_eq!( + assert!(matches!( map_instruction_error(&InstructionError::BorshIoError).unwrap(), - CompactInstructionError::BorshIoError(String::new()) - ); + CompactInstructionError::BorshIoError(ref message) if message.is_empty() + )); } #[test] From b264010e283c00312cb9225cd772982885e18295 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Tue, 3 Nov 2026 17:30:18 +0100 Subject: [PATCH 10/38] add english report --- hivezilla/ledger-archive/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index 3d022c5..be67acf 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -68,7 +68,7 @@ fn run() -> Result<()> { let (view, inspected) = load_and_inspect(&ledger, start_slot, end_slot)?; write_archive(&output, &view, &inspected)?; print_report(&inspected); - println!("Sortie : {}", output.display()); + println!("Output: {}", output.display()); Ok(()) } } @@ -94,9 +94,9 @@ fn load_and_inspect( fn print_report(inspected: &check::Inspected) { let first = inspected.block_slots.first().copied().unwrap_or(0); let last = inspected.block_slots.last().copied().unwrap_or(0); - println!("Blocs : {}", inspected.block_slots.len()); - println!("Sauts prouvés : {}", inspected.skips.len()); - println!("Premier bloc : {first}"); - println!("Dernier bloc : {last}"); - println!("Hash de frontière : {}", bs58::encode(inspected.boundary_hash).into_string()); + println!("Blocks: {}", inspected.block_slots.len()); + println!("Proven skips: {}", inspected.skips.len()); + println!("First block: {first}"); + println!("Last block: {last}"); + println!("Boundary hash: {}", bs58::encode(inspected.boundary_hash).into_string()); } From 681759aeec640e7ca3adb538c96c3a0f0b335525 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 4 Nov 2026 16:18:17 +0100 Subject: [PATCH 11/38] read and compress directly --- hivezilla/ledger-archive/src/check.rs | 49 ++- hivezilla/ledger-archive/src/convert.rs | 473 ++++++++++++++++-------- hivezilla/ledger-archive/src/ledger.rs | 298 +++++++++------ hivezilla/ledger-archive/src/main.rs | 35 +- 4 files changed, 550 insertions(+), 305 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index cb51262..d3859ee 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -5,9 +5,9 @@ use crate::source::{LedgerView, REWARD_SLOT_DISTANCE}; /// Blocks and proven skips for one inclusive slot window. /// -/// `check-ledger-range` and `build-archive-v2-from-ledger` both call this -/// function. A check that passes on a snapshot will not later stop the build -/// of that same snapshot for a hole, a missing status, or a missing parent. +/// The check command and the build command apply the same slot rules. A check +/// that passes on a snapshot will not later stop the build of that same +/// snapshot for a hole, a missing status, or a missing parent. #[derive(Debug, Clone)] pub(crate) struct Inspected { pub block_slots: Vec, @@ -15,6 +15,13 @@ pub(crate) struct Inspected { pub boundary_hash: [u8; 32], } +/// The previous produced block. The chain check needs its slot and blockhash. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PreviousBlock { + pub slot: u64, + pub blockhash: [u8; 32], +} + pub(crate) fn inspect(view: &LedgerView) -> Result { if view.start_slot > view.end_slot { bail!( @@ -26,12 +33,20 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { let mut block_slots = Vec::new(); let mut skips = Vec::new(); + let mut previous = None; for slot in view.start_slot..=view.end_slot { if view.slots.contains_key(&slot) { - accept_block(view, slot, &block_slots)?; + let data = &view.slots[&slot]; + accept_block_against(data, view.start_slot, view.end_slot, previous.as_ref())?; + previous = Some(PreviousBlock { + slot: data.slot, + blockhash: data.blockhash, + }); block_slots.push(slot); } else { - classify_hole(view, slot, &mut skips)?; + let carrier = slot.checked_add(REWARD_SLOT_DISTANCE); + let markers = carrier.and_then(|carrier| view.slots.get(&carrier).map(|data| data.markers.as_slice())); + classify_hole(slot, markers, &mut skips)?; } } @@ -53,8 +68,13 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { }) } -fn accept_block(view: &LedgerView, slot: u64, produced: &[u64]) -> Result<()> { - let data = &view.slots[&slot]; +pub(crate) fn accept_block_against( + data: &crate::source::SlotData, + start_slot: u64, + end_slot: u64, + previous: Option<&PreviousBlock>, +) -> Result<()> { + let slot = data.slot; if !data.is_root { bail!("slot {slot} has data and is not rooted"); } @@ -74,8 +94,7 @@ fn accept_block(view: &LedgerView, slot: u64, produced: &[u64]) -> Result<()> { bail!("slot {slot} has block markers and no consensus block id"); } - if let Some(previous_slot) = produced.last().copied() { - let previous = &view.slots[&previous_slot]; + if let Some(previous) = previous { if parent_slot != previous.slot { bail!( "slot {slot} parent {parent_slot} is not the previous block {}", @@ -88,7 +107,7 @@ fn accept_block(view: &LedgerView, slot: u64, produced: &[u64]) -> Result<()> { previous.slot ); } - } else if (view.start_slot..=view.end_slot).contains(&parent_slot) { + } else if (start_slot..=end_slot).contains(&parent_slot) { bail!("slot {slot} parent {parent_slot} is inside the range and is not a block"); } @@ -100,14 +119,18 @@ fn accept_block(view: &LedgerView, slot: u64, produced: &[u64]) -> Result<()> { Ok(()) } -fn classify_hole(view: &LedgerView, slot: u64, skips: &mut Vec) -> Result<()> { +pub(crate) fn classify_hole( + slot: u64, + carrier_markers: Option<&[Vec]>, + skips: &mut Vec, +) -> Result<()> { let carrier = slot .checked_add(REWARD_SLOT_DISTANCE) .context("slot + 8 overflows")?; - let Some(data) = view.slots.get(&carrier) else { + let Some(markers) = carrier_markers else { bail!("slot {slot} has no SlotMeta and slot {carrier} has no footer proof"); }; - let names = reward_names(&data.markers) + let names = reward_names(markers) .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; if names.notars.contains(&slot) { bail!("slot {slot} is missing and slot {carrier} has a notarization reward for it"); diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 6fe515d..c4aea93 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -1,202 +1,291 @@ use std::fs::{self, File}; use std::io::{BufWriter, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use blockzilla_archive_v2::{ - write_archive_v2_hot_block_index, ArchiveV2HotBlockBlob, ArchiveV2HotBlockHeader, - ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction, ArchiveV2HotInstructionData, - ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload, ArchiveV2HotRewards, - ArchiveV2HotTxRow, ArchiveV2HotV0Message, ArchiveV2HotV1Message, WincodeArchiveV2PohRecord, - WincodeArchiveV2ShreddingRecord, ARCHIVE_V2_BLOCKS_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, - ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_POH_FILE, ARCHIVE_V2_SHREDDING_FILE, - ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_TX_FLAG_HAS_ERROR, ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, + ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, ARCHIVE_V2_BLOCKS_FILE, + ARCHIVE_V2_POH_FILE, ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, + ARCHIVE_V2_TX_FLAG_HAS_ERROR, ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES, ARCHIVE_V2_TX_FLAG_HAS_LOGS, ARCHIVE_V2_TX_FLAG_HAS_METADATA, ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA, ARCHIVE_V2_TX_FLAG_HAS_TOKEN_BALANCES, ARCHIVE_V2_TX_FLAG_MESSAGE_V0, - ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, + ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, ArchiveV2HotBlockBlob, ArchiveV2HotBlockHeader, + ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction, ArchiveV2HotInstructionData, + ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload, ArchiveV2HotRewards, ArchiveV2HotTxRow, + ArchiveV2HotV0Message, ArchiveV2HotV1Message, WincodeArchiveV2PohRecord, + WincodeArchiveV2ShreddingRecord, write_archive_v2_hot_block_index, }; +use blockzilla_compact::parse_logs_with_compactor; use blockzilla_compact::{ CompactInnerInstruction, CompactInnerInstructions, CompactMessageHeader, CompactMetaV1, - CompactPohEntry, CompactReturnData, CompactReward, CompactShredding, - CompactTokenBalance, CompactTransactionConfig, OwnedCompactAddressTableLookup, - OwnedCompactRecentBlockhash, + CompactPohEntry, CompactReturnData, CompactReward, CompactShredding, CompactTokenBalance, + CompactTransactionConfig, OwnedCompactAddressTableLookup, OwnedCompactRecentBlockhash, }; -use blockzilla_primitives::{wincode_leb128_config, CompactPubkey, WincodeLeb128FramedWriter}; +use blockzilla_primitives::{CompactPubkey, WincodeLeb128FramedWriter, wincode_leb128_config}; use blockzilla_registry::RawPubkeyCompactor; -use blockzilla_compact::parse_logs_with_compactor; use crate::check::Inspected; +use crate::ledger::{open_read_only, scan_range}; use crate::markers::footer_bank_hash; use crate::source::{ - LedgerView, SlotData, SourceInnerGroup, SourceMessage, SourceMessageBody, SourceReward, - SourceStatus, SourceTokenBalance, SourceTx, SourceV1Message, + SlotData, SourceInnerGroup, SourceMessage, SourceMessageBody, SourceReward, SourceStatus, + SourceTokenBalance, SourceTx, SourceV1Message, }; pub(crate) const ZSTD_LEVEL: i32 = 3; const INDEX_FLAGS: u32 = 0; const VOTE_HASH_ROW_LEN: usize = 65; -pub(crate) fn write_archive(output: &Path, view: &LedgerView, inspected: &Inspected) -> Result<()> { - fs::create_dir_all(output) - .with_context(|| format!("create {}", output.display()))?; - - let blocks_tmp = output.join(format!("{ARCHIVE_V2_BLOCKS_FILE}.tmp")); - let signatures_tmp = output.join(format!("{ARCHIVE_V2_SIGNATURES_FILE}.tmp")); - let blockhash_tmp = output.join(format!("{ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE}.tmp")); - let vote_tmp = output.join(format!("{ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE}.tmp")); - let poh_tmp = output.join(format!("{ARCHIVE_V2_POH_FILE}.tmp")); - let shredding_tmp = output.join(format!("{ARCHIVE_V2_SHREDDING_FILE}.tmp")); - let temps = [ - &blocks_tmp, - &signatures_tmp, - &blockhash_tmp, - &vote_tmp, - &poh_tmp, - &shredding_tmp, - ]; - - let write_result = write_archive_files( - output, - view, - inspected, - &blocks_tmp, - &signatures_tmp, - &blockhash_tmp, - &vote_tmp, - &poh_tmp, - &shredding_tmp, - ); - if write_result.is_err() { - for path in temps { - let _ = fs::remove_file(path); +pub(crate) fn build_archive( + ledger: &Path, + output: &Path, + start_slot: u64, + end_slot: u64, +) -> Result { + let blockstore = open_read_only(ledger)?; + fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; + let temps = temp_paths(output); + let result = (|| { + let mut writer = ArchiveWriter::create(&temps)?; + let inspected = scan_range(&blockstore, start_slot, end_slot, |block| { + writer.write_block(block) + })?; + writer.finish(output)?; + Ok(inspected) + })(); + if result.is_err() { + remove_temps(&temps); + } + result +} + +pub(crate) fn write_archive(output: &Path, inspected: &Inspected, mut load_block: F) -> Result<()> +where + F: FnMut(u64) -> Result, +{ + fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; + let temps = temp_paths(output); + let result = (|| { + let mut writer = ArchiveWriter::create(&temps)?; + for slot in &inspected.block_slots { + let block = load_block(*slot)?; + writer.write_block(&block)?; } + writer.finish(output) + })(); + if result.is_err() { + remove_temps(&temps); } - write_result + result } -fn write_archive_files( - output: &Path, - view: &LedgerView, - inspected: &Inspected, - blocks_tmp: &Path, - signatures_tmp: &Path, - blockhash_tmp: &Path, - vote_tmp: &Path, - poh_tmp: &Path, - shredding_tmp: &Path, -) -> Result<()> { - let mut blocks = BufWriter::new( - File::create(blocks_tmp).with_context(|| format!("create {}", blocks_tmp.display()))?, - ); - let mut signatures = Vec::new(); - let mut blockhashes = Vec::with_capacity(32 * (inspected.block_slots.len() + 1)); - blockhashes.extend(inspected.boundary_hash); - let mut vote_hashes = Vec::with_capacity(VOTE_HASH_ROW_LEN * inspected.block_slots.len()); - let mut poh = WincodeLeb128FramedWriter::new(BufWriter::new( - File::create(poh_tmp).with_context(|| format!("create {}", poh_tmp.display()))?, - )); - let mut shredding = WincodeLeb128FramedWriter::new(BufWriter::new( - File::create(shredding_tmp) - .with_context(|| format!("create {}", shredding_tmp.display()))?, - )); - - let mut rows = Vec::with_capacity(inspected.block_slots.len()); - let mut compressed_offset = 0u64; - let mut next_tx_ordinal = 0u64; - let mut next_signature_ordinal = 0u64; - - for (ordinal, slot) in inspected.block_slots.iter().copied().enumerate() { - let block = &view.slots[&slot]; - let block_id = u32::try_from(ordinal).context("archive block id exceeds u32")?; +struct ArchivePaths { + blocks: PathBuf, + signatures: PathBuf, + blockhash: PathBuf, + vote: PathBuf, + poh: PathBuf, + shredding: PathBuf, +} + +fn temp_paths(output: &Path) -> ArchivePaths { + ArchivePaths { + blocks: output.join(format!("{ARCHIVE_V2_BLOCKS_FILE}.tmp")), + signatures: output.join(format!("{ARCHIVE_V2_SIGNATURES_FILE}.tmp")), + blockhash: output.join(format!("{ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE}.tmp")), + vote: output.join(format!("{ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE}.tmp")), + poh: output.join(format!("{ARCHIVE_V2_POH_FILE}.tmp")), + shredding: output.join(format!("{ARCHIVE_V2_SHREDDING_FILE}.tmp")), + } +} + +fn remove_temps(temps: &ArchivePaths) { + for path in [ + &temps.blocks, + &temps.signatures, + &temps.blockhash, + &temps.vote, + &temps.poh, + &temps.shredding, + ] { + let _ = fs::remove_file(path); + } +} + +struct ArchiveWriter { + blocks: BufWriter, + signatures: BufWriter, + poh: WincodeLeb128FramedWriter>, + shredding: WincodeLeb128FramedWriter>, + blockhashes: Vec, + vote_hashes: Vec, + rows: Vec, + compressed_offset: u64, + next_tx_ordinal: u64, + next_signature_ordinal: u64, + paths: ArchivePaths, +} + +impl ArchiveWriter { + fn create(temps: &ArchivePaths) -> Result { + Ok(Self { + blocks: BufWriter::new( + File::create(&temps.blocks) + .with_context(|| format!("create {}", temps.blocks.display()))?, + ), + signatures: BufWriter::new( + File::create(&temps.signatures) + .with_context(|| format!("create {}", temps.signatures.display()))?, + ), + poh: WincodeLeb128FramedWriter::new(BufWriter::new( + File::create(&temps.poh) + .with_context(|| format!("create {}", temps.poh.display()))?, + )), + shredding: WincodeLeb128FramedWriter::new(BufWriter::new( + File::create(&temps.shredding) + .with_context(|| format!("create {}", temps.shredding.display()))?, + )), + blockhashes: Vec::new(), + vote_hashes: Vec::new(), + rows: Vec::new(), + compressed_offset: 0, + next_tx_ordinal: 0, + next_signature_ordinal: 0, + paths: ArchivePaths { + blocks: temps.blocks.clone(), + signatures: temps.signatures.clone(), + blockhash: temps.blockhash.clone(), + vote: temps.vote.clone(), + poh: temps.poh.clone(), + shredding: temps.shredding.clone(), + }, + }) + } + + fn write_block(&mut self, block: &SlotData) -> Result<()> { + let slot = block.slot; + let block_id = u32::try_from(self.rows.len()).context("archive block id exceeds u32")?; + if self.rows.is_empty() { + let boundary = block + .previous_blockhash + .context("first block is missing parent entries")?; + self.blockhashes.extend(boundary); + } let uncompressed = encode_hot_block(block, block_id)?; let compressed = zstd::bulk::compress(&uncompressed, ZSTD_LEVEL) .with_context(|| format!("compress slot {slot}"))?; - let compressed_len = u32::try_from(compressed.len()).context("compressed block exceeds u32")?; + let compressed_len = + u32::try_from(compressed.len()).context("compressed block exceeds u32")?; let uncompressed_len = u32::try_from(uncompressed.len()).context("block bytes exceed u32")?; - blocks + self.blocks .write_all(&compressed) .with_context(|| format!("write slot {slot}"))?; - - let signature_count = append_signatures(&mut signatures, block)?; - blockhashes.extend(block.blockhash); - vote_hashes.extend(vote_hash_row(block)?); - poh.write(&poh_record(block, block_id)) + let signature_count = append_signatures(&mut self.signatures, block)?; + self.blockhashes.extend(block.blockhash); + self.vote_hashes.extend(vote_hash_row(block)?); + self.poh + .write(&poh_record(block, block_id)) .with_context(|| format!("write PoH for slot {slot}"))?; - shredding + self.shredding .write(&shredding_record(block, block_id)) .with_context(|| format!("write shredding for slot {slot}"))?; - - let tx_count = u32::try_from(block.transactions.len()).context("transaction count exceeds u32")?; - rows.push(ArchiveV2HotBlockIndexRow { + let tx_count = + u32::try_from(block.transactions.len()).context("transaction count exceeds u32")?; + self.rows.push(ArchiveV2HotBlockIndexRow { block_id, slot, - compressed_offset, + compressed_offset: self.compressed_offset, compressed_len, uncompressed_len, tx_count, - first_tx_ordinal: next_tx_ordinal, - first_signature_ordinal: next_signature_ordinal, + first_tx_ordinal: self.next_tx_ordinal, + first_signature_ordinal: self.next_signature_ordinal, signature_count, }); - compressed_offset = compressed_offset + self.compressed_offset = self + .compressed_offset .checked_add(u64::from(compressed_len)) .context("compressed archive exceeds u64")?; - next_tx_ordinal = next_tx_ordinal + self.next_tx_ordinal = self + .next_tx_ordinal .checked_add(u64::from(tx_count)) .context("transaction ordinal exceeds u64")?; - next_signature_ordinal = next_signature_ordinal + self.next_signature_ordinal = self + .next_signature_ordinal .checked_add(u64::from(signature_count)) .context("signature ordinal exceeds u64")?; + let written = self.rows.len(); + if written % 1000 == 0 { + progress(format!("wrote {written} blocks, latest slot {slot}")); + } + Ok(()) + } + + fn finish(mut self, output: &Path) -> Result<()> { + progress(format!("writing finished: {} blocks", self.rows.len())); + self.blocks + .flush() + .with_context(|| format!("flush {}", self.paths.blocks.display()))?; + self.signatures + .flush() + .with_context(|| format!("flush {}", self.paths.signatures.display()))?; + self.poh.flush().context("flush PoH sidecar")?; + let poh_writer = self.poh.into_inner(); + poh_writer + .into_inner() + .map_err(|err| anyhow::anyhow!(err)) + .context("flush PoH sidecar")?; + self.shredding.flush().context("flush shredding sidecar")?; + let shredding_writer = self.shredding.into_inner(); + shredding_writer + .into_inner() + .map_err(|err| anyhow::anyhow!(err)) + .context("flush shredding sidecar")?; + fs::write(&self.paths.blockhash, &self.blockhashes) + .with_context(|| format!("write {}", self.paths.blockhash.display()))?; + fs::write(&self.paths.vote, &self.vote_hashes) + .with_context(|| format!("write {}", self.paths.vote.display()))?; + rename_finished(&self.paths.blocks, &output.join(ARCHIVE_V2_BLOCKS_FILE))?; + rename_finished( + &self.paths.signatures, + &output.join(ARCHIVE_V2_SIGNATURES_FILE), + )?; + rename_finished( + &self.paths.blockhash, + &output.join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE), + )?; + rename_finished( + &self.paths.vote, + &output.join(ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE), + )?; + rename_finished(&self.paths.poh, &output.join(ARCHIVE_V2_POH_FILE))?; + rename_finished( + &self.paths.shredding, + &output.join(ARCHIVE_V2_SHREDDING_FILE), + )?; + write_archive_v2_hot_block_index( + &output.join(ARCHIVE_V2_BLOCK_INDEX_FILE), + self.compressed_offset, + ZSTD_LEVEL, + INDEX_FLAGS, + &self.rows, + )?; + Ok(()) } +} - blocks - .flush() - .with_context(|| format!("flush {}", blocks_tmp.display()))?; - drop(blocks); - poh.flush().context("flush PoH sidecar")?; - let poh_writer = poh.into_inner(); - poh_writer - .into_inner() - .map_err(|err| anyhow::anyhow!(err)) - .context("flush PoH sidecar")?; - shredding.flush().context("flush shredding sidecar")?; - let shredding_writer = shredding.into_inner(); - shredding_writer - .into_inner() - .map_err(|err| anyhow::anyhow!(err)) - .context("flush shredding sidecar")?; - - fs::write(signatures_tmp, &signatures) - .with_context(|| format!("write {}", signatures_tmp.display()))?; - fs::write(blockhash_tmp, &blockhashes) - .with_context(|| format!("write {}", blockhash_tmp.display()))?; - fs::write(vote_tmp, &vote_hashes) - .with_context(|| format!("write {}", vote_tmp.display()))?; - - rename_finished(blocks_tmp, &output.join(ARCHIVE_V2_BLOCKS_FILE))?; - rename_finished(signatures_tmp, &output.join(ARCHIVE_V2_SIGNATURES_FILE))?; - rename_finished( - blockhash_tmp, - &output.join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE), - )?; - rename_finished(vote_tmp, &output.join(ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE))?; - rename_finished(poh_tmp, &output.join(ARCHIVE_V2_POH_FILE))?; - rename_finished(shredding_tmp, &output.join(ARCHIVE_V2_SHREDDING_FILE))?; - write_archive_v2_hot_block_index( - &output.join(ARCHIVE_V2_BLOCK_INDEX_FILE), - compressed_offset, - ZSTD_LEVEL, - INDEX_FLAGS, - &rows, - )?; - Ok(()) +fn progress(message: impl std::fmt::Display) { + if cfg!(test) { + return; + } + eprintln!("{message}"); + let _ = std::io::stderr().flush(); } fn rename_finished(from: &Path, to: &Path) -> Result<()> { - fs::rename(from, to) - .with_context(|| format!("rename {} to {}", from.display(), to.display())) + fs::rename(from, to).with_context(|| format!("rename {} to {}", from.display(), to.display())) } pub(crate) fn encode_hot_block(block: &SlotData, block_id: u32) -> Result> { @@ -218,7 +307,10 @@ pub(crate) fn encode_hot_block(block: &SlotData, block_id: u32) -> Result Result Vec { keys.iter().copied().map(CompactPubkey::raw).collect() } -fn raw_instructions(instructions: &[crate::source::SourceInstruction]) -> Vec { +fn raw_instructions( + instructions: &[crate::source::SourceInstruction], +) -> Vec { instructions .iter() .map(|instruction| ArchiveV2HotInstruction { @@ -467,12 +563,15 @@ fn compact_reward(reward: &SourceReward) -> CompactReward { } } -fn append_signatures(out: &mut Vec, block: &SlotData) -> Result { +fn append_signatures(out: &mut impl Write, block: &SlotData) -> Result { let mut count = 0u32; for tx in &block.transactions { for signature in &tx.signatures { - out.extend(signature); - count = count.checked_add(1).context("signature count exceeds u32")?; + out.write_all(signature) + .with_context(|| format!("write a signature for slot {}", block.slot))?; + count = count + .checked_add(1) + .context("signature count exceeds u32")?; } } Ok(count) @@ -528,7 +627,7 @@ fn vote_hash_row(block: &SlotData) -> Result<[u8; VOTE_HASH_ROW_LEN]> { mod tests { use super::*; use crate::check::inspect; - use crate::source::{test_block, LedgerView, SourceShredGroup, REWARD_SLOT_DISTANCE}; + use crate::source::{LedgerView, REWARD_SLOT_DISTANCE, SourceShredGroup, test_block}; use blockzilla_archive_v2::deserialize_archive_v2_hot_block_blob; use blockzilla_primitives::WincodeLeb128FramedReader; use of_car_reader::blockmarker::parent_consensus_block_id; @@ -613,12 +712,24 @@ mod tests { }; let inspected = inspect(&view).unwrap(); let dir = tempfile::tempdir().unwrap(); - write_archive(dir.path(), &view, &inspected).unwrap(); + let mut loaded = Vec::new(); + write_archive(dir.path(), &inspected, |slot| { + loaded.push(slot); + view.slots + .get(&slot) + .cloned() + .with_context(|| format!("test block {slot} is missing")) + }) + .unwrap(); + assert_eq!(loaded, vec![10, 12]); let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); assert_eq!(&index[..8], b"BZV2HIX1"); assert_eq!(u64::from_le_bytes(index[12..20].try_into().unwrap()), 2); - assert_eq!(i32::from_le_bytes(index[28..32].try_into().unwrap()), ZSTD_LEVEL); + assert_eq!( + i32::from_le_bytes(index[28..32].try_into().unwrap()), + ZSTD_LEVEL + ); assert_eq!(u32::from_le_bytes(index[32..36].try_into().unwrap()), 0); let row_len = 52; let slots: Vec = index[36..] @@ -646,14 +757,54 @@ mod tests { .unwrap() .unwrap() .1; - assert!(reader - .read::() - .unwrap() - .is_none()); + assert!( + reader + .read::() + .unwrap() + .is_none() + ); assert_eq!(first_record.slot, 10); assert_eq!(second_record.slot, 12); } + #[test] + fn a_block_load_failure_removes_temp_files_and_skips_the_index() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let second = test_block(11, 10, [5; 32], [4; 32]); + let view = LedgerView { + start_slot: 10, + end_slot: 11, + slots: [first, second] + .into_iter() + .map(|slot| (slot.slot, slot)) + .collect(), + }; + let inspected = inspect(&view).unwrap(); + let dir = tempfile::tempdir().unwrap(); + + let error = write_archive(dir.path(), &inspected, |slot| { + if slot == 11 { + bail!("slot 11 is missing transaction status"); + } + view.slots + .get(&slot) + .cloned() + .with_context(|| format!("test block {slot} is missing")) + }) + .unwrap_err(); + + assert!( + error.to_string().contains("missing transaction status"), + "{error:#}" + ); + assert!(!dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE).exists()); + let names: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + assert!(names.is_empty(), "{names:?}"); + } + fn skip_footer(skipped_slot: u64) -> Vec { let mut certificates = vec![0, 1]; certificates.extend_from_slice(&skipped_slot.to_le_bytes()); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 1bd12a5..8032b27 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -1,25 +1,27 @@ +use std::io::{self, Write}; use std::path::Path; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{Context, Result, anyhow, bail}; use blockzilla_compact::{CompactInstructionError, CompactTransactionError}; use solana_entry::block_component::BlockComponent; use solana_instruction_error::InstructionError; -use solana_ledger::blockstore::error::BlockstoreError; use solana_ledger::blockstore::Blockstore; +use solana_ledger::blockstore::error::BlockstoreError; use solana_ledger::blockstore_meta::BlockLocation; use solana_ledger::blockstore_options::{ - AccessType, BlockstoreOptions, BLOCKSTORE_DIRECTORY_ROCKS_LEVEL, + AccessType, BLOCKSTORE_DIRECTORY_ROCKS_LEVEL, BlockstoreOptions, }; use solana_message::VersionedMessage; use solana_reward_info::RewardType; use solana_transaction_error::TransactionError; use solana_transaction_status::{Reward, TransactionStatusMeta}; +use crate::check::{accept_block_against, classify_hole, Inspected, PreviousBlock}; use crate::source::{ - LedgerView, RawComponent, SlotData, SourceEntry, SourceHeader, SourceInnerGroup, + REWARD_SLOT_DISTANCE, RawComponent, SlotData, SourceEntry, SourceHeader, SourceInnerGroup, SourceInnerInstruction, SourceInstruction, SourceLookup, SourceMessage, SourceMessageBody, SourceReturnData, SourceReward, SourceStatus, SourceTokenBalance, SourceTx, SourceTxConfig, - SourceV1Message, REWARD_SLOT_DISTANCE, + SourceV1Message, }; /// Open a ledger only in `AccessType::ReadOnly`. @@ -45,47 +47,158 @@ pub(crate) fn reject_non_readonly(access: &AccessType) -> Result<()> { } } -pub(crate) fn load_range(ledger: &Path, start_slot: u64, end_slot: u64) -> Result { - if start_slot > end_slot { - bail!("start slot {start_slot} is after end slot {end_slot}"); - } +pub(crate) fn open_read_only(ledger: &Path) -> Result { let options = read_only_options(ledger)?; - let blockstore = Blockstore::open_with_options(ledger, options) - .with_context(|| format!("open ledger {}", ledger.display()))?; - load_blockstore(&blockstore, start_slot, end_slot) + Blockstore::open_with_options(ledger, options) + .with_context(|| format!("open ledger {}", ledger.display())) } -fn load_blockstore(blockstore: &Blockstore, start_slot: u64, end_slot: u64) -> Result { - let last_carrier = end_slot - .checked_add(REWARD_SLOT_DISTANCE) - .context("end slot + 8 overflows")?; - let mut slots = std::collections::BTreeMap::new(); - for slot in start_slot..=last_carrier { - let Some(meta) = blockstore +/// Read each slot once. A rooted full block is checked, passed to `on_block`, +/// then dropped. A hole is proved from the footer of slot N+8. +/// +/// When that footer belongs to a later block in the window, the block is kept +/// until the scan reaches it, so its shreds are not read a second time. +pub(crate) fn scan_range( + blockstore: &Blockstore, + start_slot: u64, + end_slot: u64, + mut on_block: F, +) -> Result +where + F: FnMut(&SlotData) -> Result<()>, +{ + if start_slot > end_slot { + bail!("start slot {start_slot} is after end slot {end_slot}"); + } + progress(format!("reading slots {start_slot} to {end_slot}")); + let mut ahead = std::collections::BTreeMap::::new(); + let mut block_slots = Vec::new(); + let mut skips = Vec::new(); + let mut previous = None; + let mut boundary_hash = None; + let mut scanned = 0u64; + for slot in start_slot..=end_slot { + scanned += 1; + if let Some(data) = ahead.remove(&slot) { + accept_one( + data, + start_slot, + end_slot, + &mut previous, + &mut boundary_hash, + &mut block_slots, + &mut on_block, + )?; + } else if let Some(meta) = blockstore .meta(slot) .with_context(|| format!("read SlotMeta for slot {slot}"))? - else { - continue; - }; - let is_root = blockstore.is_root(slot); - let data = if slot <= end_slot && is_root && meta.is_full() { - load_rooted_block(blockstore, slot)? - } else if slot > end_slot { - let mut data = empty_slot(slot, meta.parent_slot, is_root, meta.is_full()); - data.markers = read_markers(blockstore, slot)?; - data + { + let is_root = blockstore.is_root(slot); + if !is_root { + bail!("slot {slot} has data and is not rooted"); + } + if !meta.is_full() { + bail!("slot {slot} is rooted and is not full"); + } + let data = load_rooted_block(blockstore, slot)?; + accept_one( + data, + start_slot, + end_slot, + &mut previous, + &mut boundary_hash, + &mut block_slots, + &mut on_block, + )?; } else { - empty_slot(slot, meta.parent_slot, is_root, meta.is_full()) - }; - slots.insert(slot, data); + let markers = proof_markers(blockstore, slot, end_slot, &mut ahead)?; + classify_hole(slot, markers.as_deref(), &mut skips)?; + } + if scanned % 1000 == 0 { + progress(format!( + "read {scanned} slots, latest slot {slot}, {} blocks accepted", + block_slots.len() + )); + } } - Ok(LedgerView { - start_slot, - end_slot, - slots, + progress(format!( + "read finished: {scanned} slots scanned, {} blocks accepted", + block_slots.len() + )); + let Some(boundary_hash) = boundary_hash else { + bail!("slots {start_slot} to {end_slot} have no block"); + }; + Ok(Inspected { + block_slots, + skips, + boundary_hash, }) } +fn accept_one( + data: SlotData, + start_slot: u64, + end_slot: u64, + previous: &mut Option, + boundary_hash: &mut Option<[u8; 32]>, + block_slots: &mut Vec, + on_block: &mut F, +) -> Result<()> +where + F: FnMut(&SlotData) -> Result<()>, +{ + accept_block_against(&data, start_slot, end_slot, previous.as_ref())?; + if boundary_hash.is_none() { + *boundary_hash = data.previous_blockhash; + } + on_block(&data)?; + *previous = Some(PreviousBlock { + slot: data.slot, + blockhash: data.blockhash, + }); + block_slots.push(data.slot); + Ok(()) +} + +fn proof_markers( + blockstore: &Blockstore, + slot: u64, + end_slot: u64, + ahead: &mut std::collections::BTreeMap, +) -> Result>>> { + let carrier = slot + .checked_add(REWARD_SLOT_DISTANCE) + .context("slot + 8 overflows")?; + if let Some(data) = ahead.get(&carrier) { + return Ok(Some(data.markers.clone())); + } + let Some(meta) = blockstore + .meta(carrier) + .with_context(|| format!("read SlotMeta for slot {carrier}"))? + else { + return Ok(None); + }; + let is_root = blockstore.is_root(carrier); + if carrier > end_slot { + return Ok(Some(read_markers(blockstore, carrier)?)); + } + if is_root && meta.is_full() { + let data = load_rooted_block(blockstore, carrier)?; + let markers = data.markers.clone(); + ahead.insert(carrier, data); + return Ok(Some(markers)); + } + Ok(Some(Vec::new())) +} + +fn progress(message: impl std::fmt::Display) { + if cfg!(test) { + return; + } + eprintln!("{message}"); + let _ = io::stderr().flush(); +} + fn load_rooted_block(blockstore: &Blockstore, slot: u64) -> Result { let (components, ranges, is_full) = blockstore .get_slot_components_with_shred_info(slot, 0, false) @@ -139,9 +252,12 @@ fn load_rooted_block(blockstore: &Blockstore, slot: u64) -> Result { let mut entries = Vec::with_capacity(raw_entries.len()); for entry in &raw_entries { let tx_count = entry.transactions.len(); - let rooted = block.transactions.get(tx_cursor..tx_cursor + tx_count).with_context(|| { - format!("slot {slot} entry transactions do not match the rooted block") - })?; + let rooted = block + .transactions + .get(tx_cursor..tx_cursor + tx_count) + .with_context(|| { + format!("slot {slot} entry transactions do not match the rooted block") + })?; let mut signature_count = 0u32; for (entry_tx, rooted_tx) in entry.transactions.iter().zip(rooted) { if entry_tx.signatures != rooted_tx.transaction.signatures { @@ -230,26 +346,6 @@ fn read_markers(blockstore: &Blockstore, slot: u64) -> Result>> { Ok(markers) } -fn empty_slot(slot: u64, parent_slot: Option, is_root: bool, is_full: bool) -> SlotData { - SlotData { - slot, - is_root, - is_full, - parent_slot, - previous_blockhash: None, - blockhash: [0; 32], - block_time: None, - block_height: None, - num_partitions: None, - rewards: Vec::new(), - consensus_block_id: None, - markers: Vec::new(), - entries: Vec::new(), - shredding: Vec::new(), - transactions: Vec::new(), - } -} - fn map_transaction( slot: u64, index: usize, @@ -281,11 +377,7 @@ fn map_message(message: &VersionedMessage) -> Result { .map(|key| copy32(key.as_ref())) .collect::>>()?, recent_blockhash: copy32(message.recent_blockhash.as_ref())?, - instructions: message - .instructions - .iter() - .map(map_instruction) - .collect(), + instructions: message.instructions.iter().map(map_instruction).collect(), lookups: Vec::new(), })), VersionedMessage::V0(message) => Ok(SourceMessage::V0(SourceMessageBody { @@ -296,11 +388,7 @@ fn map_message(message: &VersionedMessage) -> Result { .map(|key| copy32(key.as_ref())) .collect::>>()?, recent_blockhash: copy32(message.recent_blockhash.as_ref())?, - instructions: message - .instructions - .iter() - .map(map_instruction) - .collect(), + instructions: message.instructions.iter().map(map_instruction).collect(), lookups: message .address_table_lookups .iter() @@ -327,11 +415,7 @@ fn map_message(message: &VersionedMessage) -> Result { .map(|key| copy32(key.as_ref())) .collect::>>()?, recent_blockhash: copy32(message.lifetime_specifier.as_ref())?, - instructions: message - .instructions - .iter() - .map(map_instruction) - .collect(), + instructions: message.instructions.iter().map(map_instruction).collect(), })), } } @@ -344,7 +428,9 @@ fn map_header(header: &solana_message::MessageHeader) -> SourceHeader { } } -fn map_instruction(instruction: &solana_message::compiled_instruction::CompiledInstruction) -> SourceInstruction { +fn map_instruction( + instruction: &solana_message::compiled_instruction::CompiledInstruction, +) -> SourceInstruction { SourceInstruction { program_id_index: instruction.program_id_index, accounts: instruction.accounts.clone(), @@ -365,27 +451,24 @@ fn map_status(slot: u64, index: usize, meta: &TransactionStatusMeta) -> Result Result { } if let (Some(commission), Some(bps)) = (reward.commission, reward.commission_bps) { if u16::from(commission).checked_mul(100) != Some(bps) { - bail!( - "slot {slot} reward commission {commission} does not match commission_bps {bps}" - ); + bail!("slot {slot} reward commission {commission} does not match commission_bps {bps}"); } } Ok(SourceReward { @@ -494,14 +575,15 @@ pub(crate) fn map_transaction_error(err: &TransactionError) -> Result CompactTransactionError::AccountLoadedTwice, TransactionError::AccountNotFound => CompactTransactionError::AccountNotFound, TransactionError::ProgramAccountNotFound => CompactTransactionError::ProgramAccountNotFound, - TransactionError::InsufficientFundsForFee => CompactTransactionError::InsufficientFundsForFee, + TransactionError::InsufficientFundsForFee => { + CompactTransactionError::InsufficientFundsForFee + } TransactionError::InvalidAccountForFee => CompactTransactionError::InvalidAccountForFee, TransactionError::AlreadyProcessed => CompactTransactionError::AlreadyProcessed, TransactionError::BlockhashNotFound => CompactTransactionError::BlockhashNotFound, - TransactionError::InstructionError(index, err) => CompactTransactionError::InstructionError( - *index, - map_instruction_error(err)?, - ), + TransactionError::InstructionError(index, err) => { + CompactTransactionError::InstructionError(*index, map_instruction_error(err)?) + } TransactionError::CallChainTooDeep => CompactTransactionError::CallChainTooDeep, TransactionError::MissingSignatureForFee => CompactTransactionError::MissingSignatureForFee, TransactionError::InvalidAccountIndex => CompactTransactionError::InvalidAccountIndex, @@ -568,7 +650,9 @@ pub(crate) fn map_transaction_error(err: &TransactionError) -> Result CompactTransactionError::UnbalancedTransaction, - TransactionError::ProgramCacheHitMaxLimit => CompactTransactionError::ProgramCacheHitMaxLimit, + TransactionError::ProgramCacheHitMaxLimit => { + CompactTransactionError::ProgramCacheHitMaxLimit + } TransactionError::CommitCancelled => CompactTransactionError::CommitCancelled, TransactionError::BailOut => { bail!("transaction error BailOut has no compact encoding"); @@ -640,7 +724,9 @@ pub(crate) fn map_instruction_error(err: &InstructionError) -> Result { CompactInstructionError::ProgramEnvironmentSetupFailure } - InstructionError::ProgramFailedToComplete => CompactInstructionError::ProgramFailedToComplete, + InstructionError::ProgramFailedToComplete => { + CompactInstructionError::ProgramFailedToComplete + } InstructionError::ProgramFailedToCompile => CompactInstructionError::ProgramFailedToCompile, InstructionError::Immutable => CompactInstructionError::Immutable, InstructionError::IncorrectAuthority => CompactInstructionError::IncorrectAuthority, diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index be67acf..5c64934 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -6,12 +6,11 @@ mod source; use std::path::PathBuf; -use anyhow::{bail, Result}; +use anyhow::Result; use clap::{Parser, Subcommand}; -use crate::check::inspect; -use crate::convert::write_archive; -use crate::ledger::load_range; +use crate::convert::build_archive; +use crate::ledger::{open_read_only, scan_range}; #[derive(Parser)] #[command(name = "hivezilla-ledger-archive")] @@ -55,7 +54,8 @@ fn run() -> Result<()> { start_slot, end_slot, } => { - let inspected = inspect_ledger(&ledger, start_slot, end_slot)?; + let blockstore = open_read_only(&ledger)?; + let inspected = scan_range(&blockstore, start_slot, end_slot, |_| Ok(()))?; print_report(&inspected); Ok(()) } @@ -65,8 +65,7 @@ fn run() -> Result<()> { start_slot, end_slot, } => { - let (view, inspected) = load_and_inspect(&ledger, start_slot, end_slot)?; - write_archive(&output, &view, &inspected)?; + let inspected = build_archive(&ledger, &output, start_slot, end_slot)?; print_report(&inspected); println!("Output: {}", output.display()); Ok(()) @@ -74,23 +73,6 @@ fn run() -> Result<()> { } } -fn inspect_ledger(ledger: &std::path::Path, start_slot: u64, end_slot: u64) -> Result { - Ok(load_and_inspect(ledger, start_slot, end_slot)?.1) -} - -fn load_and_inspect( - ledger: &std::path::Path, - start_slot: u64, - end_slot: u64, -) -> Result<(source::LedgerView, check::Inspected)> { - if start_slot > end_slot { - bail!("start slot {start_slot} is after end slot {end_slot}"); - } - let view = load_range(ledger, start_slot, end_slot)?; - let inspected = inspect(&view)?; - Ok((view, inspected)) -} - fn print_report(inspected: &check::Inspected) { let first = inspected.block_slots.first().copied().unwrap_or(0); let last = inspected.block_slots.last().copied().unwrap_or(0); @@ -98,5 +80,8 @@ fn print_report(inspected: &check::Inspected) { println!("Proven skips: {}", inspected.skips.len()); println!("First block: {first}"); println!("Last block: {last}"); - println!("Boundary hash: {}", bs58::encode(inspected.boundary_hash).into_string()); + println!( + "Boundary hash: {}", + bs58::encode(inspected.boundary_hash).into_string() + ); } From a6561b43ddf76f047cf42d96b99f2c2ff41279bb Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 4 Nov 2026 17:41:40 +0100 Subject: [PATCH 12/38] solve fork issue --- hivezilla/ledger-archive/src/check.rs | 210 +++++++++++++++++++++++-- hivezilla/ledger-archive/src/ledger.rs | 85 ++++++++-- hivezilla/ledger-archive/src/main.rs | 1 + 3 files changed, 269 insertions(+), 27 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index d3859ee..a44d6f1 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -3,7 +3,7 @@ use anyhow::{bail, Context, Result}; use crate::markers::reward_names; use crate::source::{LedgerView, REWARD_SLOT_DISTANCE}; -/// Blocks and proven skips for one inclusive slot window. +/// Blocks, proven skips, and abandoned forks for one inclusive slot window. /// /// The check command and the build command apply the same slot rules. A check /// that passes on a snapshot will not later stop the build of that same @@ -12,9 +12,21 @@ use crate::source::{LedgerView, REWARD_SLOT_DISTANCE}; pub(crate) struct Inspected { pub block_slots: Vec, pub skips: Vec, + /// Slots that have data, are not root, and are left out of the archive. + /// The next root jumps over each one, and slot N+8 has a skip certificate. + pub forks: Vec, pub boundary_hash: [u8; 32], } +/// Where `slot` sits relative to the next rooted slot. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ChainRelation { + /// `next_root`'s parent chain reaches a slot below `slot` without visiting it. + JumpsOver { next_root: u64 }, + /// `next_root`'s parent chain visits `slot`. + DescendsFrom { next_root: u64 }, +} + /// The previous produced block. The chain check needs its slot and blockhash. #[derive(Debug, Clone, Copy)] pub(crate) struct PreviousBlock { @@ -33,19 +45,35 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { let mut block_slots = Vec::new(); let mut skips = Vec::new(); + let mut forks = Vec::new(); let mut previous = None; for slot in view.start_slot..=view.end_slot { - if view.slots.contains_key(&slot) { - let data = &view.slots[&slot]; - accept_block_against(data, view.start_slot, view.end_slot, previous.as_ref())?; - previous = Some(PreviousBlock { - slot: data.slot, - blockhash: data.blockhash, - }); - block_slots.push(slot); + if let Some(data) = view.slots.get(&slot) { + if data.is_root { + accept_block_against(data, view.start_slot, view.end_slot, previous.as_ref())?; + previous = Some(PreviousBlock { + slot: data.slot, + blockhash: data.blockhash, + }); + block_slots.push(slot); + } else { + let relation = chain_relation_in_view(view, slot)?; + let markers = match relation { + ChainRelation::JumpsOver { .. } => { + let carrier = slot.checked_add(REWARD_SLOT_DISTANCE); + carrier.and_then(|carrier| { + view.slots.get(&carrier).map(|data| data.markers.as_slice()) + }) + } + ChainRelation::DescendsFrom { .. } => None, + }; + classify_abandoned_fork(slot, relation, markers, &mut forks)?; + } } else { let carrier = slot.checked_add(REWARD_SLOT_DISTANCE); - let markers = carrier.and_then(|carrier| view.slots.get(&carrier).map(|data| data.markers.as_slice())); + let markers = carrier.and_then(|carrier| { + view.slots.get(&carrier).map(|data| data.markers.as_slice()) + }); classify_hole(slot, markers, &mut skips)?; } } @@ -64,10 +92,89 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { Ok(Inspected { block_slots, skips, + forks, boundary_hash, }) } +/// Follow parent links from `next_root` until the walk passes `slot`. +/// +/// The walk stops when a parent is below `slot`. A parent equal to `slot`, or +/// a cursor equal to `slot`, means the rooted chain includes that slot. +pub(crate) fn parent_chain_relation( + slot: u64, + next_root: u64, + mut parent_of: F, +) -> Result +where + F: FnMut(u64) -> Result>, +{ + if next_root <= slot { + bail!("next root {next_root} is not after slot {slot}"); + } + let mut cursor = next_root; + loop { + if cursor == slot { + return Ok(ChainRelation::DescendsFrom { next_root }); + } + let Some(parent) = parent_of(cursor)? else { + bail!("slot {cursor} has no parent slot"); + }; + if parent == slot { + return Ok(ChainRelation::DescendsFrom { next_root }); + } + if parent < slot { + return Ok(ChainRelation::JumpsOver { next_root }); + } + if parent >= cursor { + bail!("slot {cursor} parent {parent} is not before the slot"); + } + cursor = parent; + } +} + +fn chain_relation_in_view(view: &LedgerView, slot: u64) -> Result { + let start = slot.checked_add(1).context("slot + 1 overflows")?; + let Some(next_root) = view + .slots + .range(start..) + .find(|(_, data)| data.is_root) + .map(|(next_root, _)| *next_root) + else { + bail!("slot {slot} has data and is not rooted"); + }; + parent_chain_relation(slot, next_root, |cursor| { + Ok(view.slots.get(&cursor).and_then(|data| data.parent_slot)) + }) +} + +/// Leave out a non-root slot when the rooted chain jumps over it and slot N+8 +/// carries a skip certificate. A notarization certificate does not cancel that +/// skip. A slot the next root descends from stays an error. +pub(crate) fn classify_abandoned_fork( + slot: u64, + chain: ChainRelation, + carrier_markers: Option<&[Vec]>, + forks: &mut Vec, +) -> Result<()> { + if let ChainRelation::DescendsFrom { next_root } = chain { + bail!("slot {slot} has data and is not rooted, and slot {next_root} descends from it"); + } + let carrier = slot + .checked_add(REWARD_SLOT_DISTANCE) + .context("slot + 8 overflows")?; + let Some(markers) = carrier_markers else { + bail!("slot {slot} has data and is not rooted, and slot {carrier} has no footer proof"); + }; + let names = reward_names(markers) + .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; + if names.skips.contains(&slot) { + forks.push(slot); + return Ok(()); + } + bail!("slot {slot} has data and is not rooted, and slot {carrier} does not prove a skip"); +} + pub(crate) fn accept_block_against( data: &crate::source::SlotData, start_slot: u64, @@ -248,9 +355,92 @@ mod tests { let inspected = inspect(&view(10, 12, vec![first, second, carrier])).unwrap(); assert_eq!(inspected.block_slots, vec![10, 12]); assert_eq!(inspected.skips, vec![11]); + assert_eq!(inspected.forks, Vec::::new()); assert_eq!(inspected.boundary_hash, [3; 32]); } + #[test] + fn an_abandoned_fork_with_skip_and_notar_is_omitted() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut fork = test_block(11, 10, [8; 32], [4; 32]); + fork.is_root = false; + let next = test_block(13, 10, [5; 32], [4; 32]); + let mut fork_carrier = test_block(19, 18, [6; 32], [5; 32]); + fork_carrier.is_root = false; + fork_carrier.markers = vec![skip_footer(11), notar_footer(11)]; + let mut hole_carrier = test_block(20, 19, [7; 32], [6; 32]); + hole_carrier.is_root = false; + hole_carrier.markers = vec![skip_footer(12)]; + + let inspected = inspect(&view( + 10, + 13, + vec![first, fork, next, fork_carrier, hole_carrier], + )) + .unwrap(); + assert_eq!(inspected.block_slots, vec![10, 13]); + assert_eq!(inspected.forks, vec![11]); + assert_eq!(inspected.skips, vec![12]); + assert_eq!(inspected.boundary_hash, [3; 32]); + } + + #[test] + fn a_parent_walk_jumps_over_a_slot_after_one_intermediate() { + let parents = [(14u64, 13u64), (13, 10)]; + let relation = parent_chain_relation(11, 14, |cursor| { + Ok(parents + .iter() + .find(|(slot, _)| *slot == cursor) + .map(|(_, parent)| *parent)) + }) + .unwrap(); + assert_eq!(relation, ChainRelation::JumpsOver { next_root: 14 }); + } + + #[test] + fn a_parent_walk_finds_the_slot_one_step_down() { + let relation = parent_chain_relation(11, 13, |cursor| { + Ok(match cursor { + 13 => Some(12), + 12 => Some(11), + _ => None, + }) + }) + .unwrap(); + assert_eq!(relation, ChainRelation::DescendsFrom { next_root: 13 }); + } + + #[test] + fn a_non_root_ancestor_of_the_next_root_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut fork = test_block(11, 10, [8; 32], [4; 32]); + fork.is_root = false; + let next = test_block(12, 11, [5; 32], [8; 32]); + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_footer(11)]; + + let error = inspect(&view(10, 12, vec![first, fork, next, carrier])).unwrap_err(); + assert!(error.to_string().contains("descends from it"), "{error:#}"); + } + + #[test] + fn a_chain_jump_without_a_skip_certificate_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut fork = test_block(11, 10, [8; 32], [4; 32]); + fork.is_root = false; + let next = test_block(12, 10, [5; 32], [4; 32]); + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![notar_footer(11)]; + + let error = inspect(&view(10, 12, vec![first, fork, next, carrier])).unwrap_err(); + assert!( + error.to_string().contains("does not prove a skip"), + "{error:#}" + ); + } + #[test] fn a_notarized_missing_slot_is_an_error() { let first = test_block(10, 9, [4; 32], [3; 32]); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 8032b27..d473f78 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -16,7 +16,10 @@ use solana_reward_info::RewardType; use solana_transaction_error::TransactionError; use solana_transaction_status::{Reward, TransactionStatusMeta}; -use crate::check::{accept_block_against, classify_hole, Inspected, PreviousBlock}; +use crate::check::{ + accept_block_against, classify_abandoned_fork, classify_hole, parent_chain_relation, + ChainRelation, Inspected, PreviousBlock, +}; use crate::source::{ REWARD_SLOT_DISTANCE, RawComponent, SlotData, SourceEntry, SourceHeader, SourceInnerGroup, SourceInnerInstruction, SourceInstruction, SourceLookup, SourceMessage, SourceMessageBody, @@ -54,7 +57,9 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { } /// Read each slot once. A rooted full block is checked, passed to `on_block`, -/// then dropped. A hole is proved from the footer of slot N+8. +/// then dropped. A non-root slot is left out when the next root jumps over it +/// and slot N+8 has a skip certificate. A hole is proved from the footer of +/// slot N+8. /// /// When that footer belongs to a later block in the window, the block is kept /// until the scan reaches it, so its shreds are not read a second time. @@ -74,6 +79,7 @@ where let mut ahead = std::collections::BTreeMap::::new(); let mut block_slots = Vec::new(); let mut skips = Vec::new(); + let mut forks = Vec::new(); let mut previous = None; let mut boundary_hash = None; let mut scanned = 0u64; @@ -93,23 +99,22 @@ where .meta(slot) .with_context(|| format!("read SlotMeta for slot {slot}"))? { - let is_root = blockstore.is_root(slot); - if !is_root { - bail!("slot {slot} has data and is not rooted"); - } - if !meta.is_full() { + if !blockstore.is_root(slot) { + leave_out_abandoned_fork(blockstore, slot, end_slot, &mut ahead, &mut forks)?; + } else if !meta.is_full() { bail!("slot {slot} is rooted and is not full"); + } else { + let data = load_rooted_block(blockstore, slot)?; + accept_one( + data, + start_slot, + end_slot, + &mut previous, + &mut boundary_hash, + &mut block_slots, + &mut on_block, + )?; } - let data = load_rooted_block(blockstore, slot)?; - accept_one( - data, - start_slot, - end_slot, - &mut previous, - &mut boundary_hash, - &mut block_slots, - &mut on_block, - )?; } else { let markers = proof_markers(blockstore, slot, end_slot, &mut ahead)?; classify_hole(slot, markers.as_deref(), &mut skips)?; @@ -131,10 +136,56 @@ where Ok(Inspected { block_slots, skips, + forks, boundary_hash, }) } +fn leave_out_abandoned_fork( + blockstore: &Blockstore, + slot: u64, + end_slot: u64, + ahead: &mut std::collections::BTreeMap, + forks: &mut Vec, +) -> Result<()> { + let relation = rooted_chain_relation(blockstore, slot)?; + let markers = match relation { + ChainRelation::JumpsOver { .. } => proof_markers(blockstore, slot, end_slot, ahead)?, + ChainRelation::DescendsFrom { .. } => None, + }; + let next_root = match relation { + ChainRelation::JumpsOver { next_root } => Some(next_root), + ChainRelation::DescendsFrom { .. } => None, + }; + classify_abandoned_fork(slot, relation, markers.as_deref(), forks)?; + if let Some(next_root) = next_root { + progress(format!( + "slot {slot} left out, rooted chain jumps to {next_root}" + )); + } + Ok(()) +} + +fn rooted_chain_relation(blockstore: &Blockstore, slot: u64) -> Result { + let start = slot.checked_add(1).context("slot + 1 overflows")?; + let Some(next_root) = blockstore + .rooted_slot_iterator(start) + .with_context(|| format!("read roots after slot {slot}"))? + .next() + else { + bail!("slot {slot} has data and is not rooted"); + }; + parent_chain_relation(slot, next_root, |cursor| { + let meta = blockstore + .meta(cursor) + .with_context(|| format!("read SlotMeta for slot {cursor}"))?; + let Some(meta) = meta else { + bail!("slot {cursor} has no SlotMeta"); + }; + Ok(meta.parent_slot) + }) +} + fn accept_one( data: SlotData, start_slot: u64, diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index 5c64934..1cacc11 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -78,6 +78,7 @@ fn print_report(inspected: &check::Inspected) { let last = inspected.block_slots.last().copied().unwrap_or(0); println!("Blocks: {}", inspected.block_slots.len()); println!("Proven skips: {}", inspected.skips.len()); + println!("Abandoned forks: {}", inspected.forks.len()); println!("First block: {first}"); println!("Last block: {last}"); println!( From 0e5c89ef2bb64909efc9015d7d87b18f0bf70a29 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 13:20:35 +0100 Subject: [PATCH 13/38] add registry.bin in ledger archive --- hivezilla/ledger-archive/src/convert.rs | 343 +++++++++++++++++++++--- 1 file changed, 306 insertions(+), 37 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index c4aea93..037bfe6 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::fs::{self, File}; use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; @@ -5,7 +6,8 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use blockzilla_archive_v2::{ ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, ARCHIVE_V2_BLOCKS_FILE, - ARCHIVE_V2_POH_FILE, ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, + ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, ARCHIVE_V2_SHREDDING_FILE, + ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_TX_FLAG_HAS_ERROR, ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES, ARCHIVE_V2_TX_FLAG_HAS_LOGS, ARCHIVE_V2_TX_FLAG_HAS_METADATA, ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA, @@ -23,7 +25,7 @@ use blockzilla_compact::{ CompactTransactionConfig, OwnedCompactAddressTableLookup, OwnedCompactRecentBlockhash, }; use blockzilla_primitives::{CompactPubkey, WincodeLeb128FramedWriter, wincode_leb128_config}; -use blockzilla_registry::RawPubkeyCompactor; +use blockzilla_registry::{KeyIndex, write_registry}; use crate::check::Inspected; use crate::ledger::{open_read_only, scan_range}; @@ -36,6 +38,7 @@ use crate::source::{ pub(crate) const ZSTD_LEVEL: i32 = 3; const INDEX_FLAGS: u32 = 0; const VOTE_HASH_ROW_LEN: usize = 65; +type PubkeyCounts = HashMap<[u8; 32], u32>; pub(crate) fn build_archive( ledger: &Path, @@ -47,8 +50,21 @@ pub(crate) fn build_archive( fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; let temps = temp_paths(output); let result = (|| { - let mut writer = ArchiveWriter::create(&temps)?; + create_temp_files(&temps)?; + progress(format!( + "counting pubkeys in slots {start_slot} to {end_slot}" + )); + let mut counts = PubkeyCounts::new(); let inspected = scan_range(&blockstore, start_slot, end_slot, |block| { + count_slot_pubkeys(block, &mut counts); + Ok(()) + })?; + let key_index = prepare_registry(&temps, counts)?; + progress(format!( + "writing blocks for slots {start_slot} to {end_slot}" + )); + let mut writer = ArchiveWriter::create(&temps, key_index)?; + scan_range(&blockstore, start_slot, end_slot, |block| { writer.write_block(block) })?; writer.finish(output)?; @@ -67,7 +83,14 @@ where fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; let temps = temp_paths(output); let result = (|| { - let mut writer = ArchiveWriter::create(&temps)?; + create_temp_files(&temps)?; + let mut counts = PubkeyCounts::new(); + for slot in &inspected.block_slots { + let block = load_block(*slot)?; + count_slot_pubkeys(&block, &mut counts); + } + let key_index = prepare_registry(&temps, counts)?; + let mut writer = ArchiveWriter::create(&temps, key_index)?; for slot in &inspected.block_slots { let block = load_block(*slot)?; writer.write_block(&block)?; @@ -87,6 +110,7 @@ struct ArchivePaths { vote: PathBuf, poh: PathBuf, shredding: PathBuf, + registry: PathBuf, } fn temp_paths(output: &Path) -> ArchivePaths { @@ -97,18 +121,31 @@ fn temp_paths(output: &Path) -> ArchivePaths { vote: output.join(format!("{ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE}.tmp")), poh: output.join(format!("{ARCHIVE_V2_POH_FILE}.tmp")), shredding: output.join(format!("{ARCHIVE_V2_SHREDDING_FILE}.tmp")), + registry: output.join(format!("{ARCHIVE_V2_PUBKEY_REGISTRY_FILE}.tmp")), } } -fn remove_temps(temps: &ArchivePaths) { - for path in [ +fn temp_file_list(temps: &ArchivePaths) -> [&Path; 7] { + [ &temps.blocks, &temps.signatures, &temps.blockhash, &temps.vote, &temps.poh, &temps.shredding, - ] { + &temps.registry, + ] +} + +fn create_temp_files(temps: &ArchivePaths) -> Result<()> { + for path in temp_file_list(temps) { + File::create(path).with_context(|| format!("create {}", path.display()))?; + } + Ok(()) +} + +fn remove_temps(temps: &ArchivePaths) { + for path in temp_file_list(temps) { let _ = fs::remove_file(path); } } @@ -124,11 +161,12 @@ struct ArchiveWriter { compressed_offset: u64, next_tx_ordinal: u64, next_signature_ordinal: u64, + key_index: KeyIndex, paths: ArchivePaths, } impl ArchiveWriter { - fn create(temps: &ArchivePaths) -> Result { + fn create(temps: &ArchivePaths, key_index: KeyIndex) -> Result { Ok(Self { blocks: BufWriter::new( File::create(&temps.blocks) @@ -152,6 +190,7 @@ impl ArchiveWriter { compressed_offset: 0, next_tx_ordinal: 0, next_signature_ordinal: 0, + key_index, paths: ArchivePaths { blocks: temps.blocks.clone(), signatures: temps.signatures.clone(), @@ -159,6 +198,7 @@ impl ArchiveWriter { vote: temps.vote.clone(), poh: temps.poh.clone(), shredding: temps.shredding.clone(), + registry: temps.registry.clone(), }, }) } @@ -172,7 +212,7 @@ impl ArchiveWriter { .context("first block is missing parent entries")?; self.blockhashes.extend(boundary); } - let uncompressed = encode_hot_block(block, block_id)?; + let uncompressed = encode_hot_block(block, block_id, &self.key_index)?; let compressed = zstd::bulk::compress(&uncompressed, ZSTD_LEVEL) .with_context(|| format!("compress slot {slot}"))?; let compressed_len = @@ -265,6 +305,10 @@ impl ArchiveWriter { &self.paths.shredding, &output.join(ARCHIVE_V2_SHREDDING_FILE), )?; + rename_finished( + &self.paths.registry, + &output.join(ARCHIVE_V2_PUBKEY_REGISTRY_FILE), + )?; write_archive_v2_hot_block_index( &output.join(ARCHIVE_V2_BLOCK_INDEX_FILE), self.compressed_offset, @@ -288,7 +332,96 @@ fn rename_finished(from: &Path, to: &Path) -> Result<()> { fs::rename(from, to).with_context(|| format!("rename {} to {}", from.display(), to.display())) } -pub(crate) fn encode_hot_block(block: &SlotData, block_id: u32) -> Result> { +fn prepare_registry(temps: &ArchivePaths, counts: PubkeyCounts) -> Result { + let keys = ordered_registry_keys(counts); + write_registry(&temps.registry, &keys) + .with_context(|| format!("write {}", temps.registry.display()))?; + Ok(KeyIndex::build_from_slice_low_memory(&keys)) +} + +fn ordered_registry_keys(counts: PubkeyCounts) -> Vec<[u8; 32]> { + let mut items: Vec<([u8; 32], u32)> = counts.into_iter().collect(); + items.sort_unstable_by(|(left_key, left_count), (right_key, right_count)| { + right_count + .cmp(left_count) + .then_with(|| left_key.cmp(right_key)) + }); + let compute_budget = compute_budget_program_id(); + let mut keys = Vec::with_capacity(items.len() + 1); + if !items.iter().any(|(key, _)| *key == compute_budget) { + keys.push(compute_budget); + } + keys.extend(items.into_iter().map(|(key, _)| key)); + keys +} + +fn compute_budget_program_id() -> [u8; 32] { + let bytes = bs58::decode("ComputeBudget111111111111111111111111111111") + .into_vec() + .expect("ComputeBudget program id is valid base58"); + bytes + .as_slice() + .try_into() + .expect("ComputeBudget program id is 32 bytes") +} + +fn count_slot_pubkeys(block: &SlotData, counts: &mut PubkeyCounts) { + for tx in &block.transactions { + match &tx.message { + SourceMessage::Legacy(body) => count_keys(&body.account_keys, counts), + SourceMessage::V0(body) => { + count_keys(&body.account_keys, counts); + for lookup in &body.lookups { + bump_key(counts, lookup.account_key); + } + } + SourceMessage::V1(body) => count_keys(&body.account_keys, counts), + } + let Some(status) = &tx.status else { + continue; + }; + count_keys(&status.loaded_writable, counts); + count_keys(&status.loaded_readonly, counts); + for balance in status + .pre_token_balances + .iter() + .chain(status.post_token_balances.iter()) + { + bump_key(counts, balance.mint); + if let Some(owner) = balance.owner { + bump_key(counts, owner); + } + if let Some(program_id) = balance.program_id { + bump_key(counts, program_id); + } + } + for reward in &status.rewards { + bump_key(counts, reward.pubkey); + } + if let Some(return_data) = &status.return_data { + bump_key(counts, return_data.program_id); + } + } +} + +fn count_keys(keys: &[[u8; 32]], counts: &mut PubkeyCounts) { + for key in keys { + bump_key(counts, *key); + } +} + +fn bump_key(counts: &mut PubkeyCounts, key: [u8; 32]) { + counts + .entry(key) + .and_modify(|count| *count = count.saturating_add(1)) + .or_insert(1); +} + +pub(crate) fn encode_hot_block( + block: &SlotData, + block_id: u32, + key_index: &KeyIndex, +) -> Result> { let entry_txs = block.entries.iter().try_fold(0u32, |total, entry| { total .checked_add(entry.tx_count) @@ -316,6 +449,7 @@ pub(crate) fn encode_hot_block(block: &SlotData, block_id: u32) -> Result Result, metadata_bytes: &mut Vec, ) -> Result { - let (payload, mut flags) = encode_message(&tx.message)?; + let (payload, mut flags) = encode_message(&tx.message, key_index)?; flags |= ARCHIVE_V2_TX_FLAG_HAS_METADATA; if status.err.is_some() { flags |= ARCHIVE_V2_TX_FLAG_HAS_ERROR; @@ -385,7 +524,8 @@ fn encode_tx_row( let message = wincode::config::serialize(&payload, wincode_leb128_config()) .context("serialize transaction message")?; - let metadata = wincode::config::serialize(&compact_meta(status), wincode_leb128_config()) + let metadata = + wincode::config::serialize(&compact_meta(status, key_index), wincode_leb128_config()) .context("serialize transaction metadata")?; let message_offset = u32::try_from(message_bytes.len()).context("message region exceeds u32")?; @@ -410,12 +550,15 @@ fn encode_tx_row( }) } -fn encode_message(message: &SourceMessage) -> Result<(ArchiveV2HotMessagePayload, u32)> { +fn encode_message( + message: &SourceMessage, + key_index: &KeyIndex, +) -> Result<(ArchiveV2HotMessagePayload, u32)> { match message { SourceMessage::Legacy(body) => Ok(( ArchiveV2HotMessagePayload::Legacy(ArchiveV2HotLegacyMessage { header: compact_header(body), - account_keys: raw_keys(&body.account_keys), + account_keys: compact_keys(&body.account_keys, key_index), recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), instructions: raw_instructions(&body.instructions), }), @@ -424,14 +567,14 @@ fn encode_message(message: &SourceMessage) -> Result<(ArchiveV2HotMessagePayload SourceMessage::V0(body) => Ok(( ArchiveV2HotMessagePayload::V0(ArchiveV2HotV0Message { header: compact_header(body), - account_keys: raw_keys(&body.account_keys), + account_keys: compact_keys(&body.account_keys, key_index), recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), instructions: raw_instructions(&body.instructions), address_table_lookups: body .lookups .iter() .map(|lookup| OwnedCompactAddressTableLookup { - account_key: CompactPubkey::raw(lookup.account_key), + account_key: key_index.compact(&lookup.account_key), writable_indexes: lookup.writable_indexes.clone(), readonly_indexes: lookup.readonly_indexes.clone(), }) @@ -448,7 +591,7 @@ fn encode_message(message: &SourceMessage) -> Result<(ArchiveV2HotMessagePayload loaded_accounts_data_size_limit: body.config.loaded_accounts_data_size_limit, heap_size: body.config.heap_size, }, - account_keys: raw_keys(&body.account_keys), + account_keys: compact_keys(&body.account_keys, key_index), recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), instructions: raw_instructions(&body.instructions), }), @@ -473,8 +616,8 @@ fn compact_header_v1(body: &SourceV1Message) -> CompactMessageHeader { } } -fn raw_keys(keys: &[[u8; 32]]) -> Vec { - keys.iter().copied().map(CompactPubkey::raw).collect() +fn compact_keys(keys: &[[u8; 32]], key_index: &KeyIndex) -> Vec { + keys.iter().map(|key| key_index.compact(key)).collect() } fn raw_instructions( @@ -490,7 +633,7 @@ fn raw_instructions( .collect() } -fn compact_meta(status: &SourceStatus) -> CompactMetaV1 { +fn compact_meta(status: &SourceStatus, key_index: &KeyIndex) -> CompactMetaV1 { CompactMetaV1 { err: status.err.clone(), fee: status.fee, @@ -503,22 +646,26 @@ fn compact_meta(status: &SourceStatus) -> CompactMetaV1 { logs: status .logs .as_ref() - .map(|lines| parse_logs_with_compactor(lines, &RawPubkeyCompactor)), + .map(|lines| parse_logs_with_compactor(lines, key_index)), pre_token_balances: status .pre_token_balances .iter() - .map(compact_token_balance) + .map(|balance| compact_token_balance(balance, key_index)) .collect(), post_token_balances: status .post_token_balances .iter() - .map(compact_token_balance) + .map(|balance| compact_token_balance(balance, key_index)) .collect(), - rewards: status.rewards.iter().map(compact_reward).collect(), - loaded_writable_addresses: raw_keys(&status.loaded_writable), - loaded_readonly_addresses: raw_keys(&status.loaded_readonly), + rewards: status + .rewards + .iter() + .map(|reward| compact_reward(reward, key_index)) + .collect(), + loaded_writable_addresses: compact_keys(&status.loaded_writable, key_index), + loaded_readonly_addresses: compact_keys(&status.loaded_readonly, key_index), return_data: status.return_data.as_ref().map(|data| CompactReturnData { - program_id: CompactPubkey::raw(data.program_id), + program_id: key_index.compact(&data.program_id), data: data.data.clone(), }), compute_units_consumed: status.compute_units_consumed, @@ -542,20 +689,20 @@ fn compact_inner_group(group: &SourceInnerGroup) -> CompactInnerInstructions { } } -fn compact_token_balance(balance: &SourceTokenBalance) -> CompactTokenBalance { +fn compact_token_balance(balance: &SourceTokenBalance, key_index: &KeyIndex) -> CompactTokenBalance { CompactTokenBalance { account_index: balance.account_index, - mint: Some(CompactPubkey::raw(balance.mint)), - owner: balance.owner.map(CompactPubkey::raw), - program_id: balance.program_id.map(CompactPubkey::raw), + mint: Some(key_index.compact(&balance.mint)), + owner: balance.owner.map(|owner| key_index.compact(&owner)), + program_id: balance.program_id.map(|program_id| key_index.compact(&program_id)), amount: balance.amount, decimals: balance.decimals, } } -fn compact_reward(reward: &SourceReward) -> CompactReward { +fn compact_reward(reward: &SourceReward, key_index: &KeyIndex) -> CompactReward { CompactReward { - pubkey: CompactPubkey::raw(reward.pubkey), + pubkey: key_index.compact(&reward.pubkey), lamports: reward.lamports, post_balance: reward.post_balance, reward_type: reward.reward_type, @@ -627,7 +774,10 @@ fn vote_hash_row(block: &SlotData) -> Result<[u8; VOTE_HASH_ROW_LEN]> { mod tests { use super::*; use crate::check::inspect; - use crate::source::{LedgerView, REWARD_SLOT_DISTANCE, SourceShredGroup, test_block}; + use crate::source::{ + LedgerView, REWARD_SLOT_DISTANCE, SourceHeader, SourceMessage, SourceMessageBody, + SourceReward, SourceShredGroup, SourceStatus, SourceTx, test_block, + }; use blockzilla_archive_v2::deserialize_archive_v2_hot_block_blob; use blockzilla_primitives::WincodeLeb128FramedReader; use of_car_reader::blockmarker::parent_consensus_block_id; @@ -648,7 +798,7 @@ mod tests { #[test] fn empty_tower_tail_is_two_zero_bytes_and_has_no_consensus_id() { let block = test_block(10, 9, [2; 32], [1; 32]); - let bytes = encode_hot_block(&block, 0).unwrap(); + let bytes = encode_hot_block(&block, 0, &index_for(&block)).unwrap(); assert_eq!(&bytes[bytes.len() - 2..], &[0, 0]); let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); assert_eq!(decoded.consensus_block_id, None); @@ -665,7 +815,7 @@ mod tests { block.consensus_block_id = Some(consensus_id); block.markers = vec![header_marker(9, parent_id)]; - let bytes = encode_hot_block(&block, 0).unwrap(); + let bytes = encode_hot_block(&block, 0, &index_for(&block)).unwrap(); let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); assert_eq!(decoded.consensus_block_id, Some(consensus_id)); assert_eq!( @@ -744,6 +894,9 @@ mod tests { assert_eq!(&hashes[32..64], &[4; 32]); assert_eq!(&hashes[64..], &[5; 32]); + let registry = std::fs::read(dir.path().join(ARCHIVE_V2_PUBKEY_REGISTRY_FILE)).unwrap(); + assert_eq!(registry.as_slice(), compute_budget_program_id().as_slice()); + let mut reader = WincodeLeb128FramedReader::new(Cursor::new( std::fs::read(dir.path().join(ARCHIVE_V2_SHREDDING_FILE)).unwrap(), )); @@ -805,6 +958,122 @@ mod tests { assert!(names.is_empty(), "{names:?}"); } + #[test] + fn repeated_account_key_gets_the_lowest_counted_id() { + let repeated = [4u8; 32]; + let once = [9u8; 32]; + let block = block_with_keys(vec![ + vec![repeated, repeated], + vec![once], + ]); + let keys = registry_keys_for(&block); + assert_eq!(keys.len(), 3); + assert_eq!(keys[0], compute_budget_program_id()); + assert_eq!(keys[1], repeated); + assert_eq!(keys[2], once); + assert_eq!(keys.len() * 32, 96); + + let (message, _) = encode_message(&block.transactions[0].message, &index_for(&block)).unwrap(); + let ArchiveV2HotMessagePayload::Legacy(message) = message else { + panic!("expected a legacy message"); + }; + assert_eq!(message.account_keys[0], CompactPubkey::id(2)); + assert_eq!(message.account_keys[1], CompactPubkey::id(2)); + let (other, _) = encode_message(&block.transactions[1].message, &index_for(&block)).unwrap(); + let ArchiveV2HotMessagePayload::Legacy(other) = other else { + panic!("expected a legacy message"); + }; + assert_eq!(other.account_keys[0], CompactPubkey::id(3)); + } + + #[test] + fn a_seen_compute_budget_key_is_not_inserted_again() { + let compute_budget = compute_budget_program_id(); + let other = [5u8; 32]; + let block = block_with_keys(vec![vec![compute_budget, compute_budget], vec![other]]); + let keys = registry_keys_for(&block); + assert_eq!(keys, vec![compute_budget, other]); + } + + #[test] + fn a_block_reward_absent_from_transactions_stays_raw() { + let reward_key = [6u8; 32]; + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.rewards.push(SourceReward { + pubkey: reward_key, + lamports: 1, + post_balance: 2, + reward_type: 1, + commission: None, + }); + let keys = registry_keys_for(&block); + assert_eq!(keys, vec![compute_budget_program_id()]); + let decoded = deserialize_archive_v2_hot_block_blob( + &encode_hot_block(&block, 0, &index_for(&block)).unwrap(), + ) + .unwrap(); + assert_eq!( + decoded.header.rewards.unwrap().decoded[0].pubkey, + CompactPubkey::raw(reward_key) + ); + } + + #[test] + fn a_block_reward_seen_in_a_transaction_uses_its_id() { + let reward_key = [6u8; 32]; + let mut block = block_with_keys(vec![vec![reward_key]]); + block.rewards.push(SourceReward { + pubkey: reward_key, + lamports: 1, + post_balance: 2, + reward_type: 1, + commission: None, + }); + let decoded = deserialize_archive_v2_hot_block_blob( + &encode_hot_block(&block, 0, &index_for(&block)).unwrap(), + ) + .unwrap(); + assert_eq!( + decoded.header.rewards.unwrap().decoded[0].pubkey, + CompactPubkey::id(2) + ); + } + + fn registry_keys_for(block: &SlotData) -> Vec<[u8; 32]> { + let mut counts = PubkeyCounts::new(); + count_slot_pubkeys(block, &mut counts); + ordered_registry_keys(counts) + } + + fn index_for(block: &SlotData) -> KeyIndex { + KeyIndex::build_from_slice_low_memory(®istry_keys_for(block)) + } + + fn block_with_keys(key_groups: Vec>) -> SlotData { + let mut block = test_block(10, 9, [2; 32], [1; 32]); + block.transactions = key_groups + .into_iter() + .map(|account_keys| SourceTx { + signatures: vec![[7; 64]], + message: SourceMessage::Legacy(SourceMessageBody { + header: SourceHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys, + recent_blockhash: [8; 32], + instructions: Vec::new(), + lookups: Vec::new(), + }), + status: Some(SourceStatus::success()), + }) + .collect(); + block.entries[0].tx_count = u32::try_from(block.transactions.len()).unwrap(); + block.entries[0].signature_count = block.entries[0].tx_count; + block + } + fn skip_footer(skipped_slot: u64) -> Vec { let mut certificates = vec![0, 1]; certificates.extend_from_slice(&skipped_slot.to_le_bytes()); From abfe637927cfe7c14637be096c6a8ba9d303e506 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 13:22:43 +0100 Subject: [PATCH 14/38] fix test issues --- hivezilla/ledger-archive/src/convert.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 037bfe6..7def1d3 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -871,7 +871,7 @@ mod tests { .with_context(|| format!("test block {slot} is missing")) }) .unwrap(); - assert_eq!(loaded, vec![10, 12]); + assert_eq!(loaded, vec![10, 12, 10, 12]); let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); assert_eq!(&index[..8], b"BZV2HIX1"); From 05b857eb43ab56632273301c48efea2ce0376a2c Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 13:55:31 +0100 Subject: [PATCH 15/38] replace recent_blockhash by an id --- hivezilla/ledger-archive/src/check.rs | 21 +- hivezilla/ledger-archive/src/convert.rs | 564 +++++++++++++++++++++--- hivezilla/ledger-archive/src/ledger.rs | 4 +- 3 files changed, 526 insertions(+), 63 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index a44d6f1..5786e4b 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use crate::markers::reward_names; use crate::source::{LedgerView, REWARD_SLOT_DISTANCE}; @@ -71,9 +71,8 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { } } else { let carrier = slot.checked_add(REWARD_SLOT_DISTANCE); - let markers = carrier.and_then(|carrier| { - view.slots.get(&carrier).map(|data| data.markers.as_slice()) - }); + let markers = carrier + .and_then(|carrier| view.slots.get(&carrier).map(|data| data.markers.as_slice())); classify_hole(slot, markers, &mut skips)?; } } @@ -252,7 +251,7 @@ pub(crate) fn classify_hole( #[cfg(test)] mod tests { use super::*; - use crate::source::{test_block, SlotData}; + use crate::source::{SlotData, test_block}; fn view(start: u64, end: u64, slots: Vec) -> LedgerView { LedgerView { @@ -326,10 +325,7 @@ mod tests { block.entries[0].tx_count = 1; let error = inspect(&view(10, 10, vec![block])).unwrap_err(); - assert!( - error.to_string().contains("missing status"), - "{error:#}" - ); + assert!(error.to_string().contains("missing status"), "{error:#}"); } #[test] @@ -459,7 +455,11 @@ mod tests { fn an_unproven_hole_is_an_error() { let block = test_block(10, 9, [4; 32], [3; 32]); let error = inspect(&view(10, 11, vec![block])).unwrap_err(); - assert!(error.to_string().contains("does not prove a skip") || error.to_string().contains("no footer proof"), "{error:#}"); + assert!( + error.to_string().contains("does not prove a skip") + || error.to_string().contains("no footer proof"), + "{error:#}" + ); } #[test] @@ -488,5 +488,4 @@ mod tests { "{error:#}" ); } - } diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 7def1d3..a785679 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::fs::{self, File}; use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; @@ -6,17 +6,16 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use blockzilla_archive_v2::{ ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, ARCHIVE_V2_BLOCKS_FILE, - ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, ARCHIVE_V2_SHREDDING_FILE, - ARCHIVE_V2_SIGNATURES_FILE, - ARCHIVE_V2_TX_FLAG_HAS_ERROR, ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, - ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES, ARCHIVE_V2_TX_FLAG_HAS_LOGS, - ARCHIVE_V2_TX_FLAG_HAS_METADATA, ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA, - ARCHIVE_V2_TX_FLAG_HAS_TOKEN_BALANCES, ARCHIVE_V2_TX_FLAG_MESSAGE_V0, - ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, ArchiveV2HotBlockBlob, ArchiveV2HotBlockHeader, - ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction, ArchiveV2HotInstructionData, - ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload, ArchiveV2HotRewards, ArchiveV2HotTxRow, - ArchiveV2HotV0Message, ArchiveV2HotV1Message, WincodeArchiveV2PohRecord, - WincodeArchiveV2ShreddingRecord, write_archive_v2_hot_block_index, + ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, + ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_TX_FLAG_HAS_ERROR, + ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES, + ARCHIVE_V2_TX_FLAG_HAS_LOGS, ARCHIVE_V2_TX_FLAG_HAS_METADATA, + ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA, ARCHIVE_V2_TX_FLAG_HAS_TOKEN_BALANCES, + ARCHIVE_V2_TX_FLAG_MESSAGE_V0, ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, ArchiveV2HotBlockBlob, + ArchiveV2HotBlockHeader, ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction, + ArchiveV2HotInstructionData, ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload, + ArchiveV2HotRewards, ArchiveV2HotTxRow, ArchiveV2HotV0Message, ArchiveV2HotV1Message, + WincodeArchiveV2PohRecord, WincodeArchiveV2ShreddingRecord, write_archive_v2_hot_block_index, }; use blockzilla_compact::parse_logs_with_compactor; use blockzilla_compact::{ @@ -26,6 +25,8 @@ use blockzilla_compact::{ }; use blockzilla_primitives::{CompactPubkey, WincodeLeb128FramedWriter, wincode_leb128_config}; use blockzilla_registry::{KeyIndex, write_registry}; +use solana_entry::block_component::BlockComponent; +use solana_ledger::blockstore::Blockstore; use crate::check::Inspected; use crate::ledger::{open_read_only, scan_range}; @@ -38,6 +39,8 @@ use crate::source::{ pub(crate) const ZSTD_LEVEL: i32 = 3; const INDEX_FLAGS: u32 = 0; const VOTE_HASH_ROW_LEN: usize = 65; +const RECENT_BLOCKHASH_SLOT_WINDOW: u64 = 150; +const ROLLING_BLOCKHASH_CAPACITY: usize = 300; type PubkeyCounts = HashMap<[u8; 32], u32>; pub(crate) fn build_archive( @@ -60,10 +63,26 @@ pub(crate) fn build_archive( Ok(()) })?; let key_index = prepare_registry(&temps, counts)?; + let first_slot = inspected + .block_slots + .first() + .copied() + .context("range has no block")?; + progress(format!( + "reading previous blockhashes before slot {first_slot}" + )); + let tail = read_previous_tail(&blockstore, first_slot)?; + let newest = tail + .last() + .context("first block is missing parent entries")?; + if newest.hash != inspected.boundary_hash { + bail!("slot {first_slot} parent blockhash does not match the rooted block"); + } + progress(format!("read {} previous blockhashes", tail.len())); progress(format!( "writing blocks for slots {start_slot} to {end_slot}" )); - let mut writer = ArchiveWriter::create(&temps, key_index)?; + let mut writer = ArchiveWriter::create(&temps, key_index, tail, false)?; scan_range(&blockstore, start_slot, end_slot, |block| { writer.write_block(block) })?; @@ -76,7 +95,11 @@ pub(crate) fn build_archive( result } -pub(crate) fn write_archive(output: &Path, inspected: &Inspected, mut load_block: F) -> Result<()> +pub(crate) fn write_archive( + output: &Path, + inspected: &Inspected, + mut load_block: F, +) -> Result<()> where F: FnMut(u64) -> Result, { @@ -90,7 +113,7 @@ where count_slot_pubkeys(&block, &mut counts); } let key_index = prepare_registry(&temps, counts)?; - let mut writer = ArchiveWriter::create(&temps, key_index)?; + let mut writer = ArchiveWriter::create(&temps, key_index, Vec::new(), true)?; for slot in &inspected.block_slots { let block = load_block(*slot)?; writer.write_block(&block)?; @@ -111,6 +134,7 @@ struct ArchivePaths { poh: PathBuf, shredding: PathBuf, registry: PathBuf, + tail: PathBuf, } fn temp_paths(output: &Path) -> ArchivePaths { @@ -122,10 +146,11 @@ fn temp_paths(output: &Path) -> ArchivePaths { poh: output.join(format!("{ARCHIVE_V2_POH_FILE}.tmp")), shredding: output.join(format!("{ARCHIVE_V2_SHREDDING_FILE}.tmp")), registry: output.join(format!("{ARCHIVE_V2_PUBKEY_REGISTRY_FILE}.tmp")), + tail: output.join(format!("{ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE}.tmp")), } } -fn temp_file_list(temps: &ArchivePaths) -> [&Path; 7] { +fn temp_file_list(temps: &ArchivePaths) -> [&Path; 8] { [ &temps.blocks, &temps.signatures, @@ -134,6 +159,7 @@ fn temp_file_list(temps: &ArchivePaths) -> [&Path; 7] { &temps.poh, &temps.shredding, &temps.registry, + &temps.tail, ] } @@ -157,6 +183,9 @@ struct ArchiveWriter { shredding: WincodeLeb128FramedWriter>, blockhashes: Vec, vote_hashes: Vec, + tail: Vec, + rolling: RollingBlockhashes, + seed_parent_on_first_block: bool, rows: Vec, compressed_offset: u64, next_tx_ordinal: u64, @@ -166,7 +195,14 @@ struct ArchiveWriter { } impl ArchiveWriter { - fn create(temps: &ArchivePaths, key_index: KeyIndex) -> Result { + fn create( + temps: &ArchivePaths, + key_index: KeyIndex, + tail: Vec, + seed_parent_on_first_block: bool, + ) -> Result { + let mut rolling = RollingBlockhashes::default(); + rolling.seed(&tail)?; Ok(Self { blocks: BufWriter::new( File::create(&temps.blocks) @@ -186,6 +222,9 @@ impl ArchiveWriter { )), blockhashes: Vec::new(), vote_hashes: Vec::new(), + tail, + rolling, + seed_parent_on_first_block, rows: Vec::new(), compressed_offset: 0, next_tx_ordinal: 0, @@ -199,6 +238,7 @@ impl ArchiveWriter { poh: temps.poh.clone(), shredding: temps.shredding.clone(), registry: temps.registry.clone(), + tail: temps.tail.clone(), }, }) } @@ -206,13 +246,21 @@ impl ArchiveWriter { fn write_block(&mut self, block: &SlotData) -> Result<()> { let slot = block.slot; let block_id = u32::try_from(self.rows.len()).context("archive block id exceeds u32")?; - if self.rows.is_empty() { - let boundary = block + if self.seed_parent_on_first_block && self.rows.is_empty() { + let parent_slot = block + .parent_slot + .with_context(|| format!("slot {slot} has no parent slot"))?; + let hash = block .previous_blockhash .context("first block is missing parent entries")?; - self.blockhashes.extend(boundary); + self.tail = vec![PreviousBlockhash { + hash, + slot: parent_slot, + }]; + self.rolling.seed(&self.tail)?; } - let uncompressed = encode_hot_block(block, block_id, &self.key_index)?; + self.rolling.prune_for_slot(slot)?; + let uncompressed = encode_hot_block(block, block_id, &self.key_index, &self.rolling)?; let compressed = zstd::bulk::compress(&uncompressed, ZSTD_LEVEL) .with_context(|| format!("compress slot {slot}"))?; let compressed_len = @@ -224,6 +272,11 @@ impl ArchiveWriter { .with_context(|| format!("write slot {slot}"))?; let signature_count = append_signatures(&mut self.signatures, block)?; self.blockhashes.extend(block.blockhash); + self.rolling.insert( + block.blockhash, + i32::try_from(block_id).context("blockhash id exceeds i32")?, + slot, + )?; self.vote_hashes.extend(vote_hash_row(block)?); self.poh .write(&poh_record(block, block_id)) @@ -287,6 +340,8 @@ impl ArchiveWriter { .with_context(|| format!("write {}", self.paths.blockhash.display()))?; fs::write(&self.paths.vote, &self.vote_hashes) .with_context(|| format!("write {}", self.paths.vote.display()))?; + fs::write(&self.paths.tail, tail_bytes(&self.tail)) + .with_context(|| format!("write {}", self.paths.tail.display()))?; rename_finished(&self.paths.blocks, &output.join(ARCHIVE_V2_BLOCKS_FILE))?; rename_finished( &self.paths.signatures, @@ -296,6 +351,10 @@ impl ArchiveWriter { &self.paths.blockhash, &output.join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE), )?; + rename_finished( + &self.paths.tail, + &output.join(ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE), + )?; rename_finished( &self.paths.vote, &output.join(ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE), @@ -332,6 +391,213 @@ fn rename_finished(from: &Path, to: &Path) -> Result<()> { fs::rename(from, to).with_context(|| format!("rename {} to {}", from.display(), to.display())) } +#[derive(Clone, Copy)] +struct PreviousBlockhash { + hash: [u8; 32], + slot: u64, +} + +#[derive(Clone, Copy)] +struct RollingBlockhash { + hash: [u8; 32], + block_id: i32, + slot: u64, +} + +struct RollingBlockhashes { + map: HashMap<[u8; 32], (i32, u64)>, + order: VecDeque, +} + +impl Default for RollingBlockhashes { + fn default() -> Self { + Self { + map: HashMap::with_capacity(ROLLING_BLOCKHASH_CAPACITY), + order: VecDeque::with_capacity(ROLLING_BLOCKHASH_CAPACITY), + } + } +} + +impl RollingBlockhashes { + fn seed(&mut self, tail: &[PreviousBlockhash]) -> Result<()> { + for (index, item) in tail.iter().enumerate() { + let distance_from_newest = + i32::try_from(tail.len() - index).context("previous blockhash tail exceeds i32")?; + self.insert(item.hash, -distance_from_newest, item.slot)?; + } + Ok(()) + } + + fn insert(&mut self, hash: [u8; 32], block_id: i32, slot: u64) -> Result<()> { + if self.map.contains_key(&hash) { + bail!( + "duplicate blockhash {} at block_id {block_id} slot {slot}", + hash_text(&hash) + ); + } + self.map.insert(hash, (block_id, slot)); + self.order.push_back(RollingBlockhash { + hash, + block_id, + slot, + }); + self.enforce_capacity(); + Ok(()) + } + + fn prune_for_slot(&mut self, current_slot: u64) -> Result<()> { + loop { + let Some(entry) = self.order.front().copied() else { + return Ok(()); + }; + if entry.slot > current_slot { + bail!( + "block slots went backwards: rolling blockhash slot {} is ahead of current slot {current_slot}", + entry.slot + ); + } + if current_slot - entry.slot <= RECENT_BLOCKHASH_SLOT_WINDOW { + return Ok(()); + } + self.remove_front(); + } + } + + fn resolve( + &self, + hash: &[u8; 32], + slot: u64, + tx_index: usize, + ) -> Result { + let Some((block_id, hash_slot)) = self.map.get(hash).copied() else { + return Ok(OwnedCompactRecentBlockhash::Nonce(*hash)); + }; + if hash_slot > slot { + bail!( + "slot {slot} tx#{tx_index} recent_blockhash {} points at future block slot {hash_slot}", + hash_text(hash) + ); + } + if slot - hash_slot > RECENT_BLOCKHASH_SLOT_WINDOW { + return Ok(OwnedCompactRecentBlockhash::Nonce(*hash)); + } + Ok(OwnedCompactRecentBlockhash::Id(block_id)) + } + + fn enforce_capacity(&mut self) { + while self.order.len() > ROLLING_BLOCKHASH_CAPACITY { + self.remove_front(); + } + } + + fn remove_front(&mut self) { + if let Some(old) = self.order.pop_front() + && self.map.get(&old.hash).copied() == Some((old.block_id, old.slot)) + { + self.map.remove(&old.hash); + } + } +} + +fn tail_bytes(tail: &[PreviousBlockhash]) -> Vec { + let mut bytes = Vec::with_capacity(tail.len() * 40); + for item in tail { + bytes.extend_from_slice(&item.hash); + bytes.extend_from_slice(&item.slot.to_le_bytes()); + } + bytes +} + +fn hash_text(hash: &[u8; 32]) -> String { + bs58::encode(hash).into_string() +} + +fn copy32(bytes: &[u8]) -> Result<[u8; 32]> { + <[u8; 32]>::try_from(bytes).context("hash is not 32 bytes") +} + +/// Blockhashes of at most 300 rooted blocks before `first_slot`, oldest first. +/// +/// The parent of `first_slot` is required. An older missing block ends the walk. +fn read_previous_tail(blockstore: &Blockstore, first_slot: u64) -> Result> { + let meta = blockstore + .meta(first_slot) + .with_context(|| format!("read SlotMeta for slot {first_slot}"))?; + let Some(meta) = meta else { + bail!("slot {first_slot} has no SlotMeta"); + }; + let Some(mut slot) = meta.parent_slot else { + bail!("slot {first_slot} has no parent slot"); + }; + + let mut newest_first = Vec::new(); + let mut required = true; + while newest_first.len() < ROLLING_BLOCKHASH_CAPACITY { + let Some(ancestor) = read_ancestor(blockstore, slot)? else { + if required { + bail!("slot {slot} is missing parent entries"); + } + break; + }; + newest_first.push(PreviousBlockhash { + hash: ancestor.hash, + slot, + }); + let Some(parent) = ancestor.parent_slot else { + break; + }; + if parent >= slot { + bail!("slot {slot} parent {parent} does not move backward"); + } + slot = parent; + required = false; + } + newest_first.reverse(); + Ok(newest_first) +} + +struct AncestorBlock { + hash: [u8; 32], + parent_slot: Option, +} + +fn read_ancestor(blockstore: &Blockstore, slot: u64) -> Result> { + let Some(meta) = blockstore + .meta(slot) + .with_context(|| format!("read SlotMeta for slot {slot}"))? + else { + return Ok(None); + }; + if !blockstore.is_root(slot) || !meta.is_full() { + return Ok(None); + } + let Some(hash) = blockhash_from_shreds(blockstore, slot)? else { + return Ok(None); + }; + Ok(Some(AncestorBlock { + hash, + parent_slot: meta.parent_slot, + })) +} + +fn blockhash_from_shreds(blockstore: &Blockstore, slot: u64) -> Result> { + let (components, _, is_full) = blockstore + .get_slot_components_with_shred_info(slot, 0, false) + .with_context(|| format!("read shreds for slot {slot}"))?; + if !is_full { + return Ok(None); + } + let mut hash = None; + for component in components { + if let BlockComponent::EntryBatch(entries) = component + && let Some(entry) = entries.last() + { + hash = Some(copy32(entry.hash.as_ref())?); + } + } + Ok(hash) +} + fn prepare_registry(temps: &ArchivePaths, counts: PubkeyCounts) -> Result { let keys = ordered_registry_keys(counts); write_registry(&temps.registry, &keys) @@ -421,6 +687,7 @@ pub(crate) fn encode_hot_block( block: &SlotData, block_id: u32, key_index: &KeyIndex, + rolling: &RollingBlockhashes, ) -> Result> { let entry_txs = block.entries.iter().try_fold(0u32, |total, entry| { total @@ -448,8 +715,10 @@ pub(crate) fn encode_hot_block( let row = encode_tx_row( tx, status, - u32::try_from(tx_index).context("transaction index exceeds u32")?, + tx_index, key_index, + rolling, + block.slot, &mut message_bytes, &mut metadata_bytes, )?; @@ -474,10 +743,8 @@ pub(crate) fn encode_hot_block( parent_slot: block .parent_slot .with_context(|| format!("slot {} has no parent slot", block.slot))?, - blockhash_id: block_id - .checked_add(1) - .context("blockhash id exceeds u32")?, - previous_blockhash_id: block_id, + blockhash_id, + previous_blockhash_id: block_id.saturating_sub(1), block_time: block.block_time, block_height: block.block_height, rewards, @@ -496,12 +763,14 @@ pub(crate) fn encode_hot_block( fn encode_tx_row( tx: &SourceTx, status: &SourceStatus, - tx_index: u32, + tx_index: usize, key_index: &KeyIndex, + rolling: &RollingBlockhashes, + slot: u64, message_bytes: &mut Vec, metadata_bytes: &mut Vec, ) -> Result { - let (payload, mut flags) = encode_message(&tx.message, key_index)?; + let (payload, mut flags) = encode_message(&tx.message, key_index, rolling, slot, tx_index)?; flags |= ARCHIVE_V2_TX_FLAG_HAS_METADATA; if status.err.is_some() { flags |= ARCHIVE_V2_TX_FLAG_HAS_ERROR; @@ -526,7 +795,7 @@ fn encode_tx_row( .context("serialize transaction message")?; let metadata = wincode::config::serialize(&compact_meta(status, key_index), wincode_leb128_config()) - .context("serialize transaction metadata")?; + .context("serialize transaction metadata")?; let message_offset = u32::try_from(message_bytes.len()).context("message region exceeds u32")?; let message_len = u32::try_from(message.len()).context("message exceeds u32")?; @@ -539,7 +808,7 @@ fn encode_tx_row( u8::try_from(tx.signatures.len()).context("transaction has more than 255 signatures")?; Ok(ArchiveV2HotTxRow { - tx_index, + tx_index: u32::try_from(tx_index).context("transaction index exceeds u32")?, flags, message_offset, message_len, @@ -553,13 +822,22 @@ fn encode_tx_row( fn encode_message( message: &SourceMessage, key_index: &KeyIndex, + rolling: &RollingBlockhashes, + slot: u64, + tx_index: usize, ) -> Result<(ArchiveV2HotMessagePayload, u32)> { + let recent_blockhash = match message { + SourceMessage::Legacy(body) => body.recent_blockhash, + SourceMessage::V0(body) => body.recent_blockhash, + SourceMessage::V1(body) => body.recent_blockhash, + }; + let recent_blockhash = rolling.resolve(&recent_blockhash, slot, tx_index)?; match message { SourceMessage::Legacy(body) => Ok(( ArchiveV2HotMessagePayload::Legacy(ArchiveV2HotLegacyMessage { header: compact_header(body), account_keys: compact_keys(&body.account_keys, key_index), - recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), + recent_blockhash, instructions: raw_instructions(&body.instructions), }), 0, @@ -568,7 +846,7 @@ fn encode_message( ArchiveV2HotMessagePayload::V0(ArchiveV2HotV0Message { header: compact_header(body), account_keys: compact_keys(&body.account_keys, key_index), - recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), + recent_blockhash, instructions: raw_instructions(&body.instructions), address_table_lookups: body .lookups @@ -592,7 +870,7 @@ fn encode_message( heap_size: body.config.heap_size, }, account_keys: compact_keys(&body.account_keys, key_index), - recent_blockhash: OwnedCompactRecentBlockhash::Nonce(body.recent_blockhash), + recent_blockhash, instructions: raw_instructions(&body.instructions), }), 0, @@ -689,12 +967,17 @@ fn compact_inner_group(group: &SourceInnerGroup) -> CompactInnerInstructions { } } -fn compact_token_balance(balance: &SourceTokenBalance, key_index: &KeyIndex) -> CompactTokenBalance { +fn compact_token_balance( + balance: &SourceTokenBalance, + key_index: &KeyIndex, +) -> CompactTokenBalance { CompactTokenBalance { account_index: balance.account_index, mint: Some(key_index.compact(&balance.mint)), owner: balance.owner.map(|owner| key_index.compact(&owner)), - program_id: balance.program_id.map(|program_id| key_index.compact(&program_id)), + program_id: balance + .program_id + .map(|program_id| key_index.compact(&program_id)), amount: balance.amount, decimals: balance.decimals, } @@ -798,13 +1081,19 @@ mod tests { #[test] fn empty_tower_tail_is_two_zero_bytes_and_has_no_consensus_id() { let block = test_block(10, 9, [2; 32], [1; 32]); - let bytes = encode_hot_block(&block, 0, &index_for(&block)).unwrap(); + let bytes = encode_hot_block( + &block, + 0, + &index_for(&block), + &RollingBlockhashes::default(), + ) + .unwrap(); assert_eq!(&bytes[bytes.len() - 2..], &[0, 0]); let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); assert_eq!(decoded.consensus_block_id, None); assert!(decoded.block_markers.is_empty()); assert_eq!(decoded.header.previous_blockhash_id, 0); - assert_eq!(decoded.header.blockhash_id, 1); + assert_eq!(decoded.header.blockhash_id, 0); } #[test] @@ -815,7 +1104,13 @@ mod tests { block.consensus_block_id = Some(consensus_id); block.markers = vec![header_marker(9, parent_id)]; - let bytes = encode_hot_block(&block, 0, &index_for(&block)).unwrap(); + let bytes = encode_hot_block( + &block, + 0, + &index_for(&block), + &RollingBlockhashes::default(), + ) + .unwrap(); let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); assert_eq!(decoded.consensus_block_id, Some(consensus_id)); assert_eq!( @@ -889,10 +1184,13 @@ mod tests { assert_eq!(slots, vec![10, 12]); let hashes = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE)).unwrap(); - assert_eq!(hashes.len(), 96); - assert_eq!(&hashes[..32], &[3; 32]); - assert_eq!(&hashes[32..64], &[4; 32]); - assert_eq!(&hashes[64..], &[5; 32]); + assert_eq!(hashes.len(), 64); + assert_eq!(&hashes[..32], &[4; 32]); + assert_eq!(&hashes[32..], &[5; 32]); + let tail = std::fs::read(dir.path().join(ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE)).unwrap(); + assert_eq!(tail.len(), 40); + assert_eq!(&tail[..32], &[3; 32]); + assert_eq!(u64::from_le_bytes(tail[32..40].try_into().unwrap()), 9); let registry = std::fs::read(dir.path().join(ARCHIVE_V2_PUBKEY_REGISTRY_FILE)).unwrap(); assert_eq!(registry.as_slice(), compute_budget_program_id().as_slice()); @@ -962,10 +1260,7 @@ mod tests { fn repeated_account_key_gets_the_lowest_counted_id() { let repeated = [4u8; 32]; let once = [9u8; 32]; - let block = block_with_keys(vec![ - vec![repeated, repeated], - vec![once], - ]); + let block = block_with_keys(vec![vec![repeated, repeated], vec![once]]); let keys = registry_keys_for(&block); assert_eq!(keys.len(), 3); assert_eq!(keys[0], compute_budget_program_id()); @@ -973,13 +1268,27 @@ mod tests { assert_eq!(keys[2], once); assert_eq!(keys.len() * 32, 96); - let (message, _) = encode_message(&block.transactions[0].message, &index_for(&block)).unwrap(); + let (message, _) = encode_message( + &block.transactions[0].message, + &index_for(&block), + &RollingBlockhashes::default(), + block.slot, + 0, + ) + .unwrap(); let ArchiveV2HotMessagePayload::Legacy(message) = message else { panic!("expected a legacy message"); }; assert_eq!(message.account_keys[0], CompactPubkey::id(2)); assert_eq!(message.account_keys[1], CompactPubkey::id(2)); - let (other, _) = encode_message(&block.transactions[1].message, &index_for(&block)).unwrap(); + let (other, _) = encode_message( + &block.transactions[1].message, + &index_for(&block), + &RollingBlockhashes::default(), + block.slot, + 1, + ) + .unwrap(); let ArchiveV2HotMessagePayload::Legacy(other) = other else { panic!("expected a legacy message"); }; @@ -1009,7 +1318,13 @@ mod tests { let keys = registry_keys_for(&block); assert_eq!(keys, vec![compute_budget_program_id()]); let decoded = deserialize_archive_v2_hot_block_blob( - &encode_hot_block(&block, 0, &index_for(&block)).unwrap(), + &encode_hot_block( + &block, + 0, + &index_for(&block), + &RollingBlockhashes::default(), + ) + .unwrap(), ) .unwrap(); assert_eq!( @@ -1030,7 +1345,13 @@ mod tests { commission: None, }); let decoded = deserialize_archive_v2_hot_block_blob( - &encode_hot_block(&block, 0, &index_for(&block)).unwrap(), + &encode_hot_block( + &block, + 0, + &index_for(&block), + &RollingBlockhashes::default(), + ) + .unwrap(), ) .unwrap(); assert_eq!( @@ -1039,6 +1360,125 @@ mod tests { ); } + #[test] + fn a_recent_blockhash_from_the_previous_block_uses_its_id() { + let mut rolling = RollingBlockhashes::default(); + rolling.insert([4; 32], 0, 10).unwrap(); + let mut block = block_with_keys(vec![vec![[1; 32]]]); + block.slot = 11; + block.parent_slot = Some(10); + set_recent_blockhash(&mut block, [4; 32]); + let bytes = encode_hot_block(&block, 1, &index_for(&block), &rolling).unwrap(); + let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); + assert_eq!(decoded.header.blockhash_id, 1); + assert_eq!(decoded.header.previous_blockhash_id, 0); + assert_eq!( + recent_blockhash_of(&bytes), + OwnedCompactRecentBlockhash::Id(0) + ); + } + + #[test] + fn an_unknown_recent_blockhash_stays_raw() { + let block = block_with_keys(vec![vec![[1; 32]]]); + let bytes = encode_hot_block( + &block, + 0, + &index_for(&block), + &RollingBlockhashes::default(), + ) + .unwrap(); + assert_eq!( + recent_blockhash_of(&bytes), + OwnedCompactRecentBlockhash::Nonce([8; 32]) + ); + } + + #[test] + fn a_blockhash_older_than_the_window_stays_raw() { + let mut rolling = RollingBlockhashes::default(); + rolling.insert([8; 32], 0, 10).unwrap(); + let mut block = block_with_keys(vec![vec![[1; 32]]]); + block.slot = 160; + block.parent_slot = Some(159); + let bytes = encode_hot_block(&block, 1, &index_for(&block), &rolling).unwrap(); + assert_eq!( + recent_blockhash_of(&bytes), + OwnedCompactRecentBlockhash::Id(0) + ); + + block.slot = 161; + block.parent_slot = Some(160); + let bytes = encode_hot_block(&block, 1, &index_for(&block), &rolling).unwrap(); + assert_eq!( + recent_blockhash_of(&bytes), + OwnedCompactRecentBlockhash::Nonce([8; 32]) + ); + } + + #[test] + fn a_parent_hash_before_the_window_uses_a_negative_id() { + let mut block = block_with_keys(vec![vec![[1; 32]]]); + set_recent_blockhash(&mut block, [1; 32]); + let view = LedgerView { + start_slot: 10, + end_slot: 10, + slots: [(block.slot, block.clone())].into_iter().collect(), + }; + let inspected = inspect(&view).unwrap(); + let dir = tempfile::tempdir().unwrap(); + write_archive(dir.path(), &inspected, |_| Ok(block.clone())).unwrap(); + + let tail = std::fs::read(dir.path().join(ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE)).unwrap(); + assert_eq!(tail.len(), 40); + assert_eq!(&tail[..32], &[1; 32]); + assert_eq!(u64::from_le_bytes(tail[32..40].try_into().unwrap()), 9); + let hashes = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE)).unwrap(); + assert_eq!(hashes.as_slice(), &[2; 32]); + let compressed = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCKS_FILE)).unwrap(); + let raw = zstd::bulk::decompress(&compressed, 1_000_000).unwrap(); + assert_eq!( + recent_blockhash_of(&raw), + OwnedCompactRecentBlockhash::Id(-1) + ); + } + + #[test] + fn a_duplicate_blockhash_stops_the_archive() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let second = test_block(11, 10, [4; 32], [4; 32]); + let view = LedgerView { + start_slot: 10, + end_slot: 11, + slots: [(10, first.clone()), (11, second.clone())] + .into_iter() + .collect(), + }; + let inspected = inspect(&view).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let error = write_archive(dir.path(), &inspected, |slot| { + Ok(if slot == 10 { + first.clone() + } else { + second.clone() + }) + }) + .unwrap_err(); + assert!( + error.to_string().contains("duplicate blockhash"), + "{error:#}" + ); + } + + #[test] + fn a_recent_blockhash_from_a_future_slot_is_an_error() { + let mut rolling = RollingBlockhashes::default(); + rolling.insert([8; 32], 1, 20).unwrap(); + let block = block_with_keys(vec![vec![[1; 32]]]); + let error = encode_hot_block(&block, 0, &index_for(&block), &rolling).unwrap_err(); + assert!(error.to_string().contains("future block slot"), "{error:#}"); + } + fn registry_keys_for(block: &SlotData) -> Vec<[u8; 32]> { let mut counts = PubkeyCounts::new(); count_slot_pubkeys(block, &mut counts); @@ -1074,6 +1514,30 @@ mod tests { block } + fn set_recent_blockhash(block: &mut SlotData, hash: [u8; 32]) { + let SourceMessage::Legacy(body) = &mut block.transactions[0].message else { + panic!("expected a legacy message"); + }; + body.recent_blockhash = hash; + } + + fn recent_blockhash_of(bytes: &[u8]) -> OwnedCompactRecentBlockhash { + let decoded = deserialize_archive_v2_hot_block_blob(bytes).unwrap(); + let row = &decoded.tx_rows[0]; + let start = row.message_offset as usize; + let end = start + row.message_len as usize; + let message: ArchiveV2HotMessagePayload = wincode::config::deserialize( + &decoded.message_bytes[start..end], + wincode_leb128_config(), + ) + .unwrap(); + match message { + ArchiveV2HotMessagePayload::Legacy(message) => message.recent_blockhash, + ArchiveV2HotMessagePayload::V0(message) => message.recent_blockhash, + ArchiveV2HotMessagePayload::V1(message) => message.recent_blockhash, + } + } + fn skip_footer(skipped_slot: u64) -> Vec { let mut certificates = vec![0, 1]; certificates.extend_from_slice(&skipped_slot.to_le_bytes()); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index d473f78..b8a74d6 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -17,8 +17,8 @@ use solana_transaction_error::TransactionError; use solana_transaction_status::{Reward, TransactionStatusMeta}; use crate::check::{ - accept_block_against, classify_abandoned_fork, classify_hole, parent_chain_relation, - ChainRelation, Inspected, PreviousBlock, + ChainRelation, Inspected, PreviousBlock, accept_block_against, classify_abandoned_fork, + classify_hole, parent_chain_relation, }; use crate::source::{ REWARD_SLOT_DISTANCE, RawComponent, SlotData, SourceEntry, SourceHeader, SourceInnerGroup, From 32de7092b6399bb744f1aa4b700d6682380b135e Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 13:56:58 +0100 Subject: [PATCH 16/38] fix issues with tests --- hivezilla/ledger-archive/src/convert.rs | 43 +++++++++++++------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index a785679..98d9aec 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -743,7 +743,7 @@ pub(crate) fn encode_hot_block( parent_slot: block .parent_slot .with_context(|| format!("slot {} has no parent slot", block.slot))?, - blockhash_id, + blockhash_id: block_id, previous_blockhash_id: block_id.saturating_sub(1), block_time: block.block_time, block_height: block.block_height, @@ -1372,10 +1372,7 @@ mod tests { let decoded = deserialize_archive_v2_hot_block_blob(&bytes).unwrap(); assert_eq!(decoded.header.blockhash_id, 1); assert_eq!(decoded.header.previous_blockhash_id, 0); - assert_eq!( - recent_blockhash_of(&bytes), - OwnedCompactRecentBlockhash::Id(0) - ); + assert_recent_id(&bytes, 0); } #[test] @@ -1388,10 +1385,7 @@ mod tests { &RollingBlockhashes::default(), ) .unwrap(); - assert_eq!( - recent_blockhash_of(&bytes), - OwnedCompactRecentBlockhash::Nonce([8; 32]) - ); + assert_recent_nonce(&bytes, [8; 32]); } #[test] @@ -1402,18 +1396,12 @@ mod tests { block.slot = 160; block.parent_slot = Some(159); let bytes = encode_hot_block(&block, 1, &index_for(&block), &rolling).unwrap(); - assert_eq!( - recent_blockhash_of(&bytes), - OwnedCompactRecentBlockhash::Id(0) - ); + assert_recent_id(&bytes, 0); block.slot = 161; block.parent_slot = Some(160); let bytes = encode_hot_block(&block, 1, &index_for(&block), &rolling).unwrap(); - assert_eq!( - recent_blockhash_of(&bytes), - OwnedCompactRecentBlockhash::Nonce([8; 32]) - ); + assert_recent_nonce(&bytes, [8; 32]); } #[test] @@ -1437,10 +1425,7 @@ mod tests { assert_eq!(hashes.as_slice(), &[2; 32]); let compressed = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCKS_FILE)).unwrap(); let raw = zstd::bulk::decompress(&compressed, 1_000_000).unwrap(); - assert_eq!( - recent_blockhash_of(&raw), - OwnedCompactRecentBlockhash::Id(-1) - ); + assert_recent_id(&raw, -1); } #[test] @@ -1521,6 +1506,22 @@ mod tests { body.recent_blockhash = hash; } + fn assert_recent_id(bytes: &[u8], id: i32) { + match recent_blockhash_of(bytes) { + OwnedCompactRecentBlockhash::Id(found) => assert_eq!(found, id), + OwnedCompactRecentBlockhash::Nonce(_) => panic!("expected recent blockhash id {id}"), + } + } + + fn assert_recent_nonce(bytes: &[u8], hash: [u8; 32]) { + match recent_blockhash_of(bytes) { + OwnedCompactRecentBlockhash::Nonce(found) => assert_eq!(found, hash), + OwnedCompactRecentBlockhash::Id(id) => { + panic!("expected a raw recent blockhash, found id {id}") + } + } + } + fn recent_blockhash_of(bytes: &[u8]) -> OwnedCompactRecentBlockhash { let decoded = deserialize_archive_v2_hot_block_blob(bytes).unwrap(); let row = &decoded.tx_rows[0]; From 455992cc15ba1dc0b2fff013b700e06d8298b185 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 14:27:47 +0100 Subject: [PATCH 17/38] empty vote data after AG --- hivezilla/ledger-archive/src/convert.rs | 20 ++++++-------------- hivezilla/ledger-archive/src/markers.rs | 15 --------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 98d9aec..5dfe255 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -30,7 +30,6 @@ use solana_ledger::blockstore::Blockstore; use crate::check::Inspected; use crate::ledger::{open_read_only, scan_range}; -use crate::markers::footer_bank_hash; use crate::source::{ SlotData, SourceInnerGroup, SourceMessage, SourceMessageBody, SourceReward, SourceStatus, SourceTokenBalance, SourceTx, SourceV1Message, @@ -277,7 +276,7 @@ impl ArchiveWriter { i32::try_from(block_id).context("blockhash id exceeds i32")?, slot, )?; - self.vote_hashes.extend(vote_hash_row(block)?); + self.vote_hashes.extend(empty_vote_hash_row()); self.poh .write(&poh_record(block, block_id)) .with_context(|| format!("write PoH for slot {slot}"))?; @@ -1039,18 +1038,8 @@ pub(crate) fn shredding_record(block: &SlotData, block_id: u32) -> WincodeArchiv } } -fn vote_hash_row(block: &SlotData) -> Result<[u8; VOTE_HASH_ROW_LEN]> { - let bank_hash = footer_bank_hash(&block.markers) - .with_context(|| format!("slot {} footer does not parse", block.slot))?; - let mut row = [0u8; VOTE_HASH_ROW_LEN]; - row[0] = u8::from(bank_hash.is_some()) | (u8::from(block.consensus_block_id.is_some()) << 1); - if let Some(hash) = bank_hash { - row[1..33].copy_from_slice(&hash); - } - if let Some(hash) = block.consensus_block_id { - row[33..65].copy_from_slice(&hash); - } - Ok(row) +fn empty_vote_hash_row() -> [u8; VOTE_HASH_ROW_LEN] { + [0; VOTE_HASH_ROW_LEN] } #[cfg(test)] @@ -1192,6 +1181,9 @@ mod tests { assert_eq!(&tail[..32], &[3; 32]); assert_eq!(u64::from_le_bytes(tail[32..40].try_into().unwrap()), 9); + let votes = std::fs::read(dir.path().join(ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE)).unwrap(); + assert_eq!(votes, vec![0u8; 65 * 2]); + let registry = std::fs::read(dir.path().join(ARCHIVE_V2_PUBKEY_REGISTRY_FILE)).unwrap(); assert_eq!(registry.as_slice(), compute_budget_program_id().as_slice()); diff --git a/hivezilla/ledger-archive/src/markers.rs b/hivezilla/ledger-archive/src/markers.rs index 01a028a..7b43ee5 100644 --- a/hivezilla/ledger-archive/src/markers.rs +++ b/hivezilla/ledger-archive/src/markers.rs @@ -26,18 +26,3 @@ pub(crate) fn reward_names(markers: &[Vec]) -> Result { } Ok(names) } - -/// Bank hash from the last footer. A block with no footer returns `None`. -pub(crate) fn footer_bank_hash(markers: &[Vec]) -> Result> { - let mut bank_hash = None; - for marker in markers { - let parsed = blockmarker::parse(marker).context("block marker does not parse")?; - let Marker::Footer(footer) = parsed else { - continue; - }; - blockmarker::decode_footer_certificates(&footer.certificates) - .context("footer certificates do not parse")?; - bank_hash = Some(footer.bank_hash); - } - Ok(bank_hash) -} From 72b978ec8a396a47c9e25baeb809124e6b5f7489 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 15:59:40 +0100 Subject: [PATCH 18/38] add ztsd option --- hivezilla/ledger-archive/src/convert.rs | 53 ++++++++++++++++++++----- hivezilla/ledger-archive/src/main.rs | 6 ++- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 5dfe255..2a967cb 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -35,7 +35,15 @@ use crate::source::{ SourceTokenBalance, SourceTx, SourceV1Message, }; -pub(crate) const ZSTD_LEVEL: i32 = 3; +pub(crate) const ZSTD_LEVEL: i32 = 1; + +fn check_zstd_level(level: i32) -> Result<()> { + let range = zstd::compression_level_range(); + if range.contains(&level) { + return Ok(()); + } + bail!("zstd level {level} is outside {} to {}", range.start(), range.end()); +} const INDEX_FLAGS: u32 = 0; const VOTE_HASH_ROW_LEN: usize = 65; const RECENT_BLOCKHASH_SLOT_WINDOW: u64 = 150; @@ -47,7 +55,9 @@ pub(crate) fn build_archive( output: &Path, start_slot: u64, end_slot: u64, + zstd_level: i32, ) -> Result { + check_zstd_level(zstd_level)?; let blockstore = open_read_only(ledger)?; fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; let temps = temp_paths(output); @@ -79,9 +89,9 @@ pub(crate) fn build_archive( } progress(format!("read {} previous blockhashes", tail.len())); progress(format!( - "writing blocks for slots {start_slot} to {end_slot}" + "writing blocks for slots {start_slot} to {end_slot} at zstd level {zstd_level}" )); - let mut writer = ArchiveWriter::create(&temps, key_index, tail, false)?; + let mut writer = ArchiveWriter::create(&temps, key_index, tail, false, zstd_level)?; scan_range(&blockstore, start_slot, end_slot, |block| { writer.write_block(block) })?; @@ -97,11 +107,13 @@ pub(crate) fn build_archive( pub(crate) fn write_archive( output: &Path, inspected: &Inspected, + zstd_level: i32, mut load_block: F, ) -> Result<()> where F: FnMut(u64) -> Result, { + check_zstd_level(zstd_level)?; fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; let temps = temp_paths(output); let result = (|| { @@ -112,7 +124,7 @@ where count_slot_pubkeys(&block, &mut counts); } let key_index = prepare_registry(&temps, counts)?; - let mut writer = ArchiveWriter::create(&temps, key_index, Vec::new(), true)?; + let mut writer = ArchiveWriter::create(&temps, key_index, Vec::new(), true, zstd_level)?; for slot in &inspected.block_slots { let block = load_block(*slot)?; writer.write_block(&block)?; @@ -185,6 +197,7 @@ struct ArchiveWriter { tail: Vec, rolling: RollingBlockhashes, seed_parent_on_first_block: bool, + zstd_level: i32, rows: Vec, compressed_offset: u64, next_tx_ordinal: u64, @@ -199,6 +212,7 @@ impl ArchiveWriter { key_index: KeyIndex, tail: Vec, seed_parent_on_first_block: bool, + zstd_level: i32, ) -> Result { let mut rolling = RollingBlockhashes::default(); rolling.seed(&tail)?; @@ -224,6 +238,7 @@ impl ArchiveWriter { tail, rolling, seed_parent_on_first_block, + zstd_level, rows: Vec::new(), compressed_offset: 0, next_tx_ordinal: 0, @@ -260,7 +275,7 @@ impl ArchiveWriter { } self.rolling.prune_for_slot(slot)?; let uncompressed = encode_hot_block(block, block_id, &self.key_index, &self.rolling)?; - let compressed = zstd::bulk::compress(&uncompressed, ZSTD_LEVEL) + let compressed = zstd::bulk::compress(&uncompressed, self.zstd_level) .with_context(|| format!("compress slot {slot}"))?; let compressed_len = u32::try_from(compressed.len()).context("compressed block exceeds u32")?; @@ -370,7 +385,7 @@ impl ArchiveWriter { write_archive_v2_hot_block_index( &output.join(ARCHIVE_V2_BLOCK_INDEX_FILE), self.compressed_offset, - ZSTD_LEVEL, + self.zstd_level, INDEX_FLAGS, &self.rows, )?; @@ -1147,7 +1162,7 @@ mod tests { let inspected = inspect(&view).unwrap(); let dir = tempfile::tempdir().unwrap(); let mut loaded = Vec::new(); - write_archive(dir.path(), &inspected, |slot| { + write_archive(dir.path(), &inspected, ZSTD_LEVEL, |slot| { loaded.push(slot); view.slots .get(&slot) @@ -1225,7 +1240,7 @@ mod tests { let inspected = inspect(&view).unwrap(); let dir = tempfile::tempdir().unwrap(); - let error = write_archive(dir.path(), &inspected, |slot| { + let error = write_archive(dir.path(), &inspected, ZSTD_LEVEL, |slot| { if slot == 11 { bail!("slot 11 is missing transaction status"); } @@ -1407,7 +1422,7 @@ mod tests { }; let inspected = inspect(&view).unwrap(); let dir = tempfile::tempdir().unwrap(); - write_archive(dir.path(), &inspected, |_| Ok(block.clone())).unwrap(); + write_archive(dir.path(), &inspected, ZSTD_LEVEL, |_| Ok(block.clone())).unwrap(); let tail = std::fs::read(dir.path().join(ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE)).unwrap(); assert_eq!(tail.len(), 40); @@ -1433,7 +1448,7 @@ mod tests { }; let inspected = inspect(&view).unwrap(); let dir = tempfile::tempdir().unwrap(); - let error = write_archive(dir.path(), &inspected, |slot| { + let error = write_archive(dir.path(), &inspected, ZSTD_LEVEL, |slot| { Ok(if slot == 10 { first.clone() } else { @@ -1498,6 +1513,24 @@ mod tests { body.recent_blockhash = hash; } + #[test] + fn the_requested_zstd_level_is_stored_in_the_index() { + let block = test_block(10, 9, [2; 32], [1; 32]); + let view = LedgerView { + start_slot: 10, + end_slot: 10, + slots: [(block.slot, block.clone())].into_iter().collect(), + }; + let inspected = inspect(&view).unwrap(); + let dir = tempfile::tempdir().unwrap(); + write_archive(dir.path(), &inspected, 3, |_| Ok(block.clone())).unwrap(); + let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); + assert_eq!(i32::from_le_bytes(index[28..32].try_into().unwrap()), 3); + + let error = write_archive(dir.path(), &inspected, 10_000, |_| Ok(block)).unwrap_err(); + assert!(error.to_string().contains("zstd level")); + } + fn assert_recent_id(bytes: &[u8], id: i32) { match recent_blockhash_of(bytes) { OwnedCompactRecentBlockhash::Id(found) => assert_eq!(found, id), diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index 1cacc11..d2e936c 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -37,6 +37,9 @@ enum Command { start_slot: u64, #[arg(long)] end_slot: u64, + /// zstd compression level for each independent block frame. + #[arg(long, default_value_t = convert::ZSTD_LEVEL)] + level: i32, }, } @@ -64,8 +67,9 @@ fn run() -> Result<()> { output, start_slot, end_slot, + level, } => { - let inspected = build_archive(&ledger, &output, start_slot, end_slot)?; + let inspected = build_archive(&ledger, &output, start_slot, end_slot, level)?; print_report(&inspected); println!("Output: {}", output.display()); Ok(()) From dece7ffa0d64ca27d0dc3af2180c643fd651b215 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 5 Nov 2026 16:01:31 +0100 Subject: [PATCH 19/38] fix test issue --- hivezilla/ledger-archive/src/convert.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 2a967cb..dccd0d9 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -1527,7 +1527,7 @@ mod tests { let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); assert_eq!(i32::from_le_bytes(index[28..32].try_into().unwrap()), 3); - let error = write_archive(dir.path(), &inspected, 10_000, |_| Ok(block)).unwrap_err(); + let error = write_archive(dir.path(), &inspected, 10_000, |_| Ok(block.clone())).unwrap_err(); assert!(error.to_string().contains("zstd level")); } From 1c87babd30bba3566ed3c49fbf553c7616153f16 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 6 Nov 2026 13:29:51 +0100 Subject: [PATCH 20/38] add multithread for reading --- hivezilla/ledger-archive/src/convert.rs | 17 +- hivezilla/ledger-archive/src/ledger.rs | 340 +++++++++++++++++++++++- hivezilla/ledger-archive/src/main.rs | 9 +- 3 files changed, 350 insertions(+), 16 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index dccd0d9..bd3c33d 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -56,8 +56,13 @@ pub(crate) fn build_archive( start_slot: u64, end_slot: u64, zstd_level: i32, + read_threads: u8, ) -> Result { check_zstd_level(zstd_level)?; + let read_threads = usize::from(read_threads); + if !(1..=16).contains(&read_threads) { + bail!("read threads {read_threads} is outside 1 to 16"); + } let blockstore = open_read_only(ledger)?; fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; let temps = temp_paths(output); @@ -67,7 +72,7 @@ pub(crate) fn build_archive( "counting pubkeys in slots {start_slot} to {end_slot}" )); let mut counts = PubkeyCounts::new(); - let inspected = scan_range(&blockstore, start_slot, end_slot, |block| { + let inspected = scan_range(&blockstore, start_slot, end_slot, read_threads, |block| { count_slot_pubkeys(block, &mut counts); Ok(()) })?; @@ -92,9 +97,13 @@ pub(crate) fn build_archive( "writing blocks for slots {start_slot} to {end_slot} at zstd level {zstd_level}" )); let mut writer = ArchiveWriter::create(&temps, key_index, tail, false, zstd_level)?; - scan_range(&blockstore, start_slot, end_slot, |block| { - writer.write_block(block) - })?; + scan_range( + &blockstore, + start_slot, + end_slot, + read_threads, + |block| writer.write_block(block), + )?; writer.finish(output)?; Ok(inspected) })(); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index b8a74d6..30b95e0 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -1,5 +1,8 @@ +use std::collections::{BTreeMap, HashSet}; use std::io::{self, Write}; use std::path::Path; +use std::sync::{Mutex, mpsc}; +use std::thread; use anyhow::{Context, Result, anyhow, bail}; use blockzilla_compact::{CompactInstructionError, CompactTransactionError}; @@ -56,6 +59,201 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { .with_context(|| format!("open ledger {}", ledger.display())) } +const MAX_READ_THREADS: usize = 16; + +struct LoadPool { + job_tx: mpsc::Sender>, + res_rx: mpsc::Receiver<(u64, Result)>, + inflight: HashSet, + ready: BTreeMap, + failed: Option<(u64, anyhow::Error)>, + threads: usize, + stopped: bool, +} + +struct OrderedLoads<'a, T> { + inline: Option<&'a dyn Fn(u64) -> Result>, + pool: Option>, +} + +impl OrderedLoads<'_, T> { + fn submit(&mut self, slot: u64) -> Result<()> { + let Some(pool) = &mut self.pool else { + return Ok(()); + }; + if pool.inflight.contains(&slot) || pool.ready.contains_key(&slot) { + return Ok(()); + } + pool.job_tx + .send(Some(slot)) + .context("read thread stopped")?; + pool.inflight.insert(slot); + Ok(()) + } + + fn inflight_count(&self) -> usize { + self.pool.as_ref().map(|pool| pool.inflight.len()).unwrap_or(0) + } + + fn thread_limit(&self) -> usize { + self.pool.as_ref().map(|pool| pool.threads).unwrap_or(1) + } + + fn is_pending(&self, slot: u64) -> bool { + self.pool.as_ref().is_some_and(|pool| { + pool.inflight.contains(&slot) || pool.ready.contains_key(&slot) + }) + } + + fn take(&mut self, slot: u64) -> Result { + if let Some(load) = self.inline { + return load(slot); + } + let failed_slot = self + .pool + .as_ref() + .and_then(|pool| pool.failed.as_ref().map(|(slot, _)| *slot)); + if let Some(failed_slot) = failed_slot + && failed_slot <= slot + { + return Err(self.pool.as_mut().unwrap().failed.take().unwrap().1); + } + if let Some(value) = self.pool.as_mut().unwrap().ready.remove(&slot) { + return Ok(value); + } + if !self.pool.as_ref().unwrap().inflight.contains(&slot) { + self.submit(slot)?; + } + self.wait(slot) + } + + fn wait(&mut self, slot: u64) -> Result { + loop { + if let Some(value) = self.pool.as_mut().unwrap().ready.remove(&slot) { + return Ok(value); + } + if !self.pool.as_ref().unwrap().inflight.contains(&slot) { + bail!("slot {slot} was not loaded"); + } + let (done, result) = self.recv()?; + match result { + Ok(value) if done == slot => return Ok(value), + Ok(value) => { + self.pool.as_mut().unwrap().ready.insert(done, value); + } + Err(err) => { + self.note_failure(done, err); + if done == slot { + return Err(self.pool.as_mut().unwrap().failed.take().unwrap().1); + } + } + } + } + } + + fn note_failure(&mut self, slot: u64, err: anyhow::Error) { + let pool = self.pool.as_mut().unwrap(); + match pool.failed.as_ref() { + Some((previous, _)) if *previous <= slot => {} + _ => pool.failed = Some((slot, err)), + } + } + + fn recv(&mut self) -> Result<(u64, Result)> { + let pool = self.pool.as_mut().unwrap(); + let (slot, result) = pool.res_rx.recv().context("read thread stopped")?; + pool.inflight.remove(&slot); + Ok((slot, result)) + } + + fn stop(&mut self) { + let Some(pool) = &mut self.pool else { + return; + }; + if pool.stopped { + return; + } + pool.stopped = true; + for _ in 0..pool.threads { + let _ = pool.job_tx.send(None); + } + while !pool.inflight.is_empty() { + match pool.res_rx.recv() { + Ok((slot, _)) => { + pool.inflight.remove(&slot); + } + Err(_) => break, + } + } + } +} + +impl Drop for OrderedLoads<'_, T> { + fn drop(&mut self) { + self.stop(); + } +} + +fn with_ordered_loads( + threads: usize, + load: &(impl Fn(u64) -> Result + Sync), + body: impl FnOnce(&mut OrderedLoads<'_, T>) -> Result, +) -> Result +where + T: Send, +{ + if threads == 1 { + let mut loads = OrderedLoads { + inline: Some(load), + pool: None, + }; + return body(&mut loads); + } + thread::scope(|scope| { + let (job_tx, job_rx) = mpsc::channel::>(); + let job_rx = Mutex::new(job_rx); + let (res_tx, res_rx) = mpsc::channel::<(u64, Result)>(); + for _ in 0..threads { + let res_tx = res_tx.clone(); + let job_rx = &job_rx; + let load = load; + scope.spawn(move || { + loop { + let job = { + let rx = job_rx.lock().expect("read thread lock"); + rx.recv() + }; + match job { + Ok(Some(slot)) => { + let result = load(slot); + if res_tx.send((slot, result)).is_err() { + break; + } + } + _ => break, + } + } + }); + } + drop(res_tx); + let mut loads = OrderedLoads { + inline: None, + pool: Some(LoadPool { + job_tx, + res_rx, + inflight: HashSet::new(), + ready: BTreeMap::new(), + failed: None, + threads, + stopped: false, + }), + }; + let result = body(&mut loads); + loads.stop(); + result + }) +} + /// Read each slot once. A rooted full block is checked, passed to `on_block`, /// then dropped. A non-root slot is left out when the next root jumps over it /// and slot N+8 has a skip certificate. A hole is proved from the footer of @@ -63,19 +261,51 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { /// /// When that footer belongs to a later block in the window, the block is kept /// until the scan reaches it, so its shreds are not read a second time. +/// `read_threads` above 1 reconstructs later rooted slots while the current +/// slot is still checked. Results are accepted in slot order. pub(crate) fn scan_range( blockstore: &Blockstore, start_slot: u64, end_slot: u64, + read_threads: usize, mut on_block: F, ) -> Result where F: FnMut(&SlotData) -> Result<()>, { + if !(1..=MAX_READ_THREADS).contains(&read_threads) { + bail!("read threads {read_threads} is outside 1 to {MAX_READ_THREADS}"); + } if start_slot > end_slot { bail!("start slot {start_slot} is after end slot {end_slot}"); } - progress(format!("reading slots {start_slot} to {end_slot}")); + progress(format!( + "reading slots {start_slot} to {end_slot} with {read_threads} threads" + )); + let load = |slot: u64| load_rooted_block(blockstore, slot); + with_ordered_loads(read_threads, &load, |loads| { + scan_loaded( + blockstore, + loads, + start_slot, + end_slot, + read_threads, + &mut on_block, + ) + }) +} + +fn scan_loaded( + blockstore: &Blockstore, + loads: &mut OrderedLoads, + start_slot: u64, + end_slot: u64, + read_threads: usize, + on_block: &mut F, +) -> Result +where + F: FnMut(&SlotData) -> Result<()>, +{ let mut ahead = std::collections::BTreeMap::::new(); let mut block_slots = Vec::new(); let mut skips = Vec::new(); @@ -85,6 +315,12 @@ where let mut scanned = 0u64; for slot in start_slot..=end_slot { scanned += 1; + let prefetch_end = slot + .saturating_add(u64::try_from(read_threads).unwrap_or(u64::MAX).saturating_sub(1)) + .min(end_slot); + if slot < end_slot { + prefetch_rooted_slots(blockstore, loads, &ahead, slot + 1, prefetch_end)?; + } if let Some(data) = ahead.remove(&slot) { accept_one( data, @@ -93,18 +329,25 @@ where &mut previous, &mut boundary_hash, &mut block_slots, - &mut on_block, + on_block, )?; } else if let Some(meta) = blockstore .meta(slot) .with_context(|| format!("read SlotMeta for slot {slot}"))? { if !blockstore.is_root(slot) { - leave_out_abandoned_fork(blockstore, slot, end_slot, &mut ahead, &mut forks)?; + leave_out_abandoned_fork( + blockstore, + loads, + slot, + end_slot, + &mut ahead, + &mut forks, + )?; } else if !meta.is_full() { bail!("slot {slot} is rooted and is not full"); } else { - let data = load_rooted_block(blockstore, slot)?; + let data = loads.take(slot)?; accept_one( data, start_slot, @@ -112,11 +355,11 @@ where &mut previous, &mut boundary_hash, &mut block_slots, - &mut on_block, + on_block, )?; } } else { - let markers = proof_markers(blockstore, slot, end_slot, &mut ahead)?; + let markers = proof_markers(blockstore, loads, slot, end_slot, &mut ahead)?; classify_hole(slot, markers.as_deref(), &mut skips)?; } if scanned % 1000 == 0 { @@ -141,16 +384,50 @@ where }) } +fn prefetch_rooted_slots( + blockstore: &Blockstore, + loads: &mut OrderedLoads, + ahead: &BTreeMap, + start: u64, + end: u64, +) -> Result<()> { + if loads.pool.is_none() { + return Ok(()); + } + let mut slot = start; + while slot <= end && loads.inflight_count() < loads.thread_limit() { + if ahead.contains_key(&slot) || loads.is_pending(slot) { + slot += 1; + continue; + } + let Some(meta) = blockstore + .meta(slot) + .with_context(|| format!("read SlotMeta for slot {slot}"))? + else { + slot += 1; + continue; + }; + if blockstore.is_root(slot) && meta.is_full() { + loads.submit(slot)?; + } + slot += 1; + } + Ok(()) +} + fn leave_out_abandoned_fork( blockstore: &Blockstore, + loads: &mut OrderedLoads, slot: u64, end_slot: u64, - ahead: &mut std::collections::BTreeMap, + ahead: &mut BTreeMap, forks: &mut Vec, ) -> Result<()> { let relation = rooted_chain_relation(blockstore, slot)?; let markers = match relation { - ChainRelation::JumpsOver { .. } => proof_markers(blockstore, slot, end_slot, ahead)?, + ChainRelation::JumpsOver { .. } => { + proof_markers(blockstore, loads, slot, end_slot, ahead)? + } ChainRelation::DescendsFrom { .. } => None, }; let next_root = match relation { @@ -213,9 +490,10 @@ where fn proof_markers( blockstore: &Blockstore, + loads: &mut OrderedLoads, slot: u64, end_slot: u64, - ahead: &mut std::collections::BTreeMap, + ahead: &mut BTreeMap, ) -> Result>>> { let carrier = slot .checked_add(REWARD_SLOT_DISTANCE) @@ -234,7 +512,7 @@ fn proof_markers( return Ok(Some(read_markers(blockstore, carrier)?)); } if is_root && meta.is_full() { - let data = load_rooted_block(blockstore, carrier)?; + let data = loads.take(carrier)?; let markers = data.markers.clone(); ahead.insert(carrier, data); return Ok(Some(markers)); @@ -840,6 +1118,48 @@ mod tests { use solana_entry::block_component::{BlockHeaderV1, VersionedBlockMarker}; use solana_hash::Hash; + #[test] + fn parallel_slot_loads_stay_in_order_and_run_once() { + use std::sync::Mutex; + use std::time::Duration; + + let seen = Mutex::new(Vec::new()); + let got = with_ordered_loads( + 4, + |slot| { + seen.lock().expect("seen").push(slot); + std::thread::sleep(Duration::from_millis((slot % 5) * 2)); + if slot == 9 { + bail!("slot 9 failed"); + } + Ok(slot) + }, + |loads| { + for slot in 0..16 { + loads.submit(slot)?; + } + let mut got = Vec::new(); + for slot in 0..9 { + got.push(loads.take(slot)?); + } + let error = loads.take(9).unwrap_err(); + assert!(error.to_string().contains("slot 9 failed"), "{error:#}"); + Ok(got) + }, + ) + .unwrap(); + assert_eq!(got, (0..9).collect::>()); + let mut seen = seen.into_inner().expect("seen"); + seen.sort(); + assert_eq!(seen, (0..16).collect::>()); + } + + #[test] + fn one_read_thread_loads_on_the_caller() { + let value = with_ordered_loads(1, |slot| Ok(slot + 1), |loads| loads.take(4)).unwrap(); + assert_eq!(value, 5); + } + #[test] fn read_only_options_do_not_create_a_ledger() { let dir = tempfile::tempdir().unwrap(); diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index d2e936c..0b85bb9 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -40,6 +40,9 @@ enum Command { /// zstd compression level for each independent block frame. #[arg(long, default_value_t = convert::ZSTD_LEVEL)] level: i32, + /// Parallel shred reads. 1 reads one slot at a time. + #[arg(long, default_value_t = 1)] + read_threads: u8, }, } @@ -58,7 +61,7 @@ fn run() -> Result<()> { end_slot, } => { let blockstore = open_read_only(&ledger)?; - let inspected = scan_range(&blockstore, start_slot, end_slot, |_| Ok(()))?; + let inspected = scan_range(&blockstore, start_slot, end_slot, 1, |_| Ok(()))?; print_report(&inspected); Ok(()) } @@ -68,8 +71,10 @@ fn run() -> Result<()> { start_slot, end_slot, level, + read_threads, } => { - let inspected = build_archive(&ledger, &output, start_slot, end_slot, level)?; + let inspected = + build_archive(&ledger, &output, start_slot, end_slot, level, read_threads)?; print_report(&inspected); println!("Output: {}", output.display()); Ok(()) From afb775ab497b3f94b5813d1fd57cb6ed5ac22aba Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 6 Nov 2026 13:36:28 +0100 Subject: [PATCH 21/38] fix issue for multithread --- hivezilla/ledger-archive/src/ledger.rs | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 30b95e0..b7b5382 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -209,13 +209,14 @@ where }; return body(&mut loads); } - thread::scope(|scope| { - let (job_tx, job_rx) = mpsc::channel::>(); - let job_rx = Mutex::new(job_rx); - let (res_tx, res_rx) = mpsc::channel::<(u64, Result)>(); + let (job_tx, job_rx) = mpsc::channel::>(); + let job_rx = Mutex::new(job_rx); + let job_rx = &job_rx; + let (res_tx, res_rx) = mpsc::channel::<(u64, Result)>(); + thread::scope(move |scope| { for _ in 0..threads { let res_tx = res_tx.clone(); - let job_rx = &job_rx; + let job_rx = job_rx; let load = load; scope.spawn(move || { loop { @@ -1124,16 +1125,17 @@ mod tests { use std::time::Duration; let seen = Mutex::new(Vec::new()); + let load = |slot| { + seen.lock().expect("seen").push(slot); + std::thread::sleep(Duration::from_millis((slot % 5) * 2)); + if slot == 9 { + bail!("slot 9 failed"); + } + Ok(slot) + }; let got = with_ordered_loads( 4, - |slot| { - seen.lock().expect("seen").push(slot); - std::thread::sleep(Duration::from_millis((slot % 5) * 2)); - if slot == 9 { - bail!("slot 9 failed"); - } - Ok(slot) - }, + &load, |loads| { for slot in 0..16 { loads.submit(slot)?; @@ -1156,7 +1158,8 @@ mod tests { #[test] fn one_read_thread_loads_on_the_caller() { - let value = with_ordered_loads(1, |slot| Ok(slot + 1), |loads| loads.take(4)).unwrap(); + let load = |slot| Ok(slot + 1); + let value = with_ordered_loads(1, &load, |loads| loads.take(4)).unwrap(); assert_eq!(value, 5); } From 1f07f099b4d1956b09c23d34ce775243b0fbebdd Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Mon, 9 Nov 2026 14:55:07 +0100 Subject: [PATCH 22/38] add skip scenario --- hivezilla/ledger-archive/src/check.rs | 99 ++++++++++++++++++++++++- hivezilla/ledger-archive/src/convert.rs | 33 +++++++++ hivezilla/ledger-archive/src/ledger.rs | 56 +++++++++++--- 3 files changed, 175 insertions(+), 13 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index 5786e4b..280d06a 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -73,7 +73,12 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { let carrier = slot.checked_add(REWARD_SLOT_DISTANCE); let markers = carrier .and_then(|carrier| view.slots.get(&carrier).map(|data| data.markers.as_slice())); - classify_hole(slot, markers, &mut skips)?; + let chain = if markers.is_none() { + full_root_relation_in_view(view, slot)? + } else { + None + }; + classify_hole(slot, markers, chain, &mut skips)?; } } @@ -225,16 +230,27 @@ pub(crate) fn accept_block_against( Ok(()) } +/// Prove a slot that has no `SlotMeta`. +/// +/// The footer of slot N+8 is the first proof. A skip certificate records a +/// skip. A notarization certificate is an error. A footer that names neither +/// is an error. `chain` is ignored while that footer exists. +/// +/// When slot N+8 has no `SlotMeta`, `chain` is the parent walk from the next +/// full rooted slot. A jump over this slot records a skip. A chain that +/// includes this slot is an error. `chain == None` keeps the missing-footer +/// error. pub(crate) fn classify_hole( slot: u64, carrier_markers: Option<&[Vec]>, + chain: Option, skips: &mut Vec, ) -> Result<()> { let carrier = slot .checked_add(REWARD_SLOT_DISTANCE) .context("slot + 8 overflows")?; let Some(markers) = carrier_markers else { - bail!("slot {slot} has no SlotMeta and slot {carrier} has no footer proof"); + return classify_hole_by_parent(slot, carrier, chain, skips); }; let names = reward_names(markers) .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; @@ -248,6 +264,41 @@ pub(crate) fn classify_hole( bail!("slot {slot} has no SlotMeta and slot {carrier} does not prove a skip"); } +fn classify_hole_by_parent( + slot: u64, + carrier: u64, + chain: Option, + skips: &mut Vec, +) -> Result<()> { + match chain { + Some(ChainRelation::JumpsOver { .. }) => { + skips.push(slot); + Ok(()) + } + Some(ChainRelation::DescendsFrom { next_root }) => { + bail!("slot {slot} has no SlotMeta and slot {next_root} descends from it"); + } + None => bail!("slot {slot} has no SlotMeta and slot {carrier} has no footer proof"), + } +} + +/// Parent walk from the first rooted slot after `slot`. +/// +/// A rooted slot that is not full is an error. No later root returns `None`. +fn full_root_relation_in_view(view: &LedgerView, slot: u64) -> Result> { + let start = slot.checked_add(1).context("slot + 1 overflows")?; + let Some((next_root, data)) = view.slots.range(start..).find(|(_, data)| data.is_root) else { + return Ok(None); + }; + if !data.is_full { + bail!("slot {next_root} is rooted and is not full"); + } + parent_chain_relation(slot, *next_root, |cursor| { + Ok(view.slots.get(&cursor).and_then(|data| data.parent_slot)) + }) + .map(Some) +} + #[cfg(test)] mod tests { use super::*; @@ -451,6 +502,50 @@ mod tests { ); } + #[test] + fn a_parent_jump_proves_a_run_of_holes() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let next = test_block(14, 10, [5; 32], [4; 32]); + let inspected = inspect(&view(10, 14, vec![first, next])).unwrap(); + assert_eq!(inspected.block_slots, vec![10, 14]); + assert_eq!(inspected.skips, vec![11, 12, 13]); + assert_eq!(inspected.forks, Vec::::new()); + } + + #[test] + fn a_hole_whose_next_block_descends_from_it_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let next = test_block(12, 11, [5; 32], [4; 32]); + let error = inspect(&view(10, 12, vec![first, next])).unwrap_err(); + assert!(error.to_string().contains("descends from it"), "{error:#}"); + } + + #[test] + fn a_hole_followed_by_a_partial_root_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut next = test_block(12, 10, [5; 32], [4; 32]); + next.is_full = false; + let error = inspect(&view(10, 12, vec![first, next])).unwrap_err(); + assert!( + error.to_string().contains("rooted and is not full"), + "{error:#}" + ); + } + + #[test] + fn a_footer_that_does_not_name_the_hole_stays_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let next = test_block(12, 10, [5; 32], [4; 32]); + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_footer(99)]; + let error = inspect(&view(10, 12, vec![first, next, carrier])).unwrap_err(); + assert!( + error.to_string().contains("does not prove a skip"), + "{error:#}" + ); + } + #[test] fn an_unproven_hole_is_an_error() { let block = test_block(10, 9, [4; 32], [3; 32]); diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index bd3c33d..fc7cb69 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -1153,6 +1153,39 @@ mod tests { assert_eq!(record.shredding[1].shred_end_idx, 95); } + #[test] + fn a_parent_jump_skip_writes_no_blob() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let next = test_block(14, 10, [5; 32], [4; 32]); + let view = LedgerView { + start_slot: 10, + end_slot: 14, + slots: [first, next] + .into_iter() + .map(|slot| (slot.slot, slot)) + .collect(), + }; + let inspected = inspect(&view).unwrap(); + assert_eq!(inspected.skips, vec![11, 12, 13]); + let dir = tempfile::tempdir().unwrap(); + let mut loaded = Vec::new(); + write_archive(dir.path(), &inspected, ZSTD_LEVEL, |slot| { + loaded.push(slot); + view.slots + .get(&slot) + .cloned() + .with_context(|| format!("test block {slot} is missing")) + }) + .unwrap(); + assert_eq!(loaded, vec![10, 14, 10, 14]); + let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); + let slots: Vec = index[36..] + .chunks_exact(52) + .map(|row| u64::from_le_bytes(row[4..12].try_into().unwrap())) + .collect(); + assert_eq!(slots, vec![10, 14]); + } + #[test] fn a_proven_skip_writes_no_blob() { let first = test_block(10, 9, [4; 32], [3; 32]); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index b7b5382..9fb913d 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -258,7 +258,8 @@ where /// Read each slot once. A rooted full block is checked, passed to `on_block`, /// then dropped. A non-root slot is left out when the next root jumps over it /// and slot N+8 has a skip certificate. A hole is proved from the footer of -/// slot N+8. +/// slot N+8. When that slot has no `SlotMeta`, the parent chain of the next +/// full rooted block can prove the skip. /// /// When that footer belongs to a later block in the window, the block is kept /// until the scan reaches it, so its shreds are not read a second time. @@ -361,7 +362,17 @@ where } } else { let markers = proof_markers(blockstore, loads, slot, end_slot, &mut ahead)?; - classify_hole(slot, markers.as_deref(), &mut skips)?; + let chain = if markers.is_none() { + full_root_relation(blockstore, slot)? + } else { + None + }; + classify_hole(slot, markers.as_deref(), chain, &mut skips)?; + if let Some(ChainRelation::JumpsOver { next_root }) = chain { + progress(format!( + "slot {slot} left out, rooted chain jumps to {next_root}" + )); + } } if scanned % 1000 == 0 { progress(format!( @@ -444,6 +455,27 @@ fn leave_out_abandoned_fork( Ok(()) } +fn full_root_relation(blockstore: &Blockstore, slot: u64) -> Result> { + let start = slot.checked_add(1).context("slot + 1 overflows")?; + let Some(next_root) = blockstore + .rooted_slot_iterator(start) + .with_context(|| format!("read roots after slot {slot}"))? + .next() + else { + return Ok(None); + }; + let meta = blockstore + .meta(next_root) + .with_context(|| format!("read SlotMeta for slot {next_root}"))?; + let Some(meta) = meta else { + bail!("slot {next_root} has no SlotMeta"); + }; + if !meta.is_full() { + bail!("slot {next_root} is rooted and is not full"); + } + parent_chain_relation(slot, next_root, |cursor| parent_slot(blockstore, cursor)).map(Some) +} + fn rooted_chain_relation(blockstore: &Blockstore, slot: u64) -> Result { let start = slot.checked_add(1).context("slot + 1 overflows")?; let Some(next_root) = blockstore @@ -453,15 +485,17 @@ fn rooted_chain_relation(blockstore: &Blockstore, slot: u64) -> Result Result> { + let meta = blockstore + .meta(slot) + .with_context(|| format!("read SlotMeta for slot {slot}"))?; + let Some(meta) = meta else { + bail!("slot {slot} has no SlotMeta"); + }; + Ok(meta.parent_slot) } fn accept_one( From e5967d21c9e55a3faf6a9287c5a51e5b0faa9589 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Mon, 9 Nov 2026 14:59:05 +0100 Subject: [PATCH 23/38] add 32 thread as max --- hivezilla/ledger-archive/src/convert.rs | 4 ++-- hivezilla/ledger-archive/src/ledger.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index fc7cb69..833641a 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -60,8 +60,8 @@ pub(crate) fn build_archive( ) -> Result { check_zstd_level(zstd_level)?; let read_threads = usize::from(read_threads); - if !(1..=16).contains(&read_threads) { - bail!("read threads {read_threads} is outside 1 to 16"); + if !(1..=32).contains(&read_threads) { + bail!("read threads {read_threads} is outside 1 to 32"); } let blockstore = open_read_only(ledger)?; fs::create_dir_all(output).with_context(|| format!("create {}", output.display()))?; diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 9fb913d..ea8890a 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -59,7 +59,7 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { .with_context(|| format!("open ledger {}", ledger.display())) } -const MAX_READ_THREADS: usize = 16; +const MAX_READ_THREADS: usize = 32; struct LoadPool { job_tx: mpsc::Sender>, From 7c42a2bc532e70d765f94ec12981e0b3e2f4852c Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Mon, 9 Nov 2026 15:37:23 +0100 Subject: [PATCH 24/38] add notar reward bug fix --- hivezilla/ledger-archive/src/check.rs | 100 +++++++++++++++++++++++- hivezilla/ledger-archive/src/convert.rs | 63 +++++++++++++++ hivezilla/ledger-archive/src/ledger.rs | 7 +- 3 files changed, 166 insertions(+), 4 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index 280d06a..07eb6a0 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -73,7 +73,7 @@ pub(crate) fn inspect(view: &LedgerView) -> Result { let carrier = slot.checked_add(REWARD_SLOT_DISTANCE); let markers = carrier .and_then(|carrier| view.slots.get(&carrier).map(|data| data.markers.as_slice())); - let chain = if markers.is_none() { + let chain = if hole_uses_parent_chain(slot, markers)? { full_root_relation_in_view(view, slot)? } else { None @@ -230,11 +230,28 @@ pub(crate) fn accept_block_against( Ok(()) } +/// A hole walks parents when slot N+8 has no footer, and when that footer +/// carries both a skip reward and a notarization reward for `slot`. +pub(crate) fn hole_uses_parent_chain(slot: u64, markers: Option<&[Vec]>) -> Result { + let Some(markers) = markers else { + return Ok(true); + }; + let carrier = slot + .checked_add(REWARD_SLOT_DISTANCE) + .context("slot + 8 overflows")?; + let names = reward_names(markers) + .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; + Ok(names.skips.contains(&slot) && names.notars.contains(&slot)) +} + /// Prove a slot that has no `SlotMeta`. /// /// The footer of slot N+8 is the first proof. A skip certificate records a /// skip. A notarization certificate is an error. A footer that names neither -/// is an error. `chain` is ignored while that footer exists. +/// is an error. Those three results ignore `chain`. +/// +/// A footer that carries both rewards uses `chain`. A jump records a skip. A +/// chain that includes this slot is an error. No later root is an error. /// /// When slot N+8 has no `SlotMeta`, `chain` is the parent walk from the next /// full rooted slot. A jump over this slot records a skip. A chain that @@ -254,6 +271,18 @@ pub(crate) fn classify_hole( }; let names = reward_names(markers) .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; + if names.skips.contains(&slot) && names.notars.contains(&slot) { + return match chain { + Some(ChainRelation::JumpsOver { .. }) => { + skips.push(slot); + Ok(()) + } + Some(ChainRelation::DescendsFrom { next_root }) => { + bail!("slot {slot} has no SlotMeta and slot {next_root} descends from it"); + } + None => bail!("slot {slot} has no SlotMeta and no rooted slot follows it"), + }; + } if names.notars.contains(&slot) { bail!("slot {slot} is missing and slot {carrier} has a notarization reward for it"); } @@ -337,6 +366,19 @@ mod tests { footer_marker(&certificates) } + fn skip_and_notar_footer(slot: u64) -> Vec { + let mut certificates = vec![0, 1]; + certificates.extend_from_slice(&slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(7u8, 96)); + certificates.push(0); + certificates.push(1); + certificates.extend_from_slice(&slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(9u8, 32)); + certificates.extend(std::iter::repeat_n(8u8, 96)); + certificates.push(0); + footer_marker(&certificates) + } + fn footer_marker(certificates: &[u8]) -> Vec { let mut payload = vec![1u8]; payload.extend(std::iter::repeat_n(0xabu8, 32)); @@ -488,6 +530,60 @@ mod tests { ); } + #[test] + fn a_skip_and_notar_reward_with_a_parent_jump_is_a_skip() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let second = test_block(12, 10, [5; 32], [4; 32]); + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_and_notar_footer(11)]; + + let inspected = inspect(&view(10, 12, vec![first, second, carrier])).unwrap(); + assert_eq!(inspected.block_slots, vec![10, 12]); + assert_eq!(inspected.skips, vec![11]); + } + + #[test] + fn a_skip_and_notar_reward_whose_next_block_descends_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let next = test_block(12, 11, [5; 32], [4; 32]); + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_and_notar_footer(11)]; + + let error = inspect(&view(10, 12, vec![first, next, carrier])).unwrap_err(); + assert!(error.to_string().contains("descends from it"), "{error:#}"); + } + + #[test] + fn a_skip_and_notar_reward_without_a_later_root_is_an_error() { + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_and_notar_footer(11)]; + + let error = inspect(&view(11, 19, vec![carrier])).unwrap_err(); + assert!( + error.to_string().contains("no rooted slot follows"), + "{error:#}" + ); + } + + #[test] + fn a_skip_and_notar_reward_followed_by_a_partial_root_is_an_error() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut next = test_block(12, 10, [5; 32], [4; 32]); + next.is_full = false; + let mut carrier = test_block(19, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_and_notar_footer(11)]; + + let error = inspect(&view(10, 12, vec![first, next, carrier])).unwrap_err(); + assert!( + error.to_string().contains("rooted and is not full"), + "{error:#}" + ); + } + #[test] fn a_notarized_missing_slot_is_an_error() { let first = test_block(10, 9, [4; 32], [3; 32]); diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 833641a..a002df4 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -1186,6 +1186,42 @@ mod tests { assert_eq!(slots, vec![10, 14]); } + #[test] + fn a_skip_and_notar_reward_writes_no_blob() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let second = test_block(12, 10, [5; 32], [4; 32]); + let mut carrier = test_block(11 + REWARD_SLOT_DISTANCE, 18, [6; 32], [5; 32]); + carrier.is_root = false; + carrier.markers = vec![skip_and_notar_footer(11)]; + let view = LedgerView { + start_slot: 10, + end_slot: 12, + slots: [first, second, carrier] + .into_iter() + .map(|slot| (slot.slot, slot)) + .collect(), + }; + let inspected = inspect(&view).unwrap(); + assert_eq!(inspected.skips, vec![11]); + let dir = tempfile::tempdir().unwrap(); + let mut loaded = Vec::new(); + write_archive(dir.path(), &inspected, ZSTD_LEVEL, |slot| { + loaded.push(slot); + view.slots + .get(&slot) + .cloned() + .with_context(|| format!("test block {slot} is missing")) + }) + .unwrap(); + assert_eq!(loaded, vec![10, 12, 10, 12]); + let index = std::fs::read(dir.path().join(ARCHIVE_V2_BLOCK_INDEX_FILE)).unwrap(); + let slots: Vec = index[36..] + .chunks_exact(52) + .map(|row| u64::from_le_bytes(row[4..12].try_into().unwrap())) + .collect(); + assert_eq!(slots, vec![10, 12]); + } + #[test] fn a_proven_skip_writes_no_blob() { let first = test_block(10, 9, [4; 32], [3; 32]); @@ -1606,6 +1642,33 @@ mod tests { } } + fn skip_and_notar_footer(slot: u64) -> Vec { + let mut certificates = vec![0, 1]; + certificates.extend_from_slice(&slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(7u8, 96)); + certificates.push(0); + certificates.push(1); + certificates.extend_from_slice(&slot.to_le_bytes()); + certificates.extend(std::iter::repeat_n(9u8, 32)); + certificates.extend(std::iter::repeat_n(8u8, 96)); + certificates.push(0); + footer_marker(&certificates) + } + + fn footer_marker(certificates: &[u8]) -> Vec { + let mut payload = vec![1u8]; + payload.extend(std::iter::repeat_n(0xabu8, 32)); + payload.extend_from_slice(&1u64.to_le_bytes()); + payload.push(0); + payload.extend_from_slice(certificates); + let mut out = Vec::new(); + out.extend_from_slice(&1u16.to_le_bytes()); + out.push(0); + out.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_le_bytes()); + out.extend(payload); + out + } + fn skip_footer(skipped_slot: u64) -> Vec { let mut certificates = vec![0, 1]; certificates.extend_from_slice(&skipped_slot.to_le_bytes()); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index ea8890a..df18ce4 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -21,7 +21,7 @@ use solana_transaction_status::{Reward, TransactionStatusMeta}; use crate::check::{ ChainRelation, Inspected, PreviousBlock, accept_block_against, classify_abandoned_fork, - classify_hole, parent_chain_relation, + classify_hole, hole_uses_parent_chain, parent_chain_relation, }; use crate::source::{ REWARD_SLOT_DISTANCE, RawComponent, SlotData, SourceEntry, SourceHeader, SourceInnerGroup, @@ -362,12 +362,15 @@ where } } else { let markers = proof_markers(blockstore, loads, slot, end_slot, &mut ahead)?; - let chain = if markers.is_none() { + let chain = if hole_uses_parent_chain(slot, markers.as_deref())? { full_root_relation(blockstore, slot)? } else { None }; classify_hole(slot, markers.as_deref(), chain, &mut skips)?; + // A jump is logged when N+8 has no footer, and when that footer + // carries both a skip reward and a notarization reward. A + // skip-only footer stays silent. if let Some(ChainRelation::JumpsOver { next_root }) = chain { progress(format!( "slot {slot} left out, rooted chain jumps to {next_root}" From e37bfc852da4f5eb14ef7b94033e33ae34e14a87 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Mon, 9 Nov 2026 16:23:22 +0100 Subject: [PATCH 25/38] add readme for ledger archive --- hivezilla/ledger-archive/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 hivezilla/ledger-archive/README.md diff --git a/hivezilla/ledger-archive/README.md b/hivezilla/ledger-archive/README.md new file mode 100644 index 0000000..b1b3587 --- /dev/null +++ b/hivezilla/ledger-archive/README.md @@ -0,0 +1,21 @@ +# hivezilla-ledger-archive + +Reads one inclusive slot range from an Agave ledger and writes a Compact V2 archive of the rooted blocks. The ledger is opened read-only and is not modified. + +```text +hivezilla-ledger-archive check-ledger-range --start-slot N --end-slot M +hivezilla-ledger-archive build-archive-v2-from-ledger \ + --start-slot N --end-slot M --level 1 --read-threads 1 +``` + +`check-ledger-range` applies the slot rules and prints a report. `build-archive-v2-from-ledger` writes one independent zstd frame per rooted block, then the index. `--level` is the zstd effort (default 1). `--read-threads` is the number of parallel shred reads, from 1 to 32. A failure removes the temporary files and does not publish the index. + +A slot is written only when it is rooted and its shreds are complete. The tool then checks that every transaction has a status, and that the parent slot and parent blockhash chain to the previous written block. + +Slots with no `SlotMeta` are not written. A skip reward in the block at slot N+8 records a skip. A notarization reward alone is an error. When that footer contains both rewards, the slot is a skip only if the next rooted block jumps over it. When slot N+8 has no footer, the same parent jump records a skip. A rooted block that is not full, or a hole that the next rooted block descends from, stops the run. + +A slot that has data but is not rooted is left out only when the next rooted block jumps over it and slot N+8 carries a skip reward. The same sentence is printed for that case and for a parent-jump hole: `slot N left out, rooted chain jumps to M`. A skip reward alone prints nothing. + +The tool does not verify BLS vote signatures and does not recompute the bank hash. Agave already did both before it rooted the slot. + +The slot rules were run on one full epoch, epoch 1077, slots 465264000–465695999. The scan finished with 432000 slots scanned and 431628 rooted blocks accepted. The other 372 slots were proven skips or abandoned forks. From 73db384c2acefb836de41af72b62d0bcf00425a6 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 11 Nov 2026 13:54:57 +0000 Subject: [PATCH 26/38] add epoch archive command --- hivezilla/ledger-archive/README.md | 4 +- hivezilla/ledger-archive/src/ledger.rs | 139 ++++++++++++++++++++++++- hivezilla/ledger-archive/src/main.rs | 47 ++++++++- 3 files changed, 187 insertions(+), 3 deletions(-) diff --git a/hivezilla/ledger-archive/README.md b/hivezilla/ledger-archive/README.md index b1b3587..82963dc 100644 --- a/hivezilla/ledger-archive/README.md +++ b/hivezilla/ledger-archive/README.md @@ -6,9 +6,11 @@ Reads one inclusive slot range from an Agave ledger and writes a Compact V2 arch hivezilla-ledger-archive check-ledger-range --start-slot N --end-slot M hivezilla-ledger-archive build-archive-v2-from-ledger \ --start-slot N --end-slot M --level 1 --read-threads 1 +hivezilla-ledger-archive build-epoch-archive \ + --epoch N --level 1 --read-threads 1 ``` -`check-ledger-range` applies the slot rules and prints a report. `build-archive-v2-from-ledger` writes one independent zstd frame per rooted block, then the index. `--level` is the zstd effort (default 1). `--read-threads` is the number of parallel shred reads, from 1 to 32. A failure removes the temporary files and does not publish the index. +`check-ledger-range` applies the slot rules and prints a report. `build-archive-v2-from-ledger` writes one independent zstd frame per rooted block, then the index. `build-epoch-archive` writes epoch N, 432000 slots, into `/epoch-N`. It creates that directory only when the ledger still stores the epoch's first slot and a root has reached its last slot. An existing final archive file in that directory stops the command. `--level` is the zstd effort (default 1). `--read-threads` is the number of parallel shred reads, from 1 to 32. A failure removes the temporary files and does not publish the index. A slot is written only when it is rooted and its shreds are complete. The tool then checks that every transaction has a status, and that the parent slot and parent blockhash chain to the previous written block. diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index df18ce4..4e3cf9c 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -1,10 +1,15 @@ use std::collections::{BTreeMap, HashSet}; use std::io::{self, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Mutex, mpsc}; use std::thread; use anyhow::{Context, Result, anyhow, bail}; +use blockzilla_archive_v2::{ + ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, ARCHIVE_V2_BLOCKS_FILE, + ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, + ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, +}; use blockzilla_compact::{CompactInstructionError, CompactTransactionError}; use solana_entry::block_component::BlockComponent; use solana_instruction_error::InstructionError; @@ -59,6 +64,64 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { .with_context(|| format!("open ledger {}", ledger.display())) } +pub(crate) const SLOTS_PER_EPOCH: u64 = 432_000; + +const EPOCH_ARCHIVE_FILES: [&str; 9] = [ + ARCHIVE_V2_BLOCKS_FILE, + ARCHIVE_V2_SIGNATURES_FILE, + ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, + ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, + ARCHIVE_V2_POH_FILE, + ARCHIVE_V2_SHREDDING_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_FILE, + ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, + ARCHIVE_V2_BLOCK_INDEX_FILE, +]; + +pub(crate) fn epoch_slot_range(epoch: u64) -> Result<(u64, u64)> { + let start_slot = epoch + .checked_mul(SLOTS_PER_EPOCH) + .context("epoch start overflows")?; + let end_slot = start_slot + .checked_add(SLOTS_PER_EPOCH - 1) + .context("epoch end overflows")?; + Ok((start_slot, end_slot)) +} + +pub(crate) fn epoch_output_dir(parent: &Path, epoch: u64) -> PathBuf { + parent.join(format!("epoch-{epoch}")) +} + +/// The epoch is stored when its first slot is still present and a root has +/// reached its last slot. This does not prove that every root in the range is full. +pub(crate) fn ensure_epoch_is_stored( + epoch: u64, + start_slot: u64, + end_slot: u64, + lowest_slot: u64, + max_root: u64, +) -> Result<()> { + if start_slot < lowest_slot { + bail!( + "epoch {epoch} starts at slot {start_slot} and the ledger starts at slot {lowest_slot}" + ); + } + if end_slot > max_root { + bail!("epoch {epoch} ends at slot {end_slot} and the last root is slot {max_root}"); + } + Ok(()) +} + +pub(crate) fn ensure_no_published_archive_file(output: &Path) -> Result<()> { + for name in EPOCH_ARCHIVE_FILES { + let path = output.join(name); + if path.exists() { + bail!("archive file {} already exists", path.display()); + } + } + Ok(()) +} + const MAX_READ_THREADS: usize = 32; struct LoadPool { @@ -1248,6 +1311,80 @@ mod tests { )); } + #[test] + fn epoch_zero_and_epoch_1079_have_fixed_slot_ranges() { + assert_eq!(epoch_slot_range(0).unwrap(), (0, 431_999)); + assert_eq!( + epoch_slot_range(1079).unwrap(), + (466_128_000, 466_559_999) + ); + assert_eq!( + epoch_output_dir(Path::new("/home/sol/ledger-archive-out"), 0), + PathBuf::from("/home/sol/ledger-archive-out/epoch-0") + ); + assert_eq!( + epoch_output_dir(Path::new("/home/sol/ledger-archive-out"), 1079), + PathBuf::from("/home/sol/ledger-archive-out/epoch-1079") + ); + } + + #[test] + fn an_epoch_that_overflows_u64_is_refused() { + let error = epoch_slot_range(u64::MAX).unwrap_err(); + assert!(error.to_string().contains("epoch start overflows"), "{error:#}"); + + let epoch = u64::MAX / SLOTS_PER_EPOCH; + let start_slot = epoch * SLOTS_PER_EPOCH; + let result = epoch_slot_range(epoch); + if start_slot <= u64::MAX - (SLOTS_PER_EPOCH - 1) { + assert_eq!( + result.unwrap(), + (start_slot, start_slot + SLOTS_PER_EPOCH - 1) + ); + } else { + let error = result.unwrap_err(); + assert!(error.to_string().contains("epoch end overflows"), "{error:#}"); + } + } + + #[test] + fn an_epoch_must_sit_between_the_first_stored_slot_and_the_last_root() { + let error = ensure_epoch_is_stored(1078, 465_696_000, 466_127_999, 465_818_301, 468_118_000) + .unwrap_err(); + assert!( + error.to_string().contains("starts at slot 465696000") + && error.to_string().contains("ledger starts at slot 465818301"), + "{error:#}" + ); + + let error = ensure_epoch_is_stored(1083, 467_856_000, 468_287_999, 465_818_301, 468_118_000) + .unwrap_err(); + assert!( + error.to_string().contains("ends at slot 468287999") + && error.to_string().contains("last root is slot 468118000"), + "{error:#}" + ); + + ensure_epoch_is_stored(1079, 466_128_000, 466_559_999, 465_818_301, 468_118_000).unwrap(); + ensure_epoch_is_stored(1079, 466_128_000, 466_559_999, 466_128_000, 466_559_999).unwrap(); + } + + #[test] + fn a_published_archive_file_blocks_a_new_epoch_directory() { + let dir = tempfile::tempdir().unwrap(); + let output = epoch_output_dir(dir.path(), 1079); + ensure_no_published_archive_file(&output).unwrap(); + std::fs::create_dir_all(&output).unwrap(); + std::fs::write(output.join(format!("{ARCHIVE_V2_BLOCKS_FILE}.tmp")), b"partial").unwrap(); + ensure_no_published_archive_file(&output).unwrap(); + std::fs::write(output.join(ARCHIVE_V2_BLOCKS_FILE), b"x").unwrap(); + let error = ensure_no_published_archive_file(&output).unwrap_err(); + assert!( + error.to_string().contains("archive-v2-blocks.zstd"), + "{error:#}" + ); + } + #[test] fn reward_type_codes_keep_the_archive_numbers() { assert_eq!(reward_type_code(Some(RewardType::Fee)), 1); diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index 0b85bb9..fbe3955 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -10,7 +10,10 @@ use anyhow::Result; use clap::{Parser, Subcommand}; use crate::convert::build_archive; -use crate::ledger::{open_read_only, scan_range}; +use crate::ledger::{ + ensure_epoch_is_stored, ensure_no_published_archive_file, epoch_output_dir, epoch_slot_range, + open_read_only, scan_range, +}; #[derive(Parser)] #[command(name = "hivezilla-ledger-archive")] @@ -44,6 +47,23 @@ enum Command { #[arg(long, default_value_t = 1)] read_threads: u8, }, + /// Write one epoch under OUTPUT/epoch-N. + /// + /// The epoch uses 432000 slots. The directory is created only when the + /// ledger still stores the first slot and a root has reached the last slot. + BuildEpochArchive { + ledger: PathBuf, + /// Parent directory. The archive is written to epoch-N inside it. + output: PathBuf, + #[arg(long)] + epoch: u64, + /// zstd compression level for each independent block frame. + #[arg(long, default_value_t = convert::ZSTD_LEVEL)] + level: i32, + /// Parallel shred reads. 1 reads one slot at a time. + #[arg(long, default_value_t = 1)] + read_threads: u8, + }, } fn main() { @@ -79,6 +99,31 @@ fn run() -> Result<()> { println!("Output: {}", output.display()); Ok(()) } + Command::BuildEpochArchive { + ledger, + output, + epoch, + level, + read_threads, + } => { + let (start_slot, end_slot) = epoch_slot_range(epoch)?; + let blockstore = open_read_only(&ledger)?; + ensure_epoch_is_stored( + epoch, + start_slot, + end_slot, + blockstore.lowest_slot(), + blockstore.max_root(), + )?; + drop(blockstore); + let output = epoch_output_dir(&output, epoch); + ensure_no_published_archive_file(&output)?; + let inspected = + build_archive(&ledger, &output, start_slot, end_slot, level, read_threads)?; + print_report(&inspected); + println!("Output: {}", output.display()); + Ok(()) + } } } From 7e2a5ef01d1f6b9db19e8dd60e36ba7ab23cf909 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 11 Nov 2026 18:41:35 +0000 Subject: [PATCH 27/38] fix issue with block footer --- hivezilla/ledger-archive/src/check.rs | 29 +++++++++++++++++++++----- hivezilla/ledger-archive/src/ledger.rs | 7 ++++--- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index 07eb6a0..ae00408 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -13,7 +13,8 @@ pub(crate) struct Inspected { pub block_slots: Vec, pub skips: Vec, /// Slots that have data, are not root, and are left out of the archive. - /// The next root jumps over each one, and slot N+8 has a skip certificate. + /// The next root jumps over each one. Slot N+8 has a skip certificate, or + /// it has no `SlotMeta`. pub forks: Vec, pub boundary_hash: [u8; 32], } @@ -152,9 +153,12 @@ fn chain_relation_in_view(view: &LedgerView, slot: u64) -> Result }) } -/// Leave out a non-root slot when the rooted chain jumps over it and slot N+8 -/// carries a skip certificate. A notarization certificate does not cancel that -/// skip. A slot the next root descends from stays an error. +/// Leave out a non-root slot when the rooted chain jumps over it. +/// +/// A skip certificate in slot N+8 is enough. A notarization certificate does +/// not cancel that skip. When slot N+8 has no `SlotMeta`, the certificate +/// cannot exist, and the parent jump is the proof. A slot the next root +/// descends from stays an error. pub(crate) fn classify_abandoned_fork( slot: u64, chain: ChainRelation, @@ -168,7 +172,8 @@ pub(crate) fn classify_abandoned_fork( .checked_add(REWARD_SLOT_DISTANCE) .context("slot + 8 overflows")?; let Some(markers) = carrier_markers else { - bail!("slot {slot} has data and is not rooted, and slot {carrier} has no footer proof"); + forks.push(slot); + return Ok(()); }; let names = reward_names(markers) .with_context(|| format!("slot {carrier} footer cannot prove slot {slot}"))?; @@ -513,6 +518,20 @@ mod tests { assert!(error.to_string().contains("descends from it"), "{error:#}"); } + #[test] + fn a_parent_jump_omits_a_fork_when_the_footer_slot_is_absent() { + let first = test_block(10, 9, [4; 32], [3; 32]); + let mut fork = test_block(11, 10, [8; 32], [4; 32]); + fork.is_root = false; + let next = test_block(14, 10, [5; 32], [4; 32]); + + let inspected = inspect(&view(10, 14, vec![first, fork, next])).unwrap(); + assert_eq!(inspected.block_slots, vec![10, 14]); + assert_eq!(inspected.forks, vec![11]); + assert_eq!(inspected.skips, vec![12, 13]); + assert_eq!(inspected.boundary_hash, [3; 32]); + } + #[test] fn a_chain_jump_without_a_skip_certificate_is_an_error() { let first = test_block(10, 9, [4; 32], [3; 32]); diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 4e3cf9c..8001eb3 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -320,9 +320,10 @@ where /// Read each slot once. A rooted full block is checked, passed to `on_block`, /// then dropped. A non-root slot is left out when the next root jumps over it -/// and slot N+8 has a skip certificate. A hole is proved from the footer of -/// slot N+8. When that slot has no `SlotMeta`, the parent chain of the next -/// full rooted block can prove the skip. +/// and slot N+8 has a skip certificate. When slot N+8 has no `SlotMeta`, that +/// parent jump is enough. A hole is proved from the footer of slot N+8. When +/// that slot has no `SlotMeta`, the parent chain of the next full rooted block +/// can prove the skip. /// /// When that footer belongs to a later block in the window, the block is kept /// until the scan reaches it, so its shreds are not read a second time. From 834978fe5a24ef1f8f29a3583f6ee27700bccf63 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Wed, 11 Nov 2026 19:41:14 +0000 Subject: [PATCH 28/38] add a missing case for skipping a block --- hivezilla/ledger-archive/src/check.rs | 16 +++++------ hivezilla/ledger-archive/src/ledger.rs | 39 +++++++++++++++++++------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/hivezilla/ledger-archive/src/check.rs b/hivezilla/ledger-archive/src/check.rs index ae00408..32fc3d5 100644 --- a/hivezilla/ledger-archive/src/check.rs +++ b/hivezilla/ledger-archive/src/check.rs @@ -14,7 +14,7 @@ pub(crate) struct Inspected { pub skips: Vec, /// Slots that have data, are not root, and are left out of the archive. /// The next root jumps over each one. Slot N+8 has a skip certificate, or - /// it has no `SlotMeta`. + /// it has no `SlotMeta`, or it is not root. pub forks: Vec, pub boundary_hash: [u8; 32], } @@ -156,9 +156,9 @@ fn chain_relation_in_view(view: &LedgerView, slot: u64) -> Result /// Leave out a non-root slot when the rooted chain jumps over it. /// /// A skip certificate in slot N+8 is enough. A notarization certificate does -/// not cancel that skip. When slot N+8 has no `SlotMeta`, the certificate -/// cannot exist, and the parent jump is the proof. A slot the next root -/// descends from stays an error. +/// not cancel that skip. When slot N+8 has no `SlotMeta` or is not root, the +/// certificate cannot be read, and the parent jump is the proof. A slot the +/// next root descends from stays an error. pub(crate) fn classify_abandoned_fork( slot: u64, chain: ChainRelation, @@ -258,10 +258,10 @@ pub(crate) fn hole_uses_parent_chain(slot: u64, markers: Option<&[Vec]>) -> /// A footer that carries both rewards uses `chain`. A jump records a skip. A /// chain that includes this slot is an error. No later root is an error. /// -/// When slot N+8 has no `SlotMeta`, `chain` is the parent walk from the next -/// full rooted slot. A jump over this slot records a skip. A chain that -/// includes this slot is an error. `chain == None` keeps the missing-footer -/// error. +/// When slot N+8 has no `SlotMeta` or is not root, `chain` is the parent walk +/// from the next full rooted slot. A jump over this slot records a skip. A +/// chain that includes this slot is an error. `chain == None` keeps the +/// missing-footer error. pub(crate) fn classify_hole( slot: u64, carrier_markers: Option<&[Vec]>, diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 8001eb3..0e82532 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -320,10 +320,10 @@ where /// Read each slot once. A rooted full block is checked, passed to `on_block`, /// then dropped. A non-root slot is left out when the next root jumps over it -/// and slot N+8 has a skip certificate. When slot N+8 has no `SlotMeta`, that -/// parent jump is enough. A hole is proved from the footer of slot N+8. When -/// that slot has no `SlotMeta`, the parent chain of the next full rooted block -/// can prove the skip. +/// and slot N+8 has a skip certificate. When slot N+8 has no `SlotMeta` or is +/// not root, that parent jump is enough. A hole is proved from the footer of +/// slot N+8. When that slot has no `SlotMeta` or is not root, the parent chain +/// of the next full rooted block can prove the skip. /// /// When that footer belongs to a later block in the window, the block is kept /// until the scan reaches it, so its shreds are not read a second time. @@ -613,13 +613,24 @@ fn proof_markers( if carrier > end_slot { return Ok(Some(read_markers(blockstore, carrier)?)); } - if is_root && meta.is_full() { - let data = loads.take(carrier)?; - let markers = data.markers.clone(); - ahead.insert(carrier, data); - return Ok(Some(markers)); + // A slot that is not root has no usable footer. An empty list would mean + // the footer was read and does not prove the skip. + if !in_window_footer_is_readable(is_root, meta.is_full()) { + if !is_root { + return Ok(None); + } + return Ok(Some(Vec::new())); } - Ok(Some(Vec::new())) + let data = loads.take(carrier)?; + let markers = data.markers.clone(); + ahead.insert(carrier, data); + Ok(Some(markers)) +} + +/// Inside the window, only a full root provides a footer. A slot that is not +/// root provides none, so the parent jump decides. +pub(crate) fn in_window_footer_is_readable(is_root: bool, is_full: bool) -> bool { + is_root && is_full } fn progress(message: impl std::fmt::Display) { @@ -1312,6 +1323,14 @@ mod tests { )); } + #[test] + fn a_slot_that_is_not_root_has_no_readable_footer() { + assert!(!in_window_footer_is_readable(false, false)); + assert!(!in_window_footer_is_readable(false, true)); + assert!(!in_window_footer_is_readable(true, false)); + assert!(in_window_footer_is_readable(true, true)); + } + #[test] fn epoch_zero_and_epoch_1079_have_fixed_slot_ranges() { assert_eq!(epoch_slot_range(0).unwrap(), (0, 431_999)); From e172cdfa919e8bddc1f493dcc5bfa571b875e226 Mon Sep 17 00:00:00 2001 From: Yves-Marie Saout Date: Thu, 12 Nov 2026 13:05:56 +0000 Subject: [PATCH 29/38] Remove the free-range ledger archive command. Epoch writes are the public path, and the epoch command now carries the write contract. --- hivezilla/ledger-archive/README.md | 27 ++++++++++++++++++---- hivezilla/ledger-archive/src/main.rs | 34 ++++------------------------ 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/hivezilla/ledger-archive/README.md b/hivezilla/ledger-archive/README.md index 82963dc..7e5b99a 100644 --- a/hivezilla/ledger-archive/README.md +++ b/hivezilla/ledger-archive/README.md @@ -1,16 +1,35 @@ # hivezilla-ledger-archive -Reads one inclusive slot range from an Agave ledger and writes a Compact V2 archive of the rooted blocks. The ledger is opened read-only and is not modified. +`check-ledger-range` checks one inclusive slot range in an Agave ledger and prints a report. `build-epoch-archive` writes a Compact V2 archive of one epoch. The ledger is opened read-only and is not modified. ```text hivezilla-ledger-archive check-ledger-range --start-slot N --end-slot M -hivezilla-ledger-archive build-archive-v2-from-ledger \ - --start-slot N --end-slot M --level 1 --read-threads 1 hivezilla-ledger-archive build-epoch-archive \ --epoch N --level 1 --read-threads 1 ``` -`check-ledger-range` applies the slot rules and prints a report. `build-archive-v2-from-ledger` writes one independent zstd frame per rooted block, then the index. `build-epoch-archive` writes epoch N, 432000 slots, into `/epoch-N`. It creates that directory only when the ledger still stores the epoch's first slot and a root has reached its last slot. An existing final archive file in that directory stops the command. `--level` is the zstd effort (default 1). `--read-threads` is the number of parallel shred reads, from 1 to 32. A failure removes the temporary files and does not publish the index. +`check-ledger-range` applies the slot rules and prints a report. + +## build-epoch-archive + +Write one epoch from a stopped ledger or a snapshot. The ledger is opened read-only. + +The epoch has 432000 slots, from `N * 432000` through `N * 432000 + 431999`. The archive is written to `/epoch-N`. That directory is created only when the ledger still stores the first slot and a root has reached the last slot. An existing final archive file in that directory stops the command. + +`--level` is the zstd effort. The default is 1. `--read-threads` is the number of parallel shred reads, from 1 to 32. The default is 1. + +Each rooted block becomes one independent zstd frame in `archive-v2-blocks.zstd`. The command then writes these files in the same directory: + +- `archive-v2-blocks.index` +- `signatures.bin` +- `blockhash_registry.bin` +- `vote_hash_registry.bin` +- `poh.wincode` +- `shredding.wincode` +- `registry.bin` +- `prev_blockhash_tail.bin` + +A failure removes the temporary files and does not publish the index. The command prints the block count, the proven skips, the abandoned forks, the first block, the last block, the boundary hash, and the output path. A slot is written only when it is rooted and its shreds are complete. The tool then checks that every transaction has a status, and that the parent slot and parent blockhash chain to the previous written block. diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index fbe3955..4e88700 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -32,25 +32,15 @@ enum Command { #[arg(long)] end_slot: u64, }, - /// Write a Compact V2 hot archive for one inclusive slot range. - BuildArchiveV2FromLedger { - ledger: PathBuf, - output: PathBuf, - #[arg(long)] - start_slot: u64, - #[arg(long)] - end_slot: u64, - /// zstd compression level for each independent block frame. - #[arg(long, default_value_t = convert::ZSTD_LEVEL)] - level: i32, - /// Parallel shred reads. 1 reads one slot at a time. - #[arg(long, default_value_t = 1)] - read_threads: u8, - }, /// Write one epoch under OUTPUT/epoch-N. /// /// The epoch uses 432000 slots. The directory is created only when the /// ledger still stores the first slot and a root has reached the last slot. + /// An existing final archive file in that directory stops the command. + /// Each rooted block is one independent zstd frame. The index is written + /// after the frames. `--level` is the zstd effort, default 1. + /// `--read-threads` is the number of parallel shred reads, from 1 to 32. + /// A failure removes the temporary files and does not publish the index. BuildEpochArchive { ledger: PathBuf, /// Parent directory. The archive is written to epoch-N inside it. @@ -85,20 +75,6 @@ fn run() -> Result<()> { print_report(&inspected); Ok(()) } - Command::BuildArchiveV2FromLedger { - ledger, - output, - start_slot, - end_slot, - level, - read_threads, - } => { - let inspected = - build_archive(&ledger, &output, start_slot, end_slot, level, read_threads)?; - print_report(&inspected); - println!("Output: {}", output.display()); - Ok(()) - } Command::BuildEpochArchive { ledger, output, From a4d1844d42e347d0593d8918bf93af5b6a327fdc Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 12 Nov 2026 13:57:35 +0000 Subject: [PATCH 30/38] add registry counts and mphf --- hivezilla/ledger-archive/src/convert.rs | 123 +++++++++++++++++++++--- hivezilla/ledger-archive/src/ledger.rs | 10 +- 2 files changed, 119 insertions(+), 14 deletions(-) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index a002df4..5dbed23 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -6,8 +6,10 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use blockzilla_archive_v2::{ ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, ARCHIVE_V2_BLOCKS_FILE, - ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, - ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_TX_FLAG_HAS_ERROR, + ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_COUNTS_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_INDEX_FILE, ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, + ARCHIVE_V2_TX_FLAG_HAS_ERROR, ARCHIVE_V2_TX_FLAG_HAS_INNER_IX, ARCHIVE_V2_TX_FLAG_HAS_LOADED_ADDRESSES, ARCHIVE_V2_TX_FLAG_HAS_LOGS, ARCHIVE_V2_TX_FLAG_HAS_METADATA, ARCHIVE_V2_TX_FLAG_HAS_RETURN_DATA, ARCHIVE_V2_TX_FLAG_HAS_TOKEN_BALANCES, @@ -23,7 +25,9 @@ use blockzilla_compact::{ CompactPohEntry, CompactReturnData, CompactReward, CompactShredding, CompactTokenBalance, CompactTransactionConfig, OwnedCompactAddressTableLookup, OwnedCompactRecentBlockhash, }; -use blockzilla_primitives::{CompactPubkey, WincodeLeb128FramedWriter, wincode_leb128_config}; +use blockzilla_primitives::{ + CompactPubkey, WincodeLeb128FramedWriter, wincode_leb128_config, write_u32_varint, +}; use blockzilla_registry::{KeyIndex, write_registry}; use solana_entry::block_component::BlockComponent; use solana_ledger::blockstore::Blockstore; @@ -154,6 +158,8 @@ struct ArchivePaths { poh: PathBuf, shredding: PathBuf, registry: PathBuf, + registry_counts: PathBuf, + registry_index: PathBuf, tail: PathBuf, } @@ -166,11 +172,13 @@ fn temp_paths(output: &Path) -> ArchivePaths { poh: output.join(format!("{ARCHIVE_V2_POH_FILE}.tmp")), shredding: output.join(format!("{ARCHIVE_V2_SHREDDING_FILE}.tmp")), registry: output.join(format!("{ARCHIVE_V2_PUBKEY_REGISTRY_FILE}.tmp")), + registry_counts: output.join(format!("{ARCHIVE_V2_PUBKEY_REGISTRY_COUNTS_FILE}.tmp")), + registry_index: output.join(format!("{ARCHIVE_V2_PUBKEY_REGISTRY_INDEX_FILE}.tmp")), tail: output.join(format!("{ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE}.tmp")), } } -fn temp_file_list(temps: &ArchivePaths) -> [&Path; 8] { +fn temp_file_list(temps: &ArchivePaths) -> [&Path; 10] { [ &temps.blocks, &temps.signatures, @@ -179,6 +187,8 @@ fn temp_file_list(temps: &ArchivePaths) -> [&Path; 8] { &temps.poh, &temps.shredding, &temps.registry, + &temps.registry_counts, + &temps.registry_index, &temps.tail, ] } @@ -391,6 +401,14 @@ impl ArchiveWriter { &self.paths.registry, &output.join(ARCHIVE_V2_PUBKEY_REGISTRY_FILE), )?; + rename_finished( + &self.paths.registry_counts, + &output.join(ARCHIVE_V2_PUBKEY_REGISTRY_COUNTS_FILE), + )?; + rename_finished( + &self.paths.registry_index, + &output.join(ARCHIVE_V2_PUBKEY_REGISTRY_INDEX_FILE), + )?; write_archive_v2_hot_block_index( &output.join(ARCHIVE_V2_BLOCK_INDEX_FILE), self.compressed_offset, @@ -621,14 +639,28 @@ fn blockhash_from_shreds(blockstore: &Blockstore, slot: u64) -> Result, + counts: Vec, +} + fn prepare_registry(temps: &ArchivePaths, counts: PubkeyCounts) -> Result { - let keys = ordered_registry_keys(counts); - write_registry(&temps.registry, &keys) + let ordered = ordered_registry(counts); + write_registry(&temps.registry, &ordered.keys) .with_context(|| format!("write {}", temps.registry.display()))?; - Ok(KeyIndex::build_from_slice_low_memory(&keys)) + write_registry_counts(&temps.registry_counts, &ordered.counts)?; + let index = KeyIndex::build_from_slice_low_memory(&ordered.keys); + index + .write(&temps.registry_index) + .with_context(|| format!("write {}", temps.registry_index.display()))?; + Ok(index) } fn ordered_registry_keys(counts: PubkeyCounts) -> Vec<[u8; 32]> { + ordered_registry(counts).keys +} + +fn ordered_registry(counts: PubkeyCounts) -> OrderedRegistry { let mut items: Vec<([u8; 32], u32)> = counts.into_iter().collect(); items.sort_unstable_by(|(left_key, left_count), (right_key, right_count)| { right_count @@ -637,11 +669,32 @@ fn ordered_registry_keys(counts: PubkeyCounts) -> Vec<[u8; 32]> { }); let compute_budget = compute_budget_program_id(); let mut keys = Vec::with_capacity(items.len() + 1); + let mut counts_in_order = Vec::with_capacity(items.len() + 1); if !items.iter().any(|(key, _)| *key == compute_budget) { keys.push(compute_budget); + counts_in_order.push(0); + } + for (key, count) in items { + keys.push(key); + counts_in_order.push(count); + } + OrderedRegistry { + keys, + counts: counts_in_order, + } +} + +fn write_registry_counts(path: &Path, counts: &[u32]) -> Result<()> { + let file = File::create(path).with_context(|| format!("create {}", path.display()))?; + let mut writer = BufWriter::new(file); + for count in counts { + write_u32_varint(&mut writer, *count) + .with_context(|| format!("write {}", path.display()))?; } - keys.extend(items.into_iter().map(|(key, _)| key)); - keys + writer + .flush() + .with_context(|| format!("flush {}", path.display()))?; + Ok(()) } fn compute_budget_program_id() -> [u8; 32] { @@ -1075,7 +1128,7 @@ mod tests { SourceReward, SourceShredGroup, SourceStatus, SourceTx, test_block, }; use blockzilla_archive_v2::deserialize_archive_v2_hot_block_blob; - use blockzilla_primitives::WincodeLeb128FramedReader; + use blockzilla_primitives::{WincodeLeb128FramedReader, read_u32_varint}; use of_car_reader::blockmarker::parent_consensus_block_id; use std::io::Cursor; @@ -1279,6 +1332,12 @@ mod tests { let registry = std::fs::read(dir.path().join(ARCHIVE_V2_PUBKEY_REGISTRY_FILE)).unwrap(); assert_eq!(registry.as_slice(), compute_budget_program_id().as_slice()); + let counts = + std::fs::read(dir.path().join(ARCHIVE_V2_PUBKEY_REGISTRY_COUNTS_FILE)).unwrap(); + assert_eq!(counts, [0]); + let key_index = + KeyIndex::load(&dir.path().join(ARCHIVE_V2_PUBKEY_REGISTRY_INDEX_FILE)).unwrap(); + assert_eq!(key_index.lookup(&compute_budget_program_id()), Some(1)); let mut reader = WincodeLeb128FramedReader::new(Cursor::new( std::fs::read(dir.path().join(ARCHIVE_V2_SHREDDING_FILE)).unwrap(), @@ -1318,9 +1377,13 @@ mod tests { let inspected = inspect(&view).unwrap(); let dir = tempfile::tempdir().unwrap(); + let mut slot_11_loads = 0u8; let error = write_archive(dir.path(), &inspected, ZSTD_LEVEL, |slot| { if slot == 11 { - bail!("slot 11 is missing transaction status"); + slot_11_loads += 1; + if slot_11_loads > 1 { + bail!("slot 11 is missing transaction status"); + } } view.slots .get(&slot) @@ -1341,6 +1404,44 @@ mod tests { assert!(names.is_empty(), "{names:?}"); } + #[test] + fn registry_counts_and_the_index_follow_the_key_order() { + let repeated = [4u8; 32]; + let once = [9u8; 32]; + let block = block_with_keys(vec![vec![repeated, repeated], vec![once]]); + let mut counts = PubkeyCounts::new(); + count_slot_pubkeys(&block, &mut counts); + let ordered = ordered_registry(counts); + assert_eq!( + ordered.keys, + vec![compute_budget_program_id(), repeated, once] + ); + assert_eq!(ordered.counts, vec![0, 2, 1]); + + let dir = tempfile::tempdir().unwrap(); + let temps = temp_paths(dir.path()); + let mut counts = PubkeyCounts::new(); + count_slot_pubkeys(&block, &mut counts); + prepare_registry(&temps, counts).unwrap(); + + let mut reader = Cursor::new(std::fs::read(&temps.registry_counts).unwrap()); + let mut stored = Vec::new(); + while let Some(count) = read_u32_varint(&mut reader).unwrap() { + stored.push(count); + } + assert_eq!(stored, vec![0, 2, 1]); + + let index = KeyIndex::load(&temps.registry_index).unwrap(); + assert_eq!(index.len(), ordered.keys.len()); + for (position, key) in ordered.keys.iter().enumerate() { + assert_eq!( + index.lookup(key), + Some(u32::try_from(position).unwrap() + 1) + ); + } + assert_eq!(index.lookup(&[3u8; 32]), None); + } + #[test] fn repeated_account_key_gets_the_lowest_counted_id() { let repeated = [4u8; 32]; diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 0e82532..25c29b6 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -7,8 +7,10 @@ use std::thread; use anyhow::{Context, Result, anyhow, bail}; use blockzilla_archive_v2::{ ARCHIVE_V2_BLOCK_INDEX_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, ARCHIVE_V2_BLOCKS_FILE, - ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, - ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, + ARCHIVE_V2_POH_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_COUNTS_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_INDEX_FILE, ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_SIGNATURES_FILE, + ARCHIVE_V2_VOTE_HASH_REGISTRY_FILE, }; use blockzilla_compact::{CompactInstructionError, CompactTransactionError}; use solana_entry::block_component::BlockComponent; @@ -66,7 +68,7 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { pub(crate) const SLOTS_PER_EPOCH: u64 = 432_000; -const EPOCH_ARCHIVE_FILES: [&str; 9] = [ +const EPOCH_ARCHIVE_FILES: [&str; 11] = [ ARCHIVE_V2_BLOCKS_FILE, ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, @@ -74,6 +76,8 @@ const EPOCH_ARCHIVE_FILES: [&str; 9] = [ ARCHIVE_V2_POH_FILE, ARCHIVE_V2_SHREDDING_FILE, ARCHIVE_V2_PUBKEY_REGISTRY_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_COUNTS_FILE, + ARCHIVE_V2_PUBKEY_REGISTRY_INDEX_FILE, ARCHIVE_V2_PREV_BLOCKHASH_TAIL_FILE, ARCHIVE_V2_BLOCK_INDEX_FILE, ]; From ef7e02038b5dc8173ba11a9ca04004294eb04a1d Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 12 Nov 2026 13:59:19 +0000 Subject: [PATCH 31/38] add missing fields --- hivezilla/ledger-archive/src/convert.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hivezilla/ledger-archive/src/convert.rs b/hivezilla/ledger-archive/src/convert.rs index 5dbed23..507974c 100644 --- a/hivezilla/ledger-archive/src/convert.rs +++ b/hivezilla/ledger-archive/src/convert.rs @@ -271,6 +271,8 @@ impl ArchiveWriter { poh: temps.poh.clone(), shredding: temps.shredding.clone(), registry: temps.registry.clone(), + registry_counts: temps.registry_counts.clone(), + registry_index: temps.registry_index.clone(), tail: temps.tail.clone(), }, }) From 8573aebf4ace6e17e08add57245eac64e84ac5d9 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Thu, 12 Nov 2026 14:22:51 +0000 Subject: [PATCH 32/38] add telegram monitoring --- hivezilla/ledger-archive/src/main.rs | 21 +++ hivezilla/ledger-archive/src/notify.rs | 192 +++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 hivezilla/ledger-archive/src/notify.rs diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index 4e88700..9f77e44 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -2,6 +2,7 @@ mod check; mod convert; mod ledger; mod markers; +mod notify; mod source; use std::path::PathBuf; @@ -41,6 +42,8 @@ enum Command { /// after the frames. `--level` is the zstd effort, default 1. /// `--read-threads` is the number of parallel shred reads, from 1 to 32. /// A failure removes the temporary files and does not publish the index. + /// A published epoch sends one Telegram line when the bot token and the + /// chat id are both set. A Telegram failure still exits 0. BuildEpochArchive { ledger: PathBuf, /// Parent directory. The archive is written to epoch-N inside it. @@ -53,6 +56,16 @@ enum Command { /// Parallel shred reads. 1 reads one slot at a time. #[arg(long, default_value_t = 1)] read_threads: u8, + /// Telegram bot token. The default is empty and sends nothing. + #[arg(long, default_value = notify::DEFAULT_TELEGRAM_BOT_TOKEN)] + telegram_bot_token: String, + /// Telegram channel id. The default is empty and sends nothing. + #[arg( + long, + default_value = notify::DEFAULT_TELEGRAM_CHAT_ID, + allow_hyphen_values = true + )] + telegram_chat_id: String, }, } @@ -81,6 +94,8 @@ fn run() -> Result<()> { epoch, level, read_threads, + telegram_bot_token, + telegram_chat_id, } => { let (start_slot, end_slot) = epoch_slot_range(epoch)?; let blockstore = open_read_only(&ledger)?; @@ -98,6 +113,12 @@ fn run() -> Result<()> { build_archive(&ledger, &output, start_slot, end_slot, level, read_threads)?; print_report(&inspected); println!("Output: {}", output.display()); + notify::notify_published_epoch( + epoch, + &output, + &telegram_bot_token, + &telegram_chat_id, + )?; Ok(()) } } diff --git a/hivezilla/ledger-archive/src/notify.rs b/hivezilla/ledger-archive/src/notify.rs new file mode 100644 index 0000000..93c35af --- /dev/null +++ b/hivezilla/ledger-archive/src/notify.rs @@ -0,0 +1,192 @@ +//! One Telegram line after a published epoch. +//! +//! Write the two defaults below. An empty token or an empty chat id sends +//! nothing. The archive command still exits 0 when Telegram fails. + +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result}; + +/// Bot token from BotFather. Leave this empty to send nothing. +pub(crate) const DEFAULT_TELEGRAM_BOT_TOKEN: &str = "8127218375:AAEP6oECzjqNaLaH2VHY_HA_7VYDuS4SL6Y"; + +/// Channel id, for example "-100…". Leave this empty to send nothing. +pub(crate) const DEFAULT_TELEGRAM_CHAT_ID: &str = "-4561803275"; + +const CURL_ARGS: &[&str] = &["--config", "-"]; + +pub(crate) fn notify_published_epoch( + epoch: u64, + output: &Path, + bot_token: &str, + chat_id: &str, +) -> Result<()> { + let token = bot_token.trim(); + let chat = chat_id.trim(); + if !credentials_ready(token, chat) { + eprintln!("telegram: skipped, bot token or chat id is empty"); + return Ok(()); + } + let size = match directory_size(output) { + Ok(bytes) => format_size(bytes), + Err(err) => { + eprintln!("telegram: directory size: {err:#}"); + "unknown".to_string() + } + }; + let text = published_line(epoch, &size); + if let Err(err) = send_telegram(token, chat, &text) { + eprintln!("telegram: {}", redact(&format!("{err:#}"), token)); + } + Ok(()) +} + +pub(crate) fn credentials_ready(bot_token: &str, chat_id: &str) -> bool { + !bot_token.trim().is_empty() && !chat_id.trim().is_empty() +} + +pub(crate) fn published_line(epoch: u64, size: &str) -> String { + format!("✅ epoch {epoch} archived, size {size}") +} + +pub(crate) fn directory_size(path: &Path) -> Result { + let mut total = 0u64; + for entry in std::fs::read_dir(path).with_context(|| format!("read {}", path.display()))? { + let entry = entry?; + let meta = entry.metadata()?; + if meta.is_file() { + total = total.saturating_add(meta.len()); + } + } + Ok(total) +} + +pub(crate) fn format_size(bytes: u64) -> String { + const KIB: u64 = 1024; + const MIB: u64 = 1024 * 1024; + const GIB: u64 = 1024 * 1024 * 1024; + if bytes >= GIB { + format!("{:.1} GiB", bytes as f64 / GIB as f64) + } else if bytes >= MIB { + format!("{:.1} MiB", bytes as f64 / MIB as f64) + } else if bytes >= KIB { + format!("{:.1} KiB", bytes as f64 / KIB as f64) + } else { + format!("{bytes} B") + } +} + +fn send_telegram(bot_token: &str, chat_id: &str, text: &str) -> Result<()> { + let config = curl_config(bot_token, chat_id, text); + let mut child = Command::new("curl") + .args(CURL_ARGS) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .context("start curl")?; + child + .stdin + .take() + .context("curl stdin")? + .write_all(config.as_bytes()) + .context("write curl config")?; + let output = child.wait_with_output().context("wait for curl")?; + if !output.status.success() { + let err = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("curl status {}: {}", output.status, err.trim()); + } + Ok(()) +} + +pub(crate) fn curl_config(bot_token: &str, chat_id: &str, text: &str) -> String { + format!( + "\ +silent +show-error +fail +connect-timeout = 5 +max-time = 15 +request = \"POST\" +data-urlencode = \"chat_id={chat}\" +data-urlencode = \"text={text}\" +url = \"https://api.telegram.org/bot{token}/sendMessage\" +", + chat = curl_quote(chat_id), + text = curl_quote(text), + token = curl_quote(bot_token), + ) +} + +fn curl_quote(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +pub(crate) fn redact(text: &str, token: &str) -> String { + let token = token.trim(); + if token.is_empty() { + text.to_string() + } else { + text.replace(token, "[token]") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn published_line_names_the_epoch_and_the_size() { + assert_eq!( + published_line(1084, "66.2 GiB"), + "✅ epoch 1084 archived, size 66.2 GiB" + ); + } + + #[test] + fn format_size_uses_binary_units() { + assert_eq!(format_size(0), "0 B"); + assert_eq!(format_size(1536), "1.5 KiB"); + assert_eq!(format_size(66 * 1024 * 1024 * 1024), "66.0 GiB"); + } + + #[test] + fn blank_credentials_do_not_send() { + assert!(!credentials_ready(" ", "")); + assert!(!credentials_ready("token", " ")); + assert!(credentials_ready("token", "-100")); + } + + #[test] + fn curl_config_keeps_the_token_out_of_the_arguments() { + let token = "123456:ABC"; + let text = "epoch 1 slots 2-3 size 1.0 GiB"; + let config = curl_config(token, "-100123", text); + assert_eq!(CURL_ARGS, ["--config", "-"]); + assert!(CURL_ARGS.iter().all(|arg| !arg.contains(token))); + assert!(config.contains( + "url = \"https://api.telegram.org/bot123456:ABC/sendMessage\"" + )); + assert!(config.contains("data-urlencode = \"chat_id=-100123\"")); + assert!(config.contains(&format!("data-urlencode = \"text={text}\""))); + } + + #[test] + fn an_error_text_loses_the_bot_token() { + assert_eq!( + redact("failed bot 123:ABC here", "123:ABC"), + "failed bot [token] here" + ); + } + + #[test] + fn directory_size_sums_regular_files() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("a"), [1, 2, 3]).unwrap(); + std::fs::write(dir.path().join("b"), [4, 5]).unwrap(); + std::fs::create_dir(dir.path().join("nested")).unwrap(); + assert_eq!(directory_size(dir.path()).unwrap(), 5); + } +} From 8b472e54fb0c5812291fbf72f62f92b8ee566e37 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 13 Nov 2026 11:24:12 +0000 Subject: [PATCH 33/38] add publish to CF --- hivezilla/ledger-archive/Cargo.toml | 5 + hivezilla/ledger-archive/src/ledger.rs | 2 +- hivezilla/ledger-archive/src/main.rs | 74 ++- hivezilla/ledger-archive/src/notify.rs | 60 +- hivezilla/ledger-archive/src/publish.rs | 752 ++++++++++++++++++++++++ 5 files changed, 859 insertions(+), 34 deletions(-) create mode 100644 hivezilla/ledger-archive/src/publish.rs diff --git a/hivezilla/ledger-archive/Cargo.toml b/hivezilla/ledger-archive/Cargo.toml index d05c2e2..827b7be 100644 --- a/hivezilla/ledger-archive/Cargo.toml +++ b/hivezilla/ledger-archive/Cargo.toml @@ -18,7 +18,12 @@ blockzilla-compact = { path = "../../crates/compact-v2/blockzilla-compact", defa blockzilla-primitives = { path = "../../crates/blockzilla-primitives", default-features = false } blockzilla-registry = { path = "../../crates/compact-v2/blockzilla-registry" } bs58 = "0.5.1" +chrono = { version = "0.4.45", default-features = false, features = ["clock", "std"] } clap = { version = "4.6.6", features = ["derive"] } +hex = "0.4.3" +reqwest = { version = "0.13.4", default-features = false, features = ["blocking", "rustls"] } +sha2 = "0.11.0" +url = "2.5.8" of-car-reader = { path = "../../crates/old-faithful/of-car-reader", default-features = false } # Agave 4.4.0-rc.1, commit b6149a719f8fa84123ac3ce1e5f201af64bca8ee. # The local reference checkout is branch v4.4.0-rc.1 at that tag. diff --git a/hivezilla/ledger-archive/src/ledger.rs b/hivezilla/ledger-archive/src/ledger.rs index 25c29b6..b796347 100644 --- a/hivezilla/ledger-archive/src/ledger.rs +++ b/hivezilla/ledger-archive/src/ledger.rs @@ -68,7 +68,7 @@ pub(crate) fn open_read_only(ledger: &Path) -> Result { pub(crate) const SLOTS_PER_EPOCH: u64 = 432_000; -const EPOCH_ARCHIVE_FILES: [&str; 11] = [ +pub(crate) const EPOCH_ARCHIVE_FILES: [&str; 11] = [ ARCHIVE_V2_BLOCKS_FILE, ARCHIVE_V2_SIGNATURES_FILE, ARCHIVE_V2_BLOCKHASH_REGISTRY_FILE, diff --git a/hivezilla/ledger-archive/src/main.rs b/hivezilla/ledger-archive/src/main.rs index 9f77e44..ec5f5ab 100644 --- a/hivezilla/ledger-archive/src/main.rs +++ b/hivezilla/ledger-archive/src/main.rs @@ -3,6 +3,7 @@ mod convert; mod ledger; mod markers; mod notify; +mod publish; mod source; use std::path::PathBuf; @@ -42,8 +43,8 @@ enum Command { /// after the frames. `--level` is the zstd effort, default 1. /// `--read-threads` is the number of parallel shred reads, from 1 to 32. /// A failure removes the temporary files and does not publish the index. - /// A published epoch sends one Telegram line when the bot token and the - /// chat id are both set. A Telegram failure still exits 0. + /// A published epoch sends one Telegram line when TELEGRAM_BOT_TOKEN and + /// TELEGRAM_CHAT_ID are set. A Telegram failure still exits 0. BuildEpochArchive { ledger: PathBuf, /// Parent directory. The archive is written to epoch-N inside it. @@ -56,16 +57,22 @@ enum Command { /// Parallel shred reads. 1 reads one slot at a time. #[arg(long, default_value_t = 1)] read_threads: u8, - /// Telegram bot token. The default is empty and sends nothing. - #[arg(long, default_value = notify::DEFAULT_TELEGRAM_BOT_TOKEN)] - telegram_bot_token: String, - /// Telegram channel id. The default is empty and sends nothing. - #[arg( - long, - default_value = notify::DEFAULT_TELEGRAM_CHAT_ID, - allow_hyphen_values = true - )] - telegram_chat_id: String, + }, + /// Send one archived epoch to Cloudflare R2. + /// + /// The eleven published files go to s3://R2_BUCKET/R2_PREFIX/epoch-N/. + /// R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_BUCKET, R2_PREFIX, and + /// R2_ENDPOINT or R2_S3_ENDPOINT come from the environment. A missing + /// archive file stops the command before the first upload. An object of + /// the same size stays in place. A different + /// size stops the command. Telegram runs after every object is confirmed, + /// and a Telegram failure still exits 0. + #[command(name = "public-epoch-to-cf")] + PublicEpochToCf { + /// Parent directory. The archive is read from epoch-N inside it. + output: PathBuf, + #[arg(long)] + epoch: u64, }, } @@ -94,8 +101,6 @@ fn run() -> Result<()> { epoch, level, read_threads, - telegram_bot_token, - telegram_chat_id, } => { let (start_slot, end_slot) = epoch_slot_range(epoch)?; let blockstore = open_read_only(&ledger)?; @@ -113,14 +118,43 @@ fn run() -> Result<()> { build_archive(&ledger, &output, start_slot, end_slot, level, read_threads)?; print_report(&inspected); println!("Output: {}", output.display()); - notify::notify_published_epoch( - epoch, - &output, - &telegram_bot_token, - &telegram_chat_id, - )?; + notify::notify_published_epoch(epoch, &output)?; Ok(()) } + Command::PublicEpochToCf { output, epoch } => { + let settings = publish::resolve_settings(&|name| std::env::var(name).ok())?; + let output = ledger::epoch_output_dir(&output, epoch); + let files = publish::require_epoch_files(&output)?; + publish::upload_epoch(&settings, epoch, &files)?; + notify::notify_sent_to_cf(epoch, &output)?; + let folder = publish::epoch_folder(&settings.prefix, epoch); + println!("Output: s3://{}/{folder}", settings.bucket); + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn public_epoch_to_cf_accepts_the_epoch_directory() { + let cli = Cli::try_parse_from([ + "hivezilla-ledger-archive", + "public-epoch-to-cf", + "/home/sol/ledger-archive-out", + "--epoch", + "1084", + ]) + .unwrap(); + match cli.command { + Command::PublicEpochToCf { epoch, output, .. } => { + assert_eq!(epoch, 1084); + assert_eq!(output, PathBuf::from("/home/sol/ledger-archive-out")); + } + _ => panic!("wrong command"), + } } } diff --git a/hivezilla/ledger-archive/src/notify.rs b/hivezilla/ledger-archive/src/notify.rs index 93c35af..03a6d5e 100644 --- a/hivezilla/ledger-archive/src/notify.rs +++ b/hivezilla/ledger-archive/src/notify.rs @@ -1,7 +1,8 @@ //! One Telegram line after a published epoch. //! -//! Write the two defaults below. An empty token or an empty chat id sends -//! nothing. The archive command still exits 0 when Telegram fails. +//! `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` come from the process +//! environment. An empty value sends nothing. A Telegram failure still +//! returns success. use std::io::Write; use std::path::Path; @@ -9,20 +10,20 @@ use std::process::{Command, Stdio}; use anyhow::{Context, Result}; -/// Bot token from BotFather. Leave this empty to send nothing. -pub(crate) const DEFAULT_TELEGRAM_BOT_TOKEN: &str = "8127218375:AAEP6oECzjqNaLaH2VHY_HA_7VYDuS4SL6Y"; - -/// Channel id, for example "-100…". Leave this empty to send nothing. -pub(crate) const DEFAULT_TELEGRAM_CHAT_ID: &str = "-4561803275"; +pub(crate) const TELEGRAM_BOT_TOKEN_ENV: &str = "TELEGRAM_BOT_TOKEN"; +pub(crate) const TELEGRAM_CHAT_ID_ENV: &str = "TELEGRAM_CHAT_ID"; const CURL_ARGS: &[&str] = &["--config", "-"]; -pub(crate) fn notify_published_epoch( - epoch: u64, - output: &Path, - bot_token: &str, - chat_id: &str, -) -> Result<()> { +pub(crate) fn telegram_from_env() -> (String, String) { + ( + std::env::var(TELEGRAM_BOT_TOKEN_ENV).unwrap_or_default(), + std::env::var(TELEGRAM_CHAT_ID_ENV).unwrap_or_default(), + ) +} + +pub(crate) fn notify_published_epoch(epoch: u64, output: &Path) -> Result<()> { + let (bot_token, chat_id) = telegram_from_env(); let token = bot_token.trim(); let chat = chat_id.trim(); if !credentials_ready(token, chat) { @@ -51,6 +52,32 @@ pub(crate) fn published_line(epoch: u64, size: &str) -> String { format!("✅ epoch {epoch} archived, size {size}") } +pub(crate) fn sent_to_cf_line(epoch: u64, size: &str) -> String { + format!("✅ epoch {epoch} sent to CF, size {size}") +} + +pub(crate) fn notify_sent_to_cf(epoch: u64, output: &Path) -> Result<()> { + let (bot_token, chat_id) = telegram_from_env(); + let token = bot_token.trim(); + let chat = chat_id.trim(); + if !credentials_ready(token, chat) { + eprintln!("telegram: skipped, bot token or chat id is empty"); + return Ok(()); + } + let size = match directory_size(output) { + Ok(bytes) => format_size(bytes), + Err(err) => { + eprintln!("telegram: directory size: {err:#}"); + "unknown".to_string() + } + }; + let text = sent_to_cf_line(epoch, &size); + if let Err(err) = send_telegram(token, chat, &text) { + eprintln!("telegram: {}", redact(&format!("{err:#}"), token)); + } + Ok(()) +} + pub(crate) fn directory_size(path: &Path) -> Result { let mut total = 0u64; for entry in std::fs::read_dir(path).with_context(|| format!("read {}", path.display()))? { @@ -146,6 +173,13 @@ mod tests { } #[test] + fn sent_to_cf_line_names_the_epoch_and_the_size() { + assert_eq!( + sent_to_cf_line(1084, "66.2 GiB"), + "✅ epoch 1084 sent to CF, size 66.2 GiB" + ); + } + fn format_size_uses_binary_units() { assert_eq!(format_size(0), "0 B"); assert_eq!(format_size(1536), "1.5 KiB"); diff --git a/hivezilla/ledger-archive/src/publish.rs b/hivezilla/ledger-archive/src/publish.rs new file mode 100644 index 0000000..f864f22 --- /dev/null +++ b/hivezilla/ledger-archive/src/publish.rs @@ -0,0 +1,752 @@ +//! Send one archived epoch to Cloudflare R2. +//! +//! The process environment supplies `R2_ACCESS_KEY_ID`, +//! `R2_SECRET_ACCESS_KEY`, `R2_ENDPOINT` or `R2_S3_ENDPOINT`, `R2_BUCKET`, +//! and `R2_PREFIX`. The command uploads the eleven +//! published files and leaves a remote object in place when its size already +//! matches. A different remote size stops the command. Telegram runs only +//! after every object is confirmed. + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Read; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use chrono::Utc; +use reqwest::Method; +use reqwest::blocking::{Body, Client, Response}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use sha2::{Digest, Sha256}; + +use crate::ledger::EPOCH_ARCHIVE_FILES; + +pub(crate) const R2_ACCESS_KEY_ID_ENV: &str = "R2_ACCESS_KEY_ID"; +pub(crate) const R2_SECRET_ACCESS_KEY_ENV: &str = "R2_SECRET_ACCESS_KEY"; +pub(crate) const R2_ENDPOINT_ENV: &str = "R2_ENDPOINT"; +pub(crate) const R2_S3_ENDPOINT_ENV: &str = "R2_S3_ENDPOINT"; +pub(crate) const R2_BUCKET_ENV: &str = "R2_BUCKET"; +pub(crate) const R2_PREFIX_ENV: &str = "R2_PREFIX"; +pub(crate) const R2_REGION_ENV: &str = "R2_REGION"; +const DEFAULT_REGION: &str = "auto"; +const PART_SIZE: u64 = 64 * 1024 * 1024; +const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +const MAX_RESPONSE_BYTES: u64 = 1024 * 1024; +const ATTEMPTS: u32 = 4; + +pub(crate) struct PublishSettings { + pub(crate) endpoint: String, + pub(crate) bucket: String, + pub(crate) prefix: String, + region: String, + access_key: String, + secret_key: String, +} + +pub(crate) fn resolve_settings(lookup: &dyn Fn(&str) -> Option) -> Result { + let endpoint = first_present(lookup, &[R2_ENDPOINT_ENV, R2_S3_ENDPOINT_ENV]); + if endpoint.is_empty() { + bail!("{R2_ENDPOINT_ENV} is empty"); + } + validate_endpoint(&endpoint)?; + let bucket = required(lookup, R2_BUCKET_ENV)?; + let prefix = required(lookup, R2_PREFIX_ENV)?; + let region = present_or(lookup, R2_REGION_ENV, DEFAULT_REGION); + if region != DEFAULT_REGION { + bail!("R2 region must be auto"); + } + if bucket.is_empty() || bucket.contains('/') || bucket.chars().any(|value| value < '!') { + bail!("R2 bucket name is invalid"); + } + Ok(PublishSettings { + endpoint: endpoint.trim_end_matches('/').to_string(), + bucket, + prefix: normalize_prefix(&prefix)?, + region, + access_key: required(lookup, R2_ACCESS_KEY_ID_ENV)?, + secret_key: required(lookup, R2_SECRET_ACCESS_KEY_ENV)?, + }) +} + +pub(crate) fn epoch_folder(prefix: &str, epoch: u64) -> String { + format!("{prefix}/epoch-{epoch}") +} + +pub(crate) fn object_key(prefix: &str, epoch: u64, file_name: &str) -> String { + format!("{}/{file_name}", epoch_folder(prefix, epoch)) +} + +pub(crate) fn part_ranges(len: u64, part_size: u64) -> Vec<(u64, u64)> { + let mut ranges = Vec::new(); + let mut offset = 0; + while offset < len { + let size = part_size.min(len - offset); + ranges.push((offset, size)); + offset += size; + } + ranges +} + +pub(crate) fn classify_remote(local: u64, remote: Option) -> RemoteFile { + match remote { + Some(size) if size == local => RemoteFile::SameSize, + Some(remote) => RemoteFile::DifferentSize { remote }, + None => RemoteFile::Missing, + } +} + +pub(crate) enum RemoteFile { + Missing, + SameSize, + DifferentSize { remote: u64 }, +} + +pub(crate) fn require_epoch_files(dir: &Path) -> Result> { + let mut files = Vec::with_capacity(EPOCH_ARCHIVE_FILES.len()); + for name in EPOCH_ARCHIVE_FILES { + let path = dir.join(name); + let metadata = std::fs::symlink_metadata(&path) + .with_context(|| format!("archive file {} is missing", path.display()))?; + if !metadata.file_type().is_file() { + bail!("archive file {} is not a regular file", path.display()); + } + files.push(path); + } + Ok(files) +} + +pub(crate) fn upload_epoch(settings: &PublishSettings, epoch: u64, files: &[PathBuf]) -> Result<()> { + let client = R2Client::new(settings)?; + for path in files { + let name = path + .file_name() + .and_then(|name| name.to_str()) + .context("archive file name is not UTF-8")?; + let key = object_key(&settings.prefix, epoch, name); + client.upload_file(path, &key)?; + } + Ok(()) +} + +pub(crate) fn validate_endpoint(endpoint: &str) -> Result<()> { + let parsed = url::Url::parse(endpoint).context("R2 endpoint is not a URL")?; + if parsed.scheme() != "https" || parsed.host_str().is_none() { + bail!("R2 endpoint must be an absolute HTTPS URL"); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + bail!("R2 endpoint must not contain user information"); + } + if !matches!(parsed.path(), "" | "/") || parsed.query().is_some() || parsed.fragment().is_some() + { + bail!("R2 endpoint must not contain a path, query, or fragment"); + } + Ok(()) +} + +pub(crate) fn xml_element<'a>(body: &'a str, tag: &str) -> Option<&'a str> { + let open = format!("<{tag}"); + let start_tag = body.find(&open)?; + let after = body[start_tag..].find('>')? + start_tag + 1; + let close = format!(""); + let end = body[after..].find(&close)? + after; + Some(body[after..end].trim()) +} + +pub(crate) fn complete_multipart_body(parts: &[(u32, &str)]) -> String { + let mut xml = String::from(""); + for (number, etag) in parts { + xml.push_str(""); + xml.push_str(&number.to_string()); + xml.push_str(""); + xml.push_str(&xml_escape(etag)); + xml.push_str(""); + } + xml.push_str(""); + xml +} + +struct R2Client { + endpoint: String, + host: String, + bucket: String, + region: String, + access_key: String, + secret_key: String, + client: Client, +} + +impl R2Client { + fn new(settings: &PublishSettings) -> Result { + validate_endpoint(&settings.endpoint)?; + let parsed = url::Url::parse(&settings.endpoint)?; + let mut host = parsed.host_str().context("R2 endpoint has no host")?.to_string(); + if host.contains(':') && !host.starts_with('[') { + host = format!("[{host}]"); + } + if let Some(port) = parsed.port() { + host.push(':'); + host.push_str(&port.to_string()); + } + let client = Client::builder() + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(600)) + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .build() + .context("build R2 client")?; + Ok(Self { + endpoint: settings.endpoint.clone(), + host, + bucket: settings.bucket.clone(), + region: settings.region.clone(), + access_key: settings.access_key.clone(), + secret_key: settings.secret_key.clone(), + client, + }) + } + + fn upload_file(&self, path: &Path, key: &str) -> Result<()> { + let file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let metadata = file + .metadata() + .with_context(|| format!("stat {}", path.display()))?; + let local = metadata.len(); + let name = path.file_name().and_then(|value| value.to_str()).unwrap_or(key); + match classify_remote(local, self.head_len(key)?) { + RemoteFile::SameSize => { + eprintln!("{name}: already on CF, {local} bytes"); + return Ok(()); + } + RemoteFile::DifferentSize { remote } => { + bail!("{name} on CF has {remote} bytes and the local file has {local} bytes"); + } + RemoteFile::Missing => {} + } + if local == 0 { + self.put_bytes(key, &[])?; + } else { + let upload_id = self.create_multipart(key)?; + if let Err(err) = self.upload_parts(&file, key, &upload_id, local, name) { + if let Err(abort_err) = self.abort_multipart(key, &upload_id) { + eprintln!("abort {name}: {abort_err:#}"); + } + return Err(err); + } + } + let confirmed = self + .head_len(key)? + .with_context(|| format!("{name} is missing on CF after upload"))?; + if confirmed != local { + bail!("{name} on CF has {confirmed} bytes after upload and the local file has {local} bytes"); + } + let after = std::fs::symlink_metadata(path) + .with_context(|| format!("stat {}", path.display()))? + .len(); + if after != local { + bail!("{name} changed during upload"); + } + eprintln!("{name}: sent, {local} bytes"); + Ok(()) + } + + fn upload_parts( + &self, + file: &File, + key: &str, + upload_id: &str, + local: u64, + name: &str, + ) -> Result<()> { + let ranges = part_ranges(local, PART_SIZE); + let mut etags = Vec::with_capacity(ranges.len()); + for (index, (offset, size)) in ranges.iter().enumerate() { + let part_number = u32::try_from(index + 1).context("part number overflows")?; + eprintln!("{name}: part {part_number}/{}", ranges.len()); + let bytes = read_range(file, *offset, *size)?; + let etag = self.upload_part(key, upload_id, part_number, &bytes)?; + etags.push((part_number, etag)); + } + let borrowed = etags + .iter() + .map(|(number, etag)| (*number, etag.as_str())) + .collect::>(); + self.complete_multipart(key, upload_id, &borrowed) + } + + fn head_len(&self, key: &str) -> Result> { + let response = self.request(Method::HEAD, key, &BTreeMap::new(), &BTreeMap::new(), &[])?; + if response.status == 404 { + return Ok(None); + } + if response.status != 200 { + bail!("HEAD {key} returned HTTP {}", response.status); + } + Ok(Some(content_length(&response.headers, "HEAD")?)) + } + + fn put_bytes(&self, key: &str, bytes: &[u8]) -> Result<()> { + let headers = BTreeMap::from([( + "content-type".to_string(), + "application/octet-stream".to_string(), + )]); + let response = self.request(Method::PUT, key, &BTreeMap::new(), &headers, bytes)?; + if response.status != 200 { + bail!("PUT {key} returned HTTP {}", response.status); + } + Ok(()) + } + + fn create_multipart(&self, key: &str) -> Result { + let params = BTreeMap::from([("uploads".to_string(), String::new())]); + let headers = BTreeMap::from([( + "content-type".to_string(), + "application/xml".to_string(), + )]); + let response = self.request(Method::POST, key, ¶ms, &headers, &[])?; + if response.status != 200 { + bail!("create multipart {key} returned HTTP {}", response.status); + } + let body = String::from_utf8(response.body).context("multipart response is not UTF-8")?; + xml_element(&body, "UploadId") + .filter(|value| !value.is_empty()) + .map(str::to_string) + .context("multipart response has no upload id") + } + + fn upload_part(&self, key: &str, upload_id: &str, part_number: u32, bytes: &[u8]) -> Result { + let params = BTreeMap::from([ + ("partNumber".to_string(), part_number.to_string()), + ("uploadId".to_string(), upload_id.to_string()), + ]); + let headers = BTreeMap::from([( + "content-type".to_string(), + "application/octet-stream".to_string(), + )]); + let response = self.request(Method::PUT, key, ¶ms, &headers, bytes)?; + if response.status != 200 { + bail!("upload part {part_number} of {key} returned HTTP {}", response.status); + } + let etag = header_value(&response.headers, "etag", "upload part")?; + Ok(quote_etag(etag)) + } + + fn complete_multipart(&self, key: &str, upload_id: &str, parts: &[(u32, &str)]) -> Result<()> { + let params = BTreeMap::from([("uploadId".to_string(), upload_id.to_string())]); + let headers = BTreeMap::from([( + "content-type".to_string(), + "application/xml".to_string(), + )]); + let body = complete_multipart_body(parts); + let response = self.request( + Method::POST, + key, + ¶ms, + &headers, + body.as_bytes(), + )?; + if response.status != 200 { + bail!("complete multipart {key} returned HTTP {}", response.status); + } + Ok(()) + } + + fn abort_multipart(&self, key: &str, upload_id: &str) -> Result<()> { + let params = BTreeMap::from([("uploadId".to_string(), upload_id.to_string())]); + let response = self.request(Method::DELETE, key, ¶ms, &BTreeMap::new(), &[])?; + if response.status != 204 && response.status != 200 { + bail!("abort multipart {key} returned HTTP {}", response.status); + } + Ok(()) + } + + fn request( + &self, + method: Method, + key: &str, + params: &BTreeMap, + headers: &BTreeMap, + body: &[u8], + ) -> Result { + let mut last = None; + for attempt in 1..=ATTEMPTS { + match self.request_once(method.clone(), key, params, headers, body) { + Ok(response) => return Ok(response), + Err(err) if attempt < ATTEMPTS && retryable(&err) => { + eprintln!("retry {attempt}/{ATTEMPTS} for {key}"); + last = Some(err); + thread::sleep(Duration::from_secs(u64::from(attempt))); + } + Err(err) => return Err(err), + } + } + Err(last.unwrap_or_else(|| anyhow::anyhow!("R2 request failed"))) + } + + fn request_once( + &self, + method: Method, + key: &str, + params: &BTreeMap, + headers: &BTreeMap, + body: &[u8], + ) -> Result { + let now = Utc::now(); + let date = now.format("%Y%m%d").to_string(); + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let canonical_uri = format!( + "/{}/{}", + uri_encode(&self.bucket, false), + uri_encode(key.trim_start_matches('/'), true) + ); + let canonical_query = params + .iter() + .map(|(name, value)| format!("{}={}", uri_encode(name, false), uri_encode(value, false))) + .collect::>() + .join("&"); + let mut signed = BTreeMap::from([ + ("host".to_string(), self.host.clone()), + ("x-amz-content-sha256".to_string(), UNSIGNED_PAYLOAD.to_string()), + ("x-amz-date".to_string(), amz_date.clone()), + ]); + for (name, value) in headers { + signed.insert(name.clone(), value.trim().to_string()); + } + let canonical_headers = signed + .iter() + .map(|(name, value)| format!("{name}:{}\n", value.trim())) + .collect::(); + let signed_names = signed.keys().cloned().collect::>().join(";"); + let canonical_request = format!( + "{}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n{signed_names}\n{UNSIGNED_PAYLOAD}", + method.as_str() + ); + let scope = format!("{date}/{}/s3/aws4_request", self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", + hex::encode(Sha256::digest(canonical_request.as_bytes())) + ); + let signature = hex::encode(hmac_sha256( + &signing_key(&self.secret_key, &date, &self.region), + string_to_sign.as_bytes(), + )); + let authorization = format!( + "AWS4-HMAC-SHA256 Credential={}/{scope},SignedHeaders={signed_names},Signature={signature}", + self.access_key + ); + let mut url = format!("{}{}", self.endpoint, canonical_uri); + if !canonical_query.is_empty() { + url.push('?'); + url.push_str(&canonical_query); + } + let mut request = self + .client + .request(method.clone(), &url) + .body(Body::from(body.to_vec())); + for (name, value) in &signed { + request = request.header( + HeaderName::from_bytes(name.as_bytes()).context("invalid R2 header name")?, + HeaderValue::from_str(value).context("invalid R2 header value")?, + ); + } + request = request.header("authorization", authorization); + let response = request.send().with_context(|| format!("{method} {key}"))?; + let status = response.status().as_u16(); + let headers = response.headers().clone(); + if status >= 500 || status == 429 { + bail!("{method} {key} returned HTTP {status}"); + } + let mut bytes = Vec::new(); + response + .take(MAX_RESPONSE_BYTES + 1) + .read_to_end(&mut bytes) + .with_context(|| format!("read {method} {key} response"))?; + if bytes.len() as u64 > MAX_RESPONSE_BYTES { + bail!("{method} {key} response is too large"); + } + Ok(HttpResult { + status, + headers, + body: bytes, + }) + } +} + +struct HttpResult { + status: u16, + headers: HeaderMap, + body: Vec, +} + +fn present_or(lookup: &dyn Fn(&str) -> Option, key: &str, fallback: &str) -> String { + let value = first_present(lookup, &[key]); + if value.is_empty() { fallback.to_string() } else { value } +} + +fn first_present(lookup: &dyn Fn(&str) -> Option, keys: &[&str]) -> String { + for key in keys { + if let Some(value) = lookup(key).map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) + { + return value; + } + } + String::new() +} + +fn required(lookup: &dyn Fn(&str) -> Option, key: &str) -> Result { + match lookup(key).map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) { + Some(value) => Ok(value), + None => bail!("{key} is empty"), + } +} + +fn normalize_prefix(prefix: &str) -> Result { + let prefix = prefix.trim().trim_matches('/'); + if prefix.is_empty() + || prefix.split('/').any(|part| part.is_empty() || part == "." || part == "..") + { + bail!("R2 prefix is invalid"); + } + Ok(prefix.to_string()) +} + +fn content_length(headers: &HeaderMap, operation: &str) -> Result { + let text = header_value(headers, "content-length", operation)?; + if text.is_empty() || text.len() > 20 || !text.bytes().all(|byte| byte.is_ascii_digit()) { + bail!("{operation} returned an invalid content length"); + } + text.parse() + .context(format!("{operation} returned an invalid content length")) +} + +fn header_value<'a>(headers: &'a HeaderMap, name: &str, operation: &str) -> Result<&'a str> { + let header_name = HeaderName::from_bytes(name.as_bytes()) + .with_context(|| format!("{operation} requested an invalid header"))?; + let mut values = headers.get_all(&header_name).iter(); + let value = values + .next() + .with_context(|| format!("{operation} is missing {name}"))?; + if values.next().is_some() { + bail!("{operation} returned multiple {name} values"); + } + value + .to_str() + .with_context(|| format!("{operation} returned an invalid {name}")) +} + +fn quote_etag(etag: &str) -> String { + if etag.starts_with('"') && etag.ends_with('"') { + etag.to_string() + } else { + format!("\"{etag}\"") + } +} + +fn xml_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +fn read_range(file: &File, mut offset: u64, len: u64) -> Result> { + let len = usize::try_from(len).context("part is too large")?; + let mut bytes = vec![0u8; len]; + let mut filled = 0; + while filled < len { + let count = file + .read_at(&mut bytes[filled..], offset) + .context("read archive part")?; + if count == 0 { + bail!("archive part ended early"); + } + filled += count; + offset += count as u64; + } + Ok(bytes) +} + +fn retryable(err: &anyhow::Error) -> bool { + let text = format!("{err:#}"); + text.contains("HTTP 429") + || text.contains("HTTP 500") + || text.contains("HTTP 502") + || text.contains("HTTP 503") + || text.contains("HTTP 504") + || text.contains("error sending request") + || text.contains("operation timed out") +} + +fn uri_encode(value: &str, preserve_slash: bool) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() + || matches!(byte, b'-' | b'_' | b'.' | b'~') + || (preserve_slash && byte == b'/') + { + encoded.push(byte as char); + } else { + encoded.push('%'); + encoded.push(char::from(b"0123456789ABCDEF"[(byte >> 4) as usize])); + encoded.push(char::from(b"0123456789ABCDEF"[(byte & 0xf) as usize])); + } + } + encoded +} + +fn signing_key(secret: &str, date: &str, region: &str) -> [u8; 32] { + let date_key = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes()); + let region_key = hmac_sha256(&date_key, region.as_bytes()); + let service_key = hmac_sha256(®ion_key, b"s3"); + hmac_sha256(&service_key, b"aws4_request") +} + +fn hmac_sha256(key: &[u8], message: &[u8]) -> [u8; 32] { + let mut key_block = [0u8; 64]; + if key.len() > key_block.len() { + key_block[..32].copy_from_slice(&Sha256::digest(key)); + } else { + key_block[..key.len()].copy_from_slice(key); + } + let mut inner_pad = [0x36u8; 64]; + let mut outer_pad = [0x5cu8; 64]; + for index in 0..64 { + inner_pad[index] ^= key_block[index]; + outer_pad[index] ^= key_block[index]; + } + let mut inner = Sha256::new(); + inner.update(inner_pad); + inner.update(message); + let inner_hash = inner.finalize(); + let mut outer = Sha256::new(); + outer.update(outer_pad); + outer.update(inner_hash); + outer.finalize().into() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn lookup(values: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let values = values + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect::>(); + move |key| values.get(key).cloned() + } + + #[test] + fn the_epoch_object_key_uses_the_given_prefix() { + assert_eq!( + object_key("compact-v2/blockzilla-20261004", 1084, "registry.bin"), + "compact-v2/blockzilla-20261004/epoch-1084/registry.bin" + ); + } + + #[test] + fn part_ranges_cover_the_file_once() { + assert!(part_ranges(0, PART_SIZE).is_empty()); + assert_eq!(part_ranges(10, PART_SIZE), vec![(0, 10)]); + assert_eq!( + part_ranges(PART_SIZE + 1, PART_SIZE), + vec![(0, PART_SIZE), (PART_SIZE, 1)] + ); + } + + #[test] + fn a_same_size_object_stays_and_a_different_size_does_not() { + assert!(matches!(classify_remote(10, Some(10)), RemoteFile::SameSize)); + assert!(matches!(classify_remote(10, None), RemoteFile::Missing)); + assert!(matches!( + classify_remote(10, Some(11)), + RemoteFile::DifferentSize { remote: 11 } + )); + } + + #[test] + fn settings_require_the_bucket_and_the_prefix() { + let missing = lookup(&[ + ("R2_ENDPOINT", "https://account.r2.cloudflarestorage.com"), + ("R2_ACCESS_KEY_ID", "key"), + ("R2_SECRET_ACCESS_KEY", "secret"), + ("R2_PREFIX", "compact-v2/blockzilla-20261004"), + ]); + let error = resolve_settings(&missing).unwrap_err(); + assert!(error.to_string().contains(R2_BUCKET_ENV), "{error:#}"); + + let ready = lookup(&[ + ("R2_S3_ENDPOINT", "https://account.r2.cloudflarestorage.com"), + ("R2_ACCESS_KEY_ID", "key"), + ("R2_SECRET_ACCESS_KEY", "secret"), + ("R2_BUCKET", "solana-archive-data"), + ("R2_PREFIX", "compact-v2/blockzilla-20261004/"), + ]); + let settings = resolve_settings(&ready).unwrap(); + assert_eq!(settings.bucket, "solana-archive-data"); + assert_eq!(settings.prefix, "compact-v2/blockzilla-20261004"); + assert_eq!(settings.region, "auto"); + assert_eq!( + object_key(&settings.prefix, 1084, "poh.wincode"), + "compact-v2/blockzilla-20261004/epoch-1084/poh.wincode" + ); + } + + #[test] + fn an_http_endpoint_is_rejected() { + let error = validate_endpoint("http://account.r2.cloudflarestorage.com").unwrap_err(); + assert!(error.to_string().contains("HTTPS"), "{error:#}"); + } + + #[test] + fn a_missing_archive_file_stops_before_upload() { + let dir = tempfile::tempdir().unwrap(); + let error = require_epoch_files(dir.path()).unwrap_err(); + assert!(error.to_string().contains("is missing"), "{error:#}"); + } + + #[test] + fn the_eleven_archive_files_are_accepted() { + let dir = tempfile::tempdir().unwrap(); + for name in EPOCH_ARCHIVE_FILES { + File::create(dir.path().join(name)).unwrap(); + } + let files = require_epoch_files(dir.path()).unwrap(); + assert_eq!(files.len(), 11); + } + + #[test] + fn a_symlink_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + for name in EPOCH_ARCHIVE_FILES { + let path = dir.path().join(name); + if name == EPOCH_ARCHIVE_FILES[0] { + let target = dir.path().join("target"); + let mut file = File::create(&target).unwrap(); + file.write_all(b"x").unwrap(); + std::os::unix::fs::symlink(&target, &path).unwrap(); + } else { + File::create(path).unwrap(); + } + } + let error = require_epoch_files(dir.path()).unwrap_err(); + assert!(error.to_string().contains("not a regular file"), "{error:#}"); + } + + #[test] + fn the_upload_id_and_the_complete_body_keep_the_part_order() { + let body = "abc/123="; + assert_eq!(xml_element(body, "UploadId"), Some("abc/123=")); + let xml = complete_multipart_body(&[(1, "\"aaa\""), (2, "\"bbb\"")]); + assert!(xml.find("\"aaa\"").unwrap() < xml.find("\"bbb\"").unwrap()); + assert!(xml.contains("1")); + assert!(xml.contains("2")); + } +} From c6e27795461056a00590655406aca096837fb0fc Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 13 Nov 2026 11:27:32 +0000 Subject: [PATCH 34/38] correct test issues --- hivezilla/ledger-archive/src/publish.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hivezilla/ledger-archive/src/publish.rs b/hivezilla/ledger-archive/src/publish.rs index f864f22..646e3e9 100644 --- a/hivezilla/ledger-archive/src/publish.rs +++ b/hivezilla/ledger-archive/src/publish.rs @@ -18,7 +18,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use chrono::Utc; use reqwest::Method; -use reqwest::blocking::{Body, Client, Response}; +use reqwest::blocking::{Body, Client}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use sha2::{Digest, Sha256}; @@ -679,7 +679,9 @@ mod tests { ("R2_SECRET_ACCESS_KEY", "secret"), ("R2_PREFIX", "compact-v2/blockzilla-20261004"), ]); - let error = resolve_settings(&missing).unwrap_err(); + let Err(error) = resolve_settings(&missing) else { + panic!("missing R2_BUCKET was accepted"); + }; assert!(error.to_string().contains(R2_BUCKET_ENV), "{error:#}"); let ready = lookup(&[ From 5e16b347bfa28ece320131dbb6157147abcbbe26 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 13 Nov 2026 11:30:26 +0000 Subject: [PATCH 35/38] add cargo lock --- hivezilla/ledger-archive/Cargo.lock | 59 ++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/hivezilla/ledger-archive/Cargo.lock b/hivezilla/ledger-archive/Cargo.lock index 26f5b1b..a87f6f6 100644 --- a/hivezilla/ledger-archive/Cargo.lock +++ b/hivezilla/ledger-archive/Cargo.lock @@ -672,6 +672,29 @@ dependencies = [ "cc", ] +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "aya" version = "0.14.0" @@ -1321,6 +1344,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -1775,6 +1807,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn_size_of" version = "0.4.5" @@ -2439,6 +2477,12 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hivezilla-ledger-archive" version = "0.2.0" @@ -2449,8 +2493,12 @@ dependencies = [ "blockzilla-primitives", "blockzilla-registry", "bs58", + "chrono", "clap 4.6.7", + "hex", "of-car-reader", + "reqwest 0.13.5", + "sha2 0.11.1", "solana-entry", "solana-hash", "solana-instruction-error 3.0.0", @@ -2460,6 +2508,7 @@ dependencies = [ "solana-transaction-error", "solana-transaction-status", "tempfile", + "url", "wincode", "zstd 0.13.4", ] @@ -4107,6 +4156,7 @@ version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", @@ -4445,6 +4495,7 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", @@ -4452,12 +4503,16 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -4555,6 +4610,7 @@ version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -4619,6 +4675,7 @@ version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -8118,7 +8175,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", From d03e3aa9889d701607135c811ffb85230ade76bd Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 13 Nov 2026 13:58:44 +0000 Subject: [PATCH 36/38] add multithread for pushing to CF --- hivezilla/ledger-archive/src/publish.rs | 123 ++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 6 deletions(-) diff --git a/hivezilla/ledger-archive/src/publish.rs b/hivezilla/ledger-archive/src/publish.rs index 646e3e9..e725481 100644 --- a/hivezilla/ledger-archive/src/publish.rs +++ b/hivezilla/ledger-archive/src/publish.rs @@ -12,6 +12,8 @@ use std::fs::File; use std::io::Read; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; use std::time::Duration; @@ -33,6 +35,7 @@ pub(crate) const R2_PREFIX_ENV: &str = "R2_PREFIX"; pub(crate) const R2_REGION_ENV: &str = "R2_REGION"; const DEFAULT_REGION: &str = "auto"; const PART_SIZE: u64 = 64 * 1024 * 1024; +const PARALLEL_UPLOADS: usize = 4; const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; const MAX_RESPONSE_BYTES: u64 = 1024 * 1024; const ATTEMPTS: u32 = 4; @@ -155,6 +158,63 @@ pub(crate) fn xml_element<'a>(body: &'a str, tag: &str) -> Option<&'a str> { Some(body[after..end].trim()) } +fn ordered_part_etags(mut parts: Vec<(u32, String)>) -> Vec<(u32, String)> { + parts.sort_by_key(|(number, _)| *number); + parts +} + +fn run_parallel(count: usize, workers: usize, task: F) -> Result> +where + T: Send, + F: Fn(usize) -> Result + Sync, +{ + if count == 0 { + return Ok(Vec::new()); + } + let workers = workers.max(1).min(count); + let next = Mutex::new(0usize); + let failed = AtomicBool::new(false); + let results = Mutex::new((0..count).map(|_| None).collect::>>()); + let error = Mutex::new(None); + thread::scope(|scope| { + for _ in 0..workers { + scope.spawn(|| { + loop { + let index = { + let mut next = next.lock().expect("part index"); + if failed.load(Ordering::SeqCst) || *next >= count { + break; + } + let index = *next; + *next += 1; + index + }; + match task(index) { + Ok(value) => results.lock().expect("part results")[index] = Some(value), + Err(err) => { + failed.store(true, Ordering::SeqCst); + let mut saved = error.lock().expect("part error"); + if saved.is_none() { + *saved = Some(err); + } + break; + } + } + } + }); + } + }); + if let Some(err) = error.into_inner().expect("part error") { + return Err(err); + } + Ok(results + .into_inner() + .expect("part results") + .into_iter() + .map(|item| item.expect("part result")) + .collect()) +} + pub(crate) fn complete_multipart_body(parts: &[(u32, &str)]) -> String { let mut xml = String::from(""); for (number, etag) in parts { @@ -265,14 +325,16 @@ impl R2Client { name: &str, ) -> Result<()> { let ranges = part_ranges(local, PART_SIZE); - let mut etags = Vec::with_capacity(ranges.len()); - for (index, (offset, size)) in ranges.iter().enumerate() { + let total = ranges.len(); + let etags = run_parallel(total, PARALLEL_UPLOADS, |index| { + let (offset, size) = ranges[index]; let part_number = u32::try_from(index + 1).context("part number overflows")?; - eprintln!("{name}: part {part_number}/{}", ranges.len()); - let bytes = read_range(file, *offset, *size)?; + eprintln!("{name}: part {part_number}/{total}"); + let bytes = read_range(file, offset, size)?; let etag = self.upload_part(key, upload_id, part_number, &bytes)?; - etags.push((part_number, etag)); - } + Ok((part_number, etag)) + })?; + let etags = ordered_part_etags(etags); let borrowed = etags .iter() .map(|(number, etag)| (*number, etag.as_str())) @@ -634,6 +696,7 @@ fn hmac_sha256(key: &[u8], message: &[u8]) -> [u8; 32] { mod tests { use super::*; use std::io::Write; + use std::sync::atomic::AtomicUsize; fn lookup(values: &[(&str, &str)]) -> impl Fn(&str) -> Option { let values = values @@ -651,6 +714,54 @@ mod tests { ); } + #[test] + fn part_etags_return_to_part_number_order() { + let ordered = ordered_part_etags(vec![ + (3, "c".to_string()), + (1, "a".to_string()), + (2, "b".to_string()), + ]); + assert_eq!( + ordered, + vec![(1, "a".to_string()), (2, "b".to_string()), (3, "c".to_string())] + ); + let xml = complete_multipart_body( + &ordered + .iter() + .map(|(number, etag)| (*number, etag.as_str())) + .collect::>(), + ); + assert!(xml.find("1").unwrap() < xml.find("3").unwrap()); + } + + #[test] + fn parallel_uploads_stay_within_four_and_stop_after_a_failure() { + let current = AtomicUsize::new(0); + let peak = AtomicUsize::new(0); + let started = AtomicUsize::new(0); + let release = std::sync::Barrier::new(PARALLEL_UPLOADS); + let error = run_parallel(8, PARALLEL_UPLOADS, |index| { + started.fetch_add(1, Ordering::SeqCst); + let now = current.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + let outcome = (|| { + if index < PARALLEL_UPLOADS { + release.wait(); + } + if index == 0 { + bail!("part failed"); + } + thread::sleep(Duration::from_millis(40)); + Ok(index) + })(); + current.fetch_sub(1, Ordering::SeqCst); + outcome + }); + assert!(error.is_err()); + assert!(peak.load(Ordering::SeqCst) <= PARALLEL_UPLOADS); + assert_eq!(started.load(Ordering::SeqCst), PARALLEL_UPLOADS); + } + #[test] fn part_ranges_cover_the_file_once() { assert!(part_ranges(0, PART_SIZE).is_empty()); From 6b6a77ec3891f06ea452b883527c654fb2f8aab7 Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 13 Nov 2026 14:03:26 +0000 Subject: [PATCH 37/38] change to 8 threads for CF --- hivezilla/ledger-archive/src/publish.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hivezilla/ledger-archive/src/publish.rs b/hivezilla/ledger-archive/src/publish.rs index e725481..d308fa6 100644 --- a/hivezilla/ledger-archive/src/publish.rs +++ b/hivezilla/ledger-archive/src/publish.rs @@ -35,7 +35,7 @@ pub(crate) const R2_PREFIX_ENV: &str = "R2_PREFIX"; pub(crate) const R2_REGION_ENV: &str = "R2_REGION"; const DEFAULT_REGION: &str = "auto"; const PART_SIZE: u64 = 64 * 1024 * 1024; -const PARALLEL_UPLOADS: usize = 4; +const PARALLEL_UPLOADS: usize = 8; const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; const MAX_RESPONSE_BYTES: u64 = 1024 * 1024; const ATTEMPTS: u32 = 4; @@ -735,12 +735,12 @@ mod tests { } #[test] - fn parallel_uploads_stay_within_four_and_stop_after_a_failure() { + fn parallel_uploads_stay_within_the_limit_and_stop_after_a_failure() { let current = AtomicUsize::new(0); let peak = AtomicUsize::new(0); let started = AtomicUsize::new(0); let release = std::sync::Barrier::new(PARALLEL_UPLOADS); - let error = run_parallel(8, PARALLEL_UPLOADS, |index| { + let error = run_parallel(PARALLEL_UPLOADS * 2, PARALLEL_UPLOADS, |index| { started.fetch_add(1, Ordering::SeqCst); let now = current.fetch_add(1, Ordering::SeqCst) + 1; peak.fetch_max(now, Ordering::SeqCst); From bda15f79e92ec15e1a6449bd48373e658dd5e2bb Mon Sep 17 00:00:00 2001 From: Leomrqdd Date: Fri, 13 Nov 2026 14:11:32 +0000 Subject: [PATCH 38/38] change config for speed CF --- hivezilla/ledger-archive/src/publish.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hivezilla/ledger-archive/src/publish.rs b/hivezilla/ledger-archive/src/publish.rs index d308fa6..403ade2 100644 --- a/hivezilla/ledger-archive/src/publish.rs +++ b/hivezilla/ledger-archive/src/publish.rs @@ -34,8 +34,8 @@ pub(crate) const R2_BUCKET_ENV: &str = "R2_BUCKET"; pub(crate) const R2_PREFIX_ENV: &str = "R2_PREFIX"; pub(crate) const R2_REGION_ENV: &str = "R2_REGION"; const DEFAULT_REGION: &str = "auto"; -const PART_SIZE: u64 = 64 * 1024 * 1024; -const PARALLEL_UPLOADS: usize = 8; +const PART_SIZE: u64 = 128 * 1024 * 1024; +const PARALLEL_UPLOADS: usize = 16; const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; const MAX_RESPONSE_BYTES: u64 = 1024 * 1024; const ATTEMPTS: u32 = 4;