Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
e4186b4
add agave v4.3.2
Leomrqdd Oct 22, 2026
ccbbd6c
add agave v4.3.2 for blockzilla
Leomrqdd Oct 22, 2026
c122730
add AG new info on the archive V2
Leomrqdd Oct 28, 2026
a35c848
reduce size of archive with AG
Leomrqdd Oct 28, 2026
403b962
add blockmarker to parse alpenglow markers
Leomrqdd Oct 28, 2026
fafdac5
add show markers CLI command
Leomrqdd Oct 28, 2026
cc58fa4
correct warning
Leomrqdd Oct 28, 2026
d101233
test build archive from ledger on server
Leomrqdd Nov 3, 2026
fdb220c
use the Agave RocksDB fork so ledger-archive tests build
Leomrqdd Nov 3, 2026
b264010
add english report
Leomrqdd Nov 3, 2026
681759a
read and compress directly
Leomrqdd Nov 4, 2026
a6561b4
solve fork issue
Leomrqdd Nov 4, 2026
0e5c89e
add registry.bin in ledger archive
Leomrqdd Nov 5, 2026
abfe637
fix test issues
Leomrqdd Nov 5, 2026
05b857e
replace recent_blockhash by an id
Leomrqdd Nov 5, 2026
32de709
fix issues with tests
Leomrqdd Nov 5, 2026
455992c
empty vote data after AG
Leomrqdd Nov 5, 2026
72b978e
add ztsd option
Leomrqdd Nov 5, 2026
dece7ff
fix test issue
Leomrqdd Nov 5, 2026
1c87bab
add multithread for reading
Leomrqdd Nov 6, 2026
afb775a
fix issue for multithread
Leomrqdd Nov 6, 2026
1f07f09
add skip scenario
Leomrqdd Nov 9, 2026
e5967d2
add 32 thread as max
Leomrqdd Nov 9, 2026
7c42a2b
add notar reward bug fix
Leomrqdd Nov 9, 2026
e37bfc8
add readme for ledger archive
Leomrqdd Nov 9, 2026
73db384
add epoch archive command
Leomrqdd Nov 11, 2026
7e2a5ef
fix issue with block footer
Leomrqdd Nov 11, 2026
834978f
add a missing case for skipping a block
Leomrqdd Nov 11, 2026
e172cdf
Remove the free-range ledger archive command.
ysaxyz Nov 12, 2026
a4d1844
add registry counts and mphf
Leomrqdd Nov 12, 2026
ef7e020
add missing fields
Leomrqdd Nov 12, 2026
8573aeb
add telegram monitoring
Leomrqdd Nov 12, 2026
8b472e5
add publish to CF
Leomrqdd Nov 13, 2026
c6e2779
correct test issues
Leomrqdd Nov 13, 2026
5e16b34
add cargo lock
Leomrqdd Nov 13, 2026
d03e3aa
add multithread for pushing to CF
Leomrqdd Nov 13, 2026
6b6a77e
change to 8 threads for CF
Leomrqdd Nov 13, 2026
bda15f7
change config for speed CF
Leomrqdd Nov 13, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,8 @@
[the deployment record](hivezilla/deploy/direct-agave/README.md). Check live
reports when current machine status is needed. Older handoff files are history.
- Use ASD-STE100 Simplified Technical English when communicating with the user.
- Before you change code, write the plan: the files you will change, the
behavior you will keep, and the tests you will add. Wait for that plan to
be in the reply before you edit.
- After you change code, give a short record of the work: each file you
changed, what changed in it, and which tests passed.
5 changes: 3 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
[workspace]
resolver = "2"
# The validator plugin has an independent lock file for its exact Agave ABI.
exclude = ["hivezilla/direct-agave", "hivezilla/stream-relay", "hivezilla/stream-relay/decoder"]
# The validator plugin and the ledger archive tool have independent lock files
# for their exact Agave ABI.
exclude = ["hivezilla/direct-agave", "hivezilla/stream-relay", "hivezilla/stream-relay/decoder", "hivezilla/ledger-archive"]
members = [
# Shared models, primitives, and byte sources.
"crates/blockzilla-model",
Expand Down
3 changes: 3 additions & 0 deletions blockzilla/archive-gateway/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1422,6 +1422,9 @@ mod tests {
}],
message_bytes: message,
metadata_bytes: metadata,

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap();
Expand Down
48 changes: 35 additions & 13 deletions blockzilla/cli/src/archive_v2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ use blockzilla_archive_v2::{
ArchiveV2BlockAccessBlob, ArchiveV2BlockAccessBlockhash, ArchiveV2BlockAccessIndexRow,
ArchiveV2BlockAccessPubkey, ArchiveV2BlockAccessVoteHash,
ArchiveV2ComputeBudgetInstructionData, ArchiveV2GetBlockIndexRow, ArchiveV2HotBlockBlob,
ArchiveV2HotBlockHeader, ArchiveV2HotBlockIndexRow, ArchiveV2HotInstruction,
ArchiveV2HotInstructionData, ArchiveV2HotLegacyMessage, ArchiveV2HotMessagePayload,
ArchiveV2HotMetaRecord, ArchiveV2HotRewards, ArchiveV2HotTxRow, ArchiveV2HotV0Message,
ArchiveV2HotV1Message, ArchiveV2SystemInstructionData, ArchiveV2VoteHashRef,
ArchiveV2VoteLockoutOffset, ArchiveV2VoteStateUpdate, ArchiveV2VoteTowerSync,
BLOCK_TIME_GAP_FILE, WINCODE_ARCHIVE_V2_BLOCK_ACCESS_VERSION,
ArchiveV2HotBlockHeader, ArchiveV2HotBlockIndexRow, ArchiveV2HotConsensus,
ArchiveV2HotInstruction, ArchiveV2HotInstructionData, ArchiveV2HotLegacyMessage,
ArchiveV2HotMessagePayload, ArchiveV2HotMetaRecord, ArchiveV2HotRewards, ArchiveV2HotTxRow,
ArchiveV2HotV0Message, ArchiveV2HotV1Message, ArchiveV2SystemInstructionData,
ArchiveV2VoteHashRef, ArchiveV2VoteLockoutOffset, ArchiveV2VoteStateUpdate,
ArchiveV2VoteTowerSync, BLOCK_TIME_GAP_FILE, WINCODE_ARCHIVE_V2_BLOCK_ACCESS_VERSION,
WINCODE_ARCHIVE_V2_FLAG_ALL_PUBKEY_REF_COUNTS, WINCODE_ARCHIVE_V2_FLAG_FIRST_SEEN_REGISTRY,
WINCODE_ARCHIVE_V2_FLAG_LEB128, WINCODE_ARCHIVE_V2_FLAG_NO_REGISTRY,
WINCODE_ARCHIVE_V2_HOT_BLOCK_VERSION, WINCODE_ARCHIVE_V2_VERSION, WincodeArchiveV2Block,
Expand Down Expand Up @@ -687,7 +687,7 @@ pub(crate) fn build(
.checked_sub(1)
.and_then(|index| blockhashes.get(index))
.copied();
let (record, tx_count, mut sidecar) = build_block_record(
let (record, tx_count, mut sidecar, _consensus) = build_block_record(
&mut pending,
block,
&key_index,
Expand Down Expand Up @@ -1000,7 +1000,7 @@ pub(crate) fn build_hot_blocks(
.checked_sub(1)
.and_then(|index| blockhashes.get(index))
.copied();
let (mut record, tx_count, mut sidecar) = build_block_record(
let (mut record, tx_count, mut sidecar, consensus) = build_block_record(
&mut pending,
block,
&key_index,
Expand Down Expand Up @@ -1035,7 +1035,7 @@ pub(crate) fn build_hot_blocks(
} else {
None
};
let (hot_block, block_signature_count) = hot_block_from_archive_block(
let (mut hot_block, block_signature_count) = hot_block_from_archive_block(
record,
&known_program_ids,
&slot_to_block_id,
Expand All @@ -1046,6 +1046,7 @@ pub(crate) fn build_hot_blocks(
&mut timings,
)
.with_context(|| format!("slot {slot} hot block encode"))?;
apply_hot_consensus(&mut hot_block, consensus);
patch_poh_entry_signature_counts(&mut sidecar.poh_entries, &hot_block.tx_rows)
.with_context(|| format!("slot {slot} PoH entry signature counts"))?;
block_bytes.clear();
Expand Down Expand Up @@ -1736,7 +1737,7 @@ pub(crate) fn build_hot_blocks_first_seen(
timings.classify += classify_started.elapsed();
let previous_blockhash =
last_blockhash.or_else(|| previous_tail.last().map(|previous| previous.hash));
let (mut record, tx_count, mut sidecar) = build_block_record(
let (mut record, tx_count, mut sidecar, consensus) = build_block_record(
&mut pending,
block,
&raw_key_index,
Expand Down Expand Up @@ -1798,7 +1799,7 @@ pub(crate) fn build_hot_blocks_first_seen(
known_program_ids.refresh_from_first_seen(&registry);
vote_hashes.ensure_block(block_id);
let block_shredding = std::mem::take(&mut record.header.compact.shredding);
let (hot_block, block_signature_count) = hot_block_from_first_seen_archive_block(
let (mut hot_block, block_signature_count) = hot_block_from_first_seen_archive_block(
record,
&first_seen_signatures,
&known_program_ids,
Expand All @@ -1809,6 +1810,7 @@ pub(crate) fn build_hot_blocks_first_seen(
&mut timings,
)
.with_context(|| format!("slot {slot} first-seen hot block encode"))?;
apply_hot_consensus(&mut hot_block, consensus);
first_seen_signatures.record_block_write();
patch_poh_entry_signature_counts(&mut sidecar.poh_entries, &hot_block.tx_rows)
.with_context(|| format!("slot {slot} PoH entry signature counts"))?;
Expand Down Expand Up @@ -2692,7 +2694,7 @@ fn build_pre_hot_spool(
let archive_block_id = block_id.saturating_add(blockhash_id_offset);
let previous_blockhash = (archive_block_id > 0).then_some(last_blockhash).flatten();
rolling_blockhashes.prune_for_slot(block.slot)?;
let (record, tx_count, mut sidecar) = build_block_record(
let (record, tx_count, mut sidecar, _consensus) = build_block_record(
&mut pending,
block,
&raw_key_index,
Expand Down Expand Up @@ -6332,6 +6334,9 @@ fn hot_block_from_archive_block_with_signatures(
tx_rows,
message_bytes,
metadata_bytes,

consensus_block_id: None,
block_markers: Vec::new(),
},
block_signature_count,
))
Expand Down Expand Up @@ -6500,6 +6505,9 @@ fn hot_block_from_live_no_registry_block(
tx_rows,
message_bytes,
metadata_bytes,

consensus_block_id: None,
block_markers: Vec::new(),
},
block_signature_count,
))
Expand Down Expand Up @@ -6696,6 +6704,9 @@ fn hot_block_from_live_no_registry_block_signature_counts(
tx_rows,
message_bytes,
metadata_bytes,

consensus_block_id: None,
block_markers: Vec::new(),
},
block_signature_count,
))
Expand Down Expand Up @@ -17599,7 +17610,12 @@ fn build_block_record(
zstd: &mut ZstdReusableDecoder,
parallel_tx_decoder: Option<&mut HotTxDecodePool>,
mut first_seen_signatures: Option<&mut FirstSeenBlockSignatures>,
) -> Result<(WincodeArchiveV2Block, u32, ArchiveV2BlockSidecar)> {
) -> Result<(
WincodeArchiveV2Block,
u32,
ArchiveV2BlockSidecar,
ArchiveV2HotConsensus,
)> {
pending.last_slot = block.slot;
let entries = ordered_entries(pending, &block)?;
let sidecar = block_sidecar_from_entries(
Expand Down Expand Up @@ -17765,9 +17781,15 @@ fn build_block_record(
},
ordered_txs.len() as u32,
sidecar,
ArchiveV2HotConsensus::from_slot_meta(&block.meta),
))
}

fn apply_hot_consensus(block: &mut ArchiveV2HotBlockBlob, consensus: ArchiveV2HotConsensus) {
block.consensus_block_id = consensus.block_id;
block.block_markers = consensus.markers;
}

fn build_no_registry_block_record(
pending: &mut PendingBlock,
block: RawBlockNode,
Expand Down
1 change: 1 addition & 0 deletions blockzilla/cli/src/archive_v2/direct_agave.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ fn validate(r: &Receipt) -> Result<()> {
);
ensure!(
r.agave_revision == "825efd18292aff6ffcf9daa0f7612f21b3531a72"
|| r.agave_revision == "a203300045b405f86c9c84308b2eae76af69ac3b"
|| option_env!("HIVEZILLA_AGAVE_REVISION")
.is_some_and(|revision| r.agave_revision == revision),
"unsupported Agave source revision"
Expand Down
30 changes: 30 additions & 0 deletions blockzilla/cli/src/archive_v2/registry_reprocess.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1784,6 +1784,8 @@ impl From<LegacyHotBlockWithShredding> for ArchiveV2HotBlockBlob {
tx_rows: value.tx_rows,
message_bytes: value.message_bytes,
metadata_bytes: value.metadata_bytes,
consensus_block_id: None,
block_markers: Vec::new(),
}
}
}
Expand All @@ -1808,6 +1810,8 @@ impl From<LegacyHotBlockWithRewardsVec> for ArchiveV2HotBlockBlob {
tx_rows: value.tx_rows,
message_bytes: value.message_bytes,
metadata_bytes: value.metadata_bytes,
consensus_block_id: None,
block_markers: Vec::new(),
}
}
}
Expand Down Expand Up @@ -6841,6 +6845,9 @@ fn rewrite_source_block_with_scratch(
tx_rows: std::mem::take(&mut scratch.rows),
message_bytes: std::mem::take(&mut scratch.target_messages),
metadata_bytes: std::mem::take(&mut scratch.target_metadata),

consensus_block_id: None,
block_markers: Vec::new(),
};
(block, HotBlockOuterSchema::Current, access_references)
}
Expand Down Expand Up @@ -12643,6 +12650,9 @@ mod tests {
}],
message_bytes,
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
let compressed = zstd::bulk::compress(&encoded, 1).unwrap();
Expand Down Expand Up @@ -13466,6 +13476,9 @@ mod tests {
}],
message_bytes: Vec::new(),
metadata_bytes,

consensus_block_id: None,
block_markers: Vec::new(),
};
let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
let analysis = analyze_source_exclusions(compressed_test_input(encoded, 1, 0)).unwrap();
Expand All @@ -13489,6 +13502,9 @@ mod tests {
tx_rows: Vec::new(),
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let mut encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
encoded.push(0xa5);
Expand Down Expand Up @@ -13674,6 +13690,8 @@ mod tests {
],
message_bytes: [first_message, second_message, raw_message].concat(),
metadata_bytes: [successful, failed, raw_metadata].concat(),
consensus_block_id: None,
block_markers: Vec::new(),
}
}

Expand Down Expand Up @@ -14219,6 +14237,9 @@ mod tests {
],
message_bytes: raw_messages.to_vec(),
metadata_bytes: [encoded_metadata, raw_metadata.to_vec()].concat(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let encoded = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
let mut full: ArchiveV2HotBlockBlob = wincode::config::deserialize_exact(
Expand Down Expand Up @@ -14345,6 +14366,9 @@ mod tests {
],
message_bytes: [message_bytes, raw_message.to_vec()].concat(),
metadata_bytes: [metadata_bytes, raw_metadata.to_vec()].concat(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let mut references = AccessReferenceSet::default();
super::super::collect_access_blockhash_id(
Expand Down Expand Up @@ -14463,6 +14487,9 @@ mod tests {
],
message_bytes: [first, second].concat(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let mut access = AccessReferenceSet::default();

Expand Down Expand Up @@ -14552,6 +14579,9 @@ mod tests {
tx_rows: Vec::new(),
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let mut classes = Vec::new();
rewrite_block_pubkeys(&mut block, TEST_WIRE_PROFILE, |_key, class| {
Expand Down
3 changes: 3 additions & 0 deletions blockzilla/cli/src/archive_v2/trusted_metadata_normalize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1985,6 +1985,9 @@ mod tests {
)],
message_bytes: message,
metadata_bytes: metadata,

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap();
Expand Down
24 changes: 24 additions & 0 deletions blockzilla/cli/src/archive_verify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5023,6 +5023,9 @@ mod tests {
tx_rows: vec![],
message_bytes: vec![],
metadata_bytes: vec![],

consensus_block_id: None,
block_markers: Vec::new(),
};
let raw = wincode::config::serialize(&blob, blockzilla_primitives::wincode_leb128_config())
.unwrap();
Expand Down Expand Up @@ -5238,6 +5241,9 @@ mod tests {
tx_rows: Vec::new(),
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
},
ArchiveV2HotBlockBlob {
header: ArchiveV2HotBlockHeader {
Expand All @@ -5262,6 +5268,9 @@ mod tests {
}],
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
},
];
let root = poh_migration_verify_fixture_root(label);
Expand Down Expand Up @@ -5557,6 +5566,9 @@ mod tests {
tx_rows: vec![row(1, 2), row(0, 1)],
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed =
wincode::config::serialize(&hot_block, blockzilla_primitives::wincode_leb128_config())
Expand Down Expand Up @@ -5914,6 +5926,9 @@ mod tests {
}],
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed = wincode::config::serialize(
&hot_block,
Expand Down Expand Up @@ -7511,6 +7526,9 @@ mod tests {
],
message_bytes: vec![10, 11, 12, 13, 14],
metadata_bytes: vec![20, 21, 22],

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed =
wincode::config::serialize(&hot_block, blockzilla_primitives::wincode_leb128_config())
Expand All @@ -7530,6 +7548,9 @@ mod tests {
tx_rows: Vec::new(),
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let empty_uncompressed = wincode::config::serialize(
&empty_hot_block,
Expand Down Expand Up @@ -7683,6 +7704,9 @@ mod tests {
tx_rows: Vec::new(),
message_bytes: Vec::new(),
metadata_bytes: Vec::new(),

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed =
wincode::config::serialize(&hot_block, blockzilla_primitives::wincode_leb128_config())
Expand Down
3 changes: 3 additions & 0 deletions blockzilla/cli/src/bin/archive_v2_normalize_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3542,6 +3542,9 @@ mod tests {
tx_rows: vec![row],
message_bytes: message.clone(),
metadata_bytes: metadata,

consensus_block_id: None,
block_markers: Vec::new(),
};
let uncompressed = wincode::config::serialize(&block, wincode_leb128_config()).unwrap();
let compressed = zstd::bulk::compress(&uncompressed, 3).unwrap();
Expand Down
Loading
Loading