From 05a7bdc9ac9280aa7fb35ca3f8f268a34890273c Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:08:38 +0200 Subject: [PATCH 001/103] chore(provider): use `get_in_memory_or_storage` on `transactions_by_block_range` (#11453) --- .../src/providers/blockchain_provider.rs | 35 ++++--------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index f5f838293eb4..cebb88703dcd 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -873,35 +873,12 @@ impl TransactionsProvider for BlockchainProvider2 { &self, range: impl RangeBounds, ) -> ProviderResult>> { - let (start, end) = self.convert_range_bounds(range, || { - self.canonical_in_memory_state.get_canonical_block_number() - }); - - let mut transactions = Vec::new(); - let mut last_in_memory_block = None; - - for number in start..=end { - if let Some(block_state) = self.canonical_in_memory_state.state_by_number(number) { - // TODO: there might be an update between loop iterations, we - // need to handle that situation. - transactions.push(block_state.block().block().body.transactions.clone()); - last_in_memory_block = Some(number); - } else { - break - } - } - - if let Some(last_block) = last_in_memory_block { - if last_block < end { - let mut db_transactions = - self.database.transactions_by_block_range((last_block + 1)..=end)?; - transactions.append(&mut db_transactions); - } - } else { - transactions = self.database.transactions_by_block_range(start..=end)?; - } - - Ok(transactions) + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.transactions_by_block_range(range), + |block_state, _| Some(block_state.block().block().body.transactions.clone()), + |_| true, + ) } fn transactions_by_tx_range( From a6b1fee5399b33cd229fd1e23979e39b7fdfd27f Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 3 Oct 2024 16:10:07 +0300 Subject: [PATCH 002/103] chore(exex): adjust WAL gauge metric names (#11454) --- crates/exex/exex/src/wal/metrics.rs | 8 ++++---- crates/exex/exex/src/wal/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/exex/exex/src/wal/metrics.rs b/crates/exex/exex/src/wal/metrics.rs index 7726fc978d47..01837629407e 100644 --- a/crates/exex/exex/src/wal/metrics.rs +++ b/crates/exex/exex/src/wal/metrics.rs @@ -7,10 +7,10 @@ use reth_metrics::Metrics; pub(super) struct Metrics { /// Size of all notifications in WAL in bytes pub size_bytes: Gauge, - /// Total number of notifications in WAL - pub notifications_total: Gauge, - /// Total number of committed blocks in WAL - pub committed_blocks_total: Gauge, + /// Number of notifications in WAL + pub notifications_count: Gauge, + /// Number of committed blocks in WAL + pub committed_blocks_count: Gauge, /// Lowest committed block height in WAL pub lowest_committed_block_height: Gauge, /// Highest committed block height in WAL diff --git a/crates/exex/exex/src/wal/mod.rs b/crates/exex/exex/src/wal/mod.rs index 2341b56d1044..00b0ea919ef6 100644 --- a/crates/exex/exex/src/wal/mod.rs +++ b/crates/exex/exex/src/wal/mod.rs @@ -167,8 +167,8 @@ impl WalInner { fn update_metrics(&self, block_cache: &BlockCache, size_delta: i64) { self.metrics.size_bytes.increment(size_delta as f64); - self.metrics.notifications_total.set(block_cache.notification_max_blocks.len() as f64); - self.metrics.committed_blocks_total.set(block_cache.committed_blocks.len() as f64); + self.metrics.notifications_count.set(block_cache.notification_max_blocks.len() as f64); + self.metrics.committed_blocks_count.set(block_cache.committed_blocks.len() as f64); if let Some(lowest_committed_block_height) = block_cache.lowest_committed_block_height { self.metrics.lowest_committed_block_height.set(lowest_committed_block_height as f64); From 8cd052c4d9a8477bce2d7091533b006c55a07187 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:17:17 +0200 Subject: [PATCH 003/103] chore(provider): use `get_in_memory_or_storage_by_block` on `fn block_body_indices` (#11452) --- .../src/providers/blockchain_provider.rs | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index cebb88703dcd..9171f9d069c8 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -696,34 +696,34 @@ impl BlockReader for BlockchainProvider2 { &self, number: BlockNumber, ) -> ProviderResult> { - if let Some(indices) = self.database.block_body_indices(number)? { - Ok(Some(indices)) - } else if let Some(state) = self.canonical_in_memory_state.state_by_number(number) { - // we have to construct the stored indices for the in memory blocks - // - // To calculate this we will fetch the anchor block and walk forward from all parents - let mut parent_chain = state.parent_state_chain(); - parent_chain.reverse(); - let anchor_num = state.anchor().number; - let mut stored_indices = self - .database - .block_body_indices(anchor_num)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(anchor_num))?; - stored_indices.first_tx_num = stored_indices.next_tx_num(); - - for state in parent_chain { - let txs = state.block().block.body.transactions.len() as u64; - if state.block().block().number == number { - stored_indices.tx_count = txs; - } else { - stored_indices.first_tx_num += txs; + self.get_in_memory_or_storage_by_block( + number.into(), + |db_provider| db_provider.block_body_indices(number), + |block_state| { + // Find the last block indices on database + let last_storage_block_number = block_state.anchor().number; + let mut stored_indices = self + .database + .block_body_indices(last_storage_block_number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(last_storage_block_number))?; + + // Prepare our block indices + stored_indices.first_tx_num = stored_indices.next_tx_num(); + stored_indices.tx_count = 0; + + // Iterate from the lowest block in memory until our target block + for state in block_state.chain().into_iter().rev() { + let block_tx_count = state.block().block.body.transactions.len() as u64; + if state.block_ref().block().number == number { + stored_indices.tx_count = block_tx_count; + } else { + stored_indices.first_tx_num += block_tx_count; + } } - } - Ok(Some(stored_indices)) - } else { - Ok(None) - } + Ok(Some(stored_indices)) + }, + ) } /// Returns the block with senders with matching number or hash from database. From ac85fc0507b2300350cb0156461f23b7a641b06b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:19:02 +0200 Subject: [PATCH 004/103] chore(provider): find `last_database_block_number` with `BlockState` anchor instead (#11455) --- .../src/providers/blockchain_provider.rs | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 9171f9d069c8..9b56630c3a6a 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -268,12 +268,10 @@ impl BlockchainProvider2 { // Get the last block number stored in the storage which does NOT overlap with in-memory // chain. - let mut last_database_block_number = provider.last_block_number()?; - if let Some(lowest_in_mem_block) = in_mem_chain.last() { - if lowest_in_mem_block.number() <= last_database_block_number { - last_database_block_number = lowest_in_mem_block.number().saturating_sub(1); - } - } + let last_database_block_number = in_mem_chain + .last() + .map(|b| Ok(b.anchor().number)) + .unwrap_or_else(|| provider.last_block_number())?; // Get the next tx number for the last block stored in the storage, which marks the start of // the in-memory state. @@ -361,12 +359,10 @@ impl BlockchainProvider2 { // Get the last block number stored in the database which does NOT overlap with in-memory // chain. - let mut last_database_block_number = provider.last_block_number()?; - if let Some(lowest_in_mem_block) = in_mem_chain.last() { - if lowest_in_mem_block.number() <= last_database_block_number { - last_database_block_number = lowest_in_mem_block.number().saturating_sub(1); - } - } + let last_database_block_number = in_mem_chain + .last() + .map(|b| Ok(b.anchor().number)) + .unwrap_or_else(|| provider.last_block_number())?; // Get the next tx number for the last block stored in the database and consider it the // first tx number of the in-memory state From 91af1a75c2faa5d46d74aae7ebe8feba2cc76055 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Oct 2024 15:44:41 +0200 Subject: [PATCH 005/103] feat: add metrics for failed deliveries (#11456) --- crates/engine/tree/src/tree/metrics.rs | 8 ++++++++ crates/engine/tree/src/tree/mod.rs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/crates/engine/tree/src/tree/metrics.rs b/crates/engine/tree/src/tree/metrics.rs index 52dbf34173df..d46c2f05a028 100644 --- a/crates/engine/tree/src/tree/metrics.rs +++ b/crates/engine/tree/src/tree/metrics.rs @@ -34,6 +34,14 @@ pub(crate) struct EngineMetrics { pub(crate) new_payload_messages: Counter, /// Histogram of persistence operation durations (in seconds) pub(crate) persistence_duration: Histogram, + /// Tracks the how often we failed to deliver a newPayload response. + /// + /// This effectively tracks how often the message sender dropped the channel and indicates a CL + /// request timeout (e.g. it took more than 8s to send the response and the CL terminated the + /// request which resulted in a closed channel). + pub(crate) failed_new_payload_response_deliveries: Counter, + /// Tracks the how often we failed to deliver a forkchoice update response. + pub(crate) failed_forkchoice_updated_response_deliveries: Counter, // TODO add latency metrics } diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index e01b75288220..47be69a3dee4 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1219,6 +1219,10 @@ where if let Err(err) = tx.send(output.map(|o| o.outcome).map_err(Into::into)) { + self.metrics + .engine + .failed_forkchoice_updated_response_deliveries + .increment(1); error!(target: "engine::tree", "Failed to send event: {err:?}"); } } @@ -1230,6 +1234,10 @@ where ) })) { error!(target: "engine::tree", "Failed to send event: {err:?}"); + self.metrics + .engine + .failed_new_payload_response_deliveries + .increment(1); } } BeaconEngineMessage::TransitionConfigurationExchanged => { From e5678109517839ae11ee28ff6fc2dc676e4378a8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Oct 2024 15:45:32 +0200 Subject: [PATCH 006/103] chore: release 1.0.8 (#11457) --- Cargo.lock | 236 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 119 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4fa4c6a6a26..ceeb53a2e4a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2563,7 +2563,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -5252,7 +5252,7 @@ dependencies = [ [[package]] name = "op-reth" -version = "1.0.7" +version = "1.0.8" dependencies = [ "clap", "reth-cli-util", @@ -6208,7 +6208,7 @@ dependencies = [ [[package]] name = "reth" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6280,7 +6280,7 @@ dependencies = [ [[package]] name = "reth-auto-seal-consensus" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -6310,7 +6310,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6333,7 +6333,7 @@ dependencies = [ [[package]] name = "reth-beacon-consensus" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6384,7 +6384,7 @@ dependencies = [ [[package]] name = "reth-bench" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-json-rpc", @@ -6419,7 +6419,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6457,7 +6457,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-consensus", @@ -6469,7 +6469,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-chains", "alloy-eips", @@ -6520,7 +6520,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "1.0.7" +version = "1.0.8" dependencies = [ "clap", "eyre", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "1.0.7" +version = "1.0.8" dependencies = [ "ahash", "alloy-eips", @@ -6589,7 +6589,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "1.0.7" +version = "1.0.8" dependencies = [ "reth-tasks", "tokio", @@ -6598,7 +6598,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -6615,7 +6615,7 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6638,7 +6638,7 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "1.0.7" +version = "1.0.8" dependencies = [ "convert_case", "proc-macro2", @@ -6649,7 +6649,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "eyre", @@ -6665,7 +6665,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "auto_impl", @@ -6675,7 +6675,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -6690,7 +6690,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6713,7 +6713,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "arbitrary", @@ -6753,7 +6753,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6808,7 +6808,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "arbitrary", @@ -6824,7 +6824,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6850,7 +6850,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6874,7 +6874,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-chains", "alloy-primitives", @@ -6902,7 +6902,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -6939,7 +6939,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6977,7 +6977,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "1.0.7" +version = "1.0.8" dependencies = [ "aes", "alloy-primitives", @@ -7007,7 +7007,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7037,7 +7037,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-execution-types", @@ -7049,7 +7049,7 @@ dependencies = [ [[package]] name = "reth-engine-service" -version = "1.0.7" +version = "1.0.8" dependencies = [ "futures", "pin-project", @@ -7077,7 +7077,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7124,7 +7124,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7154,7 +7154,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "1.0.7" +version = "1.0.8" dependencies = [ "reth-blockchain-tree-api", "reth-consensus", @@ -7166,7 +7166,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7200,7 +7200,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7223,7 +7223,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-genesis", "clap", @@ -7237,7 +7237,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -7249,7 +7249,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7268,7 +7268,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-chains", "alloy-primitives", @@ -7287,7 +7287,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-basic-payload-builder", @@ -7311,7 +7311,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "rayon", @@ -7321,7 +7321,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7343,7 +7343,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7366,7 +7366,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7381,7 +7381,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7398,7 +7398,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7443,7 +7443,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "1.0.7" +version = "1.0.8" dependencies = [ "eyre", "futures-util", @@ -7475,7 +7475,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7491,7 +7491,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "1.0.7" +version = "1.0.8" dependencies = [ "serde", "serde_json", @@ -7500,7 +7500,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7524,7 +7524,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "1.0.7" +version = "1.0.8" dependencies = [ "async-trait", "bytes", @@ -7546,7 +7546,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "1.0.7" +version = "1.0.8" dependencies = [ "bitflags 2.6.0", "byteorder", @@ -7567,7 +7567,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "1.0.7" +version = "1.0.8" dependencies = [ "bindgen 0.70.1", "cc", @@ -7575,7 +7575,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "1.0.7" +version = "1.0.8" dependencies = [ "futures", "metrics", @@ -7586,14 +7586,14 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", ] [[package]] name = "reth-net-nat" -version = "1.0.7" +version = "1.0.8" dependencies = [ "futures-util", "if-addrs", @@ -7607,7 +7607,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7667,7 +7667,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types-admin", @@ -7689,7 +7689,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7709,7 +7709,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7725,7 +7725,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "humantime-serde", "reth-ethereum-forks", @@ -7738,7 +7738,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "1.0.7" +version = "1.0.8" dependencies = [ "anyhow", "bincode", @@ -7756,7 +7756,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "reth-engine-primitives", "reth-evm", @@ -7773,7 +7773,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -7837,7 +7837,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -7892,7 +7892,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -7928,7 +7928,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7950,7 +7950,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "1.0.7" +version = "1.0.8" dependencies = [ "eyre", "http", @@ -7976,7 +7976,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "reth-chainspec", "reth-db-api", @@ -7985,7 +7985,7 @@ dependencies = [ [[package]] name = "reth-optimism-chainspec" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-chains", "alloy-genesis", @@ -8003,7 +8003,7 @@ dependencies = [ [[package]] name = "reth-optimism-cli" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8046,7 +8046,7 @@ dependencies = [ [[package]] name = "reth-optimism-consensus" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -8061,7 +8061,7 @@ dependencies = [ [[package]] name = "reth-optimism-evm" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8087,7 +8087,7 @@ dependencies = [ [[package]] name = "reth-optimism-forks" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8098,7 +8098,7 @@ dependencies = [ [[package]] name = "reth-optimism-node" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-genesis", @@ -8151,7 +8151,7 @@ dependencies = [ [[package]] name = "reth-optimism-payload-builder" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8184,7 +8184,7 @@ dependencies = [ [[package]] name = "reth-optimism-primitives" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-primitives", @@ -8193,7 +8193,7 @@ dependencies = [ [[package]] name = "reth-optimism-rpc" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8232,7 +8232,7 @@ dependencies = [ [[package]] name = "reth-optimism-storage" -version = "1.0.7" +version = "1.0.8" dependencies = [ "reth-codecs", "reth-db-api", @@ -8243,7 +8243,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8264,7 +8264,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8285,7 +8285,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-rpc-types", "reth-chainspec", @@ -8295,7 +8295,7 @@ dependencies = [ [[package]] name = "reth-primitives" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8340,7 +8340,7 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8368,7 +8368,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8418,7 +8418,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "assert_matches", @@ -8447,7 +8447,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "arbitrary", @@ -8467,7 +8467,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -8484,7 +8484,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -8552,7 +8552,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-json-rpc", @@ -8578,7 +8578,7 @@ dependencies = [ [[package]] name = "reth-rpc-api-testing-util" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8597,7 +8597,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-network", "alloy-primitives", @@ -8649,7 +8649,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8685,7 +8685,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-dyn-abi", "alloy-eips", @@ -8725,7 +8725,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8768,7 +8768,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-rpc-types-engine", "http", @@ -8783,7 +8783,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -8798,7 +8798,7 @@ dependencies = [ [[package]] name = "reth-rpc-types-compat" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8814,7 +8814,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8864,7 +8864,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "aquamarine", @@ -8892,7 +8892,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "arbitrary", @@ -8909,7 +8909,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "assert_matches", @@ -8934,7 +8934,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "clap", @@ -8945,7 +8945,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8963,7 +8963,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8975,7 +8975,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "1.0.7" +version = "1.0.8" dependencies = [ "auto_impl", "dyn-clone", @@ -8992,7 +8992,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9005,7 +9005,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "1.0.7" +version = "1.0.8" dependencies = [ "tokio", "tokio-stream", @@ -9014,7 +9014,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "1.0.7" +version = "1.0.8" dependencies = [ "clap", "eyre", @@ -9028,7 +9028,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9073,7 +9073,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9104,7 +9104,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -9128,7 +9128,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9163,7 +9163,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "1.0.7" +version = "1.0.8" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 6a268b0684d4..8c1ee8c4514c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "1.0.7" +version = "1.0.8" edition = "2021" rust-version = "1.81" license = "MIT OR Apache-2.0" From 96f1406a14b4e9d9ef4ed22879accd34c89430ec Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:48:37 +0200 Subject: [PATCH 007/103] chore(provider): use `block_ref` instead on `BlockState` (#11458) --- .../src/providers/blockchain_provider.rs | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 9b56630c3a6a..47f502796fdc 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -381,7 +381,7 @@ impl BlockchainProvider2 { // Iterate from the lowest block to the highest for block_state in in_mem_chain.into_iter().rev() { - let executed_block = block_state.block(); + let executed_block = block_state.block_ref(); let block = executed_block.block(); for tx_index in 0..block.body.transactions.len() { @@ -482,7 +482,7 @@ impl HeaderProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( (*block_hash).into(), |db_provider| db_provider.header(block_hash), - |block_state| Ok(Some(block_state.block().block().header.header().clone())), + |block_state| Ok(Some(block_state.block_ref().block().header.header().clone())), ) } @@ -490,7 +490,7 @@ impl HeaderProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( num.into(), |db_provider| db_provider.header_by_number(num), - |block_state| Ok(Some(block_state.block().block().header.header().clone())), + |block_state| Ok(Some(block_state.block_ref().block().header.header().clone())), ) } @@ -528,7 +528,7 @@ impl HeaderProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block_range_while( range, |db_provider, range, _| db_provider.headers_range(range), - |block_state, _| Some(block_state.block().block().header.header().clone()), + |block_state, _| Some(block_state.block_ref().block().header.header().clone()), |_| true, ) } @@ -537,7 +537,7 @@ impl HeaderProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( number.into(), |db_provider| db_provider.sealed_header(number), - |block_state| Ok(Some(block_state.block().block().header.clone())), + |block_state| Ok(Some(block_state.block_ref().block().header.clone())), ) } @@ -548,7 +548,7 @@ impl HeaderProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block_range_while( range, |db_provider, range, _| db_provider.sealed_headers_range(range), - |block_state, _| Some(block_state.block().block().header.clone()), + |block_state, _| Some(block_state.block_ref().block().header.clone()), |_| true, ) } @@ -562,7 +562,8 @@ impl HeaderProvider for BlockchainProvider2 { range, |db_provider, range, predicate| db_provider.sealed_headers_while(range, predicate), |block_state, predicate| { - Some(block_state.block().block().header.clone()).filter(|header| predicate(header)) + Some(block_state.block_ref().block().header.clone()) + .filter(|header| predicate(header)) }, predicate, ) @@ -640,7 +641,7 @@ impl BlockReader for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( hash.into(), |db_provider| db_provider.find_block_by_hash(hash, source), - |block_state| Ok(Some(block_state.block().block().clone().unseal())), + |block_state| Ok(Some(block_state.block_ref().block().clone().unseal())), ) } BlockSource::Pending => { @@ -653,7 +654,7 @@ impl BlockReader for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( id, |db_provider| db_provider.block(id), - |block_state| Ok(Some(block_state.block().block().clone().unseal())), + |block_state| Ok(Some(block_state.block_ref().block().clone().unseal())), ) } @@ -683,7 +684,7 @@ impl BlockReader for BlockchainProvider2 { return Ok(Some(Vec::new())) } - Ok(Some(block_state.block().block().body.ommers.clone())) + Ok(Some(block_state.block_ref().block().body.ommers.clone())) }, ) } @@ -709,7 +710,7 @@ impl BlockReader for BlockchainProvider2 { // Iterate from the lowest block in memory until our target block for state in block_state.chain().into_iter().rev() { - let block_tx_count = state.block().block.body.transactions.len() as u64; + let block_tx_count = state.block_ref().block.body.transactions.len() as u64; if state.block_ref().block().number == number { stored_indices.tx_count = block_tx_count; } else { @@ -756,7 +757,7 @@ impl BlockReader for BlockchainProvider2 { self.get_in_memory_or_storage_by_block_range_while( range, |db_provider, range, _| db_provider.block_range(range), - |block_state, _| Some(block_state.block().block().clone().unseal()), + |block_state, _| Some(block_state.block_ref().block().clone().unseal()), |_| true, ) } @@ -800,7 +801,7 @@ impl TransactionsProvider for BlockchainProvider2 { id.into(), |provider| provider.transaction_by_id(id), |tx_index, _, block_state| { - Ok(block_state.block().block().body.transactions.get(tx_index).cloned()) + Ok(block_state.block_ref().block().body.transactions.get(tx_index).cloned()) }, ) } @@ -850,7 +851,7 @@ impl TransactionsProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_tx( id.into(), |provider| provider.transaction_block(id), - |_, _, block_state| Ok(Some(block_state.block().block().number)), + |_, _, block_state| Ok(Some(block_state.block_ref().block().number)), ) } @@ -861,7 +862,7 @@ impl TransactionsProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( id, |provider| provider.transactions_by_block(id), - |block_state| Ok(Some(block_state.block().block().body.transactions.clone())), + |block_state| Ok(Some(block_state.block_ref().block().body.transactions.clone())), ) } @@ -872,7 +873,7 @@ impl TransactionsProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block_range_while( range, |db_provider, range, _| db_provider.transactions_by_block_range(range), - |block_state, _| Some(block_state.block().block().body.transactions.clone()), + |block_state, _| Some(block_state.block_ref().block().body.transactions.clone()), |_| true, ) } @@ -909,7 +910,7 @@ impl TransactionsProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_tx( id.into(), |provider| provider.transaction_sender(id), - |tx_index, _, block_state| Ok(block_state.block().senders.get(tx_index).copied()), + |tx_index, _, block_state| Ok(block_state.block_ref().senders.get(tx_index).copied()), ) } } @@ -927,7 +928,7 @@ impl ReceiptProvider for BlockchainProvider2 { fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { for block_state in self.canonical_in_memory_state.canonical_chain() { - let executed_block = block_state.block(); + let executed_block = block_state.block_ref(); let block = executed_block.block(); let receipts = block_state.executed_block_receipts(); @@ -1014,7 +1015,7 @@ impl WithdrawalsProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( id, |db_provider| db_provider.withdrawals_by_block(id, timestamp), - |block_state| Ok(block_state.block().block().body.withdrawals.clone()), + |block_state| Ok(block_state.block_ref().block().body.withdrawals.clone()), ) } @@ -1025,7 +1026,13 @@ impl WithdrawalsProvider for BlockchainProvider2 { best_block_num.into(), |db_provider| db_provider.latest_withdrawal(), |block_state| { - Ok(block_state.block().block().body.withdrawals.clone().and_then(|mut w| w.pop())) + Ok(block_state + .block_ref() + .block() + .body + .withdrawals + .clone() + .and_then(|mut w| w.pop())) }, ) } @@ -1044,7 +1051,7 @@ impl RequestsProvider for BlockchainProvider2 { self.get_in_memory_or_storage_by_block( id, |db_provider| db_provider.requests_by_block(id, timestamp), - |block_state| Ok(block_state.block().block().body.requests.clone()), + |block_state| Ok(block_state.block_ref().block().body.requests.clone()), ) } } @@ -1450,7 +1457,7 @@ impl AccountReader for BlockchainProvider2 { impl StateReader for BlockchainProvider2 { fn get_state(&self, block: BlockNumber) -> ProviderResult> { if let Some(state) = self.canonical_in_memory_state.state_by_number(block) { - let state = state.block().execution_outcome().clone(); + let state = state.block_ref().execution_outcome().clone(); Ok(Some(state)) } else { self.database.provider()?.get_state(block..=block) From f1aae36c6950fb7c0f99d9b10f6743a515f097f0 Mon Sep 17 00:00:00 2001 From: Francis Li Date: Thu, 3 Oct 2024 07:06:34 -0700 Subject: [PATCH 008/103] feat(rpc): Add codes in execution witness return (#11443) --- crates/rpc/rpc/src/debug.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index d7ee43720c19..1e5760768990 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -621,10 +621,18 @@ where let mut hashed_state = HashedPostState::default(); let mut keys = HashMap::default(); + let mut codes = HashMap::default(); let _ = block_executor .execute_with_state_witness( (&block.clone().unseal(), block.difficulty).into(), |statedb| { + codes = statedb + .cache + .contracts + .iter() + .map(|(hash, code)| (*hash, code.bytes())) + .collect(); + for (address, account) in &statedb.cache.accounts { let hashed_address = keccak256(address); hashed_state.accounts.insert( @@ -667,7 +675,7 @@ where state_provider.witness(Default::default(), hashed_state).map_err(Into::into)?; Ok(ExecutionWitness { state: HashMap::from_iter(state.into_iter()), - codes: Default::default(), + codes, keys: include_preimages.then_some(keys), }) }) From 59c6329ffa5f04bd70e1674c626a2dac569c1962 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Oct 2024 16:57:03 +0200 Subject: [PATCH 009/103] fix: ensure the request's gas limit does not exceed the target gas limit (#11462) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 8d34020d67bc..6189b69492a2 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -1107,6 +1107,14 @@ pub trait Call: LoadState + SpawnBlocking { DB: DatabaseRef, EthApiError: From<::Error>, { + // TODO(mattsse): cleanup, by not disabling gaslimit and instead use self.call_gas_limit + if request.gas > Some(gas_limit) { + // configured gas exceeds limit + return Err( + EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh).into() + ) + } + // we want to disable this in eth_call, since this is common practice used by other node // impls and providers cfg.disable_block_gas_limit = true; From a0867a6b053586a16ac4cdc56b8b9f39590e7ec2 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 3 Oct 2024 18:12:37 +0300 Subject: [PATCH 010/103] feat(grafana): ExEx WAL (#11461) --- etc/grafana/dashboards/overview.json | 359 ++++++++++++++++++++++++++- 1 file changed, 348 insertions(+), 11 deletions(-) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 39849f20f090..76d45ead885d 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -2,7 +2,7 @@ "__inputs": [ { "name": "DS_PROMETHEUS", - "label": "Prometheus", + "label": "prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -7787,7 +7787,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The total number of canonical state notifications sent to an ExEx.", + "description": "The total number of canonical state notifications sent to ExExes.", "fieldConfig": { "defaults": { "color": { @@ -7884,7 +7884,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The total number of events an ExEx has sent to the manager.", + "description": "The total number of events ExExes have sent to the manager.", "fieldConfig": { "defaults": { "color": { @@ -7981,7 +7981,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Current and Max capacity of the internal state notifications buffer.", + "description": "Current and Maximum capacity of the internal state notifications buffer.", "fieldConfig": { "defaults": { "color": { @@ -8187,7 +8187,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Number of ExExs on the node", + "description": "Total number of ExExes installed in the node", "fieldConfig": { "defaults": { "color": { @@ -8250,7 +8250,7 @@ "refId": "A" } ], - "title": "Number of ExExs", + "title": "Number of ExExes", "type": "stat" }, { @@ -8261,6 +8261,343 @@ "x": 0, "y": 308 }, + "id": 241, + "panels": [], + "title": "Execution Extensions Write-Ahead Log", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 309 + }, + "id": 243, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_lowest_committed_block_height{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Lowest Block", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_highest_committed_block_height{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Highest Block", + "range": true, + "refId": "C" + } + ], + "title": "Current Committed Block Heights", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 309 + }, + "id": 244, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_committed_blocks_count{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Committed Blocks", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_notifications_count{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Notifications", + "range": true, + "refId": "B" + } + ], + "title": "Number of entities", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 317 + }, + "id": 245, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_size_bytes{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "Total size of all notifications", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 325 + }, "id": 226, "panels": [], "title": "Eth Requests", @@ -8357,7 +8694,7 @@ "h": 8, "w": 12, "x": 0, - "y": 309 + "y": 326 }, "id": 225, "options": { @@ -8486,7 +8823,7 @@ "h": 8, "w": 12, "x": 12, - "y": 309 + "y": 326 }, "id": 227, "options": { @@ -8615,7 +8952,7 @@ "h": 8, "w": 12, "x": 0, - "y": 317 + "y": 334 }, "id": 235, "options": { @@ -8744,7 +9081,7 @@ "h": 8, "w": 12, "x": 12, - "y": 317 + "y": 334 }, "id": 234, "options": { @@ -8821,6 +9158,6 @@ "timezone": "", "title": "Reth", "uid": "2k8BXz24x", - "version": 3, + "version": 8, "weekStart": "" } From 7d092a265d80d7ddd370d5aac07cb740777d84fb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Oct 2024 17:14:10 +0200 Subject: [PATCH 011/103] fix: use correct rpc errors (#11463) --- crates/rpc/rpc-eth-types/src/error.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/error.rs b/crates/rpc/rpc-eth-types/src/error.rs index fbb93164ce9f..10fa3b716021 100644 --- a/crates/rpc/rpc-eth-types/src/error.rs +++ b/crates/rpc/rpc-eth-types/src/error.rs @@ -488,8 +488,14 @@ impl From for RpcInvalidTransactionError { InvalidTransaction::InvalidChainId => Self::InvalidChainId, InvalidTransaction::PriorityFeeGreaterThanMaxFee => Self::TipAboveFeeCap, InvalidTransaction::GasPriceLessThanBasefee => Self::FeeCapTooLow, - InvalidTransaction::CallerGasLimitMoreThanBlock | - InvalidTransaction::CallGasCostMoreThanGasLimit => Self::GasTooHigh, + InvalidTransaction::CallerGasLimitMoreThanBlock => { + // tx.gas > block.gas_limit + Self::GasTooHigh + } + InvalidTransaction::CallGasCostMoreThanGasLimit => { + // tx.gas < cost + Self::GasTooLow + } InvalidTransaction::RejectCallerWithCode => Self::SenderNoEOA, InvalidTransaction::LackOfFundForMaxFee { fee, balance } => { Self::InsufficientFunds { cost: *fee, balance: *balance } From af465623cb416c851c5e475916bf312d51b7642b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 17:38:00 +0200 Subject: [PATCH 012/103] chore(provider): clone after filtering on `sealed_headers_while` (#11459) Co-authored-by: Matthias Seitz --- crates/storage/provider/src/providers/blockchain_provider.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 47f502796fdc..824713c93e52 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -562,8 +562,8 @@ impl HeaderProvider for BlockchainProvider2 { range, |db_provider, range, predicate| db_provider.sealed_headers_while(range, predicate), |block_state, predicate| { - Some(block_state.block_ref().block().header.clone()) - .filter(|header| predicate(header)) + let header = &block_state.block_ref().block().header; + predicate(header).then(|| header.clone()) }, predicate, ) From 6c16df236567047ea62a4750197e1847771d20d7 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Thu, 3 Oct 2024 17:32:03 +0200 Subject: [PATCH 013/103] Map `TransferKind::EofCreate` => `OperationType::OpEofCreate` (#11090) Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/otterscan.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc/src/otterscan.rs b/crates/rpc/rpc/src/otterscan.rs index 35972d9136b9..31db343a104f 100644 --- a/crates/rpc/rpc/src/otterscan.rs +++ b/crates/rpc/rpc/src/otterscan.rs @@ -106,11 +106,10 @@ where value: op.value, r#type: match op.kind { TransferKind::Call => OperationType::OpTransfer, - TransferKind::Create | TransferKind::EofCreate => { - OperationType::OpCreate - } + TransferKind::Create => OperationType::OpCreate, TransferKind::Create2 => OperationType::OpCreate2, TransferKind::SelfDestruct => OperationType::OpSelfDestruct, + TransferKind::EofCreate => OperationType::OpEofCreate, }, }) .collect::>() From d72e438c06e040e213b5decf5f29543c86cbad0f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 3 Oct 2024 18:26:45 +0200 Subject: [PATCH 014/103] fix: windows build (#11465) --- crates/fs-util/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/fs-util/src/lib.rs b/crates/fs-util/src/lib.rs index 0cfcf04539bd..d242ecc98e2d 100644 --- a/crates/fs-util/src/lib.rs +++ b/crates/fs-util/src/lib.rs @@ -307,6 +307,9 @@ where F: FnOnce(&mut File) -> std::result::Result<(), E>, E: Into>, { + #[cfg(windows)] + use std::os::windows::fs::OpenOptionsExt; + let mut tmp_path = file_path.to_path_buf(); tmp_path.set_extension("tmp"); From b4d5adeeb1a0a3dce0c6e49b102910e4e072f7a0 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 3 Oct 2024 19:32:31 +0200 Subject: [PATCH 015/103] chore: use `block_ref` on `CanonicalInMemoryState` (#11467) --- crates/chain-state/src/in_memory.rs | 42 +++++++++++-------- .../src/providers/blockchain_provider.rs | 4 +- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index f2a73d27fa21..f47417aa385d 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -177,7 +177,7 @@ impl CanonicalInMemoryState { let in_memory_state = InMemoryState::new(blocks, numbers, pending); let header = in_memory_state .head_state() - .map_or_else(SealedHeader::default, |state| state.block().block().header.clone()); + .map_or_else(SealedHeader::default, |state| state.block_ref().block().header.clone()); let chain_info_tracker = ChainInfoTracker::new(header, finalized); let (canon_state_notification_sender, _) = broadcast::channel(CANON_STATE_NOTIFICATION_CHANNEL_SIZE); @@ -219,7 +219,7 @@ impl CanonicalInMemoryState { /// Returns the header corresponding to the given hash. pub fn header_by_hash(&self, hash: B256) -> Option { - self.state_by_hash(hash).map(|block| block.block().block.header.clone()) + self.state_by_hash(hash).map(|block| block.block_ref().block.header.clone()) } /// Clears all entries in the in memory state. @@ -323,7 +323,7 @@ impl CanonicalInMemoryState { // height) let mut old_blocks = blocks .drain() - .filter(|(_, b)| b.block().block().number > persisted_height) + .filter(|(_, b)| b.block_ref().block().number > persisted_height) .map(|(_, b)| b.block.clone()) .collect::>(); @@ -345,7 +345,7 @@ impl CanonicalInMemoryState { // also shift the pending state if it exists self.inner.in_memory_state.pending.send_modify(|p| { if let Some(p) = p.as_mut() { - p.parent = blocks.get(&p.block().block.parent_hash).cloned(); + p.parent = blocks.get(&p.block_ref().block.parent_hash).cloned(); } }); } @@ -452,7 +452,7 @@ impl CanonicalInMemoryState { /// Returns the `SealedHeader` corresponding to the pending state. pub fn pending_sealed_header(&self) -> Option { - self.pending_state().map(|h| h.block().block().header.clone()) + self.pending_state().map(|h| h.block_ref().block().header.clone()) } /// Returns the `Header` corresponding to the pending state. @@ -462,20 +462,20 @@ impl CanonicalInMemoryState { /// Returns the `SealedBlock` corresponding to the pending state. pub fn pending_block(&self) -> Option { - self.pending_state().map(|block_state| block_state.block().block().clone()) + self.pending_state().map(|block_state| block_state.block_ref().block().clone()) } /// Returns the `SealedBlockWithSenders` corresponding to the pending state. pub fn pending_block_with_senders(&self) -> Option { self.pending_state() - .and_then(|block_state| block_state.block().block().clone().seal_with_senders()) + .and_then(|block_state| block_state.block_ref().block().clone().seal_with_senders()) } /// Returns a tuple with the `SealedBlock` corresponding to the pending /// state and a vector of its `Receipt`s. pub fn pending_block_and_receipts(&self) -> Option<(SealedBlock, Vec)> { self.pending_state().map(|block_state| { - (block_state.block().block().clone(), block_state.executed_block_receipts()) + (block_state.block_ref().block().clone(), block_state.executed_block_receipts()) }) } @@ -543,7 +543,7 @@ impl CanonicalInMemoryState { pub fn transaction_by_hash(&self, hash: TxHash) -> Option { for block_state in self.canonical_chain() { if let Some(tx) = - block_state.block().block().body.transactions().find(|tx| tx.hash() == hash) + block_state.block_ref().block().body.transactions().find(|tx| tx.hash() == hash) { return Some(tx.clone()) } @@ -559,7 +559,7 @@ impl CanonicalInMemoryState { ) -> Option<(TransactionSigned, TransactionMeta)> { for block_state in self.canonical_chain() { if let Some((index, tx)) = block_state - .block() + .block_ref() .block() .body .transactions() @@ -570,10 +570,10 @@ impl CanonicalInMemoryState { tx_hash, index: index as u64, block_hash: block_state.hash(), - block_number: block_state.block().block.number, - base_fee: block_state.block().block().header.base_fee_per_gas, - timestamp: block_state.block().block.timestamp, - excess_blob_gas: block_state.block().block.excess_blob_gas, + block_number: block_state.block_ref().block.number, + base_fee: block_state.block_ref().block.header.base_fee_per_gas, + timestamp: block_state.block_ref().block.timestamp, + excess_blob_gas: block_state.block_ref().block.excess_blob_gas, }; return Some((tx.clone(), meta)) } @@ -1156,13 +1156,19 @@ mod tests { let block2 = test_block_builder.get_executed_block_with_number(0, B256::random()); let chain = NewCanonicalChain::Commit { new: vec![block1.clone()] }; state.update_chain(chain); - assert_eq!(state.head_state().unwrap().block().block().hash(), block1.block().hash()); - assert_eq!(state.state_by_number(0).unwrap().block().block().hash(), block1.block().hash()); + assert_eq!(state.head_state().unwrap().block_ref().block().hash(), block1.block().hash()); + assert_eq!( + state.state_by_number(0).unwrap().block_ref().block().hash(), + block1.block().hash() + ); let chain = NewCanonicalChain::Reorg { new: vec![block2.clone()], old: vec![block1] }; state.update_chain(chain); - assert_eq!(state.head_state().unwrap().block().block().hash(), block2.block().hash()); - assert_eq!(state.state_by_number(0).unwrap().block().block().hash(), block2.block().hash()); + assert_eq!(state.head_state().unwrap().block_ref().block().hash(), block2.block().hash()); + assert_eq!( + state.state_by_number(0).unwrap().block_ref().block().hash(), + block2.block().hash() + ); assert_eq!(state.inner.in_memory_state.block_count(), 1); } diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 824713c93e52..45bdb1525690 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -815,7 +815,7 @@ impl TransactionsProvider for BlockchainProvider2 { |provider| provider.transaction_by_id_no_hash(id), |tx_index, _, block_state| { Ok(block_state - .block() + .block_ref() .block() .body .transactions @@ -1427,7 +1427,7 @@ impl ChangeSetReader for BlockchainProvider2 { ) -> ProviderResult> { if let Some(state) = self.canonical_in_memory_state.state_by_number(block_number) { let changesets = state - .block() + .block_ref() .execution_output .bundle .reverts From ab069979867087b72c1dfe24b535cd997b3b5f4b Mon Sep 17 00:00:00 2001 From: Francis Li Date: Thu, 3 Oct 2024 11:38:55 -0700 Subject: [PATCH 016/103] chore(rpc): remove include_preimage param on debug_execution_witness (#11466) --- .../engine/invalid-block-hooks/src/witness.rs | 8 ++----- crates/rpc/rpc-api/src/debug.rs | 7 ++----- crates/rpc/rpc/src/debug.rs | 21 +++++-------------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index 37d5bb08293d..51978311faad 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -170,12 +170,8 @@ where if let Some(healthy_node_client) = &self.healthy_node_client { // Compare the witness against the healthy node. let healthy_node_witness = futures::executor::block_on(async move { - DebugApiClient::debug_execution_witness( - healthy_node_client, - block.number.into(), - true, - ) - .await + DebugApiClient::debug_execution_witness(healthy_node_client, block.number.into()) + .await })?; let healthy_path = self.save_file( diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index 4ea8c0f0e4df..3e03210f1ffd 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -140,11 +140,8 @@ pub trait DebugApi { /// The first argument is the block number or block hash. The second argument is a boolean /// indicating whether to include the preimages of keys in the response. #[method(name = "executionWitness")] - async fn debug_execution_witness( - &self, - block: BlockNumberOrTag, - include_preimages: bool, - ) -> RpcResult; + async fn debug_execution_witness(&self, block: BlockNumberOrTag) + -> RpcResult; /// Sets the logging backtrace location. When a backtrace location is set and a log message is /// emitted at that location, the stack of the goroutine executing the log statement will diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 1e5760768990..b47473c34270 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -603,7 +603,6 @@ where pub async fn debug_execution_witness( &self, block_id: BlockNumberOrTag, - include_preimages: bool, ) -> Result { let this = self.clone(); let block = this @@ -622,6 +621,7 @@ where let mut hashed_state = HashedPostState::default(); let mut keys = HashMap::default(); let mut codes = HashMap::default(); + let _ = block_executor .execute_with_state_witness( (&block.clone().unseal(), block.difficulty).into(), @@ -646,24 +646,14 @@ where ); if let Some(account) = &account.account { - if include_preimages { - keys.insert( - hashed_address, - alloy_rlp::encode(address).into(), - ); - } + keys.insert(hashed_address, address.to_vec().into()); for (slot, value) in &account.storage { let slot = B256::from(*slot); let hashed_slot = keccak256(slot); storage.storage.insert(hashed_slot, *value); - if include_preimages { - keys.insert( - hashed_slot, - alloy_rlp::encode(slot).into(), - ); - } + keys.insert(hashed_slot, slot.into()); } } } @@ -676,7 +666,7 @@ where Ok(ExecutionWitness { state: HashMap::from_iter(state.into_iter()), codes, - keys: include_preimages.then_some(keys), + keys: Some(keys), }) }) .await @@ -974,10 +964,9 @@ where async fn debug_execution_witness( &self, block: BlockNumberOrTag, - include_preimages: bool, ) -> RpcResult { let _permit = self.acquire_trace_permit().await; - Self::debug_execution_witness(self, block, include_preimages).await.map_err(Into::into) + Self::debug_execution_witness(self, block).await.map_err(Into::into) } /// Handler for `debug_traceCall` From 84370b81d7a2e2cb90df1a8d5734469e9b89f5fc Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 08:34:37 +0200 Subject: [PATCH 017/103] feat: make addons stateful (#11204) Co-authored-by: Arsenii Kulikov --- bin/reth/src/main.rs | 2 +- crates/ethereum/node/src/node.rs | 7 +++++- crates/ethereum/node/tests/it/builder.rs | 4 ++-- crates/ethereum/node/tests/it/exex.rs | 2 +- crates/exex/test-utils/src/lib.rs | 4 ++++ crates/node/builder/src/builder/add_ons.rs | 2 ++ crates/node/builder/src/builder/mod.rs | 11 ++++++---- crates/node/builder/src/builder/states.rs | 4 +++- crates/node/builder/src/launch/engine.rs | 2 +- crates/node/builder/src/launch/mod.rs | 2 +- crates/node/builder/src/node.rs | 24 +++++++++++++++------ crates/optimism/bin/src/main.rs | 2 +- crates/optimism/node/src/node.rs | 20 ++++++++++++++++- crates/optimism/node/tests/it/builder.rs | 2 +- examples/custom-engine-types/src/main.rs | 4 ++++ examples/custom-evm/src/main.rs | 2 +- examples/custom-node-components/src/main.rs | 2 +- examples/custom-payload-builder/src/main.rs | 2 +- examples/stateful-precompile/src/main.rs | 2 +- 19 files changed, 75 insertions(+), 25 deletions(-) diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 578f2987d73f..1be2d0efedcc 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -60,7 +60,7 @@ fn main() { let handle = builder .with_types_and_provider::>() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch_with_fn(|builder| { let launcher = EngineNodeLauncher::new( builder.task_executor().clone(), diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index c1c4653ae6aa..f17658bb32da 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -77,7 +77,8 @@ impl NodeTypesWithEngine for EthereumNode { } /// Add-ons w.r.t. l1 ethereum. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] +#[non_exhaustive] pub struct EthereumAddOns; impl NodeAddOns for EthereumAddOns { @@ -104,6 +105,10 @@ where fn components_builder(&self) -> Self::ComponentsBuilder { Self::components() } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } } /// A regular ethereum evm and executor builder. diff --git a/crates/ethereum/node/tests/it/builder.rs b/crates/ethereum/node/tests/it/builder.rs index 379f66e814b6..218839fbe019 100644 --- a/crates/ethereum/node/tests/it/builder.rs +++ b/crates/ethereum/node/tests/it/builder.rs @@ -22,7 +22,7 @@ fn test_basic_setup() { .with_database(db) .with_types::() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .on_component_initialized(move |ctx| { let _provider = ctx.provider(); println!("{msg}"); @@ -54,7 +54,7 @@ async fn test_eth_launcher() { NodeTypesWithDBAdapter>>, >>() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch_with_fn(|builder| { let launcher = EngineNodeLauncher::new( tasks.executor(), diff --git a/crates/ethereum/node/tests/it/exex.rs b/crates/ethereum/node/tests/it/exex.rs index db19aaaf3612..856220300c2b 100644 --- a/crates/ethereum/node/tests/it/exex.rs +++ b/crates/ethereum/node/tests/it/exex.rs @@ -33,7 +33,7 @@ fn basic_exex() { .with_database(db) .with_types::() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .install_exex("dummy", move |ctx| future::ok(DummyExEx { _ctx: ctx })) .check_launch(); } diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index b8be08616b4a..f4561a6b55f2 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -154,6 +154,10 @@ where .consensus(TestConsensusBuilder::default()) .engine_validator(EthereumEngineValidatorBuilder::default()) } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } } /// A shared [`TempDatabase`] used for testing diff --git a/crates/node/builder/src/builder/add_ons.rs b/crates/node/builder/src/builder/add_ons.rs index 910cd5896efe..26d7553bb86d 100644 --- a/crates/node/builder/src/builder/add_ons.rs +++ b/crates/node/builder/src/builder/add_ons.rs @@ -14,6 +14,8 @@ pub struct AddOns> { pub exexs: Vec<(String, Box>)>, /// Additional RPC add-ons. pub rpc: RpcAddOns, + /// Additional captured addons. + pub addons: AddOns, } /// Captures node specific addons that can be installed on top of the type configured node and are diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 4989589c9f98..d2ce2a1d8e2f 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -243,7 +243,7 @@ where where N: Node, ChainSpec = ChainSpec>, { - self.with_types().with_components(node.components_builder()).with_add_ons::() + self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons()) } } @@ -311,7 +311,7 @@ where where N: Node, ChainSpec = ChainSpec>, { - self.with_types().with_components(node.components_builder()).with_add_ons::() + self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons()) } /// Launches a preconfigured [Node] @@ -375,12 +375,15 @@ where { /// Advances the state of the node builder to the next state where all customizable /// [`NodeAddOns`] types are configured. - pub fn with_add_ons(self) -> WithLaunchContext> + pub fn with_add_ons( + self, + add_ons: AO, + ) -> WithLaunchContext> where AO: NodeAddOns>, { WithLaunchContext { - builder: self.builder.with_add_ons::(), + builder: self.builder.with_add_ons(add_ons), task_executor: self.task_executor, } } diff --git a/crates/node/builder/src/builder/states.rs b/crates/node/builder/src/builder/states.rs index 30ef54c5683e..80930ef743cd 100644 --- a/crates/node/builder/src/builder/states.rs +++ b/crates/node/builder/src/builder/states.rs @@ -58,6 +58,7 @@ impl NodeBuilderWithTypes { hooks: NodeHooks::default(), rpc: RpcAddOns { hooks: RpcHooks::default() }, exexs: Vec::new(), + addons: (), }, } } @@ -168,7 +169,7 @@ where { /// Advances the state of the node builder to the next state where all customizable /// [`NodeAddOns`] types are configured. - pub fn with_add_ons(self) -> NodeBuilderWithComponents + pub fn with_add_ons(self, addons: AO) -> NodeBuilderWithComponents where AO: NodeAddOns>, { @@ -182,6 +183,7 @@ where hooks: NodeHooks::default(), rpc: RpcAddOns { hooks: RpcHooks::default() }, exexs: Vec::new(), + addons, }, } } diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index e71a0263c52c..46ffacbf717a 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -91,7 +91,7 @@ where let NodeBuilderWithComponents { adapter: NodeTypesAdapter { database }, components_builder, - add_ons: AddOns { hooks, rpc, exexs: installed_exex }, + add_ons: AddOns { hooks, rpc, exexs: installed_exex, .. }, config, } = target; let NodeHooks { on_component_initialized, on_node_started, .. } = hooks; diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index db98ffacedeb..3188cde4b157 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -126,7 +126,7 @@ where let NodeBuilderWithComponents { adapter: NodeTypesAdapter { database }, components_builder, - add_ons: AddOns { hooks, rpc, exexs: installed_exex }, + add_ons: AddOns { hooks, rpc, exexs: installed_exex, .. }, config, } = target; let NodeHooks { on_component_initialized, on_node_started, .. } = hooks; diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index 5d047f94c918..3a70c08c1031 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -34,21 +34,29 @@ pub trait Node: NodeTypesWithEngine + Clone { /// Returns a [`NodeComponentsBuilder`] for the node. fn components_builder(&self) -> Self::ComponentsBuilder; + + /// Returns the node add-ons. + fn add_ons(&self) -> Self::AddOns; } /// A [`Node`] type builder #[derive(Clone, Default, Debug)] -pub struct AnyNode(PhantomData<(N, AO)>, C); +pub struct AnyNode(PhantomData, C, AO); -impl AnyNode { +impl AnyNode { /// Configures the types of the node. - pub fn types(self) -> AnyNode { - AnyNode::(PhantomData::<(T, ())>, self.1) + pub fn types(self) -> AnyNode { + AnyNode(PhantomData, self.1, self.2) } /// Sets the node components builder. - pub const fn components_builder(&self, value: T) -> AnyNode { - AnyNode::(PhantomData::<(N, ())>, value) + pub fn components_builder(self, value: T) -> AnyNode { + AnyNode(PhantomData, value, self.2) + } + + /// Sets the node add-ons. + pub fn add_ons(self, value: T) -> AnyNode { + AnyNode(PhantomData, self.1, value) } } @@ -84,6 +92,10 @@ where fn components_builder(&self) -> Self::ComponentsBuilder { self.1.clone() } + + fn add_ons(&self) -> Self::AddOns { + self.2.clone() + } } /// The launched node with all components including RPC handlers. diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs index cecd01ea8897..6e041e240a74 100644 --- a/crates/optimism/bin/src/main.rs +++ b/crates/optimism/bin/src/main.rs @@ -35,7 +35,7 @@ fn main() { let handle = builder .with_types_and_provider::>() .with_components(OptimismNode::components(rollup_args)) - .with_add_ons::() + .with_add_ons(OptimismAddOns::new(sequencer_http_arg.clone())) .extend_rpc_modules(move |ctx| { // register sequencer tx forwarder if let Some(sequencer_http) = sequencer_http_arg { diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index d82cb70a3ffe..4e7788569f89 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -102,6 +102,10 @@ where let Self { args } = self; Self::components(args.clone()) } + + fn add_ons(&self) -> Self::AddOns { + OptimismAddOns::new(self.args.sequencer_http.clone()) + } } impl NodeTypes for OptimismNode { @@ -115,7 +119,21 @@ impl NodeTypesWithEngine for OptimismNode { /// Add-ons w.r.t. optimism. #[derive(Debug, Clone)] -pub struct OptimismAddOns; +pub struct OptimismAddOns { + sequencer_http: Option, +} + +impl OptimismAddOns { + /// Create a new instance with the given `sequencer_http` URL. + pub const fn new(sequencer_http: Option) -> Self { + Self { sequencer_http } + } + + /// Returns the sequencer HTTP URL. + pub fn sequencer_http(&self) -> Option<&str> { + self.sequencer_http.as_deref() + } +} impl NodeAddOns for OptimismAddOns { type EthApi = OpEthApi; diff --git a/crates/optimism/node/tests/it/builder.rs b/crates/optimism/node/tests/it/builder.rs index 20363828e861..f1dde4c2c0a8 100644 --- a/crates/optimism/node/tests/it/builder.rs +++ b/crates/optimism/node/tests/it/builder.rs @@ -15,7 +15,7 @@ fn test_basic_setup() { .with_database(db) .with_types::() .with_components(OptimismNode::components(Default::default())) - .with_add_ons::() + .with_add_ons(OptimismAddOns::new(None)) .on_component_initialized(move |ctx| { let _provider = ctx.provider(); Ok(()) diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 213a156af8fd..34f8186be7f8 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -253,6 +253,10 @@ where .consensus(EthereumConsensusBuilder::default()) .engine_validator(CustomEngineValidatorBuilder::default()) } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } } /// A custom payload service builder that supports the custom engine types diff --git a/examples/custom-evm/src/main.rs b/examples/custom-evm/src/main.rs index d931c3b275bf..9c421f9c6a59 100644 --- a/examples/custom-evm/src/main.rs +++ b/examples/custom-evm/src/main.rs @@ -226,7 +226,7 @@ async fn main() -> eyre::Result<()> { .executor(MyExecutorBuilder::default()) .payload(MyPayloadBuilder::default()), ) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await .unwrap(); diff --git a/examples/custom-node-components/src/main.rs b/examples/custom-node-components/src/main.rs index 1faca73d25b0..d00b8a70224a 100644 --- a/examples/custom-node-components/src/main.rs +++ b/examples/custom-node-components/src/main.rs @@ -25,7 +25,7 @@ fn main() { // Configure the components of the node // use default ethereum components but use our custom pool .with_components(EthereumNode::components().pool(CustomPoolBuilder::default())) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await?; diff --git a/examples/custom-payload-builder/src/main.rs b/examples/custom-payload-builder/src/main.rs index 5ed414eb850b..e46b969adaa1 100644 --- a/examples/custom-payload-builder/src/main.rs +++ b/examples/custom-payload-builder/src/main.rs @@ -81,7 +81,7 @@ fn main() { .with_components( EthereumNode::components().payload(CustomPayloadBuilder::default()), ) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await?; diff --git a/examples/stateful-precompile/src/main.rs b/examples/stateful-precompile/src/main.rs index 05a6fd86c935..26ebdfe4124b 100644 --- a/examples/stateful-precompile/src/main.rs +++ b/examples/stateful-precompile/src/main.rs @@ -263,7 +263,7 @@ async fn main() -> eyre::Result<()> { .with_types::() // use default ethereum components but with our executor .with_components(EthereumNode::components().executor(MyExecutorBuilder::default())) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await .unwrap(); From 1fe9f324b0be82638f52ccd24a6d11d11e4e97b9 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 4 Oct 2024 00:34:29 -0700 Subject: [PATCH 018/103] Relax Trait Bounds on TransactionPool::Transaction and EthPoolTransaction (#11079) Co-authored-by: Matthias Seitz --- crates/net/network/src/transactions/mod.rs | 13 +++--- crates/optimism/node/src/txpool.rs | 2 +- crates/optimism/rpc/src/eth/transaction.rs | 3 +- .../rpc-eth-api/src/helpers/transaction.rs | 8 ++-- crates/rpc/rpc/src/eth/filter.rs | 4 +- crates/rpc/rpc/src/txpool.rs | 8 ++-- crates/transaction-pool/src/lib.rs | 2 + crates/transaction-pool/src/maintain.rs | 14 ++++--- crates/transaction-pool/src/ordering.rs | 11 +---- crates/transaction-pool/src/pool/mod.rs | 21 ++++++++-- crates/transaction-pool/src/traits.rs | 42 +++++++------------ crates/transaction-pool/src/validate/mod.rs | 14 ++----- 12 files changed, 71 insertions(+), 71 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 7e3b71aa4365..2fa4ccfbb606 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1033,7 +1033,7 @@ where has_bad_transactions = true; } else { // this is a new transaction that should be imported into the pool - let pool_transaction = Pool::Transaction::from_pooled(tx); + let pool_transaction = Pool::Transaction::from_pooled(tx.into()); new_txs.push(pool_transaction); entry.insert(HashSet::from([peer_id])); @@ -1396,11 +1396,14 @@ impl PropagateTransaction { } /// Create a new instance from a pooled transaction - fn new>( - tx: Arc>, - ) -> Self { + fn new(tx: Arc>) -> Self + where + T: PoolTransaction>, + { let size = tx.encoded_length(); - let transaction = Arc::new(tx.transaction.clone().into_consensus().into_signed()); + let recovered: TransactionSignedEcRecovered = + tx.transaction.clone().into_consensus().into(); + let transaction = Arc::new(recovered.into_signed()); Self { size, transaction } } } diff --git a/crates/optimism/node/src/txpool.rs b/crates/optimism/node/src/txpool.rs index 811c37e91cb5..7ed2a161d0e4 100644 --- a/crates/optimism/node/src/txpool.rs +++ b/crates/optimism/node/src/txpool.rs @@ -140,7 +140,7 @@ where let l1_block_info = self.block_info.l1_block_info.read().clone(); let mut encoded = Vec::with_capacity(valid_tx.transaction().encoded_length()); - valid_tx.transaction().clone().into_consensus().encode_2718(&mut encoded); + valid_tx.transaction().clone().into_consensus().into().encode_2718(&mut encoded); let cost_addition = match l1_block_info.l1_tx_data_fee( &self.chain_spec(), diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 2556c895783e..3ccda419cad8 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -34,7 +34,8 @@ where /// Returns the hash of the transaction. async fn send_raw_transaction(&self, tx: Bytes) -> Result { let recovered = recover_raw_transaction(tx.clone())?; - let pool_transaction = ::Transaction::from_pooled(recovered); + let pool_transaction = + ::Transaction::from_pooled(recovered.into()); // On optimism, transactions are forwarded directly to the sequencer to be included in // blocks that it builds. diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index d98cb69bfc30..d70ea7e541cb 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -231,7 +231,7 @@ pub trait EthTransactions: LoadTransaction { LoadState::pool(self).get_transaction_by_sender_and_nonce(sender, nonce) { let transaction = tx.transaction.clone().into_consensus(); - return Ok(Some(from_recovered::(transaction))); + return Ok(Some(from_recovered::(transaction.into()))); } } @@ -324,7 +324,7 @@ pub trait EthTransactions: LoadTransaction { async move { let recovered = recover_raw_transaction(tx.clone())?; let pool_transaction = - ::Transaction::from_pooled(recovered); + ::Transaction::from_pooled(recovered.into()); // submit the transaction to the pool with a `Local` origin let hash = self @@ -376,7 +376,7 @@ pub trait EthTransactions: LoadTransaction { let recovered = signed_tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; - let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(recovered).map_err(|_| EthApiError::TransactionConversionError)?; + let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(recovered.into()).map_err(|_| EthApiError::TransactionConversionError)?; // submit the transaction to the pool with a `Local` origin let hash = LoadTransaction::pool(self) @@ -518,7 +518,7 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { if let Some(tx) = self.pool().get(&hash).map(|tx| tx.transaction.clone().into_consensus()) { - resp = Some(TransactionSource::Pool(tx)); + resp = Some(TransactionSource::Pool(tx.into())); } } diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 274347404c93..f4ae89264160 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -611,7 +611,7 @@ where /// Returns all new pending transactions received since the last poll. async fn drain(&self) -> FilterChanges where - T: PoolTransaction, + T: PoolTransaction>, { let mut pending_txs = Vec::new(); let mut prepared_stream = self.txs_stream.lock().await; @@ -633,7 +633,7 @@ trait FullTransactionsFilter: fmt::Debug + Send + Sync + Unpin + 'static { impl FullTransactionsFilter for FullTransactionsReceiver where - T: PoolTransaction + 'static, + T: PoolTransaction> + 'static, TxCompat: TransactionCompat + 'static, { async fn drain(&self) -> FilterChanges { diff --git a/crates/rpc/rpc/src/txpool.rs b/crates/rpc/rpc/src/txpool.rs index 4932a563409e..5e26935ca1ba 100644 --- a/crates/rpc/rpc/src/txpool.rs +++ b/crates/rpc/rpc/src/txpool.rs @@ -41,12 +41,12 @@ where tx: &Tx, content: &mut BTreeMap>, ) where - Tx: PoolTransaction, + Tx: PoolTransaction>, RpcTxB: TransactionCompat, { content.entry(tx.sender()).or_default().insert( tx.nonce().to_string(), - from_recovered::(tx.clone().into_consensus()), + from_recovered::(tx.clone().into_consensus().into()), ); } @@ -91,12 +91,12 @@ where trace!(target: "rpc::eth", "Serving txpool_inspect"); #[inline] - fn insert>( + fn insert>>( tx: &T, inspect: &mut BTreeMap>, ) { let entry = inspect.entry(tx.sender()).or_default(); - let tx = tx.clone().into_consensus(); + let tx: TransactionSignedEcRecovered = tx.clone().into_consensus().into(); entry.insert( tx.nonce().to_string(), TxpoolInspectSummary { diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 4aec8ab40858..744dd44d2f83 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -324,6 +324,7 @@ where impl TransactionPool for Pool where V: TransactionValidator, + ::Transaction: EthPoolTransaction, T: TransactionOrdering::Transaction>, S: BlobStore, { @@ -546,6 +547,7 @@ where impl TransactionPoolExt for Pool where V: TransactionValidator, + ::Transaction: EthPoolTransaction, T: TransactionOrdering::Transaction>, S: BlobStore, { diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index da416fd2d43f..523151956dd1 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -18,6 +18,7 @@ use reth_execution_types::ChangedAccount; use reth_fs_util::FsPathError; use reth_primitives::{ BlockNumberOrTag, PooledTransactionsElementEcRecovered, SealedHeader, TransactionSigned, + TransactionSignedEcRecovered, }; use reth_storage_api::{errors::provider::ProviderError, BlockReaderIdExt, StateProviderFactory}; use reth_tasks::TaskSpawner; @@ -334,11 +335,10 @@ pub async fn maintain_transaction_pool( .ok() }) .map(|tx| { - <

::Transaction as PoolTransaction>::from_pooled(tx) +

::Transaction::from_pooled(tx.into()) }) } else { - - ::try_from_consensus(tx).ok() +

::Transaction::try_from_consensus(tx.into()).ok() } }) .collect::>(); @@ -583,7 +583,7 @@ where .filter_map(|tx| tx.try_ecrecovered()) .filter_map(|tx| { // Filter out errors - ::try_from_consensus(tx).ok() + ::try_from_consensus(tx.into()).ok() }) .collect::>(); @@ -606,7 +606,11 @@ where let local_transactions = local_transactions .into_iter() - .map(|tx| tx.to_recovered_transaction().into_signed()) + .map(|tx| { + let recovered: TransactionSignedEcRecovered = + tx.transaction.clone().into_consensus().into(); + recovered.into_signed() + }) .collect::>(); let num_txs = local_transactions.len(); diff --git a/crates/transaction-pool/src/ordering.rs b/crates/transaction-pool/src/ordering.rs index 3381bb027947..0ee0c1004a13 100644 --- a/crates/transaction-pool/src/ordering.rs +++ b/crates/transaction-pool/src/ordering.rs @@ -1,6 +1,5 @@ use crate::traits::PoolTransaction; use alloy_primitives::U256; -use reth_primitives::{PooledTransactionsElementEcRecovered, TransactionSignedEcRecovered}; use std::{fmt, marker::PhantomData}; /// Priority of the transaction that can be missing. @@ -32,10 +31,7 @@ pub trait TransactionOrdering: Send + Sync + 'static { type PriorityValue: Ord + Clone + Default + fmt::Debug + Send + Sync; /// The transaction type to determine the priority of. - type Transaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - >; + type Transaction: PoolTransaction; /// Returns the priority score for the given transaction. fn priority( @@ -55,10 +51,7 @@ pub struct CoinbaseTipOrdering(PhantomData); impl TransactionOrdering for CoinbaseTipOrdering where - T: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - > + 'static, + T: PoolTransaction + 'static, { type PriorityValue = U256; type Transaction = T; diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 4c1a7f2c29bb..b78f5128686c 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -88,6 +88,7 @@ use reth_execution_types::ChangedAccount; use reth_primitives::{ BlobTransaction, BlobTransactionSidecar, PooledTransactionsElement, TransactionSigned, + TransactionSignedEcRecovered, }; use std::{ collections::{HashMap, HashSet}, @@ -318,13 +319,19 @@ where &self, tx_hashes: Vec, limit: GetPooledTransactionLimit, - ) -> Vec { + ) -> Vec + where + ::Transaction: + PoolTransaction>, + { let transactions = self.get_all(tx_hashes); let mut elements = Vec::with_capacity(transactions.len()); let mut size = 0; for transaction in transactions { let encoded_len = transaction.encoded_length(); - let tx = transaction.to_recovered_transaction().into_signed(); + let recovered: TransactionSignedEcRecovered = + transaction.transaction.clone().into_consensus().into(); + let tx = recovered.into_signed(); let pooled = if tx.is_eip4844() { // for EIP-4844 transactions, we need to fetch the blob sidecar from the blob store if let Some(blob) = self.get_blob_transaction(tx) { @@ -360,9 +367,15 @@ where pub(crate) fn get_pooled_transaction_element( &self, tx_hash: TxHash, - ) -> Option { + ) -> Option + where + ::Transaction: + PoolTransaction>, + { self.get(&tx_hash).and_then(|transaction| { - let tx = transaction.to_recovered_transaction().into_signed(); + let recovered: TransactionSignedEcRecovered = + transaction.transaction.clone().into_consensus().into(); + let tx = recovered.into_signed(); if tx.is_eip4844() { self.get_blob_transaction(tx).map(PooledTransactionsElement::BlobTransaction) } else { diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index d4eabc73bbce..e522978fb9dc 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -44,10 +44,7 @@ pub type PeerId = alloy_primitives::B512; #[auto_impl::auto_impl(&, Arc)] pub trait TransactionPool: Send + Sync + Clone { /// The transaction type of the pool - type Transaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - >; + type Transaction: EthPoolTransaction; /// Returns stats about the pool and all sub-pools. fn pool_size(&self) -> PoolSize; @@ -496,12 +493,12 @@ pub struct AllPoolTransactions { impl AllPoolTransactions { /// Returns an iterator over all pending [`TransactionSignedEcRecovered`] transactions. pub fn pending_recovered(&self) -> impl Iterator + '_ { - self.pending.iter().map(|tx| tx.transaction.clone().into_consensus()) + self.pending.iter().map(|tx| tx.transaction.clone().into()) } /// Returns an iterator over all queued [`TransactionSignedEcRecovered`] transactions. pub fn queued_recovered(&self) -> impl Iterator + '_ { - self.queued.iter().map(|tx| tx.transaction.clone().into_consensus()) + self.queued.iter().map(|tx| tx.transaction.clone().into()) } } @@ -813,19 +810,25 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + Clone { type TryFromConsensusError; /// Associated type representing the raw consensus variant of the transaction. - type Consensus: From + TryInto; + type Consensus: From + TryInto; /// Associated type representing the recovered pooled variant of the transaction. type Pooled: Into; /// Define a method to convert from the `Consensus` type to `Self` - fn try_from_consensus(tx: Self::Consensus) -> Result; + fn try_from_consensus(tx: Self::Consensus) -> Result { + tx.try_into() + } /// Define a method to convert from the `Self` type to `Consensus` - fn into_consensus(self) -> Self::Consensus; + fn into_consensus(self) -> Self::Consensus { + self.into() + } /// Define a method to convert from the `Pooled` type to `Self` - fn from_pooled(pooled: Self::Pooled) -> Self; + fn from_pooled(pooled: Self::Pooled) -> Self { + pooled.into() + } /// Hash of the transaction. fn hash(&self) -> &TxHash; @@ -921,12 +924,11 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + Clone { fn chain_id(&self) -> Option; } -/// An extension trait that provides additional interfaces for the -/// [`EthTransactionValidator`](crate::EthTransactionValidator). +/// Super trait for transactions that can be converted to and from Eth transactions pub trait EthPoolTransaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, + Consensus: From + Into, + Pooled: From + Into, > { /// Extracts the blob sidecar from the transaction. @@ -1069,18 +1071,6 @@ impl PoolTransaction for EthPooledTransaction { type Pooled = PooledTransactionsElementEcRecovered; - fn try_from_consensus(tx: Self::Consensus) -> Result { - tx.try_into() - } - - fn into_consensus(self) -> Self::Consensus { - self.into() - } - - fn from_pooled(pooled: Self::Pooled) -> Self { - pooled.into() - } - /// Returns hash of the transaction. fn hash(&self) -> &TxHash { self.transaction.hash_ref() diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index b8fe7cbb1de0..4395cc97908b 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -7,10 +7,7 @@ use crate::{ }; use alloy_primitives::{Address, TxHash, B256, U256}; use futures_util::future::Either; -use reth_primitives::{ - BlobTransactionSidecar, PooledTransactionsElementEcRecovered, SealedBlock, - TransactionSignedEcRecovered, -}; +use reth_primitives::{BlobTransactionSidecar, SealedBlock, TransactionSignedEcRecovered}; use std::{fmt, future::Future, time::Instant}; mod constants; @@ -154,10 +151,7 @@ impl ValidTransaction { /// Provides support for validating transaction at any given state of the chain pub trait TransactionValidator: Send + Sync { /// The transaction type to validate. - type Transaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - >; + type Transaction: PoolTransaction; /// Validates the transaction and returns a [`TransactionValidationOutcome`] describing the /// validity of the given transaction. @@ -380,12 +374,12 @@ impl ValidPoolTransaction { } } -impl> ValidPoolTransaction { +impl>> ValidPoolTransaction { /// Converts to this type into a [`TransactionSignedEcRecovered`]. /// /// Note: this takes `&self` since indented usage is via `Arc`. pub fn to_recovered_transaction(&self) -> TransactionSignedEcRecovered { - self.transaction.clone().into_consensus() + self.transaction.clone().into_consensus().into() } } From 2dc5f5d7459f6c7eddd0ac4dd955c67ef94842c7 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:36:24 +0200 Subject: [PATCH 019/103] test: add unit tests for `CanonicalChain` (#11472) --- crates/blockchain-tree/src/canonical_chain.rs | 176 +++++++++++++++++- 1 file changed, 167 insertions(+), 9 deletions(-) diff --git a/crates/blockchain-tree/src/canonical_chain.rs b/crates/blockchain-tree/src/canonical_chain.rs index 7dcd466f7d64..253f799fe0f8 100644 --- a/crates/blockchain-tree/src/canonical_chain.rs +++ b/crates/blockchain-tree/src/canonical_chain.rs @@ -32,15 +32,7 @@ impl CanonicalChain { /// Returns the block number of the (non-finalized) canonical block with the given hash. #[inline] pub(crate) fn canonical_number(&self, block_hash: &BlockHash) -> Option { - self.chain.iter().find_map( - |(number, hash)| { - if hash == block_hash { - Some(*number) - } else { - None - } - }, - ) + self.chain.iter().find_map(|(number, hash)| (hash == block_hash).then_some(*number)) } /// Extends all items from the given iterator to the chain. @@ -81,3 +73,169 @@ impl CanonicalChain { self.chain.into_iter() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_replace_canonical_chain() { + // Initialize a chain with some blocks + let mut initial_chain = BTreeMap::new(); + initial_chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + initial_chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + + let mut canonical_chain = CanonicalChain::new(initial_chain.clone()); + + // Verify initial chain state + assert_eq!(canonical_chain.chain.len(), 2); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(1u64)), + Some(&BlockHash::from([0x01; 32])) + ); + + // Replace with a new chain + let mut new_chain = BTreeMap::new(); + new_chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + new_chain.insert(BlockNumber::from(4u64), BlockHash::from([0x04; 32])); + new_chain.insert(BlockNumber::from(5u64), BlockHash::from([0x05; 32])); + + canonical_chain.replace(new_chain.clone()); + + // Verify replaced chain state + assert_eq!(canonical_chain.chain.len(), 3); + assert!(!canonical_chain.chain.contains_key(&BlockNumber::from(1u64))); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(3u64)), + Some(&BlockHash::from([0x03; 32])) + ); + } + + #[test] + fn test_canonical_hash_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of a canonical chain + let canonical_chain = CanonicalChain::new(chain.clone()); + + // Check that the function returns the correct hash for a given block number + let block_number = BlockNumber::from(2u64); + let expected_hash = BlockHash::from([0x02; 32]); + assert_eq!(canonical_chain.canonical_hash(&block_number), Some(expected_hash)); + + // Check that a non-existent block returns None + let non_existent_block = BlockNumber::from(5u64); + assert_eq!(canonical_chain.canonical_hash(&non_existent_block), None); + } + + #[test] + fn test_canonical_number_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of a canonical chain + let canonical_chain = CanonicalChain::new(chain.clone()); + + // Check that the function returns the correct block number for a given block hash + let block_hash = BlockHash::from([0x02; 32]); + let expected_number = BlockNumber::from(2u64); + assert_eq!(canonical_chain.canonical_number(&block_hash), Some(expected_number)); + + // Check that a non-existent block hash returns None + let non_existent_hash = BlockHash::from([0x05; 32]); + assert_eq!(canonical_chain.canonical_number(&non_existent_hash), None); + } + + #[test] + fn test_extend_canonical_chain() { + // Initialize an empty chain + let mut canonical_chain = CanonicalChain::new(BTreeMap::new()); + + // Create an iterator with some blocks + let blocks = vec![ + (BlockNumber::from(1u64), BlockHash::from([0x01; 32])), + (BlockNumber::from(2u64), BlockHash::from([0x02; 32])), + ] + .into_iter(); + + // Extend the chain with the created blocks + canonical_chain.extend(blocks); + + // Check if the blocks were added correctly + assert_eq!(canonical_chain.chain.len(), 2); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(1u64)), + Some(&BlockHash::from([0x01; 32])) + ); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(2u64)), + Some(&BlockHash::from([0x02; 32])) + ); + + // Test extending with additional blocks again + let more_blocks = vec![(BlockNumber::from(3u64), BlockHash::from([0x03; 32]))].into_iter(); + canonical_chain.extend(more_blocks); + + assert_eq!(canonical_chain.chain.len(), 3); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(3u64)), + Some(&BlockHash::from([0x03; 32])) + ); + } + + #[test] + fn test_retain_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of CanonicalChain + let mut canonical_chain = CanonicalChain::new(chain); + + // Retain only blocks with even block numbers + canonical_chain.retain(|number, _| number % 2 == 0); + + // Check if the chain only contains the block with number 2 + assert_eq!(canonical_chain.chain.len(), 1); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(2u64)), + Some(&BlockHash::from([0x02; 32])) + ); + + // Ensure that the blocks with odd numbers were removed + assert_eq!(canonical_chain.chain.get(&BlockNumber::from(1u64)), None); + assert_eq!(canonical_chain.chain.get(&BlockNumber::from(3u64)), None); + } + + #[test] + fn test_tip_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of a canonical chain + let canonical_chain = CanonicalChain::new(chain); + + // Call the tip method and verify the returned value + let tip = canonical_chain.tip(); + assert_eq!(tip.number, BlockNumber::from(3u64)); + assert_eq!(tip.hash, BlockHash::from([0x03; 32])); + + // Test with an empty chain + let empty_chain = CanonicalChain::new(BTreeMap::new()); + let empty_tip = empty_chain.tip(); + assert_eq!(empty_tip.number, BlockNumber::default()); + assert_eq!(empty_tip.hash, BlockHash::default()); + } +} From af1eb61072bc4ac5dad23f14390102d00ab66c18 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 4 Oct 2024 10:11:38 +0200 Subject: [PATCH 020/103] feat(perf): integrate OnStateHook in executor (#11345) --- crates/ethereum/evm/src/execute.rs | 56 +++++++++++++++++++++++---- crates/evm/src/either.rs | 19 ++++++++- crates/evm/src/execute.rs | 23 +++++++++++ crates/evm/src/noop.rs | 16 +++++++- crates/evm/src/system_calls/mod.rs | 16 +++++--- crates/evm/src/test_utils.rs | 18 ++++++++- crates/optimism/evm/src/execute.rs | 62 ++++++++++++++++++++++++++---- 7 files changed, 185 insertions(+), 25 deletions(-) diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index f1d7de115d57..8c84fafc25fb 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -14,7 +14,7 @@ use reth_evm::{ BatchExecutor, BlockExecutionError, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, BlockValidationError, Executor, ProviderError, }, - system_calls::SystemCaller, + system_calls::{NoopHook, OnStateHook, SystemCaller}, ConfigureEvm, }; use reth_execution_types::ExecutionOutcome; @@ -126,20 +126,25 @@ where /// This applies the pre-execution and post-execution changes that require an [EVM](Evm), and /// executes the transactions. /// + /// The optional `state_hook` will be executed with the state changes if present. + /// /// # Note /// /// It does __not__ apply post-execution changes that do not require an [EVM](Evm), for that see /// [`EthBlockExecutor::post_execution`]. - fn execute_state_transitions( + fn execute_state_transitions( &self, block: &BlockWithSenders, mut evm: Evm<'_, Ext, &mut State>, + state_hook: Option, ) -> Result where DB: Database, DB::Error: Into + Display, + F: OnStateHook, { - let mut system_caller = SystemCaller::new(&self.evm_config, &self.chain_spec); + let mut system_caller = + SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); system_caller.apply_pre_execution_changes(block, &mut evm)?; @@ -161,7 +166,7 @@ where self.evm_config.fill_tx_env(evm.tx_mut(), transaction, *sender); // Execute transaction. - let ResultAndState { result, state } = evm.transact().map_err(move |err| { + let result_and_state = evm.transact().map_err(move |err| { let new_err = err.map_db_err(|e| e.into()); // Ensure hash is calculated for error log, if not already done BlockValidationError::EVM { @@ -169,6 +174,8 @@ where error: Box::new(new_err), } })?; + system_caller.on_state(&result_and_state); + let ResultAndState { result, state } = result_and_state; evm.db_mut().commit(state); // append gas used @@ -260,17 +267,31 @@ where EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()) } + /// Convenience method to invoke `execute_without_verification_with_state_hook` setting the + /// state hook as `None`. + fn execute_without_verification( + &mut self, + block: &BlockWithSenders, + total_difficulty: U256, + ) -> Result { + self.execute_without_verification_with_state_hook(block, total_difficulty, None::) + } + /// Execute a single block and apply the state changes to the internal state. /// /// Returns the receipts of the transactions in the block, the total gas used and the list of /// EIP-7685 [requests](Request). /// /// Returns an error if execution fails. - fn execute_without_verification( + fn execute_without_verification_with_state_hook( &mut self, block: &BlockWithSenders, total_difficulty: U256, - ) -> Result { + state_hook: Option, + ) -> Result + where + F: OnStateHook, + { // 1. prepare state on new block self.on_new_block(&block.header); @@ -278,7 +299,7 @@ where let env = self.evm_env_for_block(&block.header, total_difficulty); let output = { let evm = self.executor.evm_config.evm_with_env(&mut self.state, env); - self.executor.execute_state_transitions(block, evm) + self.executor.execute_state_transitions(block, evm, state_hook) }?; // 3. apply post execution changes @@ -368,6 +389,27 @@ where witness(&self.state); Ok(BlockExecutionOutput { state: self.state.take_bundle(), receipts, requests, gas_used }) } + + fn execute_with_state_hook( + mut self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook, + { + let BlockExecutionInput { block, total_difficulty } = input; + let EthExecuteOutput { receipts, requests, gas_used } = self + .execute_without_verification_with_state_hook( + block, + total_difficulty, + Some(state_hook), + )?; + + // NOTE: we need to merge keep the reverts for the bundle retention + self.state.merge_transitions(BundleRetention::Reverts); + Ok(BlockExecutionOutput { state: self.state.take_bundle(), receipts, requests, gas_used }) + } } /// An executor for a batch of blocks. /// diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index a3fca50ec7ee..e28ec3887f80 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -2,7 +2,10 @@ use core::fmt::Display; -use crate::execute::{BatchExecutor, BlockExecutorProvider, Executor}; +use crate::{ + execute::{BatchExecutor, BlockExecutorProvider, Executor}, + system_calls::OnStateHook, +}; use alloy_primitives::BlockNumber; use reth_execution_errors::BlockExecutionError; use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput, ExecutionOutcome}; @@ -87,6 +90,20 @@ where Self::Right(b) => b.execute_with_state_witness(input, witness), } } + + fn execute_with_state_hook( + self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook, + { + match self { + Self::Left(a) => a.execute_with_state_hook(input, state_hook), + Self::Right(b) => b.execute_with_state_hook(input, state_hook), + } + } } impl BatchExecutor for Either diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index ffc08469dc8d..3fc2975f0ff7 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -12,6 +12,8 @@ use reth_prune_types::PruneModes; use revm::State; use revm_primitives::db::Database; +use crate::system_calls::OnStateHook; + /// A general purpose executor trait that executes an input (e.g. block) and produces an output /// (e.g. state changes and receipts). /// @@ -43,6 +45,16 @@ pub trait Executor { ) -> Result where F: FnMut(&State); + + /// Executes the EVM with the given input and accepts a state hook closure that is invoked with + /// the EVM state after execution. + fn execute_with_state_hook( + self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook; } /// A general purpose executor that can execute multiple inputs in sequence, validate the outputs, @@ -199,6 +211,17 @@ mod tests { { Err(BlockExecutionError::msg("execution unavailable for tests")) } + + fn execute_with_state_hook( + self, + _: Self::Input<'_>, + _: F, + ) -> Result + where + F: OnStateHook, + { + Err(BlockExecutionError::msg("execution unavailable for tests")) + } } impl BatchExecutor for TestExecutor { diff --git a/crates/evm/src/noop.rs b/crates/evm/src/noop.rs index 392bfd0bd722..3e01bfc4cc46 100644 --- a/crates/evm/src/noop.rs +++ b/crates/evm/src/noop.rs @@ -10,7 +10,10 @@ use reth_storage_errors::provider::ProviderError; use revm::State; use revm_primitives::db::Database; -use crate::execute::{BatchExecutor, BlockExecutorProvider, Executor}; +use crate::{ + execute::{BatchExecutor, BlockExecutorProvider, Executor}, + system_calls::OnStateHook, +}; const UNAVAILABLE_FOR_NOOP: &str = "execution unavailable for noop"; @@ -58,6 +61,17 @@ impl Executor for NoopBlockExecutorProvider { { Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) } + + fn execute_with_state_hook( + self, + _: Self::Input<'_>, + _: F, + ) -> Result + where + F: OnStateHook, + { + Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) + } } impl BatchExecutor for NoopBlockExecutorProvider { diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index ce5fec42184c..48429cb49596 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -49,22 +49,19 @@ pub struct SystemCaller<'a, EvmConfig, Chainspec, Hook = NoopHook> { hook: Option, } -impl<'a, EvmConfig, Chainspec> SystemCaller<'a, EvmConfig, Chainspec> { +impl<'a, EvmConfig, Chainspec> SystemCaller<'a, EvmConfig, Chainspec, NoopHook> { /// Create a new system caller with the given EVM config, database, and chain spec, and creates /// the EVM with the given initialized config and block environment. pub const fn new(evm_config: &'a EvmConfig, chain_spec: Chainspec) -> Self { Self { evm_config, chain_spec, hook: None } } -} - -impl<'a, EvmConfig, Chainspec, Hook> SystemCaller<'a, EvmConfig, Chainspec, Hook> { /// Installs a custom hook to be called after each state change. pub fn with_state_hook( self, - hook: H, + hook: Option, ) -> SystemCaller<'a, EvmConfig, Chainspec, H> { let Self { evm_config, chain_spec, .. } = self; - SystemCaller { evm_config, chain_spec, hook: Some(hook) } + SystemCaller { evm_config, chain_spec, hook } } /// Convenience method to consume the type and drop borrowed fields pub fn finish(self) {} @@ -321,4 +318,11 @@ where eip7251::post_commit(result_and_state.result) } + + /// Delegate to stored `OnStateHook`, noop if hook is `None`. + pub fn on_state(&mut self, state: &ResultAndState) { + if let Some(ref mut hook) = &mut self.hook { + hook.on_state(state); + } + } } diff --git a/crates/evm/src/test_utils.rs b/crates/evm/src/test_utils.rs index cf45930aece9..45ab2e97734e 100644 --- a/crates/evm/src/test_utils.rs +++ b/crates/evm/src/test_utils.rs @@ -1,7 +1,10 @@ //! Helpers for testing. -use crate::execute::{ - BatchExecutor, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, Executor, +use crate::{ + execute::{ + BatchExecutor, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, Executor, + }, + system_calls::OnStateHook, }; use alloy_primitives::BlockNumber; use parking_lot::Mutex; @@ -73,6 +76,17 @@ impl Executor for MockExecutorProvider { { unimplemented!() } + + fn execute_with_state_hook( + self, + _: Self::Input<'_>, + _: F, + ) -> Result + where + F: OnStateHook, + { + unimplemented!() + } } impl BatchExecutor for MockExecutorProvider { diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 6ca9cec5260e..4e5918831f76 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -10,7 +10,7 @@ use reth_evm::{ BatchExecutor, BlockExecutionError, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, BlockValidationError, Executor, ProviderError, }, - system_calls::SystemCaller, + system_calls::{NoopHook, OnStateHook, SystemCaller}, ConfigureEvm, }; use reth_execution_types::ExecutionOutcome; @@ -108,18 +108,23 @@ where /// /// This applies the pre-execution changes, and executes the transactions. /// + /// The optional `state_hook` will be executed with the state changes if present. + /// /// # Note /// /// It does __not__ apply post-execution changes. - fn execute_pre_and_transactions( + fn execute_pre_and_transactions( &self, block: &BlockWithSenders, mut evm: Evm<'_, Ext, &mut State>, + state_hook: Option, ) -> Result<(Vec, u64), BlockExecutionError> where DB: Database + Display>, + F: OnStateHook, { - let mut system_caller = SystemCaller::new(&self.evm_config, &self.chain_spec); + let mut system_caller = + SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); // apply pre execution changes system_caller.apply_beacon_root_contract_call( @@ -178,7 +183,7 @@ where self.evm_config.fill_tx_env(evm.tx_mut(), transaction, *sender); // Execute transaction. - let ResultAndState { result, state } = evm.transact().map_err(move |err| { + let result_and_state = evm.transact().map_err(move |err| { let new_err = err.map_db_err(|e| e.into()); // Ensure hash is calculated for error log, if not already done BlockValidationError::EVM { @@ -192,7 +197,8 @@ where ?transaction, "Executed transaction" ); - + system_caller.on_state(&result_and_state); + let ResultAndState { result, state } = result_and_state; evm.db_mut().commit(state); // append gas used @@ -278,16 +284,30 @@ where EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()) } + /// Convenience method to invoke `execute_without_verification_with_state_hook` setting the + /// state hook as `None`. + fn execute_without_verification( + &mut self, + block: &BlockWithSenders, + total_difficulty: U256, + ) -> Result<(Vec, u64), BlockExecutionError> { + self.execute_without_verification_with_state_hook(block, total_difficulty, None::) + } + /// Execute a single block and apply the state changes to the internal state. /// /// Returns the receipts of the transactions in the block and the total gas used. /// /// Returns an error if execution fails. - fn execute_without_verification( + fn execute_without_verification_with_state_hook( &mut self, block: &BlockWithSenders, total_difficulty: U256, - ) -> Result<(Vec, u64), BlockExecutionError> { + state_hook: Option, + ) -> Result<(Vec, u64), BlockExecutionError> + where + F: OnStateHook, + { // 1. prepare state on new block self.on_new_block(&block.header); @@ -296,7 +316,7 @@ where let (receipts, gas_used) = { let evm = self.executor.evm_config.evm_with_env(&mut self.state, env); - self.executor.execute_pre_and_transactions(block, evm) + self.executor.execute_pre_and_transactions(block, evm, state_hook) }?; // 3. apply post execution changes @@ -383,6 +403,32 @@ where gas_used, }) } + + fn execute_with_state_hook( + mut self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook, + { + let BlockExecutionInput { block, total_difficulty } = input; + let (receipts, gas_used) = self.execute_without_verification_with_state_hook( + block, + total_difficulty, + Some(state_hook), + )?; + + // NOTE: we need to merge keep the reverts for the bundle retention + self.state.merge_transitions(BundleRetention::Reverts); + + Ok(BlockExecutionOutput { + state: self.state.take_bundle(), + receipts, + requests: vec![], + gas_used, + }) + } } /// An executor for a batch of blocks. From 5fd6906b31e9ae14bdc75e9a522d8488e1449e32 Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:44:50 +0530 Subject: [PATCH 021/103] feat: cleaned up prepare_call_env() (#11469) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 29 +++------------------- crates/rpc/rpc/src/debug.rs | 2 -- crates/rpc/rpc/src/trace.rs | 2 -- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 6189b69492a2..4d12450a1c7f 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -256,7 +256,6 @@ pub trait EthCall: Call + LoadPendingBlock { )?; let block = block.ok_or(EthApiError::HeaderNotFound(target_block))?; - let gas_limit = self.call_gas_limit(); // we're essentially replaying the transactions in the block here, hence we need the // state that points to the beginning of the block, which is the state at @@ -302,14 +301,7 @@ pub trait EthCall: Call + LoadPendingBlock { let overrides = EvmOverrides::new(state_overrides, block_overrides.clone()); let env = this - .prepare_call_env( - cfg.clone(), - block_env.clone(), - tx, - gas_limit, - &mut db, - overrides, - ) + .prepare_call_env(cfg.clone(), block_env.clone(), tx, &mut db, overrides) .map(Into::into)?; let (res, _) = this.transact(&mut db, env)?; @@ -559,14 +551,7 @@ pub trait Call: LoadState + SpawnBlocking { let mut db = CacheDB::new(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))); - let env = this.prepare_call_env( - cfg, - block_env, - request, - this.call_gas_limit(), - &mut db, - overrides, - )?; + let env = this.prepare_call_env(cfg, block_env, request, &mut db, overrides)?; f(StateCacheDbRefMutWrapper(&mut db), env) }) @@ -1099,7 +1084,6 @@ pub trait Call: LoadState + SpawnBlocking { mut cfg: CfgEnvWithHandlerCfg, mut block: BlockEnv, mut request: TransactionRequest, - gas_limit: u64, db: &mut CacheDB, overrides: EvmOverrides, ) -> Result @@ -1107,18 +1091,13 @@ pub trait Call: LoadState + SpawnBlocking { DB: DatabaseRef, EthApiError: From<::Error>, { - // TODO(mattsse): cleanup, by not disabling gaslimit and instead use self.call_gas_limit - if request.gas > Some(gas_limit) { + if request.gas > Some(self.call_gas_limit()) { // configured gas exceeds limit return Err( EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh).into() ) } - // we want to disable this in eth_call, since this is common practice used by other node - // impls and providers - cfg.disable_block_gas_limit = true; - // Disabled because eth_call is sometimes used with eoa senders // See cfg.disable_eip3607 = true; @@ -1154,7 +1133,7 @@ pub trait Call: LoadState + SpawnBlocking { // trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap as the maximum gas limit"); - env.tx.gas_limit = gas_limit; + env.tx.gas_limit = self.call_gas_limit(); } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index b47473c34270..f4aba778f572 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -506,7 +506,6 @@ where let opts = opts.unwrap_or_default(); let block = block.ok_or(EthApiError::HeaderNotFound(target_block))?; let GethDebugTracingCallOptions { tracing_options, mut state_overrides, .. } = opts; - let gas_limit = self.inner.eth_api.call_gas_limit(); // we're essentially replaying the transactions in the block here, hence we need the state // that points to the beginning of the block, which is the state at the parent block @@ -570,7 +569,6 @@ where cfg.clone(), block_env.clone(), tx, - gas_limit, &mut db, overrides, )?; diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 185383d811b7..e905ff685c86 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -153,7 +153,6 @@ where let at = block_id.unwrap_or(BlockId::pending()); let (cfg, block_env, at) = self.inner.eth_api.evm_env_at(at).await?; - let gas_limit = self.inner.eth_api.call_gas_limit(); let this = self.clone(); // execute all transactions on top of each other and record the traces self.eth_api() @@ -168,7 +167,6 @@ where cfg.clone(), block_env.clone(), call, - gas_limit, &mut db, Default::default(), )?; From 399192c3471d6a88157236a0ab02b6facf45102c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 10:37:48 +0200 Subject: [PATCH 022/103] chore: use block.body directly (#11474) --- crates/exex/exex/src/backfill/job.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 7642edbac30e..2db897270bfb 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -8,7 +8,7 @@ use alloy_primitives::BlockNumber; use reth_evm::execute::{ BatchExecutor, BlockExecutionError, BlockExecutionOutput, BlockExecutorProvider, Executor, }; -use reth_primitives::{Block, BlockBody, BlockWithSenders, Receipt}; +use reth_primitives::{Block, BlockWithSenders, Receipt}; use reth_primitives_traits::format_gas_throughput; use reth_provider::{ BlockReader, Chain, HeaderProvider, ProviderError, StateProviderFactory, TransactionVariant, @@ -103,16 +103,8 @@ where // Unseal the block for execution let (block, senders) = block.into_components(); let (unsealed_header, hash) = block.header.split(); - let block = Block { - header: unsealed_header, - body: BlockBody { - transactions: block.body.transactions, - ommers: block.body.ommers, - withdrawals: block.body.withdrawals, - requests: block.body.requests, - }, - } - .with_senders_unchecked(senders); + let block = + Block { header: unsealed_header, body: block.body }.with_senders_unchecked(senders); executor.execute_and_verify_one((&block, td).into())?; execution_duration += execute_start.elapsed(); From c5cfaf116b22c0338c4fab470077cc195b233581 Mon Sep 17 00:00:00 2001 From: garwah <14845405+garwahl@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:12:31 +1000 Subject: [PATCH 023/103] feat: Add metrics to track transactions by type in txpool (#11403) Co-authored-by: garwah Co-authored-by: Matthias Seitz --- crates/transaction-pool/src/metrics.rs | 12 +++++++- crates/transaction-pool/src/pool/txpool.rs | 35 ++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/metrics.rs b/crates/transaction-pool/src/metrics.rs index d323d47e459a..f5d269b361fc 100644 --- a/crates/transaction-pool/src/metrics.rs +++ b/crates/transaction-pool/src/metrics.rs @@ -36,8 +36,18 @@ pub struct TxPoolMetrics { /// Total amount of memory used by the transactions in the blob sub-pool in bytes pub(crate) blob_pool_size_bytes: Gauge, - /// Number of all transactions of all sub-pools: pending + basefee + queued + /// Number of all transactions of all sub-pools: pending + basefee + queued + blob pub(crate) total_transactions: Gauge, + /// Number of all legacy transactions in the pool + pub(crate) total_legacy_transactions: Gauge, + /// Number of all EIP-2930 transactions in the pool + pub(crate) total_eip2930_transactions: Gauge, + /// Number of all EIP-1559 transactions in the pool + pub(crate) total_eip1559_transactions: Gauge, + /// Number of all EIP-4844 transactions in the pool + pub(crate) total_eip4844_transactions: Gauge, + /// Number of all EIP-7702 transactions in the pool + pub(crate) total_eip7702_transactions: Gauge, /// How often the pool was updated after the canonical state changed pub(crate) performed_state_updates: Counter, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 912e04506a19..10605565c85d 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -19,8 +19,12 @@ use crate::{ ValidPoolTransaction, U256, }; use alloy_primitives::{Address, TxHash, B256}; -use reth_primitives::constants::{ - eip4844::BLOB_TX_MIN_BLOB_GASPRICE, ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE, +use reth_primitives::{ + constants::{ + eip4844::BLOB_TX_MIN_BLOB_GASPRICE, ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE, + }, + EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, + LEGACY_TX_TYPE_ID, }; use rustc_hash::FxHashMap; use smallvec::SmallVec; @@ -429,6 +433,7 @@ impl TxPool { let UpdateOutcome { promoted, discarded } = self.update_accounts(changed_senders); + self.update_transaction_type_metrics(); self.metrics.performed_state_updates.increment(1); OnNewCanonicalStateOutcome { block_hash, mined: mined_transactions, promoted, discarded } @@ -448,6 +453,32 @@ impl TxPool { self.metrics.total_transactions.set(stats.total as f64); } + /// Updates transaction type metrics for the entire pool. + pub(crate) fn update_transaction_type_metrics(&self) { + let mut legacy_count = 0; + let mut eip2930_count = 0; + let mut eip1559_count = 0; + let mut eip4844_count = 0; + let mut eip7702_count = 0; + + for tx in self.all_transactions.transactions_iter() { + match tx.transaction.tx_type() { + LEGACY_TX_TYPE_ID => legacy_count += 1, + EIP2930_TX_TYPE_ID => eip2930_count += 1, + EIP1559_TX_TYPE_ID => eip1559_count += 1, + EIP4844_TX_TYPE_ID => eip4844_count += 1, + EIP7702_TX_TYPE_ID => eip7702_count += 1, + _ => {} // Ignore other types + } + } + + self.metrics.total_legacy_transactions.set(legacy_count as f64); + self.metrics.total_eip2930_transactions.set(eip2930_count as f64); + self.metrics.total_eip1559_transactions.set(eip1559_count as f64); + self.metrics.total_eip4844_transactions.set(eip4844_count as f64); + self.metrics.total_eip7702_transactions.set(eip7702_count as f64); + } + /// Adds the transaction into the pool. /// /// This pool consists of four sub-pools: `Queued`, `Pending`, `BaseFee`, and `Blob`. From 72e0ab1000b7badbf548c549b607616fe9ffac59 Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:06:06 +0200 Subject: [PATCH 024/103] chore: op chainspec (#11415) --- Cargo.lock | 2 - crates/chainspec/Cargo.toml | 9 +- crates/chainspec/src/spec.rs | 407 +++++---------------- crates/optimism/chainspec/Cargo.toml | 5 +- crates/optimism/chainspec/src/constants.rs | 10 + crates/optimism/chainspec/src/lib.rs | 254 ++++++++++++- 6 files changed, 338 insertions(+), 349 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceeb53a2e4a7..8c66fbf09b38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6508,13 +6508,11 @@ dependencies = [ "auto_impl", "derive_more 1.0.0", "once_cell", - "op-alloy-rpc-types", "reth-ethereum-forks", "reth-network-peers", "reth-optimism-forks", "reth-primitives-traits", "reth-trie-common", - "serde", "serde_json", ] diff --git a/crates/chainspec/Cargo.toml b/crates/chainspec/Cargo.toml index de04ffd1e6aa..80636d139a1b 100644 --- a/crates/chainspec/Cargo.toml +++ b/crates/chainspec/Cargo.toml @@ -27,13 +27,9 @@ alloy-genesis.workspace = true alloy-primitives = { workspace = true, features = ["rand", "rlp"] } alloy-trie.workspace = true -# op -op-alloy-rpc-types = { workspace = true, optional = true } - # misc auto_impl.workspace = true once_cell.workspace = true -serde = { workspace = true, optional = true } serde_json.workspace = true derive_more.workspace = true @@ -44,12 +40,9 @@ alloy-eips = { workspace = true, features = ["arbitrary"] } alloy-rlp = { workspace = true, features = ["arrayvec"] } alloy-genesis.workspace = true -# op -op-alloy-rpc-types.workspace = true - [features] default = ["std"] -optimism = ["serde", "dep:op-alloy-rpc-types", "reth-optimism-forks"] +optimism = ["reth-optimism-forks"] std = [ "alloy-chains/std", "alloy-eips/std", diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index 45070db197d4..d6ca92aa24eb 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -590,14 +590,89 @@ impl ChainSpec { impl From for ChainSpec { fn from(genesis: Genesis) -> Self { - #[cfg(not(feature = "optimism"))] - { - into_ethereum_chain_spec(genesis) + // Block-based hardforks + let hardfork_opts = [ + (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), + (EthereumHardfork::Dao.boxed(), genesis.config.dao_fork_block), + (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), + (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), + (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), + (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), + (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), + (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), + (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), + (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), + (EthereumHardfork::London.boxed(), genesis.config.london_block), + (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), + (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), + ]; + let mut hardforks = hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) + .collect::>(); + + // Paris + let paris_block_and_final_difficulty = + if let Some(ttd) = genesis.config.terminal_total_difficulty { + hardforks.push(( + EthereumHardfork::Paris.boxed(), + ForkCondition::TTD { + total_difficulty: ttd, + fork_block: genesis.config.merge_netsplit_block, + }, + )); + + genesis.config.merge_netsplit_block.map(|block| (block, ttd)) + } else { + None + }; + + // Time-based hardforks + let time_hardfork_opts = [ + (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), + (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), + (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), + ]; + + let mut time_hardforks = time_hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| { + opt.map(|time| (hardfork, ForkCondition::Timestamp(time))) + }) + .collect::>(); + + hardforks.append(&mut time_hardforks); + + // Ordered Hardforks + let mainnet_hardforks: ChainHardforks = EthereumHardfork::mainnet().into(); + let mainnet_order = mainnet_hardforks.forks_iter(); + + let mut ordered_hardforks = Vec::with_capacity(hardforks.len()); + for (hardfork, _) in mainnet_order { + if let Some(pos) = hardforks.iter().position(|(e, _)| **e == *hardfork) { + ordered_hardforks.push(hardforks.remove(pos)); + } } - #[cfg(feature = "optimism")] - { - into_optimism_chain_spec(genesis) + // append the remaining unknown hardforks to ensure we don't filter any out + ordered_hardforks.append(&mut hardforks); + + // NOTE: in full node, we prune all receipts except the deposit contract's. We do not + // have the deployment block in the genesis file, so we use block zero. We use the same + // deposit topic as the mainnet contract if we have the deposit contract address in the + // genesis json. + let deposit_contract = genesis.config.deposit_contract_address.map(|address| { + DepositContract { address, block: 0, topic: MAINNET_DEPOSIT_CONTRACT.topic } + }); + + Self { + chain: genesis.config.chain_id.into(), + genesis, + genesis_hash: OnceCell::new(), + hardforks: ChainHardforks::new(ordered_hardforks), + paris_block_and_final_difficulty, + deposit_contract, + ..Default::default() } } } @@ -637,194 +712,6 @@ impl EthereumHardforks for ChainSpec { #[cfg(feature = "optimism")] impl reth_optimism_forks::OptimismHardforks for ChainSpec {} -/// Convert the given [`Genesis`] into an Ethereum [`ChainSpec`]. -#[cfg(not(feature = "optimism"))] -fn into_ethereum_chain_spec(genesis: Genesis) -> ChainSpec { - // Block-based hardforks - let hardfork_opts = [ - (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), - (EthereumHardfork::Dao.boxed(), genesis.config.dao_fork_block), - (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), - (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), - (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), - (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), - (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), - (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), - (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), - (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), - (EthereumHardfork::London.boxed(), genesis.config.london_block), - (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), - (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), - ]; - let mut hardforks = hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) - .collect::>(); - - // Paris - let paris_block_and_final_difficulty = - if let Some(ttd) = genesis.config.terminal_total_difficulty { - hardforks.push(( - EthereumHardfork::Paris.boxed(), - ForkCondition::TTD { - total_difficulty: ttd, - fork_block: genesis.config.merge_netsplit_block, - }, - )); - - genesis.config.merge_netsplit_block.map(|block| (block, ttd)) - } else { - None - }; - - // Time-based hardforks - let time_hardfork_opts = [ - (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), - (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), - (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), - ]; - - let mut time_hardforks = time_hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|time| (hardfork, ForkCondition::Timestamp(time)))) - .collect::>(); - - hardforks.append(&mut time_hardforks); - - // Ordered Hardforks - let mainnet_hardforks: ChainHardforks = EthereumHardfork::mainnet().into(); - let mainnet_order = mainnet_hardforks.forks_iter(); - - let mut ordered_hardforks = Vec::with_capacity(hardforks.len()); - for (hardfork, _) in mainnet_order { - if let Some(pos) = hardforks.iter().position(|(e, _)| **e == *hardfork) { - ordered_hardforks.push(hardforks.remove(pos)); - } - } - - // append the remaining unknown hardforks to ensure we don't filter any out - ordered_hardforks.append(&mut hardforks); - - // NOTE: in full node, we prune all receipts except the deposit contract's. We do not - // have the deployment block in the genesis file, so we use block zero. We use the same - // deposit topic as the mainnet contract if we have the deposit contract address in the - // genesis json. - let deposit_contract = genesis.config.deposit_contract_address.map(|address| DepositContract { - address, - block: 0, - topic: MAINNET_DEPOSIT_CONTRACT.topic, - }); - - ChainSpec { - chain: genesis.config.chain_id.into(), - genesis, - genesis_hash: OnceCell::new(), - hardforks: ChainHardforks::new(ordered_hardforks), - paris_block_and_final_difficulty, - deposit_contract, - ..Default::default() - } -} - -#[cfg(feature = "optimism")] -/// Convert the given [`Genesis`] into an Optimism [`ChainSpec`]. -fn into_optimism_chain_spec(genesis: Genesis) -> ChainSpec { - use reth_optimism_forks::OptimismHardfork; - let optimism_genesis_info = OptimismGenesisInfo::extract_from(&genesis); - let genesis_info = optimism_genesis_info.optimism_chain_info.genesis_info.unwrap_or_default(); - - // Block-based hardforks - let hardfork_opts = [ - (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), - (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), - (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), - (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), - (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), - (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), - (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), - (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), - (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), - (EthereumHardfork::London.boxed(), genesis.config.london_block), - (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), - (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), - (OptimismHardfork::Bedrock.boxed(), genesis_info.bedrock_block), - ]; - let mut block_hardforks = hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) - .collect::>(); - - // Paris - let paris_block_and_final_difficulty = - if let Some(ttd) = genesis.config.terminal_total_difficulty { - block_hardforks.push(( - EthereumHardfork::Paris.boxed(), - ForkCondition::TTD { - total_difficulty: ttd, - fork_block: genesis.config.merge_netsplit_block, - }, - )); - - genesis.config.merge_netsplit_block.map(|block| (block, ttd)) - } else { - None - }; - - // Time-based hardforks - let time_hardfork_opts = [ - (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), - (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), - (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), - (OptimismHardfork::Regolith.boxed(), genesis_info.regolith_time), - (OptimismHardfork::Canyon.boxed(), genesis_info.canyon_time), - (OptimismHardfork::Ecotone.boxed(), genesis_info.ecotone_time), - (OptimismHardfork::Fjord.boxed(), genesis_info.fjord_time), - (OptimismHardfork::Granite.boxed(), genesis_info.granite_time), - ]; - - let mut time_hardforks = time_hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|time| (hardfork, ForkCondition::Timestamp(time)))) - .collect::>(); - - block_hardforks.append(&mut time_hardforks); - - // Ordered Hardforks - let mainnet_hardforks = OptimismHardfork::op_mainnet(); - let mainnet_order = mainnet_hardforks.forks_iter(); - - let mut ordered_hardforks = Vec::with_capacity(block_hardforks.len()); - for (hardfork, _) in mainnet_order { - if let Some(pos) = block_hardforks.iter().position(|(e, _)| **e == *hardfork) { - ordered_hardforks.push(block_hardforks.remove(pos)); - } - } - - // append the remaining unknown hardforks to ensure we don't filter any out - ordered_hardforks.append(&mut block_hardforks); - - // NOTE: in full node, we prune all receipts except the deposit contract's. We do not - // have the deployment block in the genesis file, so we use block zero. We use the same - // deposit topic as the mainnet contract if we have the deposit contract address in the - // genesis json. - let deposit_contract = genesis.config.deposit_contract_address.map(|address| DepositContract { - address, - block: 0, - topic: MAINNET_DEPOSIT_CONTRACT.topic, - }); - - ChainSpec { - chain: genesis.config.chain_id.into(), - genesis, - genesis_hash: OnceCell::new(), - hardforks: ChainHardforks::new(ordered_hardforks), - paris_block_and_final_difficulty, - deposit_contract, - base_fee_params: optimism_genesis_info.base_fee_params, - ..Default::default() - } -} - /// A trait for reading the current chainspec. #[auto_impl::auto_impl(&, Arc)] pub trait ChainSpecProvider: Send + Sync { @@ -1102,59 +989,6 @@ impl DepositContract { } } -/// Genesis info for Optimism. -#[cfg(feature = "optimism")] -#[derive(Default, Debug, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct OptimismGenesisInfo { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo, - #[serde(skip)] - base_fee_params: BaseFeeParamsKind, -} - -#[cfg(feature = "optimism")] -impl OptimismGenesisInfo { - fn extract_from(genesis: &Genesis) -> Self { - let mut info = Self { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo::extract_from( - &genesis.config.extra_fields, - ) - .unwrap_or_default(), - ..Default::default() - }; - if let Some(optimism_base_fee_info) = &info.optimism_chain_info.base_fee_info { - if let (Some(elasticity), Some(denominator)) = ( - optimism_base_fee_info.eip1559_elasticity, - optimism_base_fee_info.eip1559_denominator, - ) { - let base_fee_params = if let Some(canyon_denominator) = - optimism_base_fee_info.eip1559_denominator_canyon - { - BaseFeeParamsKind::Variable( - vec![ - ( - EthereumHardfork::London.boxed(), - BaseFeeParams::new(denominator as u128, elasticity as u128), - ), - ( - reth_optimism_forks::OptimismHardfork::Canyon.boxed(), - BaseFeeParams::new(canyon_denominator as u128, elasticity as u128), - ), - ] - .into(), - ) - } else { - BaseFeeParams::new(denominator as u128, elasticity as u128).into() - }; - - info.base_fee_params = base_fee_params; - } - } - - info - } -} - /// Verifies [`ChainSpec`] configuration against expected data in given cases. #[cfg(any(test, feature = "test-utils"))] pub fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { @@ -2477,7 +2311,6 @@ Post-merge hard forks (timestamp based): } #[test] - #[cfg(not(feature = "optimism"))] fn test_fork_order_ethereum_mainnet() { let genesis = Genesis { config: ChainConfig { @@ -2506,7 +2339,7 @@ Post-merge hard forks (timestamp based): ..Default::default() }; - let chain_spec = into_ethereum_chain_spec(genesis); + let chain_spec: ChainSpec = genesis.into(); let hardforks: Vec<_> = chain_spec.hardforks.forks_iter().map(|(h, _)| h).collect(); let expected_hardforks = vec![ @@ -2534,80 +2367,4 @@ Post-merge hard forks (timestamp based): .all(|(expected, actual)| &**expected == *actual)); assert_eq!(expected_hardforks.len(), hardforks.len()); } - - #[test] - #[cfg(feature = "optimism")] - fn test_fork_order_optimism_mainnet() { - use reth_optimism_forks::OptimismHardfork; - - let genesis = Genesis { - config: ChainConfig { - chain_id: 0, - homestead_block: Some(0), - dao_fork_block: Some(0), - dao_fork_support: false, - eip150_block: Some(0), - eip155_block: Some(0), - eip158_block: Some(0), - byzantium_block: Some(0), - constantinople_block: Some(0), - petersburg_block: Some(0), - istanbul_block: Some(0), - muir_glacier_block: Some(0), - berlin_block: Some(0), - london_block: Some(0), - arrow_glacier_block: Some(0), - gray_glacier_block: Some(0), - merge_netsplit_block: Some(0), - shanghai_time: Some(0), - cancun_time: Some(0), - terminal_total_difficulty: Some(U256::ZERO), - extra_fields: [ - (String::from("bedrockBlock"), 0.into()), - (String::from("regolithTime"), 0.into()), - (String::from("canyonTime"), 0.into()), - (String::from("ecotoneTime"), 0.into()), - (String::from("fjordTime"), 0.into()), - (String::from("graniteTime"), 0.into()), - ] - .into_iter() - .collect(), - ..Default::default() - }, - ..Default::default() - }; - - let chain_spec: ChainSpec = into_optimism_chain_spec(genesis); - - let hardforks: Vec<_> = chain_spec.hardforks.forks_iter().map(|(h, _)| h).collect(); - let expected_hardforks = vec![ - EthereumHardfork::Homestead.boxed(), - EthereumHardfork::Tangerine.boxed(), - EthereumHardfork::SpuriousDragon.boxed(), - EthereumHardfork::Byzantium.boxed(), - EthereumHardfork::Constantinople.boxed(), - EthereumHardfork::Petersburg.boxed(), - EthereumHardfork::Istanbul.boxed(), - EthereumHardfork::MuirGlacier.boxed(), - EthereumHardfork::Berlin.boxed(), - EthereumHardfork::London.boxed(), - EthereumHardfork::ArrowGlacier.boxed(), - EthereumHardfork::GrayGlacier.boxed(), - EthereumHardfork::Paris.boxed(), - OptimismHardfork::Bedrock.boxed(), - OptimismHardfork::Regolith.boxed(), - EthereumHardfork::Shanghai.boxed(), - OptimismHardfork::Canyon.boxed(), - EthereumHardfork::Cancun.boxed(), - OptimismHardfork::Ecotone.boxed(), - OptimismHardfork::Fjord.boxed(), - OptimismHardfork::Granite.boxed(), - ]; - - assert!(expected_hardforks - .iter() - .zip(hardforks.iter()) - .all(|(expected, actual)| &**expected == *actual)); - assert_eq!(expected_hardforks.len(), hardforks.len()); - } } diff --git a/crates/optimism/chainspec/Cargo.toml b/crates/optimism/chainspec/Cargo.toml index 41b54902c201..e13b95056c7f 100644 --- a/crates/optimism/chainspec/Cargo.toml +++ b/crates/optimism/chainspec/Cargo.toml @@ -26,12 +26,15 @@ alloy-chains.workspace = true alloy-genesis.workspace = true alloy-primitives.workspace = true +# op +op-alloy-rpc-types.workspace = true + # io serde_json.workspace = true # misc -once_cell.workspace = true derive_more.workspace = true +once_cell.workspace = true [dev-dependencies] reth-chainspec = { workspace = true, features = ["test-utils"] } diff --git a/crates/optimism/chainspec/src/constants.rs b/crates/optimism/chainspec/src/constants.rs index 439c2fed83c0..8fce63daafef 100644 --- a/crates/optimism/chainspec/src/constants.rs +++ b/crates/optimism/chainspec/src/constants.rs @@ -1,5 +1,8 @@ //! OP stack variation of chain spec constants. +use alloy_primitives::{address, b256}; +use reth_chainspec::DepositContract; + //------------------------------- BASE MAINNET -------------------------------// /// Max gas limit on Base: @@ -9,3 +12,10 @@ pub const BASE_MAINNET_MAX_GAS_LIMIT: u64 = 105_000_000; /// Max gas limit on Base Sepolia: pub const BASE_SEPOLIA_MAX_GAS_LIMIT: u64 = 45_000_000; + +/// Deposit contract address: `0x00000000219ab540356cbb839cbe05303d7705fa` +pub(crate) const MAINNET_DEPOSIT_CONTRACT: DepositContract = DepositContract::new( + address!("00000000219ab540356cbb839cbe05303d7705fa"), + 11052984, + b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), +); diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 2668c374500d..76a3ed838b18 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -26,11 +26,14 @@ pub use dev::OP_DEV; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; +use crate::constants::MAINNET_DEPOSIT_CONTRACT; use derive_more::{Constructor, Deref, Into}; +use once_cell::sync::OnceCell; use reth_chainspec::{ - BaseFeeParams, ChainSpec, DepositContract, EthChainSpec, EthereumHardforks, ForkFilter, ForkId, - Hardforks, Head, + BaseFeeParams, BaseFeeParamsKind, ChainSpec, DepositContract, EthChainSpec, EthereumHardforks, + ForkFilter, ForkId, Hardforks, Head, }; +use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition}; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; @@ -52,14 +55,14 @@ impl EthChainSpec for OpChainSpec { self.inner.chain() } - fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { - self.inner.base_fee_params_at_timestamp(timestamp) - } - fn base_fee_params_at_block(&self, block_number: u64) -> BaseFeeParams { self.inner.base_fee_params_at_block(block_number) } + fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { + self.inner.base_fee_params_at_timestamp(timestamp) + } + fn deposit_contract(&self) -> Option<&DepositContract> { self.inner.deposit_contract() } @@ -118,20 +121,170 @@ impl Hardforks for OpChainSpec { } impl EthereumHardforks for OpChainSpec { + fn get_final_paris_total_difficulty(&self) -> Option { + self.inner.get_final_paris_total_difficulty() + } + fn final_paris_total_difficulty(&self, block_number: u64) -> Option { self.inner.final_paris_total_difficulty(block_number) } +} - fn get_final_paris_total_difficulty(&self) -> Option { - self.inner.get_final_paris_total_difficulty() +impl From for OpChainSpec { + fn from(genesis: Genesis) -> Self { + use reth_optimism_forks::OptimismHardfork; + let optimism_genesis_info = OptimismGenesisInfo::extract_from(&genesis); + let genesis_info = + optimism_genesis_info.optimism_chain_info.genesis_info.unwrap_or_default(); + + // Block-based hardforks + let hardfork_opts = [ + (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), + (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), + (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), + (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), + (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), + (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), + (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), + (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), + (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), + (EthereumHardfork::London.boxed(), genesis.config.london_block), + (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), + (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), + (OptimismHardfork::Bedrock.boxed(), genesis_info.bedrock_block), + ]; + let mut block_hardforks = hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) + .collect::>(); + + // Paris + let paris_block_and_final_difficulty = + if let Some(ttd) = genesis.config.terminal_total_difficulty { + block_hardforks.push(( + EthereumHardfork::Paris.boxed(), + ForkCondition::TTD { + total_difficulty: ttd, + fork_block: genesis.config.merge_netsplit_block, + }, + )); + + genesis.config.merge_netsplit_block.map(|block| (block, ttd)) + } else { + None + }; + + // Time-based hardforks + let time_hardfork_opts = [ + (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), + (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), + (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), + (OptimismHardfork::Regolith.boxed(), genesis_info.regolith_time), + (OptimismHardfork::Canyon.boxed(), genesis_info.canyon_time), + (OptimismHardfork::Ecotone.boxed(), genesis_info.ecotone_time), + (OptimismHardfork::Fjord.boxed(), genesis_info.fjord_time), + (OptimismHardfork::Granite.boxed(), genesis_info.granite_time), + ]; + + let mut time_hardforks = time_hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| { + opt.map(|time| (hardfork, ForkCondition::Timestamp(time))) + }) + .collect::>(); + + block_hardforks.append(&mut time_hardforks); + + // Ordered Hardforks + let mainnet_hardforks = OptimismHardfork::op_mainnet(); + let mainnet_order = mainnet_hardforks.forks_iter(); + + let mut ordered_hardforks = Vec::with_capacity(block_hardforks.len()); + for (hardfork, _) in mainnet_order { + if let Some(pos) = block_hardforks.iter().position(|(e, _)| **e == *hardfork) { + ordered_hardforks.push(block_hardforks.remove(pos)); + } + } + + // append the remaining unknown hardforks to ensure we don't filter any out + ordered_hardforks.append(&mut block_hardforks); + + // NOTE: in full node, we prune all receipts except the deposit contract's. We do not + // have the deployment block in the genesis file, so we use block zero. We use the same + // deposit topic as the mainnet contract if we have the deposit contract address in the + // genesis json. + let deposit_contract = genesis.config.deposit_contract_address.map(|address| { + DepositContract { address, block: 0, topic: MAINNET_DEPOSIT_CONTRACT.topic } + }); + + Self { + inner: ChainSpec { + chain: genesis.config.chain_id.into(), + genesis, + genesis_hash: OnceCell::new(), + hardforks: ChainHardforks::new(ordered_hardforks), + paris_block_and_final_difficulty, + deposit_contract, + base_fee_params: optimism_genesis_info.base_fee_params, + ..Default::default() + }, + } + } +} + +#[derive(Default, Debug)] +struct OptimismGenesisInfo { + optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo, + base_fee_params: BaseFeeParamsKind, +} + +impl OptimismGenesisInfo { + fn extract_from(genesis: &Genesis) -> Self { + let mut info = Self { + optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo::extract_from( + &genesis.config.extra_fields, + ) + .unwrap_or_default(), + ..Default::default() + }; + if let Some(optimism_base_fee_info) = &info.optimism_chain_info.base_fee_info { + if let (Some(elasticity), Some(denominator)) = ( + optimism_base_fee_info.eip1559_elasticity, + optimism_base_fee_info.eip1559_denominator, + ) { + let base_fee_params = if let Some(canyon_denominator) = + optimism_base_fee_info.eip1559_denominator_canyon + { + BaseFeeParamsKind::Variable( + vec![ + ( + EthereumHardfork::London.boxed(), + BaseFeeParams::new(denominator as u128, elasticity as u128), + ), + ( + reth_optimism_forks::OptimismHardfork::Canyon.boxed(), + BaseFeeParams::new(canyon_denominator as u128, elasticity as u128), + ), + ] + .into(), + ) + } else { + BaseFeeParams::new(denominator as u128, elasticity as u128).into() + }; + + info.base_fee_params = base_fee_params; + } + } + + info } } #[cfg(test)] mod tests { - use alloy_genesis::Genesis; + use alloy_genesis::{ChainConfig, Genesis}; use alloy_primitives::b256; - use reth_chainspec::{test_fork_ids, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; + use reth_chainspec::{test_fork_ids, BaseFeeParams, BaseFeeParamsKind}; use reth_ethereum_forks::{EthereumHardfork, ForkCondition, ForkHash, ForkId, Head}; use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; @@ -383,7 +536,7 @@ mod tests { }) ); - let chain_spec: ChainSpec = genesis.into(); + let chain_spec: OpChainSpec = genesis.into(); assert_eq!( chain_spec.base_fee_params, @@ -449,7 +602,7 @@ mod tests { }) ); - let chain_spec: ChainSpec = genesis.into(); + let chain_spec: OpChainSpec = genesis.into(); assert_eq!( chain_spec.base_fee_params, @@ -511,7 +664,7 @@ mod tests { } "#; let genesis: Genesis = serde_json::from_str(geth_genesis).unwrap(); - let chainspec = ChainSpec::from(genesis.clone()); + let chainspec = OpChainSpec::from(genesis.clone()); let actual_chain_id = genesis.config.chain_id; assert_eq!(actual_chain_id, 8453); @@ -552,4 +705,79 @@ mod tests { assert!(chainspec.is_fork_active_at_timestamp(OptimismHardfork::Regolith, 20)); } + + #[test] + fn test_fork_order_optimism_mainnet() { + use reth_optimism_forks::OptimismHardfork; + + let genesis = Genesis { + config: ChainConfig { + chain_id: 0, + homestead_block: Some(0), + dao_fork_block: Some(0), + dao_fork_support: false, + eip150_block: Some(0), + eip155_block: Some(0), + eip158_block: Some(0), + byzantium_block: Some(0), + constantinople_block: Some(0), + petersburg_block: Some(0), + istanbul_block: Some(0), + muir_glacier_block: Some(0), + berlin_block: Some(0), + london_block: Some(0), + arrow_glacier_block: Some(0), + gray_glacier_block: Some(0), + merge_netsplit_block: Some(0), + shanghai_time: Some(0), + cancun_time: Some(0), + terminal_total_difficulty: Some(U256::ZERO), + extra_fields: [ + (String::from("bedrockBlock"), 0.into()), + (String::from("regolithTime"), 0.into()), + (String::from("canyonTime"), 0.into()), + (String::from("ecotoneTime"), 0.into()), + (String::from("fjordTime"), 0.into()), + (String::from("graniteTime"), 0.into()), + ] + .into_iter() + .collect(), + ..Default::default() + }, + ..Default::default() + }; + + let chain_spec: OpChainSpec = genesis.into(); + + let hardforks: Vec<_> = chain_spec.hardforks.forks_iter().map(|(h, _)| h).collect(); + let expected_hardforks = vec![ + EthereumHardfork::Homestead.boxed(), + EthereumHardfork::Tangerine.boxed(), + EthereumHardfork::SpuriousDragon.boxed(), + EthereumHardfork::Byzantium.boxed(), + EthereumHardfork::Constantinople.boxed(), + EthereumHardfork::Petersburg.boxed(), + EthereumHardfork::Istanbul.boxed(), + EthereumHardfork::MuirGlacier.boxed(), + EthereumHardfork::Berlin.boxed(), + EthereumHardfork::London.boxed(), + EthereumHardfork::ArrowGlacier.boxed(), + EthereumHardfork::GrayGlacier.boxed(), + EthereumHardfork::Paris.boxed(), + OptimismHardfork::Bedrock.boxed(), + OptimismHardfork::Regolith.boxed(), + EthereumHardfork::Shanghai.boxed(), + OptimismHardfork::Canyon.boxed(), + EthereumHardfork::Cancun.boxed(), + OptimismHardfork::Ecotone.boxed(), + OptimismHardfork::Fjord.boxed(), + OptimismHardfork::Granite.boxed(), + ]; + + assert!(expected_hardforks + .iter() + .zip(hardforks.iter()) + .all(|(expected, actual)| &**expected == *actual)); + assert_eq!(expected_hardforks.len(), hardforks.len()); + } } From 0ce1dd6794a36c6cddab15a0b6d16f2195cd89df Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 4 Oct 2024 13:08:04 +0300 Subject: [PATCH 025/103] chore(exex): more backfill debug logs (#11476) --- crates/exex/exex/src/backfill/job.rs | 6 ++++++ crates/exex/exex/src/backfill/stream.rs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 2db897270bfb..7d66ce23d609 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -64,6 +64,12 @@ where } fn execute_range(&mut self) -> Result { + debug!( + target: "exex::backfill", + range = ?self.range, + "Executing block range" + ); + let mut executor = self.executor.batch_executor(StateProviderDatabase::new( self.provider.history_by_block_number(self.range.start().saturating_sub(1))?, )); diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index 07b710b7e389..75a1e17602cf 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -15,6 +15,7 @@ use reth_primitives::{BlockWithSenders, Receipt}; use reth_provider::{BlockReader, Chain, HeaderProvider, StateProviderFactory}; use reth_prune_types::PruneModes; use reth_stages_api::ExecutionStageThresholds; +use reth_tracing::tracing::debug; use tokio::task::JoinHandle; /// The default parallelism for active tasks in [`StreamBackfillJob`]. @@ -118,6 +119,7 @@ where // If we have range bounds, then we can spawn a new task for that range if let Some((first, last)) = range_bounds { let range = first..=last; + debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?range, "Spawning new backfill task"); let mut job = BackfillJob { executor: this.executor.clone(), provider: this.provider.clone(), From 227e2933907981f5effc5ed162b61618dba6a28a Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 4 Oct 2024 14:05:57 +0300 Subject: [PATCH 026/103] fix(exex): use thresholds in stream backfill (#11478) --- crates/exex/exex/src/backfill/stream.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index 75a1e17602cf..ca4bd326daa0 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -41,6 +41,7 @@ pub struct StreamBackfillJob { tasks: BackfillTasks, parallelism: usize, batch_size: usize, + thresholds: ExecutionStageThresholds, } impl StreamBackfillJob { @@ -124,7 +125,7 @@ where executor: this.executor.clone(), provider: this.provider.clone(), prune_modes: this.prune_modes.clone(), - thresholds: ExecutionStageThresholds::default(), + thresholds: this.thresholds.clone(), range, stream_parallelism: this.parallelism, }; @@ -150,12 +151,14 @@ impl From> for StreamBackfillJob From> for StreamBackfillJob { fn from(job: BackfillJob) -> Self { + let batch_size = job.thresholds.max_blocks.map_or(DEFAULT_BATCH_SIZE, |max| max as usize); Self { executor: job.executor, provider: job.provider, @@ -163,7 +166,11 @@ impl From> for StreamBackfillJob Date: Fri, 4 Oct 2024 15:10:32 +0300 Subject: [PATCH 027/103] chore(db): capture tx opening backtrace in debug mode (#11477) --- .../storage/db/src/implementation/mdbx/tx.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 8feb6c90ab27..2ff2789ea698 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -180,7 +180,12 @@ struct MetricsHandler { /// If `true`, the backtrace of transaction has already been recorded and logged. /// See [`MetricsHandler::log_backtrace_on_long_read_transaction`]. backtrace_recorded: AtomicBool, + /// Shared database environment metrics. env_metrics: Arc, + /// Backtrace of the location where the transaction has been opened. Reported only with debug + /// assertions, because capturing the backtrace on every transaction opening is expensive. + #[cfg(debug_assertions)] + open_backtrace: Backtrace, _marker: PhantomData, } @@ -193,6 +198,8 @@ impl MetricsHandler { close_recorded: false, record_backtrace: true, backtrace_recorded: AtomicBool::new(false), + #[cfg(debug_assertions)] + open_backtrace: Backtrace::force_capture(), env_metrics, _marker: PhantomData, } @@ -232,11 +239,22 @@ impl MetricsHandler { let open_duration = self.start.elapsed(); if open_duration >= self.long_transaction_duration { self.backtrace_recorded.store(true, Ordering::Relaxed); + #[cfg(debug_assertions)] + let message = format!( + "The database read transaction has been open for too long. Open backtrace:\n{}\n\nCurrent backtrace:\n{}", + self.open_backtrace, + Backtrace::force_capture() + ); + #[cfg(not(debug_assertions))] + let message = format!( + "The database read transaction has been open for too long. Backtrace:\n{}", + Backtrace::force_capture() + ); warn!( target: "storage::db::mdbx", ?open_duration, %self.txn_id, - "The database read transaction has been open for too long. Backtrace:\n{}", Backtrace::force_capture() + "{message}" ); } } From 16c39d919e7ca064371be9d8c169065737812d5f Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 4 Oct 2024 15:29:36 +0200 Subject: [PATCH 028/103] chore(sdk): `SealedHeader` generic over header (#11429) --- crates/primitives-traits/src/header/sealed.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/primitives-traits/src/header/sealed.rs b/crates/primitives-traits/src/header/sealed.rs index f2047e079c8d..85b2cb322047 100644 --- a/crates/primitives-traits/src/header/sealed.rs +++ b/crates/primitives-traits/src/header/sealed.rs @@ -14,22 +14,24 @@ use serde::{Deserialize, Serialize}; /// to modify header. #[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Serialize, Deserialize)] #[add_arbitrary_tests(rlp)] -pub struct SealedHeader { +pub struct SealedHeader { /// Locked Header hash. hash: BlockHash, /// Locked Header fields. #[as_ref] #[deref] - header: Header, + header: H, } -impl SealedHeader { +impl SealedHeader { /// Creates the sealed header with the corresponding block hash. #[inline] - pub const fn new(header: Header, hash: BlockHash) -> Self { + pub const fn new(header: H, hash: BlockHash) -> Self { Self { header, hash } } +} +impl SealedHeader { /// Returns the sealed Header fields. #[inline] pub const fn header(&self) -> &Header { From d5ddb4129d3464d0f5cbfdd7547528bdb278cd59 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 4 Oct 2024 15:33:18 +0200 Subject: [PATCH 029/103] chore(lint): fix lint primitives (#11487) --- crates/primitives-traits/src/header/sealed.rs | 2 +- crates/primitives/src/block.rs | 6 +++--- crates/primitives/src/receipt.rs | 6 +++--- crates/primitives/src/transaction/mod.rs | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/primitives-traits/src/header/sealed.rs b/crates/primitives-traits/src/header/sealed.rs index 85b2cb322047..7119a37e742a 100644 --- a/crates/primitives-traits/src/header/sealed.rs +++ b/crates/primitives-traits/src/header/sealed.rs @@ -184,7 +184,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for SealedHeader<'a> { + impl SerializeAs for SealedHeader<'_> { fn serialize_as(source: &super::SealedHeader, serializer: S) -> Result where S: Serializer, diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index 0464c28dee0f..b0bc538e512a 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -756,7 +756,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for BlockBody<'a> { + impl SerializeAs for BlockBody<'_> { fn serialize_as(source: &super::BlockBody, serializer: S) -> Result where S: Serializer, @@ -807,7 +807,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for SealedBlock<'a> { + impl SerializeAs for SealedBlock<'_> { fn serialize_as(source: &super::SealedBlock, serializer: S) -> Result where S: Serializer, @@ -858,7 +858,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for SealedBlockWithSenders<'a> { + impl SerializeAs for SealedBlockWithSenders<'_> { fn serialize_as( source: &super::SealedBlockWithSenders, serializer: S, diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 5c794be5061e..cfd831ed0f74 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -373,7 +373,7 @@ impl<'a> ReceiptWithBloomRef<'a> { } } -impl<'a> Encodable for ReceiptWithBloomRef<'a> { +impl Encodable for ReceiptWithBloomRef<'_> { fn encode(&self, out: &mut dyn BufMut) { self.as_encoder().encode_inner(out, true) } @@ -394,7 +394,7 @@ struct ReceiptWithBloomEncoder<'a> { receipt: &'a Receipt, } -impl<'a> ReceiptWithBloomEncoder<'a> { +impl ReceiptWithBloomEncoder<'_> { /// Returns the rlp header for the receipt payload. fn receipt_rlp_header(&self) -> alloy_rlp::Header { let mut rlp_head = alloy_rlp::Header { list: true, payload_length: 0 }; @@ -481,7 +481,7 @@ impl<'a> ReceiptWithBloomEncoder<'a> { } } -impl<'a> Encodable for ReceiptWithBloomEncoder<'a> { +impl Encodable for ReceiptWithBloomEncoder<'_> { fn encode(&self, out: &mut dyn BufMut) { self.encode_inner(out, true) } diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index b16c2c88ab52..7ef2c0c1fb76 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1640,7 +1640,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for Transaction<'a> { + impl SerializeAs for Transaction<'_> { fn serialize_as(source: &super::Transaction, serializer: S) -> Result where S: Serializer, @@ -1700,7 +1700,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for TransactionSigned<'a> { + impl SerializeAs for TransactionSigned<'_> { fn serialize_as( source: &super::TransactionSigned, serializer: S, From a42cfbbae72d2e638333e0fdb99eca226f143973 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:05:17 +0200 Subject: [PATCH 030/103] chore(provider): add more test coverage on `BlockchainProvider::*by_tx_range` queries (#11480) --- .../src/providers/blockchain_provider.rs | 163 +++++++++--------- 1 file changed, 82 insertions(+), 81 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 45bdb1525690..01d8e4cd977c 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -318,11 +318,17 @@ impl BlockchainProvider2 { let block_tx_count = block_state.block_ref().block().body.transactions.len(); let remaining = (tx_range.end() - tx_range.start() + 1) as usize; - // This should only be more than 0 in the first iteration, in case of a partial range + // If the transaction range start is higher than this block last transaction, advance + if *tx_range.start() > in_memory_tx_num + block_tx_count as u64 - 1 { + in_memory_tx_num += block_tx_count as u64; + continue + } + + // This should only be more than 0 once, in case of a partial range inside a block. let skip = (tx_range.start() - in_memory_tx_num) as usize; items.extend(fetch_from_block_state( - skip..=(remaining.min(block_tx_count) - 1), + skip..=skip + (remaining.min(block_tx_count - skip) - 1), block_state, )?); @@ -4115,51 +4121,76 @@ mod tests { Ok(()) } - #[test] - fn test_transactions_by_tx_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; + macro_rules! test_by_tx_range { + ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ + let db_tx_count = + $database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + let in_mem_tx_count = + $in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - // Define a valid transaction range within the database - let start_tx_num = 0; - let end_tx_num = 1; + let db_range = 0..=(db_tx_count - 1); + let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); - // Retrieve the transactions for this transaction number range - let result = provider.transactions_by_tx_range(start_tx_num..=end_tx_num)?; + $( + // Retrieve the expected database data + let database_data = + $database_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(db_range.clone())?, database_data); - // Ensure the transactions match the expected transactions in the database - assert_eq!(result.len(), 2); - assert_eq!(result[0], database_blocks[0].body.transactions[0].clone().into()); - assert_eq!(result[1], database_blocks[0].body.transactions[1].clone().into()); + // Retrieve the expected in-memory data + let in_memory_data = + $in_memory_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); - // Define an empty range that should return no transactions - let start_tx_num = u64::MAX; - let end_tx_num = u64::MAX; + // Test partial in-memory range + assert_eq!( + &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &in_memory_data[1..in_memory_data.len() - 1] + ); - // Retrieve the transactions for this range - let result = provider.transactions_by_tx_range(start_tx_num..end_tx_num)?; + // Test range in in-memory to unbounded end + assert_eq!($provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..]); - // Ensure no transactions are returned - assert!( - result.is_empty(), - "No transactions should be found for an empty transaction range" - ); + // Test last element in-memory + assert_eq!($provider.$method(in_mem_range.end()..)?, &in_memory_data[in_memory_data.len() -1 ..]); - Ok(()) + // Test range that spans database and in-memory + assert_eq!( + $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>() + ); + + // Test range that spans database and in-memory with unbounded end + assert_eq!( + $provider.$method(in_mem_range.start() - 2..)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..]) + .cloned() + .collect::>() + ); + + // Test invalid range + let start_tx_num = u64::MAX; + let end_tx_num = u64::MAX; + let result = $provider.$method(start_tx_num..end_tx_num)?; + assert!(result.is_empty(), "No data should be found for an invalid transaction range"); + + // Test empty range + let result = $provider.$method(in_mem_range.end()+10..in_mem_range.end()+20)?; + assert!(result.is_empty(), "No data should be found for an empty transaction range"); + )* + }}; } #[test] - fn test_senders_by_tx_range() -> eyre::Result<()> { + fn test_methods_by_tx_range() -> eyre::Result<()> { let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( + let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( &mut rng, TEST_BLOCKS_COUNT, TEST_BLOCKS_COUNT, @@ -4169,51 +4200,21 @@ mod tests { }, )?; - let db_tx_count = - database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - let in_mem_tx_count = - in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - - let db_range = 0..=(db_tx_count - 1); - let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); - - // Retrieve the senders for the whole database range - let database_senders = - database_blocks.iter().flat_map(|b| b.senders().unwrap()).collect::>(); - assert_eq!(provider.senders_by_tx_range(db_range)?, database_senders); - - // Retrieve the senders for the whole in-memory range - let in_memory_senders = - in_memory_blocks.iter().flat_map(|b| b.senders().unwrap()).collect::>(); - assert_eq!(provider.senders_by_tx_range(in_mem_range.clone())?, in_memory_senders); - - // Retrieve the senders for a partial in-memory range - assert_eq!( - &provider.senders_by_tx_range(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, - &in_memory_senders[1..in_memory_senders.len() - 1] - ); - - // Retrieve the senders for a range that spans database and in-memory - assert_eq!( - provider.senders_by_tx_range(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, - database_senders[database_senders.len() - 2..] - .iter() - .chain(&in_memory_senders[..in_memory_senders.len() - 1]) - .copied() - .collect::>() - ); - - // Define an empty range that should return no sender addresses - let start_tx_num = u64::MAX; - let end_tx_num = u64::MAX; - - // Retrieve the senders for this range - let result = provider.senders_by_tx_range(start_tx_num..end_tx_num)?; - - // Ensure no sender addresses are returned - assert!( - result.is_empty(), - "No sender addresses should be found for an empty transaction range" + test_by_tx_range!( + provider, + database_blocks, + in_memory_blocks, + [ + (senders_by_tx_range, |block: &SealedBlock| block.senders().unwrap()), + (transactions_by_tx_range, |block: &SealedBlock| block + .body + .transactions + .iter() + .map(|tx| Into::::into(tx.clone())) + .collect::>()), + (receipts_by_tx_range, |block: &SealedBlock| receipts[block.number as usize] + .clone()) + ] ); Ok(()) From 62e629329ad73b9fe2e33d50c9cd6b9140b1c49c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 17:43:35 +0200 Subject: [PATCH 031/103] test: ensure default hash matches (#11486) --- crates/primitives/src/block.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index b0bc538e512a..de0817fb025d 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -1130,4 +1130,13 @@ mod tests { Some(SealedBlockWithSenders { block: sealed, senders: vec![sender] }) ); } + + #[test] + fn test_default_seal() { + let block = SealedBlock::default(); + let sealed = block.hash(); + let block = block.unseal(); + let block = block.seal_slow(); + assert_eq!(sealed, block.hash()); + } } From 1f190ad8ea2093b90afb3fa64a30af962f91a909 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 17:44:47 +0200 Subject: [PATCH 032/103] chore: rm deposit contract config for op (#11479) --- crates/optimism/chainspec/src/constants.rs | 10 ---------- crates/optimism/chainspec/src/lib.rs | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/crates/optimism/chainspec/src/constants.rs b/crates/optimism/chainspec/src/constants.rs index 8fce63daafef..439c2fed83c0 100644 --- a/crates/optimism/chainspec/src/constants.rs +++ b/crates/optimism/chainspec/src/constants.rs @@ -1,8 +1,5 @@ //! OP stack variation of chain spec constants. -use alloy_primitives::{address, b256}; -use reth_chainspec::DepositContract; - //------------------------------- BASE MAINNET -------------------------------// /// Max gas limit on Base: @@ -12,10 +9,3 @@ pub const BASE_MAINNET_MAX_GAS_LIMIT: u64 = 105_000_000; /// Max gas limit on Base Sepolia: pub const BASE_SEPOLIA_MAX_GAS_LIMIT: u64 = 45_000_000; - -/// Deposit contract address: `0x00000000219ab540356cbb839cbe05303d7705fa` -pub(crate) const MAINNET_DEPOSIT_CONTRACT: DepositContract = DepositContract::new( - address!("00000000219ab540356cbb839cbe05303d7705fa"), - 11052984, - b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), -); diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 76a3ed838b18..22316a234d9b 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -26,7 +26,6 @@ pub use dev::OP_DEV; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; -use crate::constants::MAINNET_DEPOSIT_CONTRACT; use derive_more::{Constructor, Deref, Into}; use once_cell::sync::OnceCell; use reth_chainspec::{ @@ -209,14 +208,6 @@ impl From for OpChainSpec { // append the remaining unknown hardforks to ensure we don't filter any out ordered_hardforks.append(&mut block_hardforks); - // NOTE: in full node, we prune all receipts except the deposit contract's. We do not - // have the deployment block in the genesis file, so we use block zero. We use the same - // deposit topic as the mainnet contract if we have the deposit contract address in the - // genesis json. - let deposit_contract = genesis.config.deposit_contract_address.map(|address| { - DepositContract { address, block: 0, topic: MAINNET_DEPOSIT_CONTRACT.topic } - }); - Self { inner: ChainSpec { chain: genesis.config.chain_id.into(), @@ -224,7 +215,6 @@ impl From for OpChainSpec { genesis_hash: OnceCell::new(), hardforks: ChainHardforks::new(ordered_hardforks), paris_block_and_final_difficulty, - deposit_contract, base_fee_params: optimism_genesis_info.base_fee_params, ..Default::default() }, From 16bc9c8b491878720852713c370f188b5b202091 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 4 Oct 2024 19:19:02 +0200 Subject: [PATCH 033/103] chore(lint): fix lint storage (#11485) --- crates/etl/src/lib.rs | 4 ++-- crates/storage/db-api/src/cursor.rs | 16 ++++++++-------- crates/storage/db/src/static_file/cursor.rs | 2 +- crates/storage/libmdbx-rs/src/codec.rs | 2 +- crates/storage/libmdbx-rs/src/cursor.rs | 8 ++++---- crates/storage/nippy-jar/src/compression/zstd.rs | 12 ++++++------ crates/storage/nippy-jar/src/cursor.rs | 2 +- crates/trie/trie/src/forward_cursor.rs | 2 +- crates/trie/trie/src/hashed_cursor/post_state.rs | 6 +++--- crates/trie/trie/src/trie_cursor/in_memory.rs | 4 ++-- crates/trie/trie/src/updates.rs | 4 ++-- 11 files changed, 31 insertions(+), 31 deletions(-) diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index 3f978fabeee2..d30f432f9c19 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -190,14 +190,14 @@ pub struct EtlIter<'a> { files: &'a mut Vec, } -impl<'a> EtlIter<'a> { +impl EtlIter<'_> { /// Peeks into the next element pub fn peek(&self) -> Option<&(Vec, Vec)> { self.heap.peek().map(|(Reverse(entry), _)| entry) } } -impl<'a> Iterator for EtlIter<'a> { +impl Iterator for EtlIter<'_> { type Item = std::io::Result<(Vec, Vec)>; fn next(&mut self) -> Option { diff --git a/crates/storage/db-api/src/cursor.rs b/crates/storage/db-api/src/cursor.rs index 134819a8e2cd..585aa4947a28 100644 --- a/crates/storage/db-api/src/cursor.rs +++ b/crates/storage/db-api/src/cursor.rs @@ -149,7 +149,7 @@ where } } -impl<'cursor, T: Table, CURSOR: DbCursorRO> Iterator for Walker<'cursor, T, CURSOR> { +impl> Iterator for Walker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { let start = self.start.take(); @@ -174,7 +174,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRO> Walker<'cursor, T, CURSOR> { } } -impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> Walker<'cursor, T, CURSOR> { +impl + DbCursorRO> Walker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -217,7 +217,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRO> ReverseWalker<'cursor, T, CURSOR> } } -impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> ReverseWalker<'cursor, T, CURSOR> { +impl + DbCursorRO> ReverseWalker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -225,7 +225,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> ReverseWalker<'cu } } -impl<'cursor, T: Table, CURSOR: DbCursorRO> Iterator for ReverseWalker<'cursor, T, CURSOR> { +impl> Iterator for ReverseWalker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { @@ -266,7 +266,7 @@ where } } -impl<'cursor, T: Table, CURSOR: DbCursorRO> Iterator for RangeWalker<'cursor, T, CURSOR> { +impl> Iterator for RangeWalker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { @@ -316,7 +316,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRO> RangeWalker<'cursor, T, CURSOR> { } } -impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> RangeWalker<'cursor, T, CURSOR> { +impl + DbCursorRO> RangeWalker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -349,7 +349,7 @@ where } } -impl<'cursor, T: DupSort, CURSOR: DbCursorRW + DbDupCursorRO> DupWalker<'cursor, T, CURSOR> { +impl + DbDupCursorRO> DupWalker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -357,7 +357,7 @@ impl<'cursor, T: DupSort, CURSOR: DbCursorRW + DbDupCursorRO> DupWalker<'c } } -impl<'cursor, T: DupSort, CURSOR: DbDupCursorRO> Iterator for DupWalker<'cursor, T, CURSOR> { +impl> Iterator for DupWalker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { let start = self.start.take(); diff --git a/crates/storage/db/src/static_file/cursor.rs b/crates/storage/db/src/static_file/cursor.rs index f14e02308773..a9eadd9c12a6 100644 --- a/crates/storage/db/src/static_file/cursor.rs +++ b/crates/storage/db/src/static_file/cursor.rs @@ -115,7 +115,7 @@ impl<'a> From<&'a B256> for KeyOrNumber<'a> { } } -impl<'a> From for KeyOrNumber<'a> { +impl From for KeyOrNumber<'_> { fn from(value: u64) -> Self { KeyOrNumber::Number(value) } diff --git a/crates/storage/libmdbx-rs/src/codec.rs b/crates/storage/libmdbx-rs/src/codec.rs index a97ea28ca2ae..c78f79db9f94 100644 --- a/crates/storage/libmdbx-rs/src/codec.rs +++ b/crates/storage/libmdbx-rs/src/codec.rs @@ -22,7 +22,7 @@ pub trait TableObject: Sized { } } -impl<'tx> TableObject for Cow<'tx, [u8]> { +impl TableObject for Cow<'_, [u8]> { fn decode(_: &[u8]) -> Result { unreachable!() } diff --git a/crates/storage/libmdbx-rs/src/cursor.rs b/crates/storage/libmdbx-rs/src/cursor.rs index d007cc03e490..3deff0c249bc 100644 --- a/crates/storage/libmdbx-rs/src/cursor.rs +++ b/crates/storage/libmdbx-rs/src/cursor.rs @@ -539,7 +539,7 @@ where }, } -impl<'cur, K, Key, Value> IntoIter<'cur, K, Key, Value> +impl IntoIter<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, @@ -551,7 +551,7 @@ where } } -impl<'cur, K, Key, Value> Iterator for IntoIter<'cur, K, Key, Value> +impl Iterator for IntoIter<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, @@ -646,7 +646,7 @@ where } } -impl<'cur, K, Key, Value> Iterator for Iter<'cur, K, Key, Value> +impl Iterator for Iter<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, @@ -736,7 +736,7 @@ where } } -impl<'cur, K, Key, Value> fmt::Debug for IterDup<'cur, K, Key, Value> +impl fmt::Debug for IterDup<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, diff --git a/crates/storage/nippy-jar/src/compression/zstd.rs b/crates/storage/nippy-jar/src/compression/zstd.rs index 494d79de52f6..500247d17677 100644 --- a/crates/storage/nippy-jar/src/compression/zstd.rs +++ b/crates/storage/nippy-jar/src/compression/zstd.rs @@ -266,13 +266,13 @@ mod dictionaries_serde { #[derive(Serialize, Deserialize, Deref)] pub(crate) struct ZstdDictionaries<'a>(Vec>); -impl<'a> std::fmt::Debug for ZstdDictionaries<'a> { +impl std::fmt::Debug for ZstdDictionaries<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ZstdDictionaries").field("num", &self.len()).finish_non_exhaustive() } } -impl<'a> ZstdDictionaries<'a> { +impl ZstdDictionaries<'_> { #[cfg(test)] /// Creates [`ZstdDictionaries`]. pub(crate) fn new(raw: Vec) -> Self { @@ -321,7 +321,7 @@ pub(crate) enum ZstdDictionary<'a> { Loaded(DecoderDictionary<'a>), } -impl<'a> ZstdDictionary<'a> { +impl ZstdDictionary<'_> { /// Returns a reference to the expected `RawDictionary` pub(crate) const fn raw(&self) -> Option<&RawDictionary> { match self { @@ -339,7 +339,7 @@ impl<'a> ZstdDictionary<'a> { } } -impl<'de, 'a> Deserialize<'de> for ZstdDictionary<'a> { +impl<'de> Deserialize<'de> for ZstdDictionary<'_> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -349,7 +349,7 @@ impl<'de, 'a> Deserialize<'de> for ZstdDictionary<'a> { } } -impl<'a> Serialize for ZstdDictionary<'a> { +impl Serialize for ZstdDictionary<'_> { fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -362,7 +362,7 @@ impl<'a> Serialize for ZstdDictionary<'a> { } #[cfg(test)] -impl<'a> PartialEq for ZstdDictionary<'a> { +impl PartialEq for ZstdDictionary<'_> { fn eq(&self, other: &Self) -> bool { if let (Self::Raw(a), Self::Raw(b)) = (self, &other) { return a == b diff --git a/crates/storage/nippy-jar/src/cursor.rs b/crates/storage/nippy-jar/src/cursor.rs index 7af55fd436e4..267764827299 100644 --- a/crates/storage/nippy-jar/src/cursor.rs +++ b/crates/storage/nippy-jar/src/cursor.rs @@ -18,7 +18,7 @@ pub struct NippyJarCursor<'a, H = ()> { row: u64, } -impl<'a, H: NippyJarHeader> std::fmt::Debug for NippyJarCursor<'a, H> { +impl std::fmt::Debug for NippyJarCursor<'_, H> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("NippyJarCursor").field("config", &self.jar).finish_non_exhaustive() } diff --git a/crates/trie/trie/src/forward_cursor.rs b/crates/trie/trie/src/forward_cursor.rs index da71326cea90..6db214bb51a3 100644 --- a/crates/trie/trie/src/forward_cursor.rs +++ b/crates/trie/trie/src/forward_cursor.rs @@ -21,7 +21,7 @@ impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> { } } -impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> +impl ForwardInMemoryCursor<'_, K, V> where K: PartialOrd + Clone, V: Clone, diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index 53a2cdb3bb64..678914191527 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -132,7 +132,7 @@ where } } -impl<'a, C> HashedCursor for HashedPostStateAccountCursor<'a, C> +impl HashedCursor for HashedPostStateAccountCursor<'_, C> where C: HashedCursor, { @@ -276,7 +276,7 @@ where } } -impl<'a, C> HashedCursor for HashedPostStateStorageCursor<'a, C> +impl HashedCursor for HashedPostStateStorageCursor<'_, C> where C: HashedStorageCursor, { @@ -304,7 +304,7 @@ where } } -impl<'a, C> HashedStorageCursor for HashedPostStateStorageCursor<'a, C> +impl HashedStorageCursor for HashedPostStateStorageCursor<'_, C> where C: HashedStorageCursor, { diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 82ffcfb428f9..c313231e66dc 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -188,7 +188,7 @@ impl<'a, C> InMemoryStorageTrieCursor<'a, C> { } } -impl<'a, C: TrieCursor> InMemoryStorageTrieCursor<'a, C> { +impl InMemoryStorageTrieCursor<'_, C> { fn seek_inner( &mut self, key: Nibbles, @@ -237,7 +237,7 @@ impl<'a, C: TrieCursor> InMemoryStorageTrieCursor<'a, C> { } } -impl<'a, C: TrieCursor> TrieCursor for InMemoryStorageTrieCursor<'a, C> { +impl TrieCursor for InMemoryStorageTrieCursor<'_, C> { fn seek_exact( &mut self, key: Nibbles, diff --git a/crates/trie/trie/src/updates.rs b/crates/trie/trie/src/updates.rs index a2d3d67363ea..03e80cf52e53 100644 --- a/crates/trie/trie/src/updates.rs +++ b/crates/trie/trie/src/updates.rs @@ -455,7 +455,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for TrieUpdates<'a> { + impl SerializeAs for TrieUpdates<'_> { fn serialize_as(source: &super::TrieUpdates, serializer: S) -> Result where S: Serializer, @@ -515,7 +515,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for StorageTrieUpdates<'a> { + impl SerializeAs for StorageTrieUpdates<'_> { fn serialize_as( source: &super::StorageTrieUpdates, serializer: S, From 093b1a16dfca2def7b1bf76cb12e554bbe7d967b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:57:44 +0200 Subject: [PATCH 034/103] chore(provider): add more test coverage on `BlockchainProvider::*by_block_range` queries (#11488) --- .../src/providers/blockchain_provider.rs | 555 +++--------------- 1 file changed, 90 insertions(+), 465 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 01d8e4cd977c..9a2a3d0a77ee 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1502,8 +1502,8 @@ mod tests { use reth_db::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ - BlockWithSenders, Receipt, SealedBlock, SealedBlockWithSenders, StaticFileSegment, - TransactionMeta, TransactionSignedNoHash, Withdrawals, + Receipt, SealedBlock, StaticFileSegment, TransactionMeta, TransactionSignedNoHash, + Withdrawals, }; use reth_storage_api::{ BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, @@ -2250,459 +2250,6 @@ mod tests { Ok(()) } - #[test] - fn test_block_range_in_memory_only() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the range of in-memory blocks - let start_block_number = in_memory_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - // Check if the retrieved blocks match the in-memory blocks - assert_eq!(blocks.len(), in_memory_blocks.len()); - // Check if the blocks are equal - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter()) { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - // Check for partial in-memory ranges - let blocks = provider.block_range(start_block_number + 1..=end_block_number)?; - assert_eq!(blocks.len(), in_memory_blocks.len() - 1); - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter().skip(1)) - { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - let blocks = provider.block_range(start_block_number + 1..=end_block_number - 1)?; - assert_eq!(blocks.len(), in_memory_blocks.len() - 2); - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter().skip(1)) - { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - let blocks = provider.block_range(start_block_number + 1..=end_block_number + 1)?; - assert_eq!(blocks.len(), in_memory_blocks.len() - 1); - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter().skip(1)) - { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - Ok(()) - } - - #[test] - fn test_block_range_in_database_only() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, // No blocks in memory - BlockRangeParams::default(), - )?; - - // Get the range of database blocks - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = database_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - // Check if the retrieved blocks match the database blocks - assert_eq!(blocks.len(), database_blocks.len()); - // Check if the blocks are equal - for (retrieved_block, expected_block) in blocks.iter().zip(database_blocks.iter()) { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - Ok(()) - } - - #[test] - fn test_block_range_across_memory_and_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - TEST_BLOCKS_COUNT - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - // Check if the retrieved blocks match the database and in-memory blocks - assert_eq!(blocks.len(), TEST_BLOCKS_COUNT); - let all_expected_blocks = - database_blocks.iter().chain(in_memory_blocks.iter()).collect::>(); - // Check if the blocks are equal - for (retrieved_block, expected_block) in blocks.iter().zip(all_expected_blocks.iter()) { - assert_eq!(retrieved_block.clone(), (*expected_block).clone().unseal()); - } - - Ok(()) - } - - #[test] - fn test_block_range_non_existent_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Generate a non-existent range - let non_existent_range = 9999..=10000; - let blocks = provider.block_range(non_existent_range)?; - - // The range is non-existent, so the blocks should be empty - assert!(blocks.is_empty()); - - Ok(()) - } - - #[test] - fn test_block_range_partial_overlap() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.last().unwrap().number; - let end_block_number = in_memory_blocks.first().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - assert_eq!(blocks.len(), 2); // Only one block from each side of the overlap - assert_eq!(blocks[0], database_blocks.last().unwrap().clone().unseal()); - assert_eq!(blocks[1], in_memory_blocks.first().unwrap().clone().unseal()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_across_memory_and_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - TEST_BLOCKS_COUNT - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database and in-memory blocks - assert_eq!(blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let all_expected_blocks_with_senders = database_blocks - .iter() - .chain(in_memory_blocks.iter()) - .map(|sealed_block| BlockWithSenders { - block: sealed_block.clone().unseal(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_block_with_senders, expected_block_with_senders) in - blocks_with_senders.iter().zip(all_expected_blocks_with_senders.iter()) - { - assert_eq!(retrieved_block_with_senders.block, expected_block_with_senders.block); - assert_eq!(retrieved_block_with_senders.senders, expected_block_with_senders.senders); - } - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_only_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the range of in-memory blocks - let start_block_number = in_memory_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // Check if the retrieved blocks match the in-memory blocks - assert_eq!(blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_blocks_with_senders = in_memory_blocks - .iter() - .map(|sealed_block| BlockWithSenders { - block: sealed_block.clone().unseal(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_block_with_senders, expected_block_with_senders) in - blocks_with_senders.iter().zip(expected_blocks_with_senders.iter()) - { - assert_eq!(retrieved_block_with_senders.block, expected_block_with_senders.block); - assert_eq!(retrieved_block_with_senders.senders, expected_block_with_senders.senders); - } - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_only_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, - BlockRangeParams::default(), - )?; - - // Get the range of database blocks - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = database_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database blocks - assert_eq!(blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_blocks_with_senders = database_blocks - .iter() - .map(|sealed_block| BlockWithSenders { - block: sealed_block.clone().unseal(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_block_with_senders, expected_block_with_senders) in - blocks_with_senders.iter().zip(expected_blocks_with_senders.iter()) - { - assert_eq!(retrieved_block_with_senders.block, expected_block_with_senders.block); - assert_eq!(retrieved_block_with_senders.senders, expected_block_with_senders.senders); - } - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_non_existent_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Assuming this range does not exist - let start_block_number = 1000; - let end_block_number = 2000; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // The range is non-existent, so the blocks should be empty - assert!(blocks_with_senders.is_empty()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_across_memory_and_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - TEST_BLOCKS_COUNT - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database and in-memory blocks - assert_eq!(sealed_blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let all_expected_sealed_blocks_with_senders = database_blocks - .iter() - .chain(in_memory_blocks.iter()) - .map(|sealed_block| SealedBlockWithSenders { - block: sealed_block.clone(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_sealed_block_with_senders, expected_sealed_block_with_senders) in - sealed_blocks_with_senders.iter().zip(all_expected_sealed_blocks_with_senders.iter()) - { - assert_eq!( - retrieved_sealed_block_with_senders.block, - expected_sealed_block_with_senders.block - ); - assert_eq!( - retrieved_sealed_block_with_senders.senders, - expected_sealed_block_with_senders.senders - ); - } - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_only_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the range of in-memory blocks - let start_block_number = in_memory_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // Check if the retrieved blocks match the in-memory blocks - assert_eq!(sealed_blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_sealed_blocks_with_senders = in_memory_blocks - .iter() - .map(|sealed_block| SealedBlockWithSenders { - block: sealed_block.clone(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_sealed_block_with_senders, expected_sealed_block_with_senders) in - sealed_blocks_with_senders.iter().zip(expected_sealed_blocks_with_senders.iter()) - { - assert_eq!( - retrieved_sealed_block_with_senders.block, - expected_sealed_block_with_senders.block - ); - assert_eq!( - retrieved_sealed_block_with_senders.senders, - expected_sealed_block_with_senders.senders - ); - } - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_only_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, - BlockRangeParams::default(), - )?; - - // Get the range of database blocks - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = database_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database blocks - assert_eq!(sealed_blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_sealed_blocks_with_senders = database_blocks - .iter() - .map(|sealed_block| SealedBlockWithSenders { - block: sealed_block.clone(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_sealed_block_with_senders, expected_sealed_block_with_senders) in - sealed_blocks_with_senders.iter().zip(expected_sealed_blocks_with_senders.iter()) - { - assert_eq!( - retrieved_sealed_block_with_senders.block, - expected_sealed_block_with_senders.block - ); - assert_eq!( - retrieved_sealed_block_with_senders.senders, - expected_sealed_block_with_senders.senders - ); - } - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_non_existent_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Assuming this range does not exist - let start_block_number = 1000; - let end_block_number = 2000; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // The range is non-existent, so the blocks should be empty - assert!(sealed_blocks_with_senders.is_empty()); - - Ok(()) - } - #[test] fn test_block_hash_reader() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -2773,11 +2320,6 @@ mod tests { Some(in_memory_block.difficulty) ); - assert_eq!( - provider.headers_range(0..=10)?, - blocks.iter().map(|b| b.header().clone()).collect::>() - ); - assert_eq!( provider.sealed_header(database_block.number)?, Some(database_block.header.clone()) @@ -2787,11 +2329,6 @@ mod tests { Some(in_memory_block.header.clone()) ); - assert_eq!( - provider.sealed_headers_range(0..=10)?, - blocks.iter().map(|b| b.header.clone()).collect::>() - ); - assert_eq!( provider.sealed_headers_while(0..=10, |header| header.number <= 8)?, blocks @@ -4220,6 +3757,94 @@ mod tests { Ok(()) } + macro_rules! test_by_block_range { + ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ + let db_block_count = $database_blocks.len() as u64; + let in_mem_block_count = $in_memory_blocks.len() as u64; + + let db_range = 0..=db_block_count - 1; + let in_mem_range = db_block_count..=(in_mem_block_count + db_range.end()); + + $( + // Retrieve the expected database data + let database_data = + $database_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(db_range.clone())?, database_data); + + // Retrieve the expected in-memory data + let in_memory_data = + $in_memory_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); + + // Test partial in-memory range + assert_eq!( + &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &in_memory_data[1..in_memory_data.len() - 1] + ); + + // Test range that spans database and in-memory + assert_eq!( + $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>() + ); + + // Test invalid range + let start_block_num = u64::MAX; + let end_block_num = u64::MAX; + let result = $provider.$method(start_block_num..=end_block_num-1)?; + assert!(result.is_empty(), "No data should be found for an invalid block range"); + + // Test valid range with empty results + let result = $provider.$method(in_mem_range.end() + 10..=in_mem_range.end() + 20)?; + assert!(result.is_empty(), "No data should be found for an empty block range"); + )* + }}; + } + + #[test] + fn test_methods_by_block_range() -> eyre::Result<()> { + let mut rng = generators::rng(); + let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + + // todo(joshie) add canonical_hashes_range below after changing its interface into range + // instead start end + test_by_block_range!( + provider, + database_blocks, + in_memory_blocks, + [ + (headers_range, |block: &SealedBlock| block.header().clone()), + (sealed_headers_range, |block: &SealedBlock| block.header.clone()), + (block_range, |block: &SealedBlock| block.clone().unseal()), + (block_with_senders_range, |block: &SealedBlock| block + .clone() + .unseal() + .with_senders_unchecked(vec![])), + (sealed_block_with_senders_range, |block: &SealedBlock| block + .clone() + .with_senders_unchecked(vec![])), + (transactions_by_block_range, |block: &SealedBlock| block + .body + .transactions + .clone()), + ] + ); + + Ok(()) + } + #[test] fn transaction_sender_found_in_memory() -> eyre::Result<()> { let mut rng = generators::rng(); From 72865f1d8344b192ec7af1a672384800bcb060d5 Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 5 Oct 2024 16:48:18 +0800 Subject: [PATCH 035/103] fix(rpc-eth-types): incorrect error msg(; -> :) (#11503) Signed-off-by: jsvisa --- crates/rpc/rpc-eth-types/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc-eth-types/src/error.rs b/crates/rpc/rpc-eth-types/src/error.rs index 10fa3b716021..212fca36d9c9 100644 --- a/crates/rpc/rpc-eth-types/src/error.rs +++ b/crates/rpc/rpc-eth-types/src/error.rs @@ -365,7 +365,7 @@ pub enum RpcInvalidTransactionError { PrecompileOutOfGas(u64), /// An operand to an opcode was invalid or out of range. /// Contains the gas limit. - #[error("out of gas: invalid operand to an opcode; {0}")] + #[error("out of gas: invalid operand to an opcode: {0}")] InvalidOperandOutOfGas(u64), /// Thrown if executing a transaction failed during estimate/call #[error(transparent)] From 08c4065482afc0fbf4cdcd07445b93379c8057b2 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Sat, 5 Oct 2024 02:07:14 -0700 Subject: [PATCH 036/103] Reexport optimism specific crates from `op-reth` (#11499) --- Cargo.lock | 6 +++ crates/optimism/bin/Cargo.toml | 6 +++ crates/optimism/bin/src/lib.rs | 73 +++++++++++++++++++++++++++++++++ crates/optimism/bin/src/main.rs | 1 - 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 crates/optimism/bin/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8c66fbf09b38..73f500d9d22f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5257,8 +5257,14 @@ dependencies = [ "clap", "reth-cli-util", "reth-node-builder", + "reth-optimism-chainspec", "reth-optimism-cli", + "reth-optimism-consensus", + "reth-optimism-evm", + "reth-optimism-forks", "reth-optimism-node", + "reth-optimism-payload-builder", + "reth-optimism-primitives", "reth-optimism-rpc", "reth-provider", "tracing", diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index 13e7b1aa6d4d..ecb92dfc454b 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -15,6 +15,12 @@ reth-optimism-cli.workspace = true reth-provider.workspace = true reth-optimism-rpc.workspace = true reth-optimism-node.workspace = true +reth-optimism-chainspec.workspace = true +reth-optimism-consensus.workspace = true +reth-optimism-evm.workspace = true +reth-optimism-payload-builder.workspace = true +reth-optimism-primitives.workspace = true +reth-optimism-forks.workspace = true clap = { workspace = true, features = ["derive", "env"] } tracing.workspace = true diff --git a/crates/optimism/bin/src/lib.rs b/crates/optimism/bin/src/lib.rs new file mode 100644 index 000000000000..21c28f7c5470 --- /dev/null +++ b/crates/optimism/bin/src/lib.rs @@ -0,0 +1,73 @@ +//! Rust Optimism (op-reth) binary executable. +//! +//! ## Feature Flags +//! +//! - `jemalloc`: Uses [jemallocator](https://github.com/tikv/jemallocator) as the global allocator. +//! This is **not recommended on Windows**. See [here](https://rust-lang.github.io/rfcs/1974-global-allocators.html#jemalloc) +//! for more info. +//! - `jemalloc-prof`: Enables [jemallocator's](https://github.com/tikv/jemallocator) heap profiling +//! and leak detection functionality. See [jemalloc's opt.prof](https://jemalloc.net/jemalloc.3.html#opt.prof) +//! documentation for usage details. This is **not recommended on Windows**. See [here](https://rust-lang.github.io/rfcs/1974-global-allocators.html#jemalloc) +//! for more info. +//! - `asm-keccak`: replaces the default, pure-Rust implementation of Keccak256 with one implemented +//! in assembly; see [the `keccak-asm` crate](https://github.com/DaniPopes/keccak-asm) for more +//! details and supported targets +//! - `min-error-logs`: Disables all logs below `error` level. +//! - `min-warn-logs`: Disables all logs below `warn` level. +//! - `min-info-logs`: Disables all logs below `info` level. This can speed up the node, since fewer +//! calls to the logging component is made. +//! - `min-debug-logs`: Disables all logs below `debug` level. +//! - `min-trace-logs`: Disables all logs below `trace` level. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +// The `optimism` feature must be enabled to use this crate. +#![cfg(feature = "optimism")] + +/// Re-exported from `reth_optimism_cli`. +pub mod cli { + pub use reth_optimism_cli::*; +} + +/// Re-exported from `reth_optimism_chainspec`. +pub mod chainspec { + pub use reth_optimism_chainspec::*; +} + +/// Re-exported from `reth_optimism_consensus`. +pub mod consensus { + pub use reth_optimism_consensus::*; +} + +/// Re-exported from `reth_optimism_evm`. +pub mod evm { + pub use reth_optimism_evm::*; +} + +/// Re-exported from `reth_optimism_forks`. +pub mod forks { + pub use reth_optimism_forks::*; +} + +/// Re-exported from `reth_optimism_node`. +pub mod node { + pub use reth_optimism_node::*; +} + +/// Re-exported from `reth_optimism_payload_builder`. +pub mod payload { + pub use reth_optimism_payload_builder::*; +} + +/// Re-exported from `reth_optimism_primitives`. +pub mod primitives { + pub use reth_optimism_primitives::*; +} + +/// Re-exported from `reth_optimism_rpc`. +pub mod rpc { + pub use reth_optimism_rpc::*; +} diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs index 6e041e240a74..6822a6a50ec4 100644 --- a/crates/optimism/bin/src/main.rs +++ b/crates/optimism/bin/src/main.rs @@ -1,7 +1,6 @@ #![allow(missing_docs, rustdoc::missing_crate_level_docs)] // The `optimism` feature must be enabled to use this crate. #![cfg(feature = "optimism")] -#![cfg_attr(not(test), warn(unused_crate_dependencies))] use clap::Parser; use reth_node_builder::{engine_tree_config::TreeConfig, EngineNodeLauncher}; From 2ca6141233868984caa5b91a2f5076041022bd4f Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Sat, 5 Oct 2024 14:44:06 +0530 Subject: [PATCH 037/103] feat: rpc replace function created (#11501) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-builder/src/lib.rs | 260 +++++++++++++++++++++++------- 1 file changed, 201 insertions(+), 59 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 113cb93c4630..e49105ab7d7d 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1832,6 +1832,13 @@ impl TransportRpcModules { } } + /// Removes the given methods from the configured http methods. + pub fn remove_http_methods(&mut self, methods: impl IntoIterator) { + for name in methods { + self.remove_http_method(name); + } + } + /// Removes the method with the given name from the configured ws methods. /// /// Returns `true` if the method was found and removed, `false` otherwise. @@ -1847,6 +1854,13 @@ impl TransportRpcModules { } } + /// Removes the given methods from the configured ws methods. + pub fn remove_ws_methods(&mut self, methods: impl IntoIterator) { + for name in methods { + self.remove_ws_method(name); + } + } + /// Removes the method with the given name from the configured ipc methods. /// /// Returns `true` if the method was found and removed, `false` otherwise. @@ -1862,6 +1876,13 @@ impl TransportRpcModules { } } + /// Removes the given methods from the configured ipc methods. + pub fn remove_ipc_methods(&mut self, methods: impl IntoIterator) { + for name in methods { + self.remove_ipc_method(name); + } + } + /// Removes the method with the given name from all configured transports. /// /// Returns `true` if the method was found and removed, `false` otherwise. @@ -1872,6 +1893,56 @@ impl TransportRpcModules { http_removed || ws_removed || ipc_removed } + + /// Replace the given [Methods] in the configured http methods. + /// + /// Fails if any of the methods in other is present already or if the method being removed is + /// not present + /// + /// Returns [Ok(false)] if no http transport is configured. + pub fn replace_http(&mut self, other: impl Into) -> Result { + let other = other.into(); + self.remove_http_methods(other.method_names()); + self.merge_http(other) + } + + /// Replace the given [Methods] in the configured ipc methods. + /// + /// Fails if any of the methods in other is present already or if the method being removed is + /// not present + /// + /// Returns [Ok(false)] if no ipc transport is configured. + pub fn replace_ipc(&mut self, other: impl Into) -> Result { + let other = other.into(); + self.remove_ipc_methods(other.method_names()); + self.merge_ipc(other) + } + + /// Replace the given [Methods] in the configured ws methods. + /// + /// Fails if any of the methods in other is present already or if the method being removed is + /// not present + /// + /// Returns [Ok(false)] if no ws transport is configured. + pub fn replace_ws(&mut self, other: impl Into) -> Result { + let other = other.into(); + self.remove_ws_methods(other.method_names()); + self.merge_ws(other) + } + + /// Replaces the method with the given name from all configured transports. + /// + /// Returns `true` if the method was found and replaced, `false` otherwise + pub fn replace_configured( + &mut self, + other: impl Into, + ) -> Result { + let other = other.into(); + self.replace_http(other.clone())?; + self.replace_ws(other.clone())?; + self.replace_ipc(other)?; + Ok(true) + } } /// A handle to the spawned servers. @@ -2121,81 +2192,152 @@ mod tests { ) } - mod remove_methods { - use super::*; + fn create_test_module() -> RpcModule<()> { + let mut module = RpcModule::new(()); + module.register_method("anything", |_, _, _| "succeed").unwrap(); + module + } - fn create_test_module() -> RpcModule<()> { - let mut module = RpcModule::new(()); - module.register_method("anything", |_, _, _| "succeed").unwrap(); - module - } + #[test] + fn test_remove_http_method() { + let mut modules = + TransportRpcModules { http: Some(create_test_module()), ..Default::default() }; + // Remove a method that exists + assert!(modules.remove_http_method("anything")); + + // Remove a method that does not exist + assert!(!modules.remove_http_method("non_existent_method")); - #[test] - fn test_remove_http_method() { - let mut modules = - TransportRpcModules { http: Some(create_test_module()), ..Default::default() }; - // Remove a method that exists - assert!(modules.remove_http_method("anything")); + // Verify that the method was removed + assert!(modules.http.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_http_method("non_existent_method")); + #[test] + fn test_remove_ws_method() { + let mut modules = + TransportRpcModules { ws: Some(create_test_module()), ..Default::default() }; - // Verify that the method was removed - assert!(modules.http.as_ref().unwrap().method("anything").is_none()); - } + // Remove a method that exists + assert!(modules.remove_ws_method("anything")); - #[test] - fn test_remove_ws_method() { - let mut modules = - TransportRpcModules { ws: Some(create_test_module()), ..Default::default() }; + // Remove a method that does not exist + assert!(!modules.remove_ws_method("non_existent_method")); - // Remove a method that exists - assert!(modules.remove_ws_method("anything")); + // Verify that the method was removed + assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_ws_method("non_existent_method")); + #[test] + fn test_remove_ipc_method() { + let mut modules = + TransportRpcModules { ipc: Some(create_test_module()), ..Default::default() }; - // Verify that the method was removed - assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); - } + // Remove a method that exists + assert!(modules.remove_ipc_method("anything")); - #[test] - fn test_remove_ipc_method() { - let mut modules = - TransportRpcModules { ipc: Some(create_test_module()), ..Default::default() }; + // Remove a method that does not exist + assert!(!modules.remove_ipc_method("non_existent_method")); - // Remove a method that exists - assert!(modules.remove_ipc_method("anything")); + // Verify that the method was removed + assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_ipc_method("non_existent_method")); + #[test] + fn test_remove_method_from_configured() { + let mut modules = TransportRpcModules { + http: Some(create_test_module()), + ws: Some(create_test_module()), + ipc: Some(create_test_module()), + ..Default::default() + }; - // Verify that the method was removed - assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); - } + // Remove a method that exists + assert!(modules.remove_method_from_configured("anything")); - #[test] - fn test_remove_method_from_configured() { - let mut modules = TransportRpcModules { - http: Some(create_test_module()), - ws: Some(create_test_module()), - ipc: Some(create_test_module()), - ..Default::default() - }; + // Remove a method that was just removed (it does not exist anymore) + assert!(!modules.remove_method_from_configured("anything")); - // Remove a method that exists - assert!(modules.remove_method_from_configured("anything")); + // Remove a method that does not exist + assert!(!modules.remove_method_from_configured("non_existent_method")); - // Remove a method that was just removed (it does not exist anymore) - assert!(!modules.remove_method_from_configured("anything")); + // Verify that the method was removed from all transports + assert!(modules.http.as_ref().unwrap().method("anything").is_none()); + assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); + assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_method_from_configured("non_existent_method")); + #[test] + fn test_replace_http_method() { + let mut modules = + TransportRpcModules { http: Some(create_test_module()), ..Default::default() }; - // Verify that the method was removed from all transports - assert!(modules.http.as_ref().unwrap().method("anything").is_none()); - assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); - assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); - } + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_http(other_module.clone()).unwrap()); + + assert!(modules.http.as_ref().unwrap().method("something").is_some()); + + other_module.register_method("anything", |_, _, _| "fails").unwrap(); + assert!(modules.replace_http(other_module.clone()).unwrap()); + + assert!(modules.http.as_ref().unwrap().method("anything").is_some()); + } + #[test] + fn test_replace_ipc_method() { + let mut modules = + TransportRpcModules { ipc: Some(create_test_module()), ..Default::default() }; + + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_ipc(other_module.clone()).unwrap()); + + assert!(modules.ipc.as_ref().unwrap().method("something").is_some()); + + other_module.register_method("anything", |_, _, _| "fails").unwrap(); + assert!(modules.replace_ipc(other_module.clone()).unwrap()); + + assert!(modules.ipc.as_ref().unwrap().method("anything").is_some()); + } + #[test] + fn test_replace_ws_method() { + let mut modules = + TransportRpcModules { ws: Some(create_test_module()), ..Default::default() }; + + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_ws(other_module.clone()).unwrap()); + + assert!(modules.ws.as_ref().unwrap().method("something").is_some()); + + other_module.register_method("anything", |_, _, _| "fails").unwrap(); + assert!(modules.replace_ws(other_module.clone()).unwrap()); + + assert!(modules.ws.as_ref().unwrap().method("anything").is_some()); + } + + #[test] + fn test_replace_configured() { + let mut modules = TransportRpcModules { + http: Some(create_test_module()), + ws: Some(create_test_module()), + ipc: Some(create_test_module()), + ..Default::default() + }; + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_configured(other_module).unwrap()); + + // Verify that the other_method was added + assert!(modules.http.as_ref().unwrap().method("something").is_some()); + assert!(modules.ipc.as_ref().unwrap().method("something").is_some()); + assert!(modules.ws.as_ref().unwrap().method("something").is_some()); + + assert!(modules.http.as_ref().unwrap().method("anything").is_some()); + assert!(modules.ipc.as_ref().unwrap().method("anything").is_some()); + assert!(modules.ws.as_ref().unwrap().method("anything").is_some()); } } From a846cbdcee94d00ed1b022708f19e8ed20ffbb29 Mon Sep 17 00:00:00 2001 From: Parikalp Bhardwaj <53660958+Parikalp-Bhardwaj@users.noreply.github.com> Date: Sat, 5 Oct 2024 14:11:29 +0400 Subject: [PATCH 038/103] Add metrics for failed deliveries to Grafana dashboard (#11481) --- etc/grafana/dashboards/overview.json | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 76d45ead885d..15786764f429 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -5732,6 +5732,79 @@ "title": "Engine API getPayloadBodies Latency", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counts the number of failed response deliveries due to client request termination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 198 + }, + "id": 213, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "consensus_engine_beacon_failed_new_payload_response_deliveries{instance=~\"$instance\"}", + "legendFormat": "Failed NewPayload Deliveries", + "refId": "A" + }, + { + "expr": "consensus_engine_beacon_failed_forkchoice_updated_response_deliveries{instance=~\"$instance\"}", + "legendFormat": "Failed ForkchoiceUpdated Deliveries", + "refId": "B" + } + ], + "title": "Failed Engine API Response Deliveries", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { From 28750e5b0e2789eb62c2edd11e7b3a9769988aba Mon Sep 17 00:00:00 2001 From: garwah <14845405+garwahl@users.noreply.github.com> Date: Sat, 5 Oct 2024 20:52:08 +1000 Subject: [PATCH 039/103] chore: Remove duplicate EthereumChainSpecParser in favor of existing EthChainSpecParser (#11412) Co-authored-by: garwah Co-authored-by: Matthias Seitz --- Cargo.lock | 13 +-- bin/reth/Cargo.toml | 1 + bin/reth/src/cli/mod.rs | 7 +- bin/reth/src/lib.rs | 1 + bin/reth/src/main.rs | 3 +- crates/cli/cli/Cargo.toml | 4 + crates/cli/cli/src/chainspec.rs | 20 +++- crates/cli/commands/Cargo.toml | 1 + crates/cli/commands/src/db/mod.rs | 2 +- crates/cli/commands/src/dump_genesis.rs | 2 +- crates/cli/commands/src/import.rs | 2 +- crates/cli/commands/src/node.rs | 7 +- crates/cli/commands/src/stage/unwind.rs | 2 +- crates/ethereum/cli/Cargo.toml | 7 -- crates/ethereum/cli/src/chainspec.rs | 43 +++----- crates/node/core/Cargo.toml | 6 -- crates/node/core/src/args/mod.rs | 2 - crates/node/core/src/args/utils.rs | 99 ------------------- crates/optimism/cli/src/chainspec.rs | 37 ++++--- crates/optimism/cli/src/lib.rs | 5 +- .../beacon-api-sidecar-fetcher/src/main.rs | 2 +- examples/beacon-api-sse/src/main.rs | 2 +- examples/custom-inspector/src/main.rs | 2 +- examples/node-custom-rpc/src/main.rs | 2 +- examples/txpool-tracing/src/main.rs | 2 +- 25 files changed, 85 insertions(+), 189 deletions(-) delete mode 100644 crates/node/core/src/args/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 73f500d9d22f..e75ea0ec095c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6243,6 +6243,7 @@ dependencies = [ "reth-downloaders", "reth-engine-util", "reth-errors", + "reth-ethereum-cli", "reth-ethereum-payload-builder", "reth-evm", "reth-execution-types", @@ -6526,9 +6527,12 @@ dependencies = [ name = "reth-cli" version = "1.0.8" dependencies = [ + "alloy-genesis", "clap", "eyre", "reth-cli-runner", + "serde_json", + "shellexpand", ] [[package]] @@ -6565,6 +6569,7 @@ dependencies = [ "reth-downloaders", "reth-ecies", "reth-eth-wire", + "reth-ethereum-cli", "reth-evm", "reth-exex", "reth-fs-util", @@ -7229,14 +7234,11 @@ dependencies = [ name = "reth-ethereum-cli" version = "1.0.8" dependencies = [ - "alloy-genesis", "clap", "eyre", "reth-chainspec", "reth-cli", "reth-cli-commands", - "serde_json", - "shellexpand", ] [[package]] @@ -7843,7 +7845,6 @@ dependencies = [ name = "reth-node-core" version = "1.0.8" dependencies = [ - "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", "clap", @@ -7856,19 +7857,16 @@ dependencies = [ "proptest", "rand 0.8.5", "reth-chainspec", - "reth-cli", "reth-cli-util", "reth-config", "reth-consensus-common", "reth-db", "reth-discv4", "reth-discv5", - "reth-fs-util", "reth-net-nat", "reth-network", "reth-network-p2p", "reth-network-peers", - "reth-optimism-chainspec", "reth-primitives", "reth-prune-types", "reth-rpc-api", @@ -7883,7 +7881,6 @@ dependencies = [ "reth-transaction-pool", "secp256k1", "serde", - "serde_json", "shellexpand", "strum", "tempfile", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 3d3cbd06f4d7..476f9cd5cec7 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] # reth reth-cli.workspace = true +reth-ethereum-cli.workspace = true reth-chainspec.workspace = true reth-config.workspace = true reth-primitives.workspace = true diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 8bb0971ec081..cca801da36b2 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -15,8 +15,8 @@ use reth_cli_commands::{ }; use reth_cli_runner::CliRunner; use reth_db::DatabaseEnv; +use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use reth_node_builder::{NodeBuilder, WithLaunchContext}; -use reth_node_core::args::utils::EthereumChainSpecParser; use reth_node_ethereum::{EthExecutorProvider, EthereumNode}; use reth_tracing::FileWorkerGuard; use std::{ffi::OsString, fmt, future::Future, sync::Arc}; @@ -117,7 +117,8 @@ impl, Ext: clap::Args + fmt::Debug> Cl /// /// ```no_run /// use clap::Parser; - /// use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; + /// use reth::cli::Cli; + /// use reth_ethereum_cli::chainspec::EthereumChainSpecParser; /// /// #[derive(Debug, Parser)] /// pub struct MyArgs { @@ -238,7 +239,7 @@ mod tests { use super::*; use crate::args::ColorMode; use clap::CommandFactory; - use reth_node_core::args::utils::SUPPORTED_CHAINS; + use reth_ethereum_cli::chainspec::SUPPORTED_CHAINS; #[test] fn parse_color_mode() { diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 4ff2faa1d65f..6b71f48de123 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -90,6 +90,7 @@ pub mod dirs { /// Re-exported from `reth_chainspec` pub mod chainspec { pub use reth_chainspec::*; + pub use reth_ethereum_cli::chainspec::*; } /// Re-exported from `reth_provider`. diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 1be2d0efedcc..7153cdcc6c88 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -4,7 +4,8 @@ static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); use clap::{Args, Parser}; -use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; +use reth::cli::Cli; +use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use reth_node_builder::{ engine_tree_config::{ TreeConfig, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, diff --git a/crates/cli/cli/Cargo.toml b/crates/cli/cli/Cargo.toml index e8f7a1dcbe15..7eb1f43b1e58 100644 --- a/crates/cli/cli/Cargo.toml +++ b/crates/cli/cli/Cargo.toml @@ -15,9 +15,13 @@ workspace = true # reth reth-cli-runner.workspace = true +alloy-genesis.workspace = true + # misc clap.workspace = true +shellexpand.workspace = true eyre.workspace = true +serde_json.workspace = true diff --git a/crates/cli/cli/src/chainspec.rs b/crates/cli/cli/src/chainspec.rs index 63705bd28f4a..8432009409f7 100644 --- a/crates/cli/cli/src/chainspec.rs +++ b/crates/cli/cli/src/chainspec.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{fs, path::PathBuf, sync::Arc}; use clap::builder::TypedValueParser; @@ -61,3 +61,21 @@ pub trait ChainSpecParser: Clone + Send + Sync + 'static { format!("The chain this node is running.\nPossible values are either a built-in chain or the path to a chain specification file.\n\nBuilt-in chains:\n {}", Self::SUPPORTED_CHAINS.join(", ")) } } + +/// A helper to parse a [`Genesis`](alloy_genesis::Genesis) as argument or from disk. +pub fn parse_genesis(s: &str) -> eyre::Result { + // try to read json from path first + let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) { + Ok(raw) => raw, + Err(io_err) => { + // valid json may start with "\n", but must contain "{" + if s.contains('{') { + s.to_string() + } else { + return Err(io_err.into()) // assume invalid path + } + } + }; + + Ok(serde_json::from_str(&raw)?) +} diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index 4835c3d0fa29..e307859dfd86 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -14,6 +14,7 @@ repository.workspace = true reth-beacon-consensus.workspace = true reth-chainspec.workspace = true reth-cli.workspace = true +reth-ethereum-cli.workspace = true reth-cli-runner.workspace = true reth-cli-util.workspace = true reth-config.workspace = true diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 1c000f56bc25..e1a9a90bacc3 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -160,7 +160,7 @@ impl> Command #[cfg(test)] mod tests { use super::*; - use reth_node_core::args::utils::{EthereumChainSpecParser, SUPPORTED_CHAINS}; + use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; use std::path::Path; #[test] diff --git a/crates/cli/commands/src/dump_genesis.rs b/crates/cli/commands/src/dump_genesis.rs index a5c0675cc7e8..c3e7bb217b57 100644 --- a/crates/cli/commands/src/dump_genesis.rs +++ b/crates/cli/commands/src/dump_genesis.rs @@ -32,7 +32,7 @@ impl> DumpGenesisCommand { #[cfg(test)] mod tests { use super::*; - use reth_node_core::args::utils::{EthereumChainSpecParser, SUPPORTED_CHAINS}; + use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; #[test] fn parse_dump_genesis_command_chain_args() { diff --git a/crates/cli/commands/src/import.rs b/crates/cli/commands/src/import.rs index 31c6cdc69157..6b750d32a3df 100644 --- a/crates/cli/commands/src/import.rs +++ b/crates/cli/commands/src/import.rs @@ -231,7 +231,7 @@ where #[cfg(test)] mod tests { use super::*; - use reth_node_core::args::utils::{EthereumChainSpecParser, SUPPORTED_CHAINS}; + use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; #[test] fn parse_common_import_command_chain_args() { diff --git a/crates/cli/commands/src/node.rs b/crates/cli/commands/src/node.rs index fe49b769a3d3..5b1a87e068b3 100644 --- a/crates/cli/commands/src/node.rs +++ b/crates/cli/commands/src/node.rs @@ -6,11 +6,12 @@ use reth_cli::chainspec::ChainSpecParser; use reth_cli_runner::CliContext; use reth_cli_util::parse_socket_address; use reth_db::{init_db, DatabaseEnv}; +use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::{ args::{ - utils::EthereumChainSpecParser, DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, NetworkArgs, - PayloadBuilderArgs, PruningArgs, RpcServerArgs, TxPoolArgs, + DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, + PruningArgs, RpcServerArgs, TxPoolArgs, }, node_config::NodeConfig, version, @@ -210,7 +211,7 @@ pub struct NoArgs; mod tests { use super::*; use reth_discv4::DEFAULT_DISCOVERY_PORT; - use reth_node_core::args::utils::SUPPORTED_CHAINS; + use reth_ethereum_cli::chainspec::SUPPORTED_CHAINS; use std::{ net::{IpAddr, Ipv4Addr}, path::Path, diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index ae3ae2500874..19305554eaa3 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -213,7 +213,7 @@ impl Subcommands { #[cfg(test)] mod tests { - use reth_node_core::args::utils::EthereumChainSpecParser; + use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use super::*; diff --git a/crates/ethereum/cli/Cargo.toml b/crates/ethereum/cli/Cargo.toml index b5fadc684b84..108f42f539b9 100644 --- a/crates/ethereum/cli/Cargo.toml +++ b/crates/ethereum/cli/Cargo.toml @@ -15,13 +15,6 @@ workspace = true reth-cli.workspace = true reth-chainspec.workspace = true -# ethereum -alloy-genesis.workspace = true - -# io -shellexpand.workspace = true -serde_json.workspace = true - # misc eyre.workspace = true diff --git a/crates/ethereum/cli/src/chainspec.rs b/crates/ethereum/cli/src/chainspec.rs index 05db177df832..cbcce9f69f61 100644 --- a/crates/ethereum/cli/src/chainspec.rs +++ b/crates/ethereum/cli/src/chainspec.rs @@ -1,48 +1,33 @@ -use alloy_genesis::Genesis; use reth_chainspec::{ChainSpec, DEV, HOLESKY, MAINNET, SEPOLIA}; -use reth_cli::chainspec::ChainSpecParser; -use std::{fs, path::PathBuf, sync::Arc}; +use reth_cli::chainspec::{parse_genesis, ChainSpecParser}; +use std::sync::Arc; + +/// Chains supported by reth. First value should be used as the default. +pub const SUPPORTED_CHAINS: &[&str] = &["mainnet", "sepolia", "holesky", "dev"]; /// Clap value parser for [`ChainSpec`]s. /// /// The value parser matches either a known chain, the path /// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { +pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { Ok(match s { "mainnet" => MAINNET.clone(), "sepolia" => SEPOLIA.clone(), "holesky" => HOLESKY.clone(), "dev" => DEV.clone(), - _ => { - // try to read json from path first - let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) { - Ok(raw) => raw, - Err(io_err) => { - // valid json may start with "\n", but must contain "{" - if s.contains('{') { - s.to_string() - } else { - return Err(io_err.into()) // assume invalid path - } - } - }; - - // both serialized Genesis and ChainSpec structs supported - let genesis: Genesis = serde_json::from_str(&raw)?; - - Arc::new(genesis.into()) - } + _ => Arc::new(parse_genesis(s)?.into()), }) } /// Ethereum chain specification parser. #[derive(Debug, Clone, Default)] -pub struct EthChainSpecParser; +#[non_exhaustive] +pub struct EthereumChainSpecParser; -impl ChainSpecParser for EthChainSpecParser { +impl ChainSpecParser for EthereumChainSpecParser { type ChainSpec = ChainSpec; - const SUPPORTED_CHAINS: &'static [&'static str] = &["mainnet", "sepolia", "holesky", "dev"]; + const SUPPORTED_CHAINS: &'static [&'static str] = SUPPORTED_CHAINS; fn parse(s: &str) -> eyre::Result> { chain_value_parser(s) @@ -56,8 +41,8 @@ mod tests { #[test] fn parse_known_chain_spec() { - for &chain in EthChainSpecParser::SUPPORTED_CHAINS { - assert!(::parse(chain).is_ok()); + for &chain in EthereumChainSpecParser::SUPPORTED_CHAINS { + assert!(::parse(chain).is_ok()); } } @@ -108,7 +93,7 @@ mod tests { } }"#; - let spec = ::parse(s).unwrap(); + let spec = ::parse(s).unwrap(); assert!(spec.is_shanghai_active_at_timestamp(0)); assert!(spec.is_cancun_active_at_timestamp(0)); assert!(spec.is_prague_active_at_timestamp(0)); diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 35802cf5165a..46476273e42d 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -14,9 +14,7 @@ workspace = true # reth reth-chainspec.workspace = true reth-primitives.workspace = true -reth-cli.workspace = true reth-cli-util.workspace = true -reth-fs-util.workspace = true reth-db = { workspace = true, features = ["mdbx"] } reth-storage-errors.workspace = true reth-storage-api.workspace = true @@ -37,10 +35,8 @@ reth-network-peers.workspace = true reth-consensus-common.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true -reth-optimism-chainspec = { workspace = true, optional = true } # ethereum -alloy-genesis.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine = { workspace = true, features = ["jwt"] } @@ -59,7 +55,6 @@ thiserror.workspace = true # io dirs-next = "2.0.0" shellexpand.workspace = true -serde_json.workspace = true # tracing tracing.workspace = true @@ -85,7 +80,6 @@ optimism = [ "reth-primitives/optimism", "reth-rpc-types-compat/optimism", "reth-rpc-eth-api/optimism", - "dep:reth-optimism-chainspec", ] # Features for vergen to generate correct env vars jemalloc = [] diff --git a/crates/node/core/src/args/mod.rs b/crates/node/core/src/args/mod.rs index 1a647ac65b03..7f1b64361515 100644 --- a/crates/node/core/src/args/mod.rs +++ b/crates/node/core/src/args/mod.rs @@ -56,7 +56,5 @@ pub use datadir_args::DatadirArgs; mod benchmark_args; pub use benchmark_args::BenchmarkArgs; -pub mod utils; - mod error; pub mod types; diff --git a/crates/node/core/src/args/utils.rs b/crates/node/core/src/args/utils.rs deleted file mode 100644 index e6ebda45b7f7..000000000000 --- a/crates/node/core/src/args/utils.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Clap parser utilities - -use std::{path::PathBuf, sync::Arc}; - -use alloy_genesis::Genesis; -use reth_chainspec::ChainSpec; -#[cfg(not(feature = "optimism"))] -use reth_chainspec::{DEV, HOLESKY, MAINNET, SEPOLIA}; -use reth_cli::chainspec::ChainSpecParser; -use reth_fs_util as fs; -#[cfg(feature = "optimism")] -use reth_optimism_chainspec::{BASE_MAINNET, BASE_SEPOLIA, OP_DEV, OP_MAINNET, OP_SEPOLIA}; - -#[cfg(feature = "optimism")] -/// Chains supported by op-reth. First value should be used as the default. -pub const SUPPORTED_CHAINS: &[&str] = - &["optimism", "optimism-sepolia", "base", "base-sepolia", "dev"]; -#[cfg(not(feature = "optimism"))] -/// Chains supported by reth. First value should be used as the default. -pub const SUPPORTED_CHAINS: &[&str] = &["mainnet", "sepolia", "holesky", "dev"]; - -/// Clap value parser for [`ChainSpec`]s. -/// -/// The value parser matches either a known chain, the path -/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -#[cfg(not(feature = "optimism"))] -pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { - Ok(match s { - "mainnet" => MAINNET.clone(), - "sepolia" => SEPOLIA.clone(), - "holesky" => HOLESKY.clone(), - "dev" => DEV.clone(), - _ => Arc::new(parse_custom_chain_spec(s)?), - }) -} - -/// Clap value parser for [`OpChainSpec`](reth_optimism_chainspec::OpChainSpec)s. -/// -/// The value parser matches either a known chain, the path -/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -#[cfg(feature = "optimism")] -pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { - Ok(Arc::new(match s { - "optimism" => OP_MAINNET.inner.clone(), - "optimism_sepolia" | "optimism-sepolia" => OP_SEPOLIA.inner.clone(), - "base" => BASE_MAINNET.inner.clone(), - "base_sepolia" | "base-sepolia" => BASE_SEPOLIA.inner.clone(), - "dev" => OP_DEV.inner.clone(), - _ => parse_custom_chain_spec(s)?, - })) -} - -/// Parses a custom [`ChainSpec`]. -pub fn parse_custom_chain_spec(s: &str) -> eyre::Result { - // try to read json from path first - let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) { - Ok(raw) => raw, - Err(io_err) => { - // valid json may start with "\n", but must contain "{" - if s.contains('{') { - s.to_string() - } else { - return Err(io_err.into()) // assume invalid path - } - } - }; - - // both serialized Genesis and ChainSpec structs supported - let genesis: Genesis = serde_json::from_str(&raw)?; - - Ok(genesis.into()) -} - -/// A chain specification parser for ethereum chains. -#[derive(Debug, Copy, Clone, Default)] -#[non_exhaustive] -pub struct EthereumChainSpecParser; - -impl ChainSpecParser for EthereumChainSpecParser { - type ChainSpec = ChainSpec; - - const SUPPORTED_CHAINS: &'static [&'static str] = SUPPORTED_CHAINS; - - fn parse(s: &str) -> eyre::Result> { - chain_value_parser(s) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_known_chain_spec() { - for chain in SUPPORTED_CHAINS { - chain_value_parser(chain).unwrap(); - } - } -} diff --git a/crates/optimism/cli/src/chainspec.rs b/crates/optimism/cli/src/chainspec.rs index e76bfd5f0656..329669ab8c9c 100644 --- a/crates/optimism/cli/src/chainspec.rs +++ b/crates/optimism/cli/src/chainspec.rs @@ -1,28 +1,12 @@ -use std::sync::Arc; - -use reth_cli::chainspec::ChainSpecParser; -use reth_node_core::args::utils::parse_custom_chain_spec; +use reth_cli::chainspec::{parse_genesis, ChainSpecParser}; use reth_optimism_chainspec::{ OpChainSpec, BASE_MAINNET, BASE_SEPOLIA, OP_DEV, OP_MAINNET, OP_SEPOLIA, }; - -/// Clap value parser for [`OpChainSpec`]s. -/// -/// The value parser matches either a known chain, the path -/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { - Ok(match s { - "dev" => OP_DEV.clone(), - "optimism" => OP_MAINNET.clone(), - "optimism_sepolia" | "optimism-sepolia" => OP_SEPOLIA.clone(), - "base" => BASE_MAINNET.clone(), - "base_sepolia" | "base-sepolia" => BASE_SEPOLIA.clone(), - _ => Arc::new(OpChainSpec { inner: parse_custom_chain_spec(s)? }), - }) -} +use std::sync::Arc; /// Optimism chain specification parser. #[derive(Debug, Clone, Default)] +#[non_exhaustive] pub struct OpChainSpecParser; impl ChainSpecParser for OpChainSpecParser { @@ -43,6 +27,21 @@ impl ChainSpecParser for OpChainSpecParser { } } +/// Clap value parser for [`OpChainSpec`]s. +/// +/// The value parser matches either a known chain, the path +/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. +pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { + Ok(match s { + "dev" => OP_DEV.clone(), + "optimism" => OP_MAINNET.clone(), + "optimism_sepolia" | "optimism-sepolia" => OP_SEPOLIA.clone(), + "base" => BASE_MAINNET.clone(), + "base_sepolia" | "base-sepolia" => BASE_SEPOLIA.clone(), + _ => Arc::new(parse_genesis(s)?.into()), + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index b17cefa63287..74bc80c426ef 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -176,13 +176,14 @@ where #[cfg(test)] mod test { + use crate::chainspec::OpChainSpecParser; use clap::Parser; - use reth_cli_commands::NodeCommand; + use reth_cli_commands::{node::NoArgs, NodeCommand}; use reth_optimism_chainspec::OP_DEV; #[test] fn parse_dev() { - let cmd: NodeCommand = NodeCommand::parse_from(["op-reth", "--dev"]); + let cmd = NodeCommand::::parse_from(["op-reth", "--dev"]); let chain = OP_DEV.clone(); assert_eq!(cmd.chain.chain, chain.chain); assert_eq!(cmd.chain.genesis_hash, chain.genesis_hash); diff --git a/examples/beacon-api-sidecar-fetcher/src/main.rs b/examples/beacon-api-sidecar-fetcher/src/main.rs index 7d8880ca185c..a0b9b6e01ec8 100644 --- a/examples/beacon-api-sidecar-fetcher/src/main.rs +++ b/examples/beacon-api-sidecar-fetcher/src/main.rs @@ -23,7 +23,7 @@ use clap::Parser; use futures_util::{stream::FuturesUnordered, StreamExt}; use mined_sidecar::MinedSidecarStream; use reth::{ - args::utils::EthereumChainSpecParser, builder::NodeHandle, cli::Cli, + builder::NodeHandle, chainspec::EthereumChainSpecParser, cli::Cli, providers::CanonStateSubscriptions, }; use reth_node_ethereum::EthereumNode; diff --git a/examples/beacon-api-sse/src/main.rs b/examples/beacon-api-sse/src/main.rs index 81535ef6140b..243511d4960e 100644 --- a/examples/beacon-api-sse/src/main.rs +++ b/examples/beacon-api-sse/src/main.rs @@ -21,7 +21,7 @@ use alloy_rpc_types_beacon::events::PayloadAttributesEvent; use clap::Parser; use futures_util::stream::StreamExt; use mev_share_sse::{client::EventStream, EventClient}; -use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; +use reth::{chainspec::EthereumChainSpecParser, cli::Cli}; use reth_node_ethereum::EthereumNode; use std::net::{IpAddr, Ipv4Addr}; use tracing::{info, warn}; diff --git a/examples/custom-inspector/src/main.rs b/examples/custom-inspector/src/main.rs index 700de274e681..12b7620f4adc 100644 --- a/examples/custom-inspector/src/main.rs +++ b/examples/custom-inspector/src/main.rs @@ -15,8 +15,8 @@ use alloy_rpc_types::state::EvmOverrides; use clap::Parser; use futures_util::StreamExt; use reth::{ - args::utils::EthereumChainSpecParser, builder::NodeHandle, + chainspec::EthereumChainSpecParser, cli::Cli, primitives::BlockNumberOrTag, revm::{ diff --git a/examples/node-custom-rpc/src/main.rs b/examples/node-custom-rpc/src/main.rs index 5aeecfd2915c..92e0bfea26e9 100644 --- a/examples/node-custom-rpc/src/main.rs +++ b/examples/node-custom-rpc/src/main.rs @@ -14,7 +14,7 @@ use clap::Parser; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; -use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; +use reth::{chainspec::EthereumChainSpecParser, cli::Cli}; use reth_node_ethereum::EthereumNode; use reth_transaction_pool::TransactionPool; diff --git a/examples/txpool-tracing/src/main.rs b/examples/txpool-tracing/src/main.rs index f8c2e19d203a..94f800987a96 100644 --- a/examples/txpool-tracing/src/main.rs +++ b/examples/txpool-tracing/src/main.rs @@ -15,7 +15,7 @@ use alloy_rpc_types_trace::{parity::TraceType, tracerequest::TraceCallRequest}; use clap::Parser; use futures_util::StreamExt; use reth::{ - args::utils::EthereumChainSpecParser, builder::NodeHandle, cli::Cli, + builder::NodeHandle, chainspec::EthereumChainSpecParser, cli::Cli, rpc::compat::transaction::transaction_to_call_request, transaction_pool::TransactionPool, }; use reth_node_ethereum::node::EthereumNode; From 36de90fdc346bc3aacbfb311fa20b623d48ef867 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 5 Oct 2024 13:08:08 +0200 Subject: [PATCH 040/103] chore(lint): fix `clippy::needles_lifetimes` (#11496) --- crates/blockchain-tree/src/bundle.rs | 4 +-- .../ethereum/engine-primitives/src/payload.rs | 2 +- crates/evm/execution-types/src/chain.rs | 8 +++--- crates/evm/src/system_calls/mod.rs | 2 +- crates/exex/types/src/notification.rs | 2 +- crates/net/network/src/peers.rs | 2 +- crates/node/builder/src/rpc.rs | 2 +- crates/optimism/payload/src/payload.rs | 2 +- crates/payload/builder/src/database.rs | 4 +-- .../prune/src/segments/static_file/headers.rs | 2 +- crates/rpc/rpc-eth-types/src/cache/db.rs | 16 +++++------ crates/rpc/rpc-layer/src/auth_layer.rs | 2 +- crates/rpc/rpc-testing-util/src/debug.rs | 8 +++--- crates/rpc/rpc-testing-util/src/trace.rs | 28 +++++++++---------- .../src/providers/state/historical.rs | 12 ++++---- .../provider/src/providers/state/latest.rs | 12 ++++---- .../provider/src/providers/static_file/jar.rs | 10 +++---- .../src/providers/static_file/writer.rs | 4 +-- .../storage/provider/src/writer/database.rs | 2 +- crates/storage/provider/src/writer/mod.rs | 10 +++---- .../provider/src/writer/static_file.rs | 2 +- crates/transaction-pool/src/traits.rs | 2 +- crates/trie/db/src/hashed_cursor.rs | 4 +-- crates/trie/db/src/prefix_set.rs | 2 +- crates/trie/db/src/trie_cursor.rs | 4 +-- crates/trie/trie/src/trie_cursor/in_memory.rs | 2 +- 26 files changed, 75 insertions(+), 75 deletions(-) diff --git a/crates/blockchain-tree/src/bundle.rs b/crates/blockchain-tree/src/bundle.rs index 6f62d4136bb7..3745753d3f47 100644 --- a/crates/blockchain-tree/src/bundle.rs +++ b/crates/blockchain-tree/src/bundle.rs @@ -18,7 +18,7 @@ pub struct BundleStateDataRef<'a> { pub canonical_fork: ForkBlock, } -impl<'a> ExecutionDataProvider for BundleStateDataRef<'a> { +impl ExecutionDataProvider for BundleStateDataRef<'_> { fn execution_outcome(&self) -> &ExecutionOutcome { self.execution_outcome } @@ -33,7 +33,7 @@ impl<'a> ExecutionDataProvider for BundleStateDataRef<'a> { } } -impl<'a> BlockExecutionForkProvider for BundleStateDataRef<'a> { +impl BlockExecutionForkProvider for BundleStateDataRef<'_> { fn canonical_fork(&self) -> ForkBlock { self.canonical_fork } diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index dd0b7b405e9f..ae370fdb9d7b 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -89,7 +89,7 @@ impl BuiltPayload for EthBuiltPayload { } } -impl<'a> BuiltPayload for &'a EthBuiltPayload { +impl BuiltPayload for &EthBuiltPayload { fn block(&self) -> &SealedBlock { (**self).block() } diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 5db5495de59f..467bd4c0ec7c 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -355,7 +355,7 @@ impl Chain { #[derive(Debug)] pub struct DisplayBlocksChain<'a>(pub &'a BTreeMap); -impl<'a> fmt::Display for DisplayBlocksChain<'a> { +impl fmt::Display for DisplayBlocksChain<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut list = f.debug_list(); let mut values = self.0.values().map(|block| block.num_hash()); @@ -376,7 +376,7 @@ pub struct ChainBlocks<'a> { blocks: Cow<'a, BTreeMap>, } -impl<'a> ChainBlocks<'a> { +impl ChainBlocks<'_> { /// Creates a consuming iterator over all blocks in the chain with increasing block number. /// /// Note: this always yields at least one block. @@ -442,7 +442,7 @@ impl<'a> ChainBlocks<'a> { } } -impl<'a> IntoIterator for ChainBlocks<'a> { +impl IntoIterator for ChainBlocks<'_> { type Item = (BlockNumber, SealedBlockWithSenders); type IntoIter = std::collections::btree_map::IntoIter; @@ -571,7 +571,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for Chain<'a> { + impl SerializeAs for Chain<'_> { fn serialize_as(source: &super::Chain, serializer: S) -> Result where S: Serializer, diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index 48429cb49596..43baa1c766c2 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -85,7 +85,7 @@ where .build() } -impl<'a, EvmConfig, Chainspec, Hook> SystemCaller<'a, EvmConfig, Chainspec, Hook> +impl SystemCaller<'_, EvmConfig, Chainspec, Hook> where EvmConfig: ConfigureEvm

, Chainspec: EthereumHardforks, diff --git a/crates/exex/types/src/notification.rs b/crates/exex/types/src/notification.rs index 53411250270d..61d42a3319be 100644 --- a/crates/exex/types/src/notification.rs +++ b/crates/exex/types/src/notification.rs @@ -136,7 +136,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for ExExNotification<'a> { + impl SerializeAs for ExExNotification<'_> { fn serialize_as( source: &super::ExExNotification, serializer: S, diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index b9196d29e8e4..3d5ff7a0d43a 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -1131,7 +1131,7 @@ mod tests { peers: &'a mut PeersManager, } - impl<'a> Future for PeerActionFuture<'a> { + impl Future for PeerActionFuture<'_> { type Output = PeerAction; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 607a33147e94..cb8d8f355dac 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -254,7 +254,7 @@ pub struct RpcContext<'a, Node: FullNodeComponents, EthApi: EthApiTypes> { pub auth_module: &'a mut AuthRpcModule, } -impl<'a, Node, EthApi> RpcContext<'a, Node, EthApi> +impl RpcContext<'_, Node, EthApi> where Node: FullNodeComponents, EthApi: EthApiTypes, diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index cb3b939136f3..f1ba24435092 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -183,7 +183,7 @@ impl BuiltPayload for OptimismBuiltPayload { } } -impl<'a> BuiltPayload for &'a OptimismBuiltPayload { +impl BuiltPayload for &OptimismBuiltPayload { fn block(&self) -> &SealedBlock { (**self).block() } diff --git a/crates/payload/builder/src/database.rs b/crates/payload/builder/src/database.rs index ea1ae0854348..d63f7322dee2 100644 --- a/crates/payload/builder/src/database.rs +++ b/crates/payload/builder/src/database.rs @@ -67,7 +67,7 @@ pub struct CachedReadsDbMut<'a, DB> { pub db: DB, } -impl<'a, DB: DatabaseRef> Database for CachedReadsDbMut<'a, DB> { +impl Database for CachedReadsDbMut<'_, DB> { type Error = ::Error; fn basic(&mut self, address: Address) -> Result, Self::Error> { @@ -130,7 +130,7 @@ pub struct CachedReadsDBRef<'a, DB> { pub inner: RefCell>, } -impl<'a, DB: DatabaseRef> DatabaseRef for CachedReadsDBRef<'a, DB> { +impl DatabaseRef for CachedReadsDBRef<'_, DB> { type Error = ::Error; fn basic_ref(&self, address: Address) -> Result, Self::Error> { diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs index a3daf504e667..8700a653b111 100644 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ b/crates/prune/prune/src/segments/static_file/headers.rs @@ -137,7 +137,7 @@ where } } -impl<'a, Provider> Iterator for HeaderTablesIter<'a, Provider> +impl Iterator for HeaderTablesIter<'_, Provider> where Provider: DBProvider, { diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index ad9804893a70..9731e3845768 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -20,7 +20,7 @@ pub type StateCacheDb<'a> = CacheDB(pub &'a dyn StateProvider); -impl<'a> reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'_> { fn state_root( &self, hashed_state: reth_trie::HashedPostState, @@ -50,7 +50,7 @@ impl<'a> reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'a } } -impl<'a> reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> { fn storage_root( &self, address: Address, @@ -60,7 +60,7 @@ impl<'a> reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper< } } -impl<'a> reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> { fn proof( &self, input: reth_trie::TrieInput, @@ -88,7 +88,7 @@ impl<'a> reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<' } } -impl<'a> reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'_> { fn basic_account( &self, address: revm_primitives::Address, @@ -97,7 +97,7 @@ impl<'a> reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'a> { } } -impl<'a> reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> { fn block_hash( &self, block_number: alloy_primitives::BlockNumber, @@ -121,7 +121,7 @@ impl<'a> reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'a> } } -impl<'a> StateProvider for StateProviderTraitObjWrapper<'a> { +impl StateProvider for StateProviderTraitObjWrapper<'_> { fn account_balance( &self, addr: revm_primitives::Address, @@ -164,7 +164,7 @@ impl<'a> StateProvider for StateProviderTraitObjWrapper<'a> { #[allow(missing_debug_implementations)] pub struct StateCacheDbRefMutWrapper<'a, 'b>(pub &'b mut StateCacheDb<'a>); -impl<'a, 'b> Database for StateCacheDbRefMutWrapper<'a, 'b> { +impl<'a> Database for StateCacheDbRefMutWrapper<'a, '_> { type Error = as Database>::Error; fn basic( &mut self, @@ -190,7 +190,7 @@ impl<'a, 'b> Database for StateCacheDbRefMutWrapper<'a, 'b> { } } -impl<'a, 'b> DatabaseRef for StateCacheDbRefMutWrapper<'a, 'b> { +impl<'a> DatabaseRef for StateCacheDbRefMutWrapper<'a, '_> { type Error = as Database>::Error; fn basic_ref( diff --git a/crates/rpc/rpc-layer/src/auth_layer.rs b/crates/rpc/rpc-layer/src/auth_layer.rs index 41ebce32dfb3..cdca181cbd06 100644 --- a/crates/rpc/rpc-layer/src/auth_layer.rs +++ b/crates/rpc/rpc-layer/src/auth_layer.rs @@ -176,7 +176,7 @@ mod tests { missing_jwt_error().await; wrong_jwt_signature_error().await; invalid_issuance_timestamp_error().await; - jwt_decode_error().await; + jwt_decode_error().await } async fn valid_jwt() { diff --git a/crates/rpc/rpc-testing-util/src/debug.rs b/crates/rpc/rpc-testing-util/src/debug.rs index cdcb454a2486..f50064e80ce9 100644 --- a/crates/rpc/rpc-testing-util/src/debug.rs +++ b/crates/rpc/rpc-testing-util/src/debug.rs @@ -304,7 +304,7 @@ impl<'a> DebugTraceTransactionsStream<'a> { } } -impl<'a> Stream for DebugTraceTransactionsStream<'a> { +impl Stream for DebugTraceTransactionsStream<'_> { type Item = TraceTransactionResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -312,7 +312,7 @@ impl<'a> Stream for DebugTraceTransactionsStream<'a> { } } -impl<'a> std::fmt::Debug for DebugTraceTransactionsStream<'a> { +impl std::fmt::Debug for DebugTraceTransactionsStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DebugTraceTransactionsStream").finish_non_exhaustive() } @@ -336,7 +336,7 @@ impl<'a> DebugTraceBlockStream<'a> { } } -impl<'a> Stream for DebugTraceBlockStream<'a> { +impl Stream for DebugTraceBlockStream<'_> { type Item = DebugTraceBlockResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -344,7 +344,7 @@ impl<'a> Stream for DebugTraceBlockStream<'a> { } } -impl<'a> std::fmt::Debug for DebugTraceBlockStream<'a> { +impl std::fmt::Debug for DebugTraceBlockStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DebugTraceBlockStream").finish_non_exhaustive() } diff --git a/crates/rpc/rpc-testing-util/src/trace.rs b/crates/rpc/rpc-testing-util/src/trace.rs index 13914a59eb3e..c6dc16cf1063 100644 --- a/crates/rpc/rpc-testing-util/src/trace.rs +++ b/crates/rpc/rpc-testing-util/src/trace.rs @@ -114,7 +114,7 @@ pub struct TraceCallStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for TraceCallStream<'a> { +impl Stream for TraceCallStream<'_> { type Item = TraceCallResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -122,7 +122,7 @@ impl<'a> Stream for TraceCallStream<'a> { } } -impl<'a> std::fmt::Debug for TraceCallStream<'a> { +impl std::fmt::Debug for TraceCallStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceCallStream").finish() } @@ -134,7 +134,7 @@ pub struct TraceFilterStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for TraceFilterStream<'a> { +impl Stream for TraceFilterStream<'_> { type Item = TraceFilterResult; /// Attempts to pull out the next value of the stream. @@ -143,7 +143,7 @@ impl<'a> Stream for TraceFilterStream<'a> { } } -impl<'a> std::fmt::Debug for TraceFilterStream<'a> { +impl std::fmt::Debug for TraceFilterStream<'_> { /// Provides a debug representation of the `TraceFilterStream`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceFilterStream").finish_non_exhaustive() @@ -157,7 +157,7 @@ pub struct TraceGetStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for TraceGetStream<'a> { +impl Stream for TraceGetStream<'_> { type Item = TraceGetResult; /// Attempts to pull out the next item of the stream @@ -166,7 +166,7 @@ impl<'a> Stream for TraceGetStream<'a> { } } -impl<'a> std::fmt::Debug for TraceGetStream<'a> { +impl std::fmt::Debug for TraceGetStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceGetStream").finish_non_exhaustive() } @@ -180,7 +180,7 @@ pub struct CallManyTraceStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for CallManyTraceStream<'a> { +impl Stream for CallManyTraceStream<'_> { type Item = CallManyTraceResult; /// Polls for the next item from the stream. @@ -189,7 +189,7 @@ impl<'a> Stream for CallManyTraceStream<'a> { } } -impl<'a> std::fmt::Debug for CallManyTraceStream<'a> { +impl std::fmt::Debug for CallManyTraceStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("CallManyTraceStream").finish() } @@ -201,7 +201,7 @@ pub struct RawTransactionTraceStream<'a> { stream: RawTransactionTraceResult<'a>, } -impl<'a> Stream for RawTransactionTraceStream<'a> { +impl Stream for RawTransactionTraceStream<'_> { type Item = Result<(TraceResults, Bytes), (RpcError, Bytes)>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -209,7 +209,7 @@ impl<'a> Stream for RawTransactionTraceStream<'a> { } } -impl<'a> std::fmt::Debug for RawTransactionTraceStream<'a> { +impl std::fmt::Debug for RawTransactionTraceStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RawTransactionTraceStream").finish() } @@ -221,7 +221,7 @@ pub struct ReplayTransactionStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for ReplayTransactionStream<'a> { +impl Stream for ReplayTransactionStream<'_> { type Item = ReplayTransactionResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -229,7 +229,7 @@ impl<'a> Stream for ReplayTransactionStream<'a> { } } -impl<'a> std::fmt::Debug for ReplayTransactionStream<'a> { +impl std::fmt::Debug for ReplayTransactionStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ReplayTransactionStream").finish() } @@ -393,7 +393,7 @@ impl<'a> TraceBlockStream<'a> { } } -impl<'a> Stream for TraceBlockStream<'a> { +impl Stream for TraceBlockStream<'_> { type Item = TraceBlockResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -401,7 +401,7 @@ impl<'a> Stream for TraceBlockStream<'a> { } } -impl<'a> std::fmt::Debug for TraceBlockStream<'a> { +impl std::fmt::Debug for TraceBlockStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceBlockStream").finish_non_exhaustive() } diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index de30f89c98ee..22f40f6f951f 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -227,7 +227,7 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> { +impl AccountReader for HistoricalStateProviderRef<'_, TX> { /// Get basic account information. fn basic_account(&self, address: Address) -> ProviderResult> { match self.account_history_lookup(address)? { @@ -249,7 +249,7 @@ impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> { +impl BlockHashReader for HistoricalStateProviderRef<'_, TX> { /// Get block hash by number. fn block_hash(&self, number: u64) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( @@ -285,7 +285,7 @@ impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateRootProvider for HistoricalStateProviderRef<'b, TX> { +impl StateRootProvider for HistoricalStateProviderRef<'_, TX> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { let mut revert_state = self.revert_state()?; revert_state.extend(hashed_state); @@ -319,7 +319,7 @@ impl<'b, TX: DbTx> StateRootProvider for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StorageRootProvider for HistoricalStateProviderRef<'b, TX> { +impl StorageRootProvider for HistoricalStateProviderRef<'_, TX> { fn storage_root( &self, address: Address, @@ -332,7 +332,7 @@ impl<'b, TX: DbTx> StorageRootProvider for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProofProvider for HistoricalStateProviderRef<'b, TX> { +impl StateProofProvider for HistoricalStateProviderRef<'_, TX> { /// Get account and storage proofs. fn proof( &self, @@ -364,7 +364,7 @@ impl<'b, TX: DbTx> StateProofProvider for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProvider for HistoricalStateProviderRef<'b, TX> { +impl StateProvider for HistoricalStateProviderRef<'_, TX> { /// Get storage. fn storage( &self, diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index f63eaee23862..9fbe00cbd5ee 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -36,14 +36,14 @@ impl<'b, TX: DbTx> LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> AccountReader for LatestStateProviderRef<'b, TX> { +impl AccountReader for LatestStateProviderRef<'_, TX> { /// Get basic account information. fn basic_account(&self, address: Address) -> ProviderResult> { self.tx.get::(address).map_err(Into::into) } } -impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> { +impl BlockHashReader for LatestStateProviderRef<'_, TX> { /// Get block hash by number. fn block_hash(&self, number: u64) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( @@ -79,7 +79,7 @@ impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateRootProvider for LatestStateProviderRef<'b, TX> { +impl StateRootProvider for LatestStateProviderRef<'_, TX> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { StateRoot::overlay_root(self.tx, hashed_state) .map_err(|err| ProviderError::Database(err.into())) @@ -107,7 +107,7 @@ impl<'b, TX: DbTx> StateRootProvider for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StorageRootProvider for LatestStateProviderRef<'b, TX> { +impl StorageRootProvider for LatestStateProviderRef<'_, TX> { fn storage_root( &self, address: Address, @@ -118,7 +118,7 @@ impl<'b, TX: DbTx> StorageRootProvider for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProofProvider for LatestStateProviderRef<'b, TX> { +impl StateProofProvider for LatestStateProviderRef<'_, TX> { fn proof( &self, input: TrieInput, @@ -146,7 +146,7 @@ impl<'b, TX: DbTx> StateProofProvider for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProvider for LatestStateProviderRef<'b, TX> { +impl StateProvider for LatestStateProviderRef<'_, TX> { /// Get storage. fn storage( &self, diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index 6372bad24424..8d1dbd117cfb 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -75,7 +75,7 @@ impl<'a> StaticFileJarProvider<'a> { } } -impl<'a> HeaderProvider for StaticFileJarProvider<'a> { +impl HeaderProvider for StaticFileJarProvider<'_> { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { Ok(self .cursor()? @@ -147,7 +147,7 @@ impl<'a> HeaderProvider for StaticFileJarProvider<'a> { } } -impl<'a> BlockHashReader for StaticFileJarProvider<'a> { +impl BlockHashReader for StaticFileJarProvider<'_> { fn block_hash(&self, number: u64) -> ProviderResult> { self.cursor()?.get_one::>(number.into()) } @@ -169,7 +169,7 @@ impl<'a> BlockHashReader for StaticFileJarProvider<'a> { } } -impl<'a> BlockNumReader for StaticFileJarProvider<'a> { +impl BlockNumReader for StaticFileJarProvider<'_> { fn chain_info(&self) -> ProviderResult { // Information on live database Err(ProviderError::UnsupportedProvider) @@ -194,7 +194,7 @@ impl<'a> BlockNumReader for StaticFileJarProvider<'a> { } } -impl<'a> TransactionsProvider for StaticFileJarProvider<'a> { +impl TransactionsProvider for StaticFileJarProvider<'_> { fn transaction_id(&self, hash: TxHash) -> ProviderResult> { let mut cursor = self.cursor()?; @@ -290,7 +290,7 @@ impl<'a> TransactionsProvider for StaticFileJarProvider<'a> { } } -impl<'a> ReceiptProvider for StaticFileJarProvider<'a> { +impl ReceiptProvider for StaticFileJarProvider<'_> { fn receipt(&self, num: TxNumber) -> ProviderResult> { self.cursor()?.get_one::>(num.into()) } diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index d086c5693ca5..3858f1b14023 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -67,14 +67,14 @@ pub struct StaticFileProviderRWRefMut<'a>( pub(crate) RwLockWriteGuard<'a, RawRwLock, Option>, ); -impl<'a> std::ops::DerefMut for StaticFileProviderRWRefMut<'a> { +impl std::ops::DerefMut for StaticFileProviderRWRefMut<'_> { fn deref_mut(&mut self) -> &mut Self::Target { // This is always created by [`StaticFileWriters::get_or_create`] self.0.as_mut().expect("static file writer provider should be init") } } -impl<'a> std::ops::Deref for StaticFileProviderRWRefMut<'a> { +impl std::ops::Deref for StaticFileProviderRWRefMut<'_> { type Target = StaticFileProviderRW; fn deref(&self) -> &Self::Target { diff --git a/crates/storage/provider/src/writer/database.rs b/crates/storage/provider/src/writer/database.rs index 3ae42b4bf1cb..1436fb8a6ab9 100644 --- a/crates/storage/provider/src/writer/database.rs +++ b/crates/storage/provider/src/writer/database.rs @@ -9,7 +9,7 @@ use reth_storage_api::ReceiptWriter; pub(crate) struct DatabaseWriter<'a, W>(pub(crate) &'a mut W); -impl<'a, W> ReceiptWriter for DatabaseWriter<'a, W> +impl ReceiptWriter for DatabaseWriter<'_, W> where W: DbCursorRO + DbCursorRW, { diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index ecb1de335559..5b16b2da4e5a 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -147,7 +147,7 @@ impl UnifiedStorageWriter<'_, (), ()> { } } -impl<'a, 'b, ProviderDB> UnifiedStorageWriter<'a, ProviderDB, &'b StaticFileProvider> +impl UnifiedStorageWriter<'_, ProviderDB, &StaticFileProvider> where ProviderDB: DBProvider + BlockWriter @@ -318,7 +318,7 @@ where } } -impl<'a, 'b, ProviderDB> UnifiedStorageWriter<'a, ProviderDB, StaticFileProviderRWRefMut<'b>> +impl UnifiedStorageWriter<'_, ProviderDB, StaticFileProviderRWRefMut<'_>> where ProviderDB: DBProvider + HeaderProvider, { @@ -429,7 +429,7 @@ where } } -impl<'a, 'b, ProviderDB> UnifiedStorageWriter<'a, ProviderDB, StaticFileProviderRWRefMut<'b>> +impl UnifiedStorageWriter<'_, ProviderDB, StaticFileProviderRWRefMut<'_>> where ProviderDB: DBProvider + HeaderProvider, { @@ -510,8 +510,8 @@ where } } -impl<'a, 'b, ProviderDB> StateWriter - for UnifiedStorageWriter<'a, ProviderDB, StaticFileProviderRWRefMut<'b>> +impl StateWriter + for UnifiedStorageWriter<'_, ProviderDB, StaticFileProviderRWRefMut<'_>> where ProviderDB: DBProvider + StateChangeWriter + HeaderProvider, { diff --git a/crates/storage/provider/src/writer/static_file.rs b/crates/storage/provider/src/writer/static_file.rs index aca226ca9b75..5514e211e58f 100644 --- a/crates/storage/provider/src/writer/static_file.rs +++ b/crates/storage/provider/src/writer/static_file.rs @@ -6,7 +6,7 @@ use reth_storage_api::ReceiptWriter; pub(crate) struct StaticFileWriter<'a, W>(pub(crate) &'a mut W); -impl<'a> ReceiptWriter for StaticFileWriter<'a, StaticFileProviderRWRefMut<'_>> { +impl ReceiptWriter for StaticFileWriter<'_, StaticFileProviderRWRefMut<'_>> { fn append_block_receipts( &mut self, first_tx_index: TxNumber, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index e522978fb9dc..f44e66e457a1 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -644,7 +644,7 @@ pub struct CanonicalStateUpdate<'a> { pub mined_transactions: Vec, } -impl<'a> CanonicalStateUpdate<'a> { +impl CanonicalStateUpdate<'_> { /// Returns the number of the tip block. pub fn number(&self) -> u64 { self.new_tip.number diff --git a/crates/trie/db/src/hashed_cursor.rs b/crates/trie/db/src/hashed_cursor.rs index bf0341c8884d..6d0b79e5a02b 100644 --- a/crates/trie/db/src/hashed_cursor.rs +++ b/crates/trie/db/src/hashed_cursor.rs @@ -11,7 +11,7 @@ use reth_trie::hashed_cursor::{HashedCursor, HashedCursorFactory, HashedStorageC #[derive(Debug)] pub struct DatabaseHashedCursorFactory<'a, TX>(&'a TX); -impl<'a, TX> Clone for DatabaseHashedCursorFactory<'a, TX> { +impl Clone for DatabaseHashedCursorFactory<'_, TX> { fn clone(&self) -> Self { Self(self.0) } @@ -24,7 +24,7 @@ impl<'a, TX> DatabaseHashedCursorFactory<'a, TX> { } } -impl<'a, TX: DbTx> HashedCursorFactory for DatabaseHashedCursorFactory<'a, TX> { +impl HashedCursorFactory for DatabaseHashedCursorFactory<'_, TX> { type AccountCursor = DatabaseHashedAccountCursor<::Cursor>; type StorageCursor = DatabaseHashedStorageCursor<::DupCursor>; diff --git a/crates/trie/db/src/prefix_set.rs b/crates/trie/db/src/prefix_set.rs index 07b87016d2b4..079fe393764d 100644 --- a/crates/trie/db/src/prefix_set.rs +++ b/crates/trie/db/src/prefix_set.rs @@ -26,7 +26,7 @@ impl<'a, TX> PrefixSetLoader<'a, TX> { } } -impl<'a, TX: DbTx> PrefixSetLoader<'a, TX> { +impl PrefixSetLoader<'_, TX> { /// Load all account and storage changes for the given block range. pub fn load(self, range: RangeInclusive) -> Result { // Initialize prefix sets. diff --git a/crates/trie/db/src/trie_cursor.rs b/crates/trie/db/src/trie_cursor.rs index 601100b3faee..bfded342ba04 100644 --- a/crates/trie/db/src/trie_cursor.rs +++ b/crates/trie/db/src/trie_cursor.rs @@ -19,7 +19,7 @@ use reth_trie_common::StorageTrieEntry; #[derive(Debug)] pub struct DatabaseTrieCursorFactory<'a, TX>(&'a TX); -impl<'a, TX> Clone for DatabaseTrieCursorFactory<'a, TX> { +impl Clone for DatabaseTrieCursorFactory<'_, TX> { fn clone(&self) -> Self { Self(self.0) } @@ -33,7 +33,7 @@ impl<'a, TX> DatabaseTrieCursorFactory<'a, TX> { } /// Implementation of the trie cursor factory for a database transaction. -impl<'a, TX: DbTx> TrieCursorFactory for DatabaseTrieCursorFactory<'a, TX> { +impl TrieCursorFactory for DatabaseTrieCursorFactory<'_, TX> { type AccountTrieCursor = DatabaseAccountTrieCursor<::Cursor>; type StorageTrieCursor = DatabaseStorageTrieCursor<::DupCursor>; diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index c313231e66dc..0f00191378ba 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -113,7 +113,7 @@ impl<'a, C: TrieCursor> InMemoryAccountTrieCursor<'a, C> { } } -impl<'a, C: TrieCursor> TrieCursor for InMemoryAccountTrieCursor<'a, C> { +impl TrieCursor for InMemoryAccountTrieCursor<'_, C> { fn seek_exact( &mut self, key: Nibbles, From 659dec9e66d6f85b1a56363442266b9e8147bf1a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 19:44:33 +0200 Subject: [PATCH 041/103] chore: rm from genesis impl (#11509) --- crates/net/eth-wire-types/Cargo.toml | 2 +- crates/net/eth-wire-types/src/status.rs | 26 ++++--------------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/crates/net/eth-wire-types/Cargo.toml b/crates/net/eth-wire-types/Cargo.toml index 6ce51786f282..82c9fe37a44d 100644 --- a/crates/net/eth-wire-types/Cargo.toml +++ b/crates/net/eth-wire-types/Cargo.toml @@ -20,7 +20,6 @@ reth-primitives.workspace = true # ethereum alloy-chains = { workspace = true, features = ["rlp"] } alloy-eips.workspace = true -alloy-genesis.workspace = true alloy-primitives.workspace = true alloy-rlp = { workspace = true, features = ["derive"] } @@ -36,6 +35,7 @@ proptest-arbitrary-interop = { workspace = true, optional = true } [dev-dependencies] reth-primitives = { workspace = true, features = ["arbitrary"] } +alloy-genesis.workspace = true alloy-chains = { workspace = true, features = ["arbitrary"] } arbitrary = { workspace = true, features = ["derive"] } proptest.workspace = true diff --git a/crates/net/eth-wire-types/src/status.rs b/crates/net/eth-wire-types/src/status.rs index baf1e2991522..a5e7530ec09a 100644 --- a/crates/net/eth-wire-types/src/status.rs +++ b/crates/net/eth-wire-types/src/status.rs @@ -1,9 +1,8 @@ use crate::EthVersion; use alloy_chains::{Chain, NamedChain}; -use alloy_genesis::Genesis; use alloy_primitives::{hex, B256, U256}; use alloy_rlp::{RlpDecodable, RlpEncodable}; -use reth_chainspec::{ChainSpec, EthChainSpec, Hardforks, MAINNET}; +use reth_chainspec::{EthChainSpec, Hardforks, MAINNET}; use reth_codecs_derive::add_arbitrary_tests; use reth_primitives::{EthereumHardfork, ForkId, Head}; use std::fmt::{Debug, Display}; @@ -43,23 +42,6 @@ pub struct Status { pub forkid: ForkId, } -impl From for Status { - fn from(genesis: Genesis) -> Self { - let chain = genesis.config.chain_id; - let total_difficulty = genesis.difficulty; - let chainspec = ChainSpec::from(genesis); - - Self { - version: EthVersion::Eth68 as u8, - chain: Chain::from_id(chain), - total_difficulty, - blockhash: chainspec.genesis_hash(), - genesis: chainspec.genesis_hash(), - forkid: chainspec.fork_id(&Head::default()), - } - } -} - impl Status { /// Helper for returning a builder for the status message. pub fn builder() -> StatusBuilder { @@ -71,10 +53,10 @@ impl Status { self.version = version as u8; } - /// Create a [`StatusBuilder`] from the given [`ChainSpec`] and head block. + /// Create a [`StatusBuilder`] from the given [`EthChainSpec`] and head block. /// - /// Sets the `chain` and `genesis`, `blockhash`, and `forkid` fields based on the [`ChainSpec`] - /// and head. + /// Sets the `chain` and `genesis`, `blockhash`, and `forkid` fields based on the + /// [`EthChainSpec`] and head. pub fn spec_builder(spec: Spec, head: &Head) -> StatusBuilder where Spec: EthChainSpec + Hardforks, From cb078f9eaef20d544af8147f7c0dbf5702671209 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 20:06:21 +0200 Subject: [PATCH 042/103] fix: cap gas limit properly (#11505) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 26 ++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 4d12450a1c7f..819aff546d42 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -1023,7 +1023,14 @@ pub trait Call: LoadState + SpawnBlocking { block_env.get_blob_gasprice().map(U256::from), )?; - let gas_limit = gas.unwrap_or_else(|| block_env.gas_limit.min(U256::from(u64::MAX)).to()); + let gas_limit = gas.unwrap_or_else(|| { + // Use maximum allowed gas limit. The reason for this + // is that both Erigon and Geth use pre-configured gas cap even if + // it's possible to derive the gas limit from the block: + // + block_env.gas_limit.saturating_to() + }); #[allow(clippy::needless_update)] let env = TxEnv { @@ -1065,7 +1072,7 @@ pub trait Call: LoadState + SpawnBlocking { Ok(EnvWithHandlerCfg::new_with_cfg_env(cfg, block, tx)) } - /// Prepares the [`EnvWithHandlerCfg`] for execution. + /// Prepares the [`EnvWithHandlerCfg`] for execution of calls. /// /// Does not commit any changes to the underlying database. /// @@ -1076,9 +1083,6 @@ pub trait Call: LoadState + SpawnBlocking { /// - `disable_eip3607` is set to `true` /// - `disable_base_fee` is set to `true` /// - `nonce` is set to `None` - /// - /// Additionally, the block gas limit so that higher tx gas limits can be used in `eth_call`. - /// - `disable_block_gas_limit` is set to `true` fn prepare_call_env( &self, mut cfg: CfgEnvWithHandlerCfg, @@ -1125,15 +1129,9 @@ pub trait Call: LoadState + SpawnBlocking { if env.tx.gas_price > U256::ZERO { // If gas price is specified, cap transaction gas limit with caller allowance trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap with caller allowance"); - cap_tx_gas_limit_with_caller_allowance(db, &mut env.tx)?; - } else { - // If no gas price is specified, use maximum allowed gas limit. The reason for this - // is that both Erigon and Geth use pre-configured gas cap even if - // it's possible to derive the gas limit from the block: - // - trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap as the maximum gas limit"); - env.tx.gas_limit = self.call_gas_limit(); + let cap = caller_gas_allowance(db, &env.tx)?; + // ensure we cap gas_limit to the block's + env.tx.gas_limit = cap.min(env.block.gas_limit).saturating_to(); } } From 51db6564d62ce61beca266199fbd55760e63e38a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 20:11:28 +0200 Subject: [PATCH 043/103] feat: add PoolBuilderConfigOverrides (#11507) --- crates/node/builder/src/components/pool.rs | 62 ++++++++++++++++++++-- crates/optimism/node/src/node.rs | 13 +++-- crates/transaction-pool/src/config.rs | 2 +- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/crates/node/builder/src/components/pool.rs b/crates/node/builder/src/components/pool.rs index af31f29307eb..234455913c6f 100644 --- a/crates/node/builder/src/components/pool.rs +++ b/crates/node/builder/src/components/pool.rs @@ -1,8 +1,8 @@ //! Pool component for the node builder. -use std::future::Future; - -use reth_transaction_pool::TransactionPool; +use alloy_primitives::Address; +use reth_transaction_pool::{PoolConfig, SubPoolLimit, TransactionPool}; +use std::{collections::HashSet, future::Future}; use crate::{BuilderContext, FullNodeTypes}; @@ -34,3 +34,59 @@ where self(ctx) } } + +/// Convenience type to override cli or default pool configuration during build. +#[derive(Debug, Clone, Default)] +pub struct PoolBuilderConfigOverrides { + /// Max number of transaction in the pending sub-pool + pub pending_limit: Option, + /// Max number of transaction in the basefee sub-pool + pub basefee_limit: Option, + /// Max number of transaction in the queued sub-pool + pub queued_limit: Option, + /// Max number of transactions in the blob sub-pool + pub blob_limit: Option, + /// Max number of executable transaction slots guaranteed per account + pub max_account_slots: Option, + /// Minimum base fee required by the protocol. + pub minimal_protocol_basefee: Option, + /// Addresses that will be considered as local. Above exemptions apply. + pub local_addresses: HashSet
, +} + +impl PoolBuilderConfigOverrides { + /// Applies the configured overrides to the given [`PoolConfig`]. + pub fn apply(self, mut config: PoolConfig) -> PoolConfig { + let Self { + pending_limit, + basefee_limit, + queued_limit, + blob_limit, + max_account_slots, + minimal_protocol_basefee, + local_addresses, + } = self; + + if let Some(pending_limit) = pending_limit { + config.pending_limit = pending_limit; + } + if let Some(basefee_limit) = basefee_limit { + config.basefee_limit = basefee_limit; + } + if let Some(queued_limit) = queued_limit { + config.queued_limit = queued_limit; + } + if let Some(blob_limit) = blob_limit { + config.blob_limit = blob_limit; + } + if let Some(max_account_slots) = max_account_slots { + config.max_account_slots = max_account_slots; + } + if let Some(minimal_protocol_basefee) = minimal_protocol_basefee { + config.minimal_protocol_basefee = minimal_protocol_basefee; + } + config.local_transactions_config.local_addresses.extend(local_addresses); + + config + } +} diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 4e7788569f89..8ae47897084b 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -9,7 +9,7 @@ use reth_node_api::{EngineValidator, FullNodeComponents, NodeAddOns}; use reth_node_builder::{ components::{ ComponentsBuilder, ConsensusBuilder, EngineValidatorBuilder, ExecutorBuilder, - NetworkBuilder, PayloadServiceBuilder, PoolBuilder, + NetworkBuilder, PayloadServiceBuilder, PoolBuilder, PoolBuilderConfigOverrides, }, node::{FullNodeTypes, NodeTypes, NodeTypesWithEngine}, BuilderContext, Node, PayloadBuilderConfig, @@ -166,9 +166,11 @@ where /// /// This contains various settings that can be configured and take precedence over the node's /// config. -#[derive(Debug, Default, Clone, Copy)] -#[non_exhaustive] -pub struct OptimismPoolBuilder; +#[derive(Debug, Default, Clone)] +pub struct OptimismPoolBuilder { + /// Enforced overrides that are applied to the pool config. + pub pool_config_overrides: PoolBuilderConfigOverrides, +} impl PoolBuilder for OptimismPoolBuilder where @@ -177,6 +179,7 @@ where type Pool = OpTransactionPool; async fn build_pool(self, ctx: &BuilderContext) -> eyre::Result { + let Self { pool_config_overrides } = self; let data_dir = ctx.config().datadir(); let blob_store = DiskFileBlobStore::open(data_dir.blobstore(), Default::default())?; @@ -198,7 +201,7 @@ where validator, CoinbaseTipOrdering::default(), blob_store, - ctx.pool_config(), + pool_config_overrides.apply(ctx.pool_config()), ); info!(target: "reth::cli", "Transaction pool initialized"); let transactions_path = data_dir.txpool_transactions(); diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 623493e6c9d4..67328c7f1be1 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -157,7 +157,7 @@ pub struct LocalTransactionConfig { /// - no price exemptions /// - no eviction exemptions pub no_exemptions: bool, - /// Addresses that will be considered as local . Above exemptions apply + /// Addresses that will be considered as local. Above exemptions apply. pub local_addresses: HashSet
, /// Flag indicating whether local transactions should be propagated. pub propagate_local_transactions: bool, From 41455dfcdd1310bdaedcbc55054cdb1ef01bf639 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 20:12:28 +0200 Subject: [PATCH 044/103] feat: expose Op node network_config helper (#11506) --- crates/net/network/src/config.rs | 8 +++++++ crates/optimism/node/src/node.rs | 41 ++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 8217a02a1bab..60b559b9dbc3 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -106,6 +106,14 @@ impl NetworkConfig { NetworkConfig::builder(secret_key).build(client) } + /// Apply a function to the config. + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + /// Sets the config to use for the discovery v4 protocol. pub fn set_discovery_v4(mut self, discovery_config: Discv4Config) -> Self { self.discovery_v4_config = Some(discovery_config); diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 8ae47897084b..caeab3741a2e 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -3,8 +3,9 @@ use std::sync::Arc; use reth_basic_payload_builder::{BasicPayloadJobGenerator, BasicPayloadJobGeneratorConfig}; +use reth_chainspec::{EthChainSpec, Hardforks}; use reth_evm::ConfigureEvm; -use reth_network::{NetworkHandle, NetworkManager}; +use reth_network::{NetworkConfig, NetworkHandle, NetworkManager}; use reth_node_api::{EngineValidator, FullNodeComponents, NodeAddOns}; use reth_node_builder::{ components::{ @@ -330,18 +331,18 @@ pub struct OptimismNetworkBuilder { pub disable_discovery_v4: bool, } -impl NetworkBuilder for OptimismNetworkBuilder -where - Node: FullNodeTypes>, - Pool: TransactionPool + Unpin + 'static, -{ - async fn build_network( - self, +impl OptimismNetworkBuilder { + /// Returns the [`NetworkConfig`] that contains the settings to launch the p2p network. + /// + /// This applies the configured [`OptimismNetworkBuilder`] settings. + pub fn network_config( + &self, ctx: &BuilderContext, - pool: Pool, - ) -> eyre::Result { - let Self { disable_txpool_gossip, disable_discovery_v4 } = self; - + ) -> eyre::Result::Provider>> + where + Node: FullNodeTypes>, + { + let Self { disable_txpool_gossip, disable_discovery_v4 } = self.clone(); let args = &ctx.config().network; let network_builder = ctx .network_config_builder()? @@ -374,8 +375,22 @@ where // gossip to prevent other parties in the network from learning about them. network_config.tx_gossip_disabled = disable_txpool_gossip; - let network = NetworkManager::builder(network_config).await?; + Ok(network_config) + } +} +impl NetworkBuilder for OptimismNetworkBuilder +where + Node: FullNodeTypes>, + Pool: TransactionPool + Unpin + 'static, +{ + async fn build_network( + self, + ctx: &BuilderContext, + pool: Pool, + ) -> eyre::Result { + let network_config = self.network_config(ctx)?; + let network = NetworkManager::builder(network_config).await?; let handle = ctx.start_network(network, pool); Ok(handle) From c08c00f2ecea90ca9b4ca37c91a053f9eb4242ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 6 Oct 2024 14:28:31 +0000 Subject: [PATCH 045/103] chore(deps): weekly `cargo update` (#11518) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 182 ++++++++++++++++++++++------------------------------- 1 file changed, 75 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e75ea0ec095c..590f01a416fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8158b4878c67837e5413721cc44298e6a2d88d39203175ea025e51892a16ba4c" +checksum = "94c225801d42099570d0674701dddd4142f0ef715282aeb5985042e2ec962df7" dependencies = [ "alloy-rlp", "arbitrary", @@ -296,7 +296,7 @@ dependencies = [ "getrandom 0.2.15", "hashbrown 0.14.5", "hex-literal", - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "k256", "keccak-asm", @@ -619,7 +619,7 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.5.0", + "indexmap 2.6.0", "proc-macro-error2", "proc-macro2", "quote", @@ -1016,9 +1016,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" dependencies = [ "brotli", "flate2", @@ -1203,26 +1203,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.6.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.79", -] - [[package]] name = "bindgen" version = "0.70.1" @@ -1333,7 +1313,7 @@ dependencies = [ "bitflags 2.6.0", "boa_interner", "boa_macros", - "indexmap 2.5.0", + "indexmap 2.6.0", "num-bigint", "rustc-hash 2.0.0", ] @@ -1359,7 +1339,7 @@ dependencies = [ "fast-float", "hashbrown 0.14.5", "icu_normalizer", - "indexmap 2.5.0", + "indexmap 2.6.0", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -1405,7 +1385,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "phf", "rustc-hash 2.0.0", @@ -1473,9 +1453,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1619,9 +1599,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" dependencies = [ "jobserver", "libc", @@ -3133,9 +3113,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3148,9 +3128,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3158,15 +3138,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -3175,9 +3155,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3196,9 +3176,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -3207,15 +3187,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -3229,9 +3209,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3311,9 +3291,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -3390,7 +3370,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3436,6 +3416,12 @@ dependencies = [ "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "hashlink" version = "0.8.4" @@ -3963,13 +3949,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -3986,7 +3972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.5.0", + "indexmap 2.6.0", "is-terminal", "itoa", "log", @@ -4084,9 +4070,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "iri-string" @@ -4124,15 +4110,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -4449,12 +4426,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.159" @@ -4498,11 +4469,11 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.8" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" +checksum = "78cca3586d5efa98fba425856ba227950fd4287525dd5317b352f476ca7d603b" dependencies = [ - "bindgen 0.69.4", + "bindgen", "errno", "libc", ] @@ -4723,7 +4694,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "metrics", "metrics-util", "quanta", @@ -5134,21 +5105,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -5494,18 +5462,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -6041,9 +6009,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -7559,7 +7527,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "derive_more 1.0.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "parking_lot 0.12.3", "pprof", "rand 0.8.5", @@ -7575,7 +7543,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "1.0.8" dependencies = [ - "bindgen 0.70.1", + "bindgen", "cc", ] @@ -9479,9 +9447,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "log", "once_cell", @@ -9780,7 +9748,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -9832,15 +9800,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -9850,9 +9818,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", @@ -10650,7 +10618,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -10999,9 +10967,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" From d2ab6352c6a1556011cab2eb81a28ce9b2edd22e Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:02:10 +0200 Subject: [PATCH 046/103] test: add unit tests for `PruneLimiter` (#11517) --- crates/prune/types/src/limiter.rs | 267 ++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) diff --git a/crates/prune/types/src/limiter.rs b/crates/prune/types/src/limiter.rs index 09f0c4cd3f2d..3a1059949300 100644 --- a/crates/prune/types/src/limiter.rs +++ b/crates/prune/types/src/limiter.rs @@ -120,3 +120,270 @@ impl PruneLimiter { self.is_deleted_entries_limit_reached() || self.is_time_limit_reached() } } + +#[cfg(test)] +mod tests { + use super::*; + use std::thread::sleep; + + #[test] + fn test_prune_deleted_entries_limit_initial_state() { + let limit_tracker = PruneDeletedEntriesLimit::new(10); + // Limit should be set properly + assert_eq!(limit_tracker.limit, 10); + // No entries should be deleted + assert_eq!(limit_tracker.deleted, 0); + assert!(!limit_tracker.is_limit_reached()); + } + + #[test] + fn test_prune_deleted_entries_limit_is_limit_reached() { + // Test when the deleted entries are less than the limit + let mut limit_tracker = PruneDeletedEntriesLimit::new(5); + limit_tracker.deleted = 3; + assert!(!limit_tracker.is_limit_reached()); + + // Test when the deleted entries are equal to the limit + limit_tracker.deleted = 5; + assert!(limit_tracker.is_limit_reached()); + + // Test when the deleted entries exceed the limit + limit_tracker.deleted = 6; + assert!(limit_tracker.is_limit_reached()); + } + + #[test] + fn test_prune_time_limit_initial_state() { + let time_limit = PruneTimeLimit::new(Duration::from_secs(10)); + // The limit should be set correctly + assert_eq!(time_limit.limit, Duration::from_secs(10)); + // The elapsed time should be very small right after creation + assert!(time_limit.start.elapsed() < Duration::from_secs(1)); + // Limit should not be reached initially + assert!(!time_limit.is_limit_reached()); + } + + #[test] + fn test_prune_time_limit_is_limit_reached() { + let time_limit = PruneTimeLimit::new(Duration::from_millis(50)); + + // Simulate waiting for some time (less than the limit) + std::thread::sleep(Duration::from_millis(30)); + assert!(!time_limit.is_limit_reached()); + + // Simulate waiting for time greater than the limit + std::thread::sleep(Duration::from_millis(30)); + assert!(time_limit.is_limit_reached()); + } + + #[test] + fn test_set_deleted_entries_limit_initial_state() { + let pruner = PruneLimiter::default().set_deleted_entries_limit(100); + // The deleted_entries_limit should be set with the correct limit + assert!(pruner.deleted_entries_limit.is_some()); + let deleted_entries_limit = pruner.deleted_entries_limit.unwrap(); + assert_eq!(deleted_entries_limit.limit, 100); + // The deleted count should be initially zero + assert_eq!(deleted_entries_limit.deleted, 0); + // The limit should not be reached initially + assert!(!deleted_entries_limit.is_limit_reached()); + } + + #[test] + fn test_set_deleted_entries_limit_overwrite_existing() { + let mut pruner = PruneLimiter::default().set_deleted_entries_limit(50); + // Overwrite the existing limit + pruner = pruner.set_deleted_entries_limit(200); + + assert!(pruner.deleted_entries_limit.is_some()); + let deleted_entries_limit = pruner.deleted_entries_limit.unwrap(); + // Check that the limit has been overwritten correctly + assert_eq!(deleted_entries_limit.limit, 200); + // Deleted count should still be zero + assert_eq!(deleted_entries_limit.deleted, 0); + assert!(!deleted_entries_limit.is_limit_reached()); + } + + #[test] + fn test_set_deleted_entries_limit_when_limit_is_reached() { + let mut pruner = PruneLimiter::default().set_deleted_entries_limit(5); + assert!(pruner.deleted_entries_limit.is_some()); + let mut deleted_entries_limit = pruner.deleted_entries_limit.clone().unwrap(); + + // Simulate deletion of entries + deleted_entries_limit.deleted = 5; + assert!(deleted_entries_limit.is_limit_reached()); + + // Overwrite the limit and check if it resets correctly + pruner = pruner.set_deleted_entries_limit(10); + deleted_entries_limit = pruner.deleted_entries_limit.unwrap(); + assert_eq!(deleted_entries_limit.limit, 10); + // Deletion count should reset + assert_eq!(deleted_entries_limit.deleted, 0); + assert!(!deleted_entries_limit.is_limit_reached()); + } + + #[test] + fn test_floor_deleted_entries_limit_to_multiple_of() { + let limiter = PruneLimiter::default().set_deleted_entries_limit(15); + let denominator = NonZeroUsize::new(4).unwrap(); + + // Floor limit to the largest multiple of 4 less than or equal to 15 (that is 12) + let updated_limiter = limiter.floor_deleted_entries_limit_to_multiple_of(denominator); + assert_eq!(updated_limiter.deleted_entries_limit.unwrap().limit, 12); + + // Test when the limit is already a multiple of the denominator + let limiter = PruneLimiter::default().set_deleted_entries_limit(16); + let updated_limiter = limiter.floor_deleted_entries_limit_to_multiple_of(denominator); + assert_eq!(updated_limiter.deleted_entries_limit.unwrap().limit, 16); + + // Test when there's no limit set (should not panic) + let limiter = PruneLimiter::default(); + let updated_limiter = limiter.floor_deleted_entries_limit_to_multiple_of(denominator); + assert!(updated_limiter.deleted_entries_limit.is_none()); + } + + #[test] + fn test_is_deleted_entries_limit_reached() { + // Limit is not set, should return false + let limiter = PruneLimiter::default(); + assert!(!limiter.is_deleted_entries_limit_reached()); + + // Limit is set but not reached, should return false + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + limiter.deleted_entries_limit.as_mut().unwrap().deleted = 5; + // 5 entries deleted out of 10 + assert!(!limiter.is_deleted_entries_limit_reached()); + + // Limit is reached, should return true + limiter.deleted_entries_limit.as_mut().unwrap().deleted = 10; + // 10 entries deleted out of 10 + assert!(limiter.is_deleted_entries_limit_reached()); + + // Deleted entries exceed the limit, should return true + limiter.deleted_entries_limit.as_mut().unwrap().deleted = 12; + // 12 entries deleted out of 10 + assert!(limiter.is_deleted_entries_limit_reached()); + } + + #[test] + fn test_increment_deleted_entries_count_by() { + // Increment when no limit is set + let mut limiter = PruneLimiter::default(); + limiter.increment_deleted_entries_count_by(5); + assert_eq!(limiter.deleted_entries_limit.as_ref().map(|l| l.deleted), None); // Still None + + // Increment when limit is set + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + limiter.increment_deleted_entries_count_by(3); + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 3); // Now 3 deleted + + // Increment again + limiter.increment_deleted_entries_count_by(2); + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 5); // Now 5 deleted + } + + #[test] + fn test_increment_deleted_entries_count() { + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(5); + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 0); // Initially 0 + + limiter.increment_deleted_entries_count(); // Increment by 1 + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 1); // Now 1 + } + + #[test] + fn test_deleted_entries_limit_left() { + // Test when limit is set and some entries are deleted + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + limiter.increment_deleted_entries_count_by(3); // Simulate 3 deleted entries + assert_eq!(limiter.deleted_entries_limit_left(), Some(7)); // 10 - 3 = 7 + + // Test when no entries are deleted + limiter = PruneLimiter::default().set_deleted_entries_limit(5); + assert_eq!(limiter.deleted_entries_limit_left(), Some(5)); // 5 - 0 = 5 + + // Test when limit is reached + limiter.increment_deleted_entries_count_by(5); // Simulate deleting 5 entries + assert_eq!(limiter.deleted_entries_limit_left(), Some(0)); // 5 - 5 = 0 + + // Test when limit is not set + limiter = PruneLimiter::default(); // No limit set + assert_eq!(limiter.deleted_entries_limit_left(), None); // Should be None + } + + #[test] + fn test_set_time_limit() { + // Create a PruneLimiter instance with no time limit set + let mut limiter = PruneLimiter::default(); + + // Set a time limit of 5 seconds + limiter = limiter.set_time_limit(Duration::new(5, 0)); + + // Verify that the time limit is set correctly + assert!(limiter.time_limit.is_some()); + let time_limit = limiter.time_limit.as_ref().unwrap(); + assert_eq!(time_limit.limit, Duration::new(5, 0)); + // Ensure the start time is recent + assert!(time_limit.start.elapsed() < Duration::new(1, 0)); + } + + #[test] + fn test_is_time_limit_reached() { + // Create a PruneLimiter instance and set a time limit of 10 milliseconds + let mut limiter = PruneLimiter::default(); + + // Time limit should not be reached initially + assert!(!limiter.is_time_limit_reached(), "Time limit should not be reached yet"); + + limiter = limiter.set_time_limit(Duration::new(0, 10_000_000)); // 10 milliseconds + + // Sleep for 5 milliseconds (less than the time limit) + sleep(Duration::new(0, 5_000_000)); // 5 milliseconds + assert!(!limiter.is_time_limit_reached(), "Time limit should not be reached yet"); + + // Sleep for an additional 10 milliseconds (totaling 15 milliseconds) + sleep(Duration::new(0, 10_000_000)); // 10 milliseconds + assert!(limiter.is_time_limit_reached(), "Time limit should be reached now"); + } + + #[test] + fn test_is_limit_reached() { + // Create a PruneLimiter instance + let mut limiter = PruneLimiter::default(); + + // Test when no limits are set + assert!(!limiter.is_limit_reached(), "Limit should not be reached with no limits set"); + + // Set a deleted entries limit + limiter = limiter.set_deleted_entries_limit(5); + assert!( + !limiter.is_limit_reached(), + "Limit should not be reached when deleted entries are less than limit" + ); + + // Increment deleted entries count to reach the limit + limiter.increment_deleted_entries_count_by(5); + assert!( + limiter.is_limit_reached(), + "Limit should be reached when deleted entries equal the limit" + ); + + // Reset the limiter + limiter = PruneLimiter::default(); + + // Set a time limit and check + limiter = limiter.set_time_limit(Duration::new(0, 10_000_000)); // 10 milliseconds + + // Sleep for 5 milliseconds (less than the time limit) + sleep(Duration::new(0, 5_000_000)); // 5 milliseconds + assert!( + !limiter.is_limit_reached(), + "Limit should not be reached when time limit not reached" + ); + + // Sleep for another 10 milliseconds (totaling 15 milliseconds) + sleep(Duration::new(0, 10_000_000)); // 10 milliseconds + assert!(limiter.is_limit_reached(), "Limit should be reached when time limit is reached"); + } +} From 78426739036e7599184f49ccf486068e1429285c Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:06:28 +0200 Subject: [PATCH 047/103] feat(provider): add `test_race` to `BlockchainProvider2` tests (#11523) --- Cargo.lock | 1 + crates/storage/db/Cargo.toml | 4 +- crates/storage/db/src/lib.rs | 46 +++++- .../src/providers/blockchain_provider.rs | 148 +++++++++++++++++- 4 files changed, 188 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 590f01a416fd..a66887900cb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6702,6 +6702,7 @@ dependencies = [ "iai-callgrind", "metrics", "page_size", + "parking_lot 0.12.3", "paste", "pprof", "proptest", diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index ba012cf68af1..a075f7724637 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -50,6 +50,7 @@ derive_more.workspace = true paste.workspace = true rustc-hash = { workspace = true, optional = true } sysinfo = { version = "0.31", default-features = false, features = ["system"] } +parking_lot = { workspace = true, optional = true } # arbitrary utils strum = { workspace = true, features = ["derive"], optional = true } @@ -61,6 +62,7 @@ rand.workspace = true serde_json.workspace = true tempfile.workspace = true test-fuzz.workspace = true +parking_lot.workspace = true pprof = { workspace = true, features = [ "flamegraph", @@ -88,7 +90,7 @@ mdbx = [ "dep:strum", "dep:rustc-hash", ] -test-utils = ["dep:tempfile", "arbitrary"] +test-utils = ["dep:tempfile", "arbitrary", "parking_lot"] bench = [] arbitrary = ["reth-primitives/arbitrary", "reth-db-api/arbitrary"] optimism = [] diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index a9f073d7b546..7090b4262fd7 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -44,6 +44,7 @@ pub use reth_db_api::*; pub mod test_utils { use super::*; use crate::mdbx::DatabaseArguments; + use parking_lot::RwLock; use reth_db_api::{ database::Database, database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics}, @@ -52,6 +53,7 @@ pub mod test_utils { use reth_fs_util; use reth_libmdbx::MaxReadTransactionDuration; use std::{ + fmt::Formatter, path::{Path, PathBuf}, sync::Arc, }; @@ -69,10 +71,19 @@ pub mod test_utils { pub const ERROR_TEMPDIR: &str = "Not able to create a temporary directory."; /// A database will delete the db dir when dropped. - #[derive(Debug)] pub struct TempDatabase { db: Option, path: PathBuf, + /// Executed right before a database transaction is created. + pre_tx_hook: RwLock>, + /// Executed right after a database transaction is created. + post_tx_hook: RwLock>, + } + + impl std::fmt::Debug for TempDatabase { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TempDatabase").field("db", &self.db).field("path", &self.path).finish() + } } impl Drop for TempDatabase { @@ -85,6 +96,16 @@ pub mod test_utils { } impl TempDatabase { + /// Create new [`TempDatabase`] instance. + pub fn new(db: DB, path: PathBuf) -> Self { + Self { + db: Some(db), + path, + pre_tx_hook: RwLock::new(Box::new(|| ())), + post_tx_hook: RwLock::new(Box::new(|| ())), + } + } + /// Returns the reference to inner db. pub fn db(&self) -> &DB { self.db.as_ref().unwrap() @@ -99,13 +120,28 @@ pub mod test_utils { pub fn into_inner_db(mut self) -> DB { self.db.take().unwrap() // take out db to avoid clean path in drop fn } + + /// Sets [`TempDatabase`] new pre transaction creation hook. + pub fn set_pre_transaction_hook(&self, hook: Box) { + let mut db_hook = self.pre_tx_hook.write(); + *db_hook = hook; + } + + /// Sets [`TempDatabase`] new post transaction creation hook. + pub fn set_post_transaction_hook(&self, hook: Box) { + let mut db_hook = self.post_tx_hook.write(); + *db_hook = hook; + } } impl Database for TempDatabase { type TX = ::TX; type TXMut = ::TXMut; fn tx(&self) -> Result { - self.db().tx() + self.pre_tx_hook.read()(); + let tx = self.db().tx()?; + self.post_tx_hook.read()(); + Ok(tx) } fn tx_mut(&self) -> Result { @@ -150,7 +186,7 @@ pub mod test_utils { ) .expect(&emsg); - Arc::new(TempDatabase { db: Some(db), path }) + Arc::new(TempDatabase::new(db, path)) } /// Create read/write database for testing @@ -162,7 +198,7 @@ pub mod test_utils { .with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)), ) .expect(ERROR_DB_CREATION); - Arc::new(TempDatabase { db: Some(db), path }) + Arc::new(TempDatabase::new(db, path)) } /// Create read only database for testing @@ -175,7 +211,7 @@ pub mod test_utils { init_db(path.as_path(), args.clone()).expect(ERROR_DB_CREATION); } let db = open_db_read_only(path.as_path(), args).expect(ERROR_DB_OPEN); - Arc::new(TempDatabase { db: Some(db), path }) + Arc::new(TempDatabase::new(db, path)) } } diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 9a2a3d0a77ee..fdd31cd848f7 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1486,20 +1486,26 @@ mod tests { MockNodeTypesWithDB, }, writer::UnifiedStorageWriter, - BlockWriter, CanonChainTracker, StaticFileProviderFactory, StaticFileWriter, + BlockWriter, CanonChainTracker, ProviderFactory, StaticFileProviderFactory, + StaticFileWriter, }; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; - use alloy_primitives::B256; + use alloy_primitives::{BlockNumber, B256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ test_utils::TestBlockBuilder, CanonStateNotification, CanonStateSubscriptions, - ExecutedBlock, NewCanonicalChain, + CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain, }; use reth_chainspec::{ ChainSpec, ChainSpecBuilder, ChainSpecProvider, EthereumHardfork, MAINNET, }; - use reth_db::models::{AccountBeforeTx, StoredBlockBodyIndices}; + use reth_db::{ + models::{AccountBeforeTx, StoredBlockBodyIndices}, + tables, + }; + use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; + use reth_errors::ProviderError; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ Receipt, SealedBlock, StaticFileSegment, TransactionMeta, TransactionSignedNoHash, @@ -1586,9 +1592,27 @@ mod tests { let factory = create_test_provider_factory_with_chain_spec(chain_spec); let provider_rw = factory.database_provider_rw()?; + let static_file_provider = factory.static_file_provider(); + + // Write transactions to static files with the right `tx_num`` + let mut bodies_cursor = provider_rw.tx_ref().cursor_read::()?; + let mut tx_num = bodies_cursor + .seek_exact(database_blocks.first().as_ref().unwrap().number.saturating_sub(1))? + .map(|(_, indices)| indices.next_tx_num()) + .unwrap_or_default(); // Insert blocks into the database for block in &database_blocks { + // TODO: this should be moved inside `insert_historical_block`: + let mut transactions_writer = + static_file_provider.latest_writer(StaticFileSegment::Transactions)?; + transactions_writer.increment_block(block.number)?; + for tx in block.body.transactions() { + let tx: TransactionSignedNoHash = tx.clone().into(); + transactions_writer.append_transaction(tx_num, &tx)?; + tx_num += 1; + } + provider_rw.insert_historical_block( block.clone().seal_with_senders().expect("failed to seal block with senders"), )?; @@ -1602,7 +1626,9 @@ mod tests { .append_receipts_from_blocks( // The initial block number is required database_blocks.first().map(|b| b.number).unwrap_or_default(), - receipts.iter().map(|vec| vec.clone().into_iter().map(Some).collect::>()), + receipts[..database_blocks.len()] + .iter() + .map(|vec| vec.clone().into_iter().map(Some).collect::>()), )?; // Commit to both storages: database and static files @@ -1668,6 +1694,42 @@ mod tests { ) } + /// This will persist the last block in-memory and delete it from + /// `canonical_in_memory_state` right after a database read transaction is created. + /// + /// This simulates a RPC method having a different view than when its database transaction was + /// created. + fn persist_block_after_db_tx_creation( + provider: Arc>, + block_number: BlockNumber, + ) { + let hook_provider = provider.clone(); + provider.database.db_ref().set_post_transaction_hook(Box::new(move || { + if let Some(state) = hook_provider.canonical_in_memory_state.head_state() { + if state.anchor().number + 1 == block_number { + let mut lowest_memory_block = + state.parent_state_chain().last().expect("qed").block(); + let num_hash = lowest_memory_block.block().num_hash(); + + let mut execution_output = (*lowest_memory_block.execution_output).clone(); + execution_output.first_block = lowest_memory_block.block().number; + lowest_memory_block.execution_output = Arc::new(execution_output); + + // Push to disk + let provider_rw = hook_provider.database_provider_rw().unwrap(); + UnifiedStorageWriter::from(&provider_rw, &hook_provider.static_file_provider()) + .save_blocks(&[lowest_memory_block]) + .unwrap(); + UnifiedStorageWriter::commit(provider_rw, hook_provider.static_file_provider()) + .unwrap(); + + // Remove from memory + hook_provider.canonical_in_memory_state.remove_persisted_blocks(num_hash); + } + } + })); + } + #[test] fn test_block_reader_find_block_by_hash() -> eyre::Result<()> { // Initialize random number generator and provider factory @@ -3908,4 +3970,80 @@ mod tests { Ok(()) } + + #[test] + fn test_race() -> eyre::Result<()> { + let mut rng = generators::rng(); + let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT - 1, + TEST_BLOCKS_COUNT + 1, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + + let provider = Arc::new(provider); + + // Old implementation was querying the database first. This is problematic, if there are + // changes AFTER the database transaction is created. + let old_transaction_hash_fn = + |hash: B256, + canonical_in_memory_state: CanonicalInMemoryState, + factory: ProviderFactory| { + assert!(factory.transaction_by_hash(hash)?.is_none(), "should not be in database"); + Ok::<_, ProviderError>(canonical_in_memory_state.transaction_by_hash(hash)) + }; + + // Correct implementation queries in-memory first + let correct_transaction_hash_fn = + |hash: B256, + canonical_in_memory_state: CanonicalInMemoryState, + _factory: ProviderFactory| { + if let Some(tx) = canonical_in_memory_state.transaction_by_hash(hash) { + return Ok::<_, ProviderError>(Some(tx)) + } + panic!("should not be in database"); + // _factory.transaction_by_hash(hash) + }; + + // OLD BEHAVIOUR + { + // This will persist block 1 AFTER a database is created. Moving it from memory to + // storage. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + let to_be_persisted_tx = in_memory_blocks[0].body.transactions[0].clone(); + + // Even though the block exists, given the order of provider queries done in the method + // above, we do not see it. + assert_eq!( + old_transaction_hash_fn( + to_be_persisted_tx.hash(), + provider.canonical_in_memory_state(), + provider.database.clone() + ), + Ok(None) + ); + } + + // CORRECT BEHAVIOUR + { + // This will persist block 1 AFTER a database is created. Moving it from memory to + // storage. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[1].number); + let to_be_persisted_tx = in_memory_blocks[1].body.transactions[0].clone(); + + assert_eq!( + correct_transaction_hash_fn( + to_be_persisted_tx.hash(), + provider.canonical_in_memory_state(), + provider.database.clone() + ), + Ok(Some(to_be_persisted_tx)) + ); + } + + Ok(()) + } } From 4b12f3285a60dd4de435dd3894e475c96487a18b Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 7 Oct 2024 05:12:35 -0400 Subject: [PATCH 048/103] fix(tree): make state methods work for historical blocks (#11265) Co-authored-by: Roman Krasiuk Co-authored-by: Federico Gimenez --- .../src/providers/blockchain_provider.rs | 239 +++++++++++++++++- .../src/providers/database/provider.rs | 17 +- crates/storage/storage-api/src/storage.rs | 11 + 3 files changed, 258 insertions(+), 9 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index fdd31cd848f7..9d075015397f 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -14,21 +14,27 @@ use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, MemoryOverlayStateProvider, }; -use reth_chainspec::ChainInfo; +use reth_chainspec::{ChainInfo, EthereumHardforks}; +use reth_db::models::BlockNumberAddress; use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_evm::ConfigureEvmEnv; -use reth_execution_types::ExecutionOutcome; +use reth_execution_types::{BundleStateInit, ExecutionOutcome, RevertsInit}; use reth_node_types::NodeTypesWithDB; use reth_primitives::{ - Account, Block, BlockWithSenders, EthereumHardforks, Header, Receipt, SealedBlock, - SealedBlockWithSenders, SealedHeader, TransactionMeta, TransactionSigned, - TransactionSignedNoHash, Withdrawal, Withdrawals, + Account, Block, BlockWithSenders, Header, Receipt, SealedBlock, SealedBlockWithSenders, + SealedHeader, StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedNoHash, + Withdrawal, Withdrawals, }; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; +use reth_storage_api::StorageChangeSetReader; use reth_storage_errors::provider::ProviderResult; -use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; +use revm::{ + db::states::PlainStorageRevert, + primitives::{BlockEnv, CfgEnvWithHandlerCfg}, +}; use std::{ + collections::{hash_map, HashMap}, ops::{Add, Bound, RangeBounds, RangeInclusive, Sub}, sync::Arc, time::Instant, @@ -122,6 +128,145 @@ impl BlockchainProvider2 { (start, end) } + /// Return the last N blocks of state, recreating the [`ExecutionOutcome`]. + /// + /// If the range is empty, or there are no blocks for the given range, then this returns `None`. + pub fn get_state( + &self, + range: RangeInclusive, + ) -> ProviderResult> { + if range.is_empty() { + return Ok(None) + } + let start_block_number = *range.start(); + let end_block_number = *range.end(); + + // We are not removing block meta as it is used to get block changesets. + let mut block_bodies = Vec::new(); + for block_num in range.clone() { + let block_body = self + .block_body_indices(block_num)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(block_num))?; + block_bodies.push((block_num, block_body)) + } + + // get transaction receipts + let Some(from_transaction_num) = block_bodies.first().map(|body| body.1.first_tx_num()) + else { + return Ok(None) + }; + let Some(to_transaction_num) = block_bodies.last().map(|body| body.1.last_tx_num()) else { + return Ok(None) + }; + + let mut account_changeset = Vec::new(); + for block_num in range.clone() { + let changeset = + self.account_block_changeset(block_num)?.into_iter().map(|elem| (block_num, elem)); + account_changeset.extend(changeset); + } + + let mut storage_changeset = Vec::new(); + for block_num in range { + let changeset = self.storage_changeset(block_num)?; + storage_changeset.extend(changeset); + } + + let (state, reverts) = + self.populate_bundle_state(account_changeset, storage_changeset, end_block_number)?; + + let mut receipt_iter = + self.receipts_by_tx_range(from_transaction_num..=to_transaction_num)?.into_iter(); + + let mut receipts = Vec::with_capacity(block_bodies.len()); + // loop break if we are at the end of the blocks. + for (_, block_body) in block_bodies { + let mut block_receipts = Vec::with_capacity(block_body.tx_count as usize); + for tx_num in block_body.tx_num_range() { + let receipt = + receipt_iter.next().ok_or(ProviderError::ReceiptNotFound(tx_num.into()))?; + block_receipts.push(Some(receipt)); + } + receipts.push(block_receipts); + } + + Ok(Some(ExecutionOutcome::new_init( + state, + reverts, + // We skip new contracts since we never delete them from the database + Vec::new(), + receipts.into(), + start_block_number, + Vec::new(), + ))) + } + + /// Populate a [`BundleStateInit`] and [`RevertsInit`] using cursors over the + /// [`reth_db::PlainAccountState`] and [`reth_db::PlainStorageState`] tables, based on the given + /// storage and account changesets. + fn populate_bundle_state( + &self, + account_changeset: Vec<(u64, AccountBeforeTx)>, + storage_changeset: Vec<(BlockNumberAddress, StorageEntry)>, + block_range_end: BlockNumber, + ) -> ProviderResult<(BundleStateInit, RevertsInit)> { + let mut state: BundleStateInit = HashMap::new(); + let mut reverts: RevertsInit = HashMap::new(); + let state_provider = self.state_by_block_number_or_tag(block_range_end.into())?; + + // add account changeset changes + for (block_number, account_before) in account_changeset.into_iter().rev() { + let AccountBeforeTx { info: old_info, address } = account_before; + match state.entry(address) { + hash_map::Entry::Vacant(entry) => { + let new_info = state_provider.basic_account(address)?; + entry.insert((old_info, new_info, HashMap::new())); + } + hash_map::Entry::Occupied(mut entry) => { + // overwrite old account state. + entry.get_mut().0 = old_info; + } + } + // insert old info into reverts. + reverts.entry(block_number).or_default().entry(address).or_default().0 = Some(old_info); + } + + // add storage changeset changes + for (block_and_address, old_storage) in storage_changeset.into_iter().rev() { + let BlockNumberAddress((block_number, address)) = block_and_address; + // get account state or insert from plain state. + let account_state = match state.entry(address) { + hash_map::Entry::Vacant(entry) => { + let present_info = state_provider.basic_account(address)?; + entry.insert((present_info, present_info, HashMap::new())) + } + hash_map::Entry::Occupied(entry) => entry.into_mut(), + }; + + // match storage. + match account_state.2.entry(old_storage.key) { + hash_map::Entry::Vacant(entry) => { + let new_storage_value = + state_provider.storage(address, old_storage.key)?.unwrap_or_default(); + entry.insert((old_storage.value, new_storage_value)); + } + hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().0 = old_storage.value; + } + }; + + reverts + .entry(block_number) + .or_default() + .entry(address) + .or_default() + .1 + .push(old_storage); + } + + Ok((state, reverts)) + } + /// Fetches a range of data from both in-memory state and persistent storage while a predicate /// is met. /// @@ -1426,6 +1571,57 @@ impl ForkChoiceSubscriptions for BlockchainProvider2 { } } +impl StorageChangeSetReader for BlockchainProvider2 { + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + if let Some(state) = self.canonical_in_memory_state.state_by_number(block_number) { + let changesets = state + .block() + .execution_output + .bundle + .reverts + .clone() + .into_plain_state_reverts() + .storage + .into_iter() + .flatten() + .flat_map(|revert: PlainStorageRevert| { + revert.storage_revert.into_iter().map(move |(key, value)| { + ( + BlockNumberAddress((block_number, revert.address)), + StorageEntry { key: key.into(), value: value.to_previous_value() }, + ) + }) + }) + .collect(); + Ok(changesets) + } else { + // Perform checks on whether or not changesets exist for the block. + let provider = self.database.provider()?; + + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let storage_history_exists = provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !storage_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + provider.storage_changeset(block_number) + } + } +} + impl ChangeSetReader for BlockchainProvider2 { fn account_block_changeset( &self, @@ -1446,7 +1642,25 @@ impl ChangeSetReader for BlockchainProvider2 { .collect(); Ok(changesets) } else { - self.database.provider()?.account_block_changeset(block_number) + // Perform checks on whether or not changesets exist for the block. + let provider = self.database.provider()?; + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let account_history_exists = provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !account_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + provider.account_block_changeset(block_number) } } } @@ -1461,12 +1675,21 @@ impl AccountReader for BlockchainProvider2 { } impl StateReader for BlockchainProvider2 { + /// Re-constructs the [`ExecutionOutcome`] from in-memory and database state, if necessary. + /// + /// If data for the block does not exist, this will return [`None`]. + /// + /// NOTE: This cannot be called safely in a loop outside of the blockchain tree thread. This is + /// because the [`CanonicalInMemoryState`] could change during a reorg, causing results to be + /// inconsistent. Currently this can safely be called within the blockchain tree thread, + /// because the tree thread is responsible for modifying the [`CanonicalInMemoryState`] in the + /// first place. fn get_state(&self, block: BlockNumber) -> ProviderResult> { if let Some(state) = self.canonical_in_memory_state.state_by_number(block) { let state = state.block_ref().execution_outcome().clone(); Ok(Some(state)) } else { - self.database.provider()?.get_state(block..=block) + self.get_state(block..=block) } } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 8627cacabb4c..1afd4da3fa8c 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -46,7 +46,7 @@ use reth_primitives::{ }; use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; -use reth_storage_api::TryIntoHistoricalStateProvider; +use reth_storage_api::{StorageChangeSetReader, TryIntoHistoricalStateProvider}; use reth_storage_errors::provider::{ProviderResult, RootMismatch}; use reth_trie::{ prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets}, @@ -1414,6 +1414,21 @@ impl AccountExtReader for DatabaseProvider StorageChangeSetReader for DatabaseProvider { + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + let range = block_number..=block_number; + let storage_range = BlockNumberAddress::range(range); + self.tx + .cursor_dup_read::()? + .walk_range(storage_range)? + .map(|result| -> ProviderResult<_> { Ok(result?) }) + .collect() + } +} + impl ChangeSetReader for DatabaseProvider { fn account_block_changeset( &self, diff --git a/crates/storage/storage-api/src/storage.rs b/crates/storage/storage-api/src/storage.rs index 91d0bc8c7353..e1443347e4bb 100644 --- a/crates/storage/storage-api/src/storage.rs +++ b/crates/storage/storage-api/src/storage.rs @@ -1,4 +1,5 @@ use alloy_primitives::{Address, BlockNumber, B256}; +use reth_db_api::models::BlockNumberAddress; use reth_primitives::StorageEntry; use reth_storage_errors::provider::ProviderResult; use std::{ @@ -30,3 +31,13 @@ pub trait StorageReader: Send + Sync { range: RangeInclusive, ) -> ProviderResult>>; } + +/// Storage ChangeSet reader +#[auto_impl::auto_impl(&, Arc, Box)] +pub trait StorageChangeSetReader: Send + Sync { + /// Iterate over storage changesets and return the storage state from before this block. + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult>; +} From 74a645339c70631023e31e4048affca1b60d8bfa Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:41:30 +0200 Subject: [PATCH 049/103] rpc: use `eth_api()` method (#11516) --- crates/rpc/rpc/src/debug.rs | 45 +++++++++++++------------------- crates/rpc/rpc/src/eth/bundle.rs | 16 +++++++----- crates/rpc/rpc/src/trace.rs | 30 +++++++++------------ 3 files changed, 40 insertions(+), 51 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index f4aba778f572..a83ba68b42f5 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -208,8 +208,8 @@ where .ok_or(EthApiError::HeaderNotFound(block_id))?; let ((cfg, block_env, _), block) = futures::try_join!( - self.inner.eth_api.evm_env_at(block_hash.into()), - self.inner.eth_api.block_with_senders(block_id), + self.eth_api().evm_env_at(block_hash.into()), + self.eth_api().block_with_senders(block_id), )?; let block = block.ok_or(EthApiError::HeaderNotFound(block_id))?; @@ -235,11 +235,11 @@ where tx_hash: B256, opts: GethDebugTracingOptions, ) -> Result { - let (transaction, block) = match self.inner.eth_api.transaction_and_block(tx_hash).await? { + let (transaction, block) = match self.eth_api().transaction_and_block(tx_hash).await? { None => return Err(EthApiError::TransactionNotFound.into()), Some(res) => res, }; - let (cfg, block_env, _) = self.inner.eth_api.evm_env_at(block.hash().into()).await?; + let (cfg, block_env, _) = self.eth_api().evm_env_at(block.hash().into()).await?; // we need to get the state of the parent block because we're essentially replaying the // block the transaction is included in @@ -248,8 +248,7 @@ where let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(state_at, move |state| { // configure env for the target transaction let tx = transaction.into_recovered(); @@ -312,8 +311,7 @@ where GethDebugBuiltInTracerType::FourByteTracer => { let mut inspector = FourByteInspector::default(); let inspector = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { this.eth_api().inspect(db, env, &mut inspector)?; Ok(inspector) @@ -331,8 +329,7 @@ where ); let frame = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { let (res, env) = this.eth_api().inspect(db, env, &mut inspector)?; let frame = inspector @@ -353,8 +350,7 @@ where ); let frame = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { // wrapper is hack to get around 'higher-ranked lifetime error', // see @@ -434,11 +430,10 @@ where GethDebugTracerType::JsTracer(code) => { let config = tracer_config.into_json(); - let (_, _, at) = self.inner.eth_api.evm_env_at(at).await?; + let (_, _, at) = self.eth_api().evm_env_at(at).await?; let res = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { // wrapper is hack to get around 'higher-ranked lifetime error', see // @@ -464,8 +459,7 @@ where let mut inspector = TracingInspector::new(inspector_config); let (res, tx_gas_limit, inspector) = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { let (res, env) = this.eth_api().inspect(db, env, &mut inspector)?; Ok((res, env.tx.gas_limit, inspector)) @@ -499,8 +493,8 @@ where let target_block = block_number.unwrap_or_default(); let ((cfg, mut block_env, _), block) = futures::try_join!( - self.inner.eth_api.evm_env_at(target_block), - self.inner.eth_api.block_with_senders(target_block), + self.eth_api().evm_env_at(target_block), + self.eth_api().block_with_senders(target_block), )?; let opts = opts.unwrap_or_default(); @@ -524,8 +518,7 @@ where let this = self.clone(); - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(at.into(), move |state| { // the outer vec for the bundles let mut all_bundles = Vec::with_capacity(bundles.len()); @@ -546,7 +539,7 @@ where ), handler_cfg: cfg.handler_cfg, }; - let (res, _) = this.inner.eth_api.transact(&mut db, env)?; + let (res, _) = this.eth_api().transact(&mut db, env)?; db.commit(res.state); } } @@ -604,14 +597,12 @@ where ) -> Result { let this = self.clone(); let block = this - .inner - .eth_api + .eth_api() .block_with_senders(block_id.into()) .await? .ok_or(EthApiError::HeaderNotFound(block_id.into()))?; - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(block.parent_hash.into(), move |state_provider| { let db = StateProviderDatabase::new(&state_provider); let block_executor = this.inner.block_executor.executor(db); @@ -868,7 +859,7 @@ where /// /// Returns the bytes of the transaction for the given hash. async fn raw_transaction(&self, hash: B256) -> RpcResult> { - self.inner.eth_api.raw_transaction_by_hash(hash).await.map_err(Into::into) + self.eth_api().raw_transaction_by_hash(hash).await.map_err(Into::into) } /// Handler for `debug_getRawTransactions` diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index bede4599e1a1..e97497786ede 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -37,6 +37,11 @@ impl EthBundle { pub fn new(eth_api: Eth, blocking_task_guard: BlockingTaskGuard) -> Self { Self { inner: Arc::new(EthBundleInner { eth_api, blocking_task_guard }) } } + + /// Access the underlying `Eth` API. + pub fn eth_api(&self) -> &Eth { + &self.inner.eth_api + } } impl EthBundle @@ -103,7 +108,7 @@ where let block_id: alloy_rpc_types::BlockId = state_block_number.into(); // Note: the block number is considered the `parent` block: - let (cfg, mut block_env, at) = self.inner.eth_api.evm_env_at(block_id).await?; + let (cfg, mut block_env, at) = self.eth_api().evm_env_at(block_id).await?; // need to adjust the timestamp for the next block if let Some(timestamp) = timestamp { @@ -125,12 +130,12 @@ where } else if cfg.handler_cfg.spec_id.is_enabled_in(SpecId::LONDON) { let parent_block = block_env.number.saturating_to::(); // here we need to fetch the _next_ block's basefee based on the parent block - let parent = LoadPendingBlock::provider(&self.inner.eth_api) + let parent = LoadPendingBlock::provider(self.eth_api()) .header_by_number(parent_block) .map_err(Eth::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(parent_block.into()))?; if let Some(base_fee) = parent.next_block_base_fee( - LoadPendingBlock::provider(&self.inner.eth_api) + LoadPendingBlock::provider(self.eth_api()) .chain_spec() .base_fee_params_at_block(parent_block), ) { @@ -142,10 +147,9 @@ where // use the block number of the request block_env.number = U256::from(block_number); - let eth_api = self.inner.eth_api.clone(); + let eth_api = self.eth_api().clone(); - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(at, move |state| { let coinbase = block_env.coinbase; let basefee = Some(block_env.basefee.to::()); diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index e905ff685c86..687762b74b5e 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -120,7 +120,7 @@ where ) -> Result { let tx = recover_raw_transaction(tx)?; - let (cfg, block, at) = self.inner.eth_api.evm_env_at(block_id.unwrap_or_default()).await?; + let (cfg, block, at) = self.eth_api().evm_env_at(block_id.unwrap_or_default()).await?; let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, @@ -130,8 +130,7 @@ where let config = TracingInspectorConfig::from_parity_config(&trace_types); - self.inner - .eth_api + self.eth_api() .spawn_trace_at_with_state(env, config, at, move |inspector, res, db| { inspector .into_parity_builder() @@ -151,7 +150,7 @@ where block_id: Option, ) -> Result, Eth::Error> { let at = block_id.unwrap_or(BlockId::pending()); - let (cfg, block_env, at) = self.inner.eth_api.evm_env_at(at).await?; + let (cfg, block_env, at) = self.eth_api().evm_env_at(at).await?; let this = self.clone(); // execute all transactions on top of each other and record the traces @@ -202,8 +201,7 @@ where trace_types: HashSet, ) -> Result { let config = TracingInspectorConfig::from_parity_config(&trace_types); - self.inner - .eth_api + self.eth_api() .spawn_trace_transaction_in_block(hash, config, move |_, inspector, res, db| { let trace_res = inspector .into_parity_builder() @@ -285,7 +283,7 @@ where let mut block_traces = Vec::with_capacity(blocks.len()); for block in &blocks { let matcher = matcher.clone(); - let traces = self.inner.eth_api.trace_block_until( + let traces = self.eth_api().trace_block_until( block.number.into(), None, TracingInspectorConfig::default_parity(), @@ -345,8 +343,7 @@ where &self, hash: B256, ) -> Result>, Eth::Error> { - self.inner - .eth_api + self.eth_api() .spawn_trace_transaction_in_block( hash, TracingInspectorConfig::default_parity(), @@ -364,7 +361,7 @@ where &self, block_id: BlockId, ) -> Result>, Eth::Error> { - let traces = self.inner.eth_api.trace_block_with( + let traces = self.eth_api().trace_block_with( block_id, TracingInspectorConfig::default_parity(), |tx_info, inspector, _, _, _| { @@ -374,7 +371,7 @@ where }, ); - let block = self.inner.eth_api.block(block_id); + let block = self.eth_api().block(block_id); let (maybe_traces, maybe_block) = futures::try_join!(traces, block)?; let mut maybe_traces = @@ -399,8 +396,7 @@ where block_id: BlockId, trace_types: HashSet, ) -> Result>, Eth::Error> { - self.inner - .eth_api + self.eth_api() .trace_block_with( block_id, TracingInspectorConfig::from_parity_config(&trace_types), @@ -431,8 +427,7 @@ where &self, tx_hash: B256, ) -> Result, Eth::Error> { - self.inner - .eth_api + self.eth_api() .spawn_trace_transaction_in_block_with_inspector( tx_hash, OpcodeGasInspector::default(), @@ -456,8 +451,7 @@ where block_id: BlockId, ) -> Result, Eth::Error> { let res = self - .inner - .eth_api + .eth_api() .trace_block_inspector( block_id, OpcodeGasInspector::default, @@ -473,7 +467,7 @@ where let Some(transactions) = res else { return Ok(None) }; - let Some(block) = self.inner.eth_api.block(block_id).await? else { return Ok(None) }; + let Some(block) = self.eth_api().block(block_id).await? else { return Ok(None) }; Ok(Some(BlockOpcodeGas { block_hash: block.hash(), From f5c9701e72ca4b51555f9aa7f87668821bef3902 Mon Sep 17 00:00:00 2001 From: crazykissshout Date: Mon, 7 Oct 2024 13:25:24 +0200 Subject: [PATCH 050/103] chore: delete rpc-types (#11528) --- docs/repo/layout.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 6ba14e652bf1..8eb0782b988c 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -132,7 +132,6 @@ The IPC transport lives in [`rpc/ipc`](../../crates/rpc/ipc). - [`rpc/rpc-api`](../../crates/rpc/rpc-api): RPC traits - Supported transports: HTTP, WS, IPC - Supported namespaces: `eth_`, `engine_`, `debug_` -- [`rpc/rpc-types`](../../crates/rpc/rpc-types): Types relevant for the RPC endpoints above, grouped by namespace - [`rpc/rpc-eth-api`](../../crates/rpc/rpc-eth-api/): Reth RPC 'eth' namespace API (including interface and implementation), this crate is re-exported by `rpc/rpc-api` - [`rpc/rpc-eth-types`](../../crates/rpc/rpc-eth-types/): Types `supporting implementation` of 'eth' namespace RPC server API - [`rpc/rpc-server-types`](../../crates/rpc/rpc-server-types/): RPC server types and constants From f8228482acd27d639f3fbd2b3b27be9c106e5d60 Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:42:56 -0400 Subject: [PATCH 051/103] feat: add get_highest_tx_by_sender to pools (#11514) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 30 ++++++++++----------- crates/transaction-pool/src/lib.rs | 7 +++++ crates/transaction-pool/src/noop.rs | 7 +++++ crates/transaction-pool/src/pool/mod.rs | 9 +++++++ crates/transaction-pool/src/traits.rs | 6 +++++ 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index dee4e5895638..4c77aec1ae15 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -13,7 +13,7 @@ use reth_provider::{ }; use reth_rpc_eth_types::{EthApiError, EthStateCache, PendingBlockEnv, RpcInvalidTransactionError}; use reth_rpc_types_compat::proof::from_primitive_account_proof; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; +use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, SpecId}; use crate::{EthApiTypes, FromEthApiError}; @@ -280,23 +280,23 @@ pub trait LoadState: EthApiTypes { if block_id == Some(BlockId::pending()) { // for pending tag we need to find the highest nonce in the pool - let address_txs = this.pool().get_transactions_by_sender(address); - if let Some(highest_pool_nonce) = - address_txs.iter().map(|item| item.transaction.nonce()).max() + if let Some(highest_pool_tx) = + this.pool().get_highest_transaction_by_sender(address) { - // and the corresponding txcount is nonce + 1 - let next_nonce = - nonce.max(highest_pool_nonce).checked_add(1).ok_or_else(|| { - Self::Error::from(EthApiError::InvalidTransaction( - RpcInvalidTransactionError::NonceMaxValue, - )) - })?; - - let tx_count = nonce.max(next_nonce); - return Ok(U256::from(tx_count)) + { + // and the corresponding txcount is nonce + 1 + let next_nonce = + nonce.max(highest_pool_tx.nonce()).checked_add(1).ok_or_else(|| { + Self::Error::from(EthApiError::InvalidTransaction( + RpcInvalidTransactionError::NonceMaxValue, + )) + })?; + + let tx_count = nonce.max(next_nonce); + return Ok(U256::from(tx_count)); + } } } - Ok(U256::from(nonce)) }) } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 744dd44d2f83..a5acd6edba5a 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -489,6 +489,13 @@ where self.pool.get_transactions_by_sender(sender) } + fn get_highest_transaction_by_sender( + &self, + sender: Address, + ) -> Option>> { + self.pool.get_highest_transaction_by_sender(sender) + } + fn get_transaction_by_sender_and_nonce( &self, sender: Address, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 0c4caa573140..ddab4f622741 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -206,6 +206,13 @@ impl TransactionPool for NoopTransactionPool { vec![] } + fn get_highest_transaction_by_sender( + &self, + _sender: Address, + ) -> Option>> { + None + } + fn get_transaction_by_sender_and_nonce( &self, _sender: Address, diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index b78f5128686c..090b92fb6594 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -744,6 +744,15 @@ where self.get_pool_data().get_transactions_by_sender(sender_id) } + /// Returns the highest transaction of the address + pub(crate) fn get_highest_transaction_by_sender( + &self, + sender: Address, + ) -> Option>> { + let sender_id = self.get_sender_id(sender); + self.get_pool_data().get_highest_transaction_by_sender(sender_id) + } + /// Returns all transactions that where submitted with the given [`TransactionOrigin`] pub(crate) fn get_transactions_by_origin( &self, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index f44e66e457a1..1598877ff3b6 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -334,6 +334,12 @@ pub trait TransactionPool: Send + Sync + Clone { sender: Address, ) -> Vec>>; + /// Returns the highest transaction sent by a given user + fn get_highest_transaction_by_sender( + &self, + sender: Address, + ) -> Option>>; + /// Returns a transaction sent by a given user and a nonce fn get_transaction_by_sender_and_nonce( &self, From 6a1ec0d1a20436a79aee612c68749abc3411ec6b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:52:01 +0200 Subject: [PATCH 052/103] ci: add `windows` cargo check (#11468) --- .github/workflows/windows.yml | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 000000000000..03c491b368a6 --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,47 @@ +# Windows build + +name: windows + +on: + push: + branches: [main] + pull_request: + branches: [main] + merge_group: + +jobs: + check-reth: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + target: x86_64-pc-windows-gnu + - uses: taiki-e/install-action@cross + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: mingw-w64 + run: sudo apt-get install -y mingw-w64 + - name: Check Reth + run: cargo check --target x86_64-pc-windows-gnu + + check-op-reth: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + target: x86_64-pc-windows-gnu + - uses: taiki-e/install-action@cross + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: mingw-w64 + run: sudo apt-get install -y mingw-w64 + - name: Check OP-Reth + run: cargo check -p op-reth --features optimism --target x86_64-pc-windows-gnu From e94d72b7bb3cf5e32a80fd7ed379de762359754e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 16:10:39 +0200 Subject: [PATCH 053/103] fix: acquire permit first (#11537) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 4c77aec1ae15..0008454c93a4 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -92,25 +92,26 @@ pub trait EthState: LoadState + SpawnBlocking { where Self: EthApiSpec, { - let chain_info = self.chain_info().map_err(Self::Error::from_eth_err)?; - let block_id = block_id.unwrap_or_default(); - - // Check whether the distance to the block exceeds the maximum configured window. - let block_number = LoadState::provider(self) - .block_number_for_id(block_id) - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(block_id))?; - let max_window = self.max_proof_window(); - if chain_info.best_number.saturating_sub(block_number) > max_window { - return Err(EthApiError::ExceedsMaxProofWindow.into()) - } - Ok(async move { let _permit = self .acquire_owned() .await .map_err(RethError::other) .map_err(EthApiError::Internal)?; + + let chain_info = self.chain_info().map_err(Self::Error::from_eth_err)?; + let block_id = block_id.unwrap_or_default(); + + // Check whether the distance to the block exceeds the maximum configured window. + let block_number = LoadState::provider(self) + .block_number_for_id(block_id) + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(block_id))?; + let max_window = self.max_proof_window(); + if chain_info.best_number.saturating_sub(block_number) > max_window { + return Err(EthApiError::ExceedsMaxProofWindow.into()) + } + self.spawn_blocking_io(move |this| { let state = this.state_at_block_id(block_id)?; let storage_keys = keys.iter().map(|key| key.0).collect::>(); From d4e9dd10e4924256b43d216525e1d2b90c967135 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 16:47:52 +0200 Subject: [PATCH 054/103] chore: dont fail on ttd (#11539) --- crates/rpc/rpc-eth-api/src/helpers/block.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 3a26536cc7cf..d7e081d80f80 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -6,7 +6,7 @@ use alloy_rpc_types::{Header, Index}; use futures::Future; use reth_primitives::{BlockId, Receipt, SealedBlock, SealedBlockWithSenders}; use reth_provider::{BlockIdReader, BlockReader, BlockReaderIdExt, HeaderProvider}; -use reth_rpc_eth_types::{EthApiError, EthStateCache}; +use reth_rpc_eth_types::EthStateCache; use reth_rpc_types_compat::block::{from_block, uncle_block_from_header}; use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; @@ -47,13 +47,20 @@ pub trait EthBlocks: LoadBlock { async move { let Some(block) = self.block_with_senders(block_id).await? else { return Ok(None) }; let block_hash = block.hash(); - let total_difficulty = EthBlocks::provider(self) + let mut total_difficulty = EthBlocks::provider(self) .header_td_by_number(block.number) - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(block_id))?; + .map_err(Self::Error::from_eth_err)?; + if total_difficulty.is_none() { + // if we failed to find td after we successfully loaded the block, try again using + // the hash this only matters if the chain is currently transitioning the merge block and there's a reorg: + total_difficulty = EthBlocks::provider(self) + .header_td(&block.hash()) + .map_err(Self::Error::from_eth_err)?; + } + let block = from_block::( block.unseal(), - total_difficulty, + total_difficulty.unwrap_or_default(), full.into(), Some(block_hash), ) From 04fa08cd4c3f5cf04bef50153e5a67a3640202fe Mon Sep 17 00:00:00 2001 From: David <39963997+No0key@users.noreply.github.com> Date: Mon, 7 Oct 2024 18:12:52 +0300 Subject: [PATCH 055/103] grafana: add metrics of all transactions in pool by type (#11515) Co-authored-by: Emilia Hane Co-authored-by: Emilia Hane --- etc/grafana/dashboards/reth-mempool.json | 172 ++++++++++++++++++++++- 1 file changed, 171 insertions(+), 1 deletion(-) diff --git a/etc/grafana/dashboards/reth-mempool.json b/etc/grafana/dashboards/reth-mempool.json index 41be1dc411eb..9e3f5ae81930 100644 --- a/etc/grafana/dashboards/reth-mempool.json +++ b/etc/grafana/dashboards/reth-mempool.json @@ -2679,6 +2679,176 @@ "title": "Fetch Hashes Pending Fetch Duration", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of all transactions of all sub-pools by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 218, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Legacy", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-2930", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-1559", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-4844", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-7702", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Transactions by Type in Pool", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -3524,6 +3694,6 @@ "timezone": "", "title": "Reth - Transaction Pool", "uid": "bee34f59-c79c-4669-a000-198057b3703d", - "version": 3, + "version": 4, "weekStart": "" } \ No newline at end of file From 8a0bcbb35620319574b7ea01312fb2cccd4c3265 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 7 Oct 2024 18:31:15 +0300 Subject: [PATCH 056/103] feat(exex): subscribe to notifications with head using `ExExContext` (#11500) --- crates/exex/exex/src/context.rs | 15 +- crates/exex/exex/src/manager.rs | 119 +++++++++----- crates/exex/exex/src/notifications.rs | 221 +++++++++++++++++--------- 3 files changed, 236 insertions(+), 119 deletions(-) diff --git a/crates/exex/exex/src/context.rs b/crates/exex/exex/src/context.rs index bdb2d4c27b99..9af12e260a72 100644 --- a/crates/exex/exex/src/context.rs +++ b/crates/exex/exex/src/context.rs @@ -1,5 +1,6 @@ use std::fmt::Debug; +use reth_exex_types::ExExHead; use reth_node_api::{FullNodeComponents, NodeTypes, NodeTypesWithEngine}; use reth_node_core::node_config::NodeConfig; use reth_primitives::Head; @@ -32,7 +33,7 @@ pub struct ExExContext { /// considered delivered by the node. pub notifications: ExExNotifications, - /// node components + /// Node components pub components: Node, } @@ -92,4 +93,16 @@ impl ExExContext { pub fn task_executor(&self) -> &TaskExecutor { self.components.task_executor() } + + /// Sets notifications stream to [`crate::ExExNotificationsWithoutHead`], a stream of + /// notifications without a head. + pub fn set_notifications_without_head(&mut self) { + self.notifications.set_without_head(); + } + + /// Sets notifications stream to [`crate::ExExNotificationsWithHead`], a stream of notifications + /// with the provided head. + pub fn set_notifications_with_head(&mut self, head: ExExHead) { + self.notifications.set_with_head(head); + } } diff --git a/crates/exex/exex/src/manager.rs b/crates/exex/exex/src/manager.rs index e8e24c09db02..31dc822222b6 100644 --- a/crates/exex/exex/src/manager.rs +++ b/crates/exex/exex/src/manager.rs @@ -610,12 +610,16 @@ impl Clone for ExExManagerHandle { mod tests { use super::*; use alloy_primitives::B256; - use eyre::OptionExt; - use futures::{FutureExt, StreamExt}; + use futures::StreamExt; use rand::Rng; + use reth_db_common::init::init_genesis; + use reth_evm_ethereum::execute::EthExecutorProvider; use reth_primitives::SealedBlockWithSenders; - use reth_provider::{test_utils::create_test_provider_factory, BlockWriter, Chain}; - use reth_testing_utils::generators::{self, random_block}; + use reth_provider::{ + providers::BlockchainProvider2, test_utils::create_test_provider_factory, BlockReader, + Chain, TransactionVariant, + }; + use reth_testing_utils::generators; fn empty_finalized_header_stream() -> ForkChoiceStream { let (tx, rx) = watch::channel(None); @@ -975,11 +979,20 @@ mod tests { #[tokio::test] async fn exex_handle_new() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); // Check initial state assert_eq!(exex_handle.id, "test_exex"); @@ -1008,7 +1021,7 @@ mod tests { // Send a notification and ensure it's received correctly match exex_handle.send(&mut cx, &(22, notification.clone())) { Poll::Ready(Ok(())) => { - let received_notification = notifications.next().await.unwrap(); + let received_notification = notifications.next().await.unwrap().unwrap(); assert_eq!(received_notification, notification); } Poll::Pending => panic!("Notification send is pending"), @@ -1021,11 +1034,20 @@ mod tests { #[tokio::test] async fn test_notification_if_finished_height_gt_chain_tip() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); // Set finished_height to a value higher than the block tip exex_handle.finished_height = Some(BlockNumHash::new(15, B256::random())); @@ -1046,11 +1068,7 @@ mod tests { poll_fn(|cx| { // The notification should be skipped, so nothing should be sent. // Check that the receiver channel is indeed empty - assert_eq!( - notifications.poll_next_unpin(cx), - Poll::Pending, - "Receiver channel should be empty" - ); + assert!(notifications.poll_next_unpin(cx).is_pending()); Poll::Ready(()) }) .await; @@ -1066,11 +1084,20 @@ mod tests { #[tokio::test] async fn test_sends_chain_reorged_notification() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); let notification = ExExNotification::ChainReorged { old: Arc::new(Chain::default()), @@ -1086,7 +1113,7 @@ mod tests { // Send the notification match exex_handle.send(&mut cx, &(22, notification.clone())) { Poll::Ready(Ok(())) => { - let received_notification = notifications.next().await.unwrap(); + let received_notification = notifications.next().await.unwrap().unwrap(); assert_eq!(received_notification, notification); } Poll::Pending | Poll::Ready(Err(_)) => { @@ -1100,11 +1127,20 @@ mod tests { #[tokio::test] async fn test_sends_chain_reverted_notification() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); let notification = ExExNotification::ChainReverted { old: Arc::new(Chain::default()) }; @@ -1117,7 +1153,7 @@ mod tests { // Send the notification match exex_handle.send(&mut cx, &(22, notification.clone())) { Poll::Ready(Ok(())) => { - let received_notification = notifications.next().await.unwrap(); + let received_notification = notifications.next().await.unwrap().unwrap(); assert_eq!(received_notification, notification); } Poll::Pending | Poll::Ready(Err(_)) => { @@ -1135,30 +1171,34 @@ mod tests { let mut rng = generators::rng(); + let provider_factory = create_test_provider_factory(); + let genesis_hash = init_genesis(&provider_factory).unwrap(); + let genesis_block = provider_factory + .sealed_block_with_senders(genesis_hash.into(), TransactionVariant::NoHash) + .unwrap() + .ok_or_else(|| eyre::eyre!("genesis block not found"))?; + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let provider_factory = create_test_provider_factory(); - - let block = random_block(&mut rng, 0, Default::default()) - .seal_with_senders() - .ok_or_eyre("failed to recover senders")?; - let provider_rw = provider_factory.provider_rw()?; - provider_rw.insert_block(block.clone())?; - provider_rw.commit()?; + let (exex_handle, events_tx, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider.clone(), + EthExecutorProvider::mainnet(), + wal.handle(), + ); let notification = ExExNotification::ChainCommitted { - new: Arc::new(Chain::new(vec![block.clone()], Default::default(), None)), + new: Arc::new(Chain::new(vec![genesis_block.clone()], Default::default(), None)), }; let (finalized_headers_tx, rx) = watch::channel(None); let finalized_header_stream = ForkChoiceStream::new(rx); - let (exex_handle, events_tx, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); - let mut exex_manager = std::pin::pin!(ExExManager::new( - provider_factory, + provider, vec![exex_handle], 1, wal, @@ -1170,16 +1210,13 @@ mod tests { exex_manager.handle().send(notification.clone())?; assert!(exex_manager.as_mut().poll(&mut cx)?.is_pending()); - assert_eq!( - notifications.next().poll_unpin(&mut cx), - Poll::Ready(Some(notification.clone())) - ); + assert_eq!(notifications.next().await.unwrap().unwrap(), notification.clone()); assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, [notification.clone()] ); - finalized_headers_tx.send(Some(block.header.clone()))?; + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx didn't emit the `FinishedHeight` event assert_eq!( @@ -1192,7 +1229,7 @@ mod tests { .send(ExExEvent::FinishedHeight((rng.gen::(), rng.gen::()).into())) .unwrap(); - finalized_headers_tx.send(Some(block.header.clone()))?; + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx emitted a `FinishedHeight` event with a // non-canonical block @@ -1202,9 +1239,9 @@ mod tests { ); // Send a `FinishedHeight` event with a canonical block - events_tx.send(ExExEvent::FinishedHeight(block.num_hash())).unwrap(); + events_tx.send(ExExEvent::FinishedHeight(genesis_block.num_hash())).unwrap(); - finalized_headers_tx.send(Some(block.header.clone()))?; + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL is finalized assert!(exex_manager.wal.iter_notifications()?.next().is_none()); diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 116dac95422b..6efdb1775cf1 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -13,27 +13,28 @@ use std::{ }; use tokio::sync::mpsc::Receiver; -/// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. +/// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. If the +/// stream is configured with a head via [`ExExNotifications::set_with_head`] or +/// [`ExExNotifications::with_head`], it will run backfill jobs to catch up to the node head. +#[derive(Debug)] pub struct ExExNotifications { - node_head: Head, - provider: P, - executor: E, - notifications: Receiver, - wal_handle: WalHandle, + inner: ExExNotificationsInner, } -impl Debug for ExExNotifications { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ExExNotifications") - .field("provider", &self.provider) - .field("executor", &self.executor) - .field("notifications", &self.notifications) - .finish() - } +#[derive(Debug)] +enum ExExNotificationsInner { + /// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. + WithoutHead(ExExNotificationsWithoutHead), + /// A stream of [`ExExNotification`]s. The stream will only emit notifications for blocks that + /// are committed or reverted after the given head. + WithHead(ExExNotificationsWithHead), + /// Internal state used when transitioning between [`ExExNotificationsInner::WithoutHead`] and + /// [`ExExNotificationsInner::WithHead`]. + Invalid, } impl ExExNotifications { - /// Creates a new instance of [`ExExNotifications`]. + /// Creates a new stream of [`ExExNotifications`] without a head. pub const fn new( node_head: Head, provider: P, @@ -41,73 +42,131 @@ impl ExExNotifications { notifications: Receiver, wal_handle: WalHandle, ) -> Self { - Self { node_head, provider, executor, notifications, wal_handle } + Self { + inner: ExExNotificationsInner::WithoutHead(ExExNotificationsWithoutHead::new( + node_head, + provider, + executor, + notifications, + wal_handle, + )), + } } - /// Receives the next value for this receiver. + /// Sets [`ExExNotifications`] to a stream of [`ExExNotification`]s without a head. /// - /// This method returns `None` if the channel has been closed and there are - /// no remaining messages in the channel's buffer. This indicates that no - /// further values can ever be received from this `Receiver`. The channel is - /// closed when all senders have been dropped, or when [`Receiver::close`] is called. + /// It's a no-op if the stream has already been configured without a head. /// - /// # Cancel safety - /// - /// This method is cancel safe. If `recv` is used as the event in a - /// [`tokio::select!`] statement and some other branch - /// completes first, it is guaranteed that no messages were received on this - /// channel. - /// - /// For full documentation, see [`Receiver::recv`]. - #[deprecated(note = "use `ExExNotifications::next` and its `Stream` implementation instead")] - pub async fn recv(&mut self) -> Option { - self.notifications.recv().await + /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. + pub fn set_without_head(&mut self) { + let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); + self.inner = ExExNotificationsInner::WithoutHead(match current { + ExExNotificationsInner::WithoutHead(notifications) => notifications, + ExExNotificationsInner::WithHead(notifications) => ExExNotificationsWithoutHead::new( + notifications.node_head, + notifications.provider, + notifications.executor, + notifications.notifications, + notifications.wal_handle, + ), + ExExNotificationsInner::Invalid => unreachable!(), + }); } - /// Polls to receive the next message on this channel. + /// Returns a new [`ExExNotifications`] without a head. /// - /// This method returns: - /// - /// * `Poll::Pending` if no messages are available but the channel is not closed, or if a - /// spurious failure happens. - /// * `Poll::Ready(Some(message))` if a message is available. - /// * `Poll::Ready(None)` if the channel has been closed and all messages sent before it was - /// closed have been received. + /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. + pub fn without_head(mut self) -> Self { + self.set_without_head(); + self + } + + /// Sets [`ExExNotifications`] to a stream of [`ExExNotification`]s with the provided head. /// - /// When the method returns `Poll::Pending`, the `Waker` in the provided - /// `Context` is scheduled to receive a wakeup when a message is sent on any - /// receiver, or when the channel is closed. Note that on multiple calls to - /// `poll_recv` or `poll_recv_many`, only the `Waker` from the `Context` - /// passed to the most recent call is scheduled to receive a wakeup. + /// It's a no-op if the stream has already been configured with a head. /// - /// If this method returns `Poll::Pending` due to a spurious failure, then - /// the `Waker` will be notified when the situation causing the spurious - /// failure has been resolved. Note that receiving such a wakeup does not - /// guarantee that the next call will succeed — it could fail with another - /// spurious failure. + /// See the documentation of [`ExExNotificationsWithHead`] for more details. + pub fn set_with_head(&mut self, exex_head: ExExHead) { + let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); + self.inner = ExExNotificationsInner::WithHead(match current { + ExExNotificationsInner::WithoutHead(notifications) => { + notifications.with_head(exex_head) + } + ExExNotificationsInner::WithHead(notifications) => ExExNotificationsWithHead::new( + notifications.node_head, + notifications.provider, + notifications.executor, + notifications.notifications, + notifications.wal_handle, + exex_head, + ), + ExExNotificationsInner::Invalid => unreachable!(), + }); + } + + /// Returns a new [`ExExNotifications`] with the provided head. /// - /// For full documentation, see [`Receiver::poll_recv`]. - #[deprecated( - note = "use `ExExNotifications::poll_next` and its `Stream` implementation instead" - )] - pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { - self.notifications.poll_recv(cx) + /// See the documentation of [`ExExNotificationsWithHead`] for more details. + pub fn with_head(mut self, exex_head: ExExHead) -> Self { + self.set_with_head(exex_head); + self } } -impl ExExNotifications +impl Stream for ExExNotifications where P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, E: BlockExecutorProvider + Clone + Unpin + 'static, { - /// Subscribe to notifications with the given head. This head is the ExEx's - /// latest view of the host chain. - /// - /// Notifications will be sent starting from the head, not inclusive. For - /// example, if `head.number == 10`, then the first notification will be - /// with `block.number == 11`. A `head.number` of 10 indicates that the ExEx - /// has processed up to block 10, and is ready to process block 11. - pub fn with_head(self, head: ExExHead) -> ExExNotificationsWithHead { + type Item = eyre::Result; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + match &mut self.get_mut().inner { + ExExNotificationsInner::WithoutHead(notifications) => { + notifications.poll_next_unpin(cx).map(|result| result.map(Ok)) + } + ExExNotificationsInner::WithHead(notifications) => notifications.poll_next_unpin(cx), + ExExNotificationsInner::Invalid => unreachable!(), + } + } +} + +/// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. +pub struct ExExNotificationsWithoutHead { + node_head: Head, + provider: P, + executor: E, + notifications: Receiver, + wal_handle: WalHandle, +} + +impl Debug for ExExNotificationsWithoutHead { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExExNotifications") + .field("provider", &self.provider) + .field("executor", &self.executor) + .field("notifications", &self.notifications) + .finish() + } +} + +impl ExExNotificationsWithoutHead { + /// Creates a new instance of [`ExExNotificationsWithoutHead`]. + const fn new( + node_head: Head, + provider: P, + executor: E, + notifications: Receiver, + wal_handle: WalHandle, + ) -> Self { + Self { node_head, provider, executor, notifications, wal_handle } + } + + /// Subscribe to notifications with the given head. + fn with_head(self, head: ExExHead) -> ExExNotificationsWithHead { ExExNotificationsWithHead::new( self.node_head, self.provider, @@ -119,7 +178,7 @@ where } } -impl Stream for ExExNotifications { +impl Stream for ExExNotificationsWithoutHead { type Item = ExExNotification; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -128,7 +187,13 @@ impl Stream for ExExNotifications { } /// A stream of [`ExExNotification`]s. The stream will only emit notifications for blocks that are -/// committed or reverted after the given head. +/// committed or reverted after the given head. The head is the ExEx's latest view of the host +/// chain. +/// +/// Notifications will be sent starting from the head, not inclusive. For example, if +/// `exex_head.number == 10`, then the first notification will be with `block.number == 11`. An +/// `exex_head.number` of 10 indicates that the ExEx has processed up to block 10, and is ready to +/// process block 11. #[derive(Debug)] pub struct ExExNotificationsWithHead { node_head: Head, @@ -147,13 +212,9 @@ pub struct ExExNotificationsWithHead { backfill_job: Option>, } -impl ExExNotificationsWithHead -where - P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, - E: BlockExecutorProvider + Clone + Unpin + 'static, -{ +impl ExExNotificationsWithHead { /// Creates a new [`ExExNotificationsWithHead`]. - pub const fn new( + const fn new( node_head: Head, provider: P, executor: E, @@ -173,7 +234,13 @@ where backfill_job: None, } } +} +impl ExExNotificationsWithHead +where + P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, + E: BlockExecutorProvider + Clone + Unpin + 'static, +{ /// Checks if the ExEx head is on the canonical chain. /// /// If the head block is not found in the database or it's ahead of the node head, it means @@ -367,7 +434,7 @@ mod tests { notifications_tx.send(notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), @@ -438,7 +505,7 @@ mod tests { notifications_tx.send(notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), @@ -528,7 +595,7 @@ mod tests { notifications_tx.send(new_notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), @@ -609,7 +676,7 @@ mod tests { notifications_tx.send(new_notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), From 54c8305fb6e94342ab90a10b022a4ba47f88c156 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 17:35:47 +0200 Subject: [PATCH 057/103] chore: enforce window (#11540) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 16 ++++++++++++++-- crates/rpc/rpc/src/eth/helpers/state.rs | 21 +++++---------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 0008454c93a4..d601e43d90a8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -9,7 +9,8 @@ use reth_errors::RethError; use reth_evm::ConfigureEvmEnv; use reth_primitives::{BlockId, Header, KECCAK_EMPTY}; use reth_provider::{ - BlockIdReader, ChainSpecProvider, StateProvider, StateProviderBox, StateProviderFactory, + BlockIdReader, BlockNumReader, ChainSpecProvider, StateProvider, StateProviderBox, + StateProviderFactory, }; use reth_rpc_eth_types::{EthApiError, EthStateCache, PendingBlockEnv, RpcInvalidTransactionError}; use reth_rpc_types_compat::proof::from_primitive_account_proof; @@ -132,10 +133,21 @@ pub trait EthState: LoadState + SpawnBlocking { ) -> impl Future, Self::Error>> + Send { self.spawn_blocking_io(move |this| { let state = this.state_at_block_id(block_id)?; - let account = state.basic_account(address).map_err(Self::Error::from_eth_err)?; let Some(account) = account else { return Ok(None) }; + // Check whether the distance to the block exceeds the maximum configured proof window. + let chain_info = + LoadState::provider(&this).chain_info().map_err(Self::Error::from_eth_err)?; + let block_number = LoadState::provider(&this) + .block_number_for_id(block_id) + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(block_id))?; + let max_window = this.max_proof_window(); + if chain_info.best_number.saturating_sub(block_number) > max_window { + return Err(EthApiError::ExceedsMaxProofWindow.into()) + } + let balance = account.balance; let nonce = account.nonce; let code_hash = account.bytecode_hash.unwrap_or(KECCAK_EMPTY); diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 006a0192f73a..8a35842798bc 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -48,7 +48,8 @@ mod tests { use alloy_primitives::{Address, StorageKey, StorageValue, U256}; use reth_chainspec::MAINNET; use reth_evm_ethereum::EthEvmConfig; - use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, KECCAK_EMPTY}; + use reth_network_api::noop::NoopNetwork; + use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}; use reth_rpc_eth_api::helpers::EthState; use reth_rpc_eth_types::{ @@ -61,7 +62,7 @@ mod tests { use reth_transaction_pool::test_utils::{testing_pool, TestPool}; use std::collections::HashMap; - fn noop_eth_api() -> EthApi { + fn noop_eth_api() -> EthApi { let pool = testing_pool(); let evm_config = EthEvmConfig::new(MAINNET.clone()); @@ -70,7 +71,7 @@ mod tests { EthApi::new( NoopProvider::default(), pool, - (), + NoopNetwork::default(), cache.clone(), GasPriceOracle::new(NoopProvider::default(), Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, @@ -102,7 +103,7 @@ mod tests { GasPriceOracle::new(mock_provider, Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, DEFAULT_MAX_SIMULATE_BLOCKS, - DEFAULT_ETH_PROOF_WINDOW, + DEFAULT_ETH_PROOF_WINDOW + 1, BlockingTaskPool::build().expect("failed to build tracing pool"), FeeHistoryCache::new(cache, FeeHistoryCacheConfig::default()), evm_config, @@ -139,16 +140,4 @@ mod tests { let account = eth_api.get_account(address, Default::default()).await.unwrap(); assert!(account.is_none()); } - - #[tokio::test] - async fn test_get_account_empty() { - let address = Address::random(); - let accounts = HashMap::from([(address, ExtendedAccount::new(0, U256::ZERO))]); - let eth_api = mock_eth_api(accounts); - - let account = eth_api.get_account(address, Default::default()).await.unwrap(); - let expected_account = - alloy_rpc_types::Account { code_hash: KECCAK_EMPTY, ..Default::default() }; - assert_eq!(Some(expected_account), account); - } } From 8ca5ec9b72b90c5c7430052c2a71fd1de153dd5b Mon Sep 17 00:00:00 2001 From: Parikalp Bhardwaj <53660958+Parikalp-Bhardwaj@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:11:59 +0400 Subject: [PATCH 058/103] Refactor get_payload_bodies_by_hash_with to be non-blocking (#11511) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-engine-api/src/engine_api.rs | 51 +++++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 907297de1776..252808c14a77 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -464,48 +464,59 @@ where } /// Called to retrieve execution payload bodies by hashes. - fn get_payload_bodies_by_hash_with( + async fn get_payload_bodies_by_hash_with( &self, hashes: Vec, f: F, ) -> EngineApiResult>> where - F: Fn(Block) -> R, + F: Fn(Block) -> R + Send + 'static, + R: Send + 'static, { let len = hashes.len() as u64; if len > MAX_PAYLOAD_BODIES_LIMIT { - return Err(EngineApiError::PayloadRequestTooLarge { len }) + return Err(EngineApiError::PayloadRequestTooLarge { len }); } - let mut result = Vec::with_capacity(hashes.len()); - for hash in hashes { - let block = self - .inner - .provider - .block(BlockHashOrNumber::Hash(hash)) - .map_err(|err| EngineApiError::Internal(Box::new(err)))?; - result.push(block.map(&f)); - } + let (tx, rx) = oneshot::channel(); + let inner = self.inner.clone(); - Ok(result) + self.inner.task_spawner.spawn_blocking(Box::pin(async move { + let mut result = Vec::with_capacity(hashes.len()); + for hash in hashes { + let block_result = inner.provider.block(BlockHashOrNumber::Hash(hash)); + match block_result { + Ok(block) => { + result.push(block.map(&f)); + } + Err(err) => { + let _ = tx.send(Err(EngineApiError::Internal(Box::new(err)))); + return; + } + } + } + tx.send(Ok(result)).ok(); + })); + + rx.await.map_err(|err| EngineApiError::Internal(Box::new(err)))? } /// Called to retrieve execution payload bodies by hashes. - pub fn get_payload_bodies_by_hash_v1( + pub async fn get_payload_bodies_by_hash_v1( &self, hashes: Vec, ) -> EngineApiResult { - self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v1) + self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v1).await } /// Called to retrieve execution payload bodies by hashes. /// /// Same as [`Self::get_payload_bodies_by_hash_v1`] but as [`ExecutionPayloadBodiesV2`]. - pub fn get_payload_bodies_by_hash_v2( + pub async fn get_payload_bodies_by_hash_v2( &self, hashes: Vec, ) -> EngineApiResult { - self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v2) + self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v2).await } /// Called to verify network configuration parameters and ensure that Consensus and Execution @@ -832,7 +843,7 @@ where let start = Instant::now(); let res = Self::get_payload_bodies_by_hash_v1(self, block_hashes); self.inner.metrics.latency.get_payload_bodies_by_hash_v1.record(start.elapsed()); - Ok(res?) + Ok(res.await?) } async fn get_payload_bodies_by_hash_v2( @@ -843,7 +854,7 @@ where let start = Instant::now(); let res = Self::get_payload_bodies_by_hash_v2(self, block_hashes); self.inner.metrics.latency.get_payload_bodies_by_hash_v2.record(start.elapsed()); - Ok(res?) + Ok(res.await?) } /// Handler for `engine_getPayloadBodiesByRangeV1` @@ -1147,7 +1158,7 @@ mod tests { .collect::>(); let hashes = blocks.iter().map(|b| b.hash()).collect(); - let res = api.get_payload_bodies_by_hash_v1(hashes).unwrap(); + let res = api.get_payload_bodies_by_hash_v1(hashes).await.unwrap(); assert_eq!(res, expected); } } From 9a732f6b7813f9ecb5f28b379d3062c287a78e70 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:30:57 -0600 Subject: [PATCH 059/103] Introduce Eth PayloadTypes Impl (#11519) Co-authored-by: Matthias Seitz --- crates/ethereum/engine-primitives/src/lib.rs | 31 ++++++++++++++++---- crates/node/types/src/lib.rs | 1 + 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/ethereum/engine-primitives/src/lib.rs b/crates/ethereum/engine-primitives/src/lib.rs index 69d73a021747..034a8c6bffbb 100644 --- a/crates/ethereum/engine-primitives/src/lib.rs +++ b/crates/ethereum/engine-primitives/src/lib.rs @@ -26,21 +26,40 @@ use reth_payload_primitives::{ /// The types used in the default mainnet ethereum beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct EthEngineTypes; +pub struct EthEngineTypes { + _marker: std::marker::PhantomData, +} -impl PayloadTypes for EthEngineTypes { - type BuiltPayload = EthBuiltPayload; - type PayloadAttributes = EthPayloadAttributes; - type PayloadBuilderAttributes = EthPayloadBuilderAttributes; +impl PayloadTypes for EthEngineTypes { + type BuiltPayload = T::BuiltPayload; + type PayloadAttributes = T::PayloadAttributes; + type PayloadBuilderAttributes = T::PayloadBuilderAttributes; } -impl EngineTypes for EthEngineTypes { +impl EngineTypes for EthEngineTypes +where + T::BuiltPayload: TryInto + + TryInto + + TryInto + + TryInto, +{ type ExecutionPayloadV1 = ExecutionPayloadV1; type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadV3 = ExecutionPayloadEnvelopeV3; type ExecutionPayloadV4 = ExecutionPayloadEnvelopeV4; } +/// A default payload type for [`EthEngineTypes`] +#[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] +pub struct EthPayloadTypes; + +impl PayloadTypes for EthPayloadTypes { + type BuiltPayload = EthBuiltPayload; + type PayloadAttributes = EthPayloadAttributes; + type PayloadBuilderAttributes = EthPayloadBuilderAttributes; +} + /// Validator for the ethereum engine API. #[derive(Debug, Clone)] pub struct EthereumEngineValidator { diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index 2c72e02d3edc..0f6d5a1a370f 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -40,6 +40,7 @@ pub trait NodeTypes: Send + Sync + Unpin + 'static { pub trait NodeTypesWithEngine: NodeTypes { /// The node's engine types, defining the interaction with the consensus engine. type Engine: EngineTypes; + // type Engine: EngineTypes; } /// A helper trait that is downstream of the [`NodeTypesWithEngine`] trait and adds database to the From a6b564e0540646bc2e0cebbd0dc04a79c6ba77dd Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:41:22 -0400 Subject: [PATCH 060/103] fix(op-reth): add jemalloc feature to optimism-cli for version (#11543) --- Cargo.lock | 1 + crates/optimism/bin/Cargo.toml | 2 +- crates/optimism/cli/Cargo.toml | 9 +++++++++ crates/optimism/cli/src/lib.rs | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a66887900cb4..93ae84221192 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7996,6 +7996,7 @@ dependencies = [ "reth-node-builder", "reth-node-core", "reth-node-events", + "reth-node-metrics", "reth-optimism-chainspec", "reth-optimism-evm", "reth-optimism-node", diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index ecb92dfc454b..2de0bb6ee181 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -31,7 +31,7 @@ workspace = true [features] default = ["jemalloc"] -jemalloc = ["reth-cli-util/jemalloc"] +jemalloc = ["reth-cli-util/jemalloc", "reth-optimism-cli/jemalloc"] jemalloc-prof = ["reth-cli-util/jemalloc-prof"] tracy-allocator = ["reth-cli-util/tracy-allocator"] diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 99d1641e3643..d53270cd62f8 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -27,6 +27,9 @@ reth-node-core.workspace = true reth-optimism-node.workspace = true reth-primitives.workspace = true +# so jemalloc metrics can be included +reth-node-metrics.workspace = true + ## optimism reth-optimism-primitives.workspace = true reth-optimism-chainspec.workspace = true @@ -82,3 +85,9 @@ asm-keccak = [ "reth-optimism-node/asm-keccak", "reth-primitives/asm-keccak", ] + +# Jemalloc feature for vergen to generate correct env vars +jemalloc = [ + "reth-node-core/jemalloc", + "reth-node-metrics/jemalloc" +] diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index 74bc80c426ef..e6eed86bf7fc 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -51,6 +51,10 @@ use reth_optimism_node::OptimismNode; use reth_tracing::FileWorkerGuard; use tracing::info; +// This allows us to manually enable node metrics features, required for proper jemalloc metric +// reporting +use reth_node_metrics as _; + /// The main op-reth cli interface. /// /// This is the entrypoint to the executable. From c35b8beb35594bb3192caec7d4335ca567d45539 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 7 Oct 2024 18:42:09 +0200 Subject: [PATCH 061/103] fix(grafana): remove rate function from panel "Transactions by Type in Pool" (#11542) --- etc/grafana/dashboards/reth-mempool.json | 52 +++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/etc/grafana/dashboards/reth-mempool.json b/etc/grafana/dashboards/reth-mempool.json index 9e3f5ae81930..ebb693184a5a 100644 --- a/etc/grafana/dashboards/reth-mempool.json +++ b/etc/grafana/dashboards/reth-mempool.json @@ -15,7 +15,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.1.3" + "version": "11.2.0" }, { "type": "panel", @@ -131,7 +131,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -201,7 +201,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -271,7 +271,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -341,7 +341,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -411,7 +411,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -481,7 +481,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -534,6 +534,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -686,6 +687,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -817,6 +819,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -969,6 +972,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1099,6 +1103,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1261,6 +1266,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1357,6 +1363,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1504,6 +1511,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1605,6 +1613,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1756,6 +1765,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1852,6 +1862,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1971,6 +1982,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2101,6 +2113,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2323,6 +2336,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2442,6 +2456,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2578,6 +2593,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2697,6 +2713,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2736,7 +2753,7 @@ } ] }, - "unit": "cps" + "unit": "none" }, "overrides": [] }, @@ -2766,8 +2783,8 @@ "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}[$__rate_interval])", + "editorMode": "code", + "expr": "reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2784,7 +2801,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2801,7 +2818,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2818,7 +2835,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2835,7 +2852,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2867,6 +2884,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3035,6 +3053,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3130,6 +3149,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3272,6 +3292,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3578,6 +3599,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3694,6 +3716,6 @@ "timezone": "", "title": "Reth - Transaction Pool", "uid": "bee34f59-c79c-4669-a000-198057b3703d", - "version": 4, + "version": 2, "weekStart": "" } \ No newline at end of file From b67f004fbe8e1b7c05f84f314c4c9f2ed9be1891 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 23:54:42 +0200 Subject: [PATCH 062/103] chore: move ethfiltererror (#11552) --- crates/rpc/rpc-eth-types/src/lib.rs | 1 - crates/rpc/rpc-eth-types/src/logs_utils.rs | 56 +--------------------- crates/rpc/rpc/src/eth/filter.rs | 53 +++++++++++++++++++- 3 files changed, 53 insertions(+), 57 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/lib.rs b/crates/rpc/rpc-eth-types/src/lib.rs index fba893c15f59..fa36dae4c881 100644 --- a/crates/rpc/rpc-eth-types/src/lib.rs +++ b/crates/rpc/rpc-eth-types/src/lib.rs @@ -36,7 +36,6 @@ pub use gas_oracle::{ GasCap, GasPriceOracle, GasPriceOracleConfig, GasPriceOracleResult, RPC_DEFAULT_GAS_CAP, }; pub use id_provider::EthSubscriptionIdProvider; -pub use logs_utils::EthFilterError; pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; pub use receipt::ReceiptBuilder; pub use transaction::TransactionSource; diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index bb44dc0e6669..f26555bb70da 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -3,64 +3,12 @@ //! Log parsing for building filter. use alloy_primitives::TxHash; -use alloy_rpc_types::{FilterId, FilteredParams, Log}; +use alloy_rpc_types::{FilteredParams, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; use reth_primitives::{BlockNumHash, Receipt}; -use reth_rpc_server_types::result::rpc_error_with_code; use reth_storage_api::BlockReader; -use crate::EthApiError; - -/// Errors that can occur in the handler implementation -#[derive(Debug, thiserror::Error)] -pub enum EthFilterError { - /// Filter not found. - #[error("filter not found")] - FilterNotFound(FilterId), - /// Invalid block range. - #[error("invalid block range params")] - InvalidBlockRangeParams, - /// Query scope is too broad. - #[error("query exceeds max block range {0}")] - QueryExceedsMaxBlocks(u64), - /// Query result is too large. - #[error("query exceeds max results {0}")] - QueryExceedsMaxResults(usize), - /// Error serving request in `eth_` namespace. - #[error(transparent)] - EthAPIError(#[from] EthApiError), - /// Error thrown when a spawned task failed to deliver a response. - #[error("internal filter error")] - InternalError, -} - -// convert the error -impl From for jsonrpsee_types::error::ErrorObject<'static> { - fn from(err: EthFilterError) -> Self { - match err { - EthFilterError::FilterNotFound(_) => { - rpc_error_with_code(jsonrpsee_types::error::INVALID_PARAMS_CODE, "filter not found") - } - err @ EthFilterError::InternalError => { - rpc_error_with_code(jsonrpsee_types::error::INTERNAL_ERROR_CODE, err.to_string()) - } - EthFilterError::EthAPIError(err) => err.into(), - err @ (EthFilterError::InvalidBlockRangeParams | - EthFilterError::QueryExceedsMaxBlocks(_) | - EthFilterError::QueryExceedsMaxResults(_)) => { - rpc_error_with_code(jsonrpsee_types::error::INVALID_PARAMS_CODE, err.to_string()) - } - } - } -} - -impl From for EthFilterError { - fn from(err: ProviderError) -> Self { - Self::EthAPIError(err.into()) - } -} - /// Returns all matching of a block's receipts when the transaction hashes are known. pub fn matching_block_logs_with_tx_hashes<'a, I>( filter: &FilteredParams, @@ -107,7 +55,7 @@ pub fn append_matching_block_logs( receipts: &[Receipt], removed: bool, block_timestamp: u64, -) -> Result<(), EthFilterError> { +) -> Result<(), ProviderError> { // Tracks the index of a log in the entire block. let mut log_index: u64 = 0; diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index f4ae89264160..c5581f42a06c 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -24,9 +24,9 @@ use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; use reth_rpc_eth_types::{ logs_utils::{self, append_matching_block_logs}, - EthApiError, EthFilterConfig, EthFilterError, EthStateCache, EthSubscriptionIdProvider, + EthApiError, EthFilterConfig, EthStateCache, EthSubscriptionIdProvider, }; -use reth_rpc_server_types::ToRpcResult; +use reth_rpc_server_types::{result::rpc_error_with_code, ToRpcResult}; use reth_rpc_types_compat::transaction::from_recovered; use reth_tasks::TaskSpawner; use reth_transaction_pool::{NewSubpoolTransactionStream, PoolTransaction, TransactionPool}; @@ -695,6 +695,55 @@ impl Iterator for BlockRangeInclusiveIter { } } +/// Errors that can occur in the handler implementation +#[derive(Debug, thiserror::Error)] +pub enum EthFilterError { + /// Filter not found. + #[error("filter not found")] + FilterNotFound(FilterId), + /// Invalid block range. + #[error("invalid block range params")] + InvalidBlockRangeParams, + /// Query scope is too broad. + #[error("query exceeds max block range {0}")] + QueryExceedsMaxBlocks(u64), + /// Query result is too large. + #[error("query exceeds max results {0}")] + QueryExceedsMaxResults(usize), + /// Error serving request in `eth_` namespace. + #[error(transparent)] + EthAPIError(#[from] EthApiError), + /// Error thrown when a spawned task failed to deliver a response. + #[error("internal filter error")] + InternalError, +} + +impl From for jsonrpsee::types::error::ErrorObject<'static> { + fn from(err: EthFilterError) -> Self { + match err { + EthFilterError::FilterNotFound(_) => rpc_error_with_code( + jsonrpsee::types::error::INVALID_PARAMS_CODE, + "filter not found", + ), + err @ EthFilterError::InternalError => { + rpc_error_with_code(jsonrpsee::types::error::INTERNAL_ERROR_CODE, err.to_string()) + } + EthFilterError::EthAPIError(err) => err.into(), + err @ (EthFilterError::InvalidBlockRangeParams | + EthFilterError::QueryExceedsMaxBlocks(_) | + EthFilterError::QueryExceedsMaxResults(_)) => { + rpc_error_with_code(jsonrpsee::types::error::INVALID_PARAMS_CODE, err.to_string()) + } + } + } +} + +impl From for EthFilterError { + fn from(err: ProviderError) -> Self { + Self::EthAPIError(err.into()) + } +} + #[cfg(test)] mod tests { use super::*; From 5e288b2469e889b41e5df16ce01951fedf4b2b55 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 03:06:16 +0200 Subject: [PATCH 063/103] chore: rm redundant type hint (#11557) --- crates/rpc/rpc/src/eth/helpers/signer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rpc/rpc/src/eth/helpers/signer.rs b/crates/rpc/rpc/src/eth/helpers/signer.rs index b5109d09017a..a5818aa494fd 100644 --- a/crates/rpc/rpc/src/eth/helpers/signer.rs +++ b/crates/rpc/rpc/src/eth/helpers/signer.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use crate::EthApi; -use alloy_consensus::TxEnvelope; use alloy_dyn_abi::TypedData; use alloy_eips::eip2718::Decodable2718; use alloy_network::{eip2718::Encodable2718, EthereumWallet, TransactionBuilder}; @@ -91,7 +90,7 @@ impl EthSigner for DevSigner { let wallet = EthereumWallet::from(signer); // build and sign transaction with signer - let txn_envelope: TxEnvelope = + let txn_envelope = request.build(&wallet).await.map_err(|_| SignError::InvalidTransactionRequest)?; // decode transaction into signed transaction type From 5eb71e32c6ca2e706a8b921dfdf64dc0b61614a4 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Tue, 8 Oct 2024 02:42:14 -0600 Subject: [PATCH 064/103] Introduce Op PayloadTypes Impl (#11558) Co-authored-by: Matthias Seitz --- crates/optimism/node/src/engine.rs | 47 +++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index a9adadd3068e..11c8f15cfdac 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -21,7 +21,39 @@ use reth_optimism_payload_builder::{OptimismBuiltPayload, OptimismPayloadBuilder /// The types used in the optimism beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct OptimismEngineTypes; +pub struct OptimismEngineTypes { + _marker: std::marker::PhantomData, +} + +impl PayloadTypes for OptimismEngineTypes { + type BuiltPayload = T::BuiltPayload; + type PayloadAttributes = T::PayloadAttributes; + type PayloadBuilderAttributes = T::PayloadBuilderAttributes; +} + +impl EngineTypes for OptimismEngineTypes +where + T::BuiltPayload: TryInto + + TryInto + + TryInto + + TryInto, +{ + type ExecutionPayloadV1 = ExecutionPayloadV1; + type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; + type ExecutionPayloadV3 = OptimismExecutionPayloadEnvelopeV3; + type ExecutionPayloadV4 = OptimismExecutionPayloadEnvelopeV4; +} + +/// A default payload type for [`OptimismEngineTypes`] +#[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] +pub struct OptimismPayloadTypes; + +impl PayloadTypes for OptimismPayloadTypes { + type BuiltPayload = OptimismBuiltPayload; + type PayloadAttributes = OptimismPayloadAttributes; + type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; +} /// Validator for Optimism engine API. #[derive(Debug, Clone)] @@ -36,19 +68,6 @@ impl OptimismEngineValidator { } } -impl PayloadTypes for OptimismEngineTypes { - type BuiltPayload = OptimismBuiltPayload; - type PayloadAttributes = OptimismPayloadAttributes; - type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; -} - -impl EngineTypes for OptimismEngineTypes { - type ExecutionPayloadV1 = ExecutionPayloadV1; - type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; - type ExecutionPayloadV3 = OptimismExecutionPayloadEnvelopeV3; - type ExecutionPayloadV4 = OptimismExecutionPayloadEnvelopeV4; -} - /// Validates the presence of the `withdrawals` field according to the payload timestamp. /// /// After Canyon, withdrawals field must be [Some]. From 33b78dcf2b65aae68803f11c220fd45d1e2abea1 Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:02:32 +0200 Subject: [PATCH 065/103] chore: chain manual serialisation implementation (#11538) --- crates/evm/execution-types/src/chain.rs | 48 ++++++++++++++++++------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 467bd4c0ec7c..25bc39ea3257 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -516,7 +516,7 @@ pub(super) mod serde_bincode_compat { use alloy_primitives::BlockNumber; use reth_primitives::serde_bincode_compat::SealedBlockWithSenders; use reth_trie::serde_bincode_compat::updates::TrieUpdates; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer}; use serde_with::{DeserializeAs, SerializeAs}; use crate::ExecutionOutcome; @@ -538,19 +538,47 @@ pub(super) mod serde_bincode_compat { /// ``` #[derive(Debug, Serialize, Deserialize)] pub struct Chain<'a> { - blocks: BTreeMap>, + blocks: SealedBlocksWithSenders<'a>, execution_outcome: Cow<'a, ExecutionOutcome>, trie_updates: Option>, } + #[derive(Debug)] + struct SealedBlocksWithSenders<'a>( + Cow<'a, BTreeMap>, + ); + + impl Serialize for SealedBlocksWithSenders<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_map(Some(self.0.len()))?; + + for (block_number, block) in self.0.iter() { + state.serialize_entry(block_number, &SealedBlockWithSenders::<'_>::from(block))?; + } + + state.end() + } + } + + impl<'de> Deserialize<'de> for SealedBlocksWithSenders<'_> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Self(Cow::Owned( + BTreeMap::>::deserialize(deserializer) + .map(|blocks| blocks.into_iter().map(|(n, b)| (n, b.into())).collect())?, + ))) + } + } + impl<'a> From<&'a super::Chain> for Chain<'a> { fn from(value: &'a super::Chain) -> Self { Self { - blocks: value - .blocks - .iter() - .map(|(block_number, block)| (*block_number, block.into())) - .collect(), + blocks: SealedBlocksWithSenders(Cow::Borrowed(&value.blocks)), execution_outcome: Cow::Borrowed(&value.execution_outcome), trie_updates: value.trie_updates.as_ref().map(Into::into), } @@ -560,11 +588,7 @@ pub(super) mod serde_bincode_compat { impl<'a> From> for super::Chain { fn from(value: Chain<'a>) -> Self { Self { - blocks: value - .blocks - .into_iter() - .map(|(block_number, block)| (block_number, block.into())) - .collect(), + blocks: value.blocks.0.into_owned(), execution_outcome: value.execution_outcome.into_owned(), trie_updates: value.trie_updates.map(Into::into), } From 6e92ab8e43fa0822c70ba7bd1fe487e7d620eac9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 11:03:50 +0200 Subject: [PATCH 066/103] chore: rm unused optimism feature (#11559) --- crates/node/core/Cargo.toml | 1 - crates/optimism/rpc/Cargo.toml | 1 - crates/rpc/rpc-eth-api/Cargo.toml | 5 ----- crates/rpc/rpc/Cargo.toml | 1 - 4 files changed, 8 deletions(-) diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 46476273e42d..f6d686451905 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -79,7 +79,6 @@ tempfile.workspace = true optimism = [ "reth-primitives/optimism", "reth-rpc-types-compat/optimism", - "reth-rpc-eth-api/optimism", ] # Features for vergen to generate correct env vars jemalloc = [] diff --git a/crates/optimism/rpc/Cargo.toml b/crates/optimism/rpc/Cargo.toml index 24c3eb02d247..65dce7510b0e 100644 --- a/crates/optimism/rpc/Cargo.toml +++ b/crates/optimism/rpc/Cargo.toml @@ -65,6 +65,5 @@ optimism = [ "reth-optimism-evm/optimism", "reth-primitives/optimism", "reth-provider/optimism", - "reth-rpc-eth-api/optimism", "revm/optimism", ] diff --git a/crates/rpc/rpc-eth-api/Cargo.toml b/crates/rpc/rpc-eth-api/Cargo.toml index 23dd46baecf1..e59ee39a694b 100644 --- a/crates/rpc/rpc-eth-api/Cargo.toml +++ b/crates/rpc/rpc-eth-api/Cargo.toml @@ -59,8 +59,3 @@ tracing.workspace = true [features] js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"] client = ["jsonrpsee/client", "jsonrpsee/async-client"] -optimism = [ - "reth-primitives/optimism", - "revm/optimism", - "reth-provider/optimism", -] diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 4665cd002ca4..b0994a33e0aa 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -102,6 +102,5 @@ optimism = [ "reth-primitives/optimism", "reth-rpc-types-compat/optimism", "reth-provider/optimism", - "reth-rpc-eth-api/optimism", "reth-revm/optimism", ] From 548f5b9c6d4a3d26f8ac7e21919c5ca83f871ff9 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 8 Oct 2024 11:47:15 +0200 Subject: [PATCH 067/103] feat(trie): expose storage proofs (#11550) --- crates/chain-state/src/in_memory.rs | 11 +++- crates/chain-state/src/memory_overlay.rs | 17 ++++++- crates/revm/src/test_utils.rs | 12 ++++- crates/rpc/rpc-eth-types/src/cache/db.rs | 9 ++++ .../src/providers/bundle_state_provider.rs | 36 +++++++++---- .../src/providers/state/historical.rs | 20 ++++++-- .../provider/src/providers/state/latest.rs | 21 ++++++-- .../provider/src/providers/state/macros.rs | 3 +- .../storage/provider/src/test_utils/mock.rs | 12 ++++- .../storage/provider/src/test_utils/noop.rs | 9 ++++ crates/storage/storage-api/src/trie.rs | 12 ++++- crates/trie/db/src/lib.rs | 2 +- crates/trie/db/src/proof.rs | 50 ++++++++++++++++++- crates/trie/trie/src/proof.rs | 33 +++++++++++- crates/trie/trie/src/witness.rs | 2 +- 15 files changed, 221 insertions(+), 28 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index f47417aa385d..07120cf8ee31 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -872,7 +872,7 @@ mod tests { AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, }; - use reth_trie::{AccountProof, HashedStorage, MultiProof, TrieInput}; + use reth_trie::{AccountProof, HashedStorage, MultiProof, StorageProof, TrieInput}; fn create_mock_state( test_block_builder: &mut TestBlockBuilder, @@ -973,6 +973,15 @@ mod tests { ) -> ProviderResult { Ok(B256::random()) } + + fn storage_proof( + &self, + _address: Address, + slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + Ok(StorageProof::new(slot)) + } } impl StateProofProvider for MockStateProvider { diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs index 2712d1259e85..eb125dad115e 100644 --- a/crates/chain-state/src/memory_overlay.rs +++ b/crates/chain-state/src/memory_overlay.rs @@ -133,11 +133,26 @@ impl StateRootProvider for MemoryOverlayStateProvider { impl StorageRootProvider for MemoryOverlayStateProvider { // TODO: Currently this does not reuse available in-memory trie nodes. fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + let state = &self.trie_state().state; let mut hashed_storage = - self.trie_state().state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); hashed_storage.extend(&storage); self.historical.storage_root(address, hashed_storage) } + + // TODO: Currently this does not reuse available in-memory trie nodes. + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + let state = &self.trie_state().state; + let mut hashed_storage = + state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + hashed_storage.extend(&storage); + self.historical.storage_proof(address, slot, hashed_storage) + } } impl StateProofProvider for MemoryOverlayStateProvider { diff --git a/crates/revm/src/test_utils.rs b/crates/revm/src/test_utils.rs index d42ec4959907..813997c72d11 100644 --- a/crates/revm/src/test_utils.rs +++ b/crates/revm/src/test_utils.rs @@ -11,7 +11,8 @@ use reth_storage_api::{ }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, + TrieInput, }; /// Mock state for testing @@ -102,6 +103,15 @@ impl StorageRootProvider for StateProviderTest { ) -> ProviderResult { unimplemented!("storage root is not supported") } + + fn storage_proof( + &self, + _address: Address, + _slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + unimplemented!("proof generation is not supported") + } } impl StateProofProvider for StateProviderTest { diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 9731e3845768..7422dcfb8a7b 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -58,6 +58,15 @@ impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> ) -> ProviderResult { self.0.storage_root(address, hashed_storage) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + self.0.storage_proof(address, slot, hashed_storage) + } } impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> { diff --git a/crates/storage/provider/src/providers/bundle_state_provider.rs b/crates/storage/provider/src/providers/bundle_state_provider.rs index 6fe3fa85cb89..296dae8c6ab7 100644 --- a/crates/storage/provider/src/providers/bundle_state_provider.rs +++ b/crates/storage/provider/src/providers/bundle_state_provider.rs @@ -31,6 +31,20 @@ impl BundleStateProvider pub const fn new(state_provider: SP, block_execution_data_provider: EDP) -> Self { Self { state_provider, block_execution_data_provider } } + + /// Retrieve hashed storage for target address. + fn get_hashed_storage(&self, address: Address) -> HashedStorage { + let bundle_state = self.block_execution_data_provider.execution_outcome().state(); + bundle_state + .account(&address) + .map(|account| { + HashedStorage::from_plain_storage( + account.status, + account.storage.iter().map(|(slot, value)| (slot, &value.present_value)), + ) + }) + .unwrap_or_else(|| HashedStorage::new(false)) + } } /* Implement StateProvider traits */ @@ -109,19 +123,21 @@ impl StorageRootProvider address: Address, hashed_storage: HashedStorage, ) -> ProviderResult { - let bundle_state = self.block_execution_data_provider.execution_outcome().state(); - let mut storage = bundle_state - .account(&address) - .map(|account| { - HashedStorage::from_plain_storage( - account.status, - account.storage.iter().map(|(slot, value)| (slot, &value.present_value)), - ) - }) - .unwrap_or_else(|| HashedStorage::new(false)); + let mut storage = self.get_hashed_storage(address); storage.extend(&hashed_storage); self.state_provider.storage_root(address, storage) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let mut storage = self.get_hashed_storage(address); + storage.extend(&hashed_storage); + self.state_provider.storage_proof(address, slot, storage) + } } impl StateProofProvider diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 22f40f6f951f..781a11f6deca 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -17,12 +17,14 @@ use reth_primitives::{constants::EPOCH_SLOTS, Account, Bytecode, StaticFileSegme use reth_storage_api::{StateProofProvider, StorageRootProvider}; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - proof::Proof, updates::TrieUpdates, witness::TrieWitness, AccountProof, HashedPostState, - HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, + proof::{Proof, StorageProof}, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, HashedPostState, HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, }; use reth_trie_db::{ DatabaseHashedPostState, DatabaseHashedStorage, DatabaseProof, DatabaseStateRoot, - DatabaseStorageRoot, DatabaseTrieWitness, + DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness, }; use std::fmt::Debug; @@ -330,6 +332,18 @@ impl StorageRootProvider for HistoricalStateProviderRef<'_, TX> { StorageRoot::overlay_root(self.tx, address, revert_storage) .map_err(|err| ProviderError::Database(err.into())) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let mut revert_storage = self.revert_storage(address)?; + revert_storage.extend(&hashed_storage); + StorageProof::overlay_storage_proof(self.tx, address, slot, revert_storage) + .map_err(Into::::into) + } } impl StateProofProvider for HistoricalStateProviderRef<'_, TX> { diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 9fbe00cbd5ee..fdcbfc4937fe 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -15,10 +15,15 @@ use reth_primitives::{Account, Bytecode, StaticFileSegment}; use reth_storage_api::{StateProofProvider, StorageRootProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie::{ - proof::Proof, updates::TrieUpdates, witness::TrieWitness, AccountProof, HashedPostState, - HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, + proof::{Proof, StorageProof}, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, HashedPostState, HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, +}; +use reth_trie_db::{ + DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, + DatabaseTrieWitness, }; -use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageRoot, DatabaseTrieWitness}; /// State provider over latest state that takes tx reference. #[derive(Debug)] @@ -116,6 +121,16 @@ impl StorageRootProvider for LatestStateProviderRef<'_, TX> { StorageRoot::overlay_root(self.tx, address, hashed_storage) .map_err(|err| ProviderError::Database(err.into())) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + StorageProof::overlay_storage_proof(self.tx, address, slot, hashed_storage) + .map_err(Into::::into) + } } impl StateProofProvider for LatestStateProviderRef<'_, TX> { diff --git a/crates/storage/provider/src/providers/state/macros.rs b/crates/storage/provider/src/providers/state/macros.rs index 388b59ab0a1e..b90924354c43 100644 --- a/crates/storage/provider/src/providers/state/macros.rs +++ b/crates/storage/provider/src/providers/state/macros.rs @@ -48,7 +48,8 @@ macro_rules! delegate_provider_impls { fn state_root_from_nodes_with_updates(&self, input: reth_trie::TrieInput) -> reth_storage_errors::provider::ProviderResult<(alloy_primitives::B256, reth_trie::updates::TrieUpdates)>; } StorageRootProvider $(where [$($generics)*])? { - fn storage_root(&self, address: alloy_primitives::Address, storage: reth_trie::HashedStorage) -> reth_storage_errors::provider::ProviderResult; + fn storage_root(&self, address: alloy_primitives::Address, storage: reth_trie::HashedStorage) -> reth_storage_errors::provider::ProviderResult; + fn storage_proof(&self, address: alloy_primitives::Address, slot: alloy_primitives::B256, storage: reth_trie::HashedStorage) -> reth_storage_errors::provider::ProviderResult; } StateProofProvider $(where [$($generics)*])? { fn proof(&self, input: reth_trie::TrieInput, address: alloy_primitives::Address, slots: &[alloy_primitives::B256]) -> reth_storage_errors::provider::ProviderResult; diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 4f2faad8abe4..3325d3ae9edb 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -31,7 +31,8 @@ use reth_storage_api::{ }; use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult}; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, + TrieInput, }; use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; use std::{ @@ -639,6 +640,15 @@ impl StorageRootProvider for MockEthProvider { ) -> ProviderResult { Ok(EMPTY_ROOT_HASH) } + + fn storage_proof( + &self, + _address: Address, + slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + Ok(StorageProof::new(slot)) + } } impl StateProofProvider for MockEthProvider { diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index e8b7760b880b..0a205389c9b6 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -356,6 +356,15 @@ impl StorageRootProvider for NoopProvider { ) -> ProviderResult { Ok(B256::default()) } + + fn storage_proof( + &self, + _address: Address, + slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + Ok(reth_trie::StorageProof::new(slot)) + } } impl StateProofProvider for NoopProvider { diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index d989def8bb0d..f7d41066d069 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -4,7 +4,8 @@ use alloy_primitives::{ }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, + TrieInput, }; /// A type that can compute the state root of a given post state. @@ -46,6 +47,15 @@ pub trait StorageRootProvider: Send + Sync { /// state. fn storage_root(&self, address: Address, hashed_storage: HashedStorage) -> ProviderResult; + + /// Returns the storage proof of the `HashedStorage` for target slot on top of the current + /// state. + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult; } /// A type that can generate state proof on top of a given post state. diff --git a/crates/trie/db/src/lib.rs b/crates/trie/db/src/lib.rs index aceea1da8660..3a9b1e328239 100644 --- a/crates/trie/db/src/lib.rs +++ b/crates/trie/db/src/lib.rs @@ -12,7 +12,7 @@ pub use hashed_cursor::{ DatabaseHashedAccountCursor, DatabaseHashedCursorFactory, DatabaseHashedStorageCursor, }; pub use prefix_set::PrefixSetLoader; -pub use proof::DatabaseProof; +pub use proof::{DatabaseProof, DatabaseStorageProof}; pub use state::{DatabaseHashedPostState, DatabaseStateRoot}; pub use storage::{DatabaseHashedStorage, DatabaseStorageRoot}; pub use trie_cursor::{ diff --git a/crates/trie/db/src/proof.rs b/crates/trie/db/src/proof.rs index 1d5fda84cc5b..9bf08fe136f7 100644 --- a/crates/trie/db/src/proof.rs +++ b/crates/trie/db/src/proof.rs @@ -1,13 +1,16 @@ use crate::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; use alloy_primitives::{ + keccak256, map::{HashMap, HashSet}, Address, B256, }; use reth_db_api::transaction::DbTx; use reth_execution_errors::StateProofError; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, proof::Proof, - trie_cursor::InMemoryTrieCursorFactory, MultiProof, TrieInput, + hashed_cursor::HashedPostStateCursorFactory, + proof::{Proof, StorageProof}, + trie_cursor::InMemoryTrieCursorFactory, + HashedPostStateSorted, HashedStorage, MultiProof, TrieInput, }; use reth_trie_common::AccountProof; @@ -81,3 +84,46 @@ impl<'a, TX: DbTx> DatabaseProof<'a, TX> .multiproof(targets) } } + +/// Extends [`StorageProof`] with operations specific for working with a database transaction. +pub trait DatabaseStorageProof<'a, TX> { + /// Create a new [`StorageProof`] from database transaction and account address. + fn from_tx(tx: &'a TX, address: Address) -> Self; + + /// Generates the storage proof for target slot based on [`TrieInput`]. + fn overlay_storage_proof( + tx: &'a TX, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result; +} + +impl<'a, TX: DbTx> DatabaseStorageProof<'a, TX> + for StorageProof, DatabaseHashedCursorFactory<'a, TX>> +{ + fn from_tx(tx: &'a TX, address: Address) -> Self { + Self::new(DatabaseTrieCursorFactory::new(tx), DatabaseHashedCursorFactory::new(tx), address) + } + + fn overlay_storage_proof( + tx: &'a TX, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let prefix_set = storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from([(hashed_address, storage.into_sorted())]), + ); + Self::from_tx(tx, address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(tx), + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + } +} diff --git a/crates/trie/trie/src/proof.rs b/crates/trie/trie/src/proof.rs index 95d9505218bf..bff681a3498d 100644 --- a/crates/trie/trie/src/proof.rs +++ b/crates/trie/trie/src/proof.rs @@ -124,7 +124,7 @@ where hashed_address, ) .with_prefix_set_mut(storage_prefix_set) - .storage_proof(proof_targets)?; + .storage_multiproof(proof_targets)?; // Encode account account_rlp.clear(); @@ -170,6 +170,26 @@ impl StorageProof { } } + /// Set the trie cursor factory. + pub fn with_trie_cursor_factory(self, trie_cursor_factory: TF) -> StorageProof { + StorageProof { + trie_cursor_factory, + hashed_cursor_factory: self.hashed_cursor_factory, + hashed_address: self.hashed_address, + prefix_set: self.prefix_set, + } + } + + /// Set the hashed cursor factory. + pub fn with_hashed_cursor_factory(self, hashed_cursor_factory: HF) -> StorageProof { + StorageProof { + trie_cursor_factory: self.trie_cursor_factory, + hashed_cursor_factory, + hashed_address: self.hashed_address, + prefix_set: self.prefix_set, + } + } + /// Set the changed prefixes. pub fn with_prefix_set_mut(mut self, prefix_set: PrefixSetMut) -> Self { self.prefix_set = prefix_set; @@ -182,8 +202,17 @@ where T: TrieCursorFactory, H: HashedCursorFactory, { - /// Generate storage proof. + /// Generate an account proof from intermediate nodes. pub fn storage_proof( + self, + slot: B256, + ) -> Result { + let targets = HashSet::from_iter([keccak256(slot)]); + Ok(self.storage_multiproof(targets)?.storage_proof(slot)?) + } + + /// Generate storage proof. + pub fn storage_multiproof( mut self, targets: HashSet, ) -> Result { diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index b0fcfb021ae1..d668946e6238 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -178,7 +178,7 @@ where hashed_address, ) .with_prefix_set_mut(storage_prefix_set) - .storage_proof(HashSet::from_iter([target_key]))?; + .storage_multiproof(HashSet::from_iter([target_key]))?; // The subtree only contains the proof for a single target. let node = From ece894bd28303fc2deb89947f1a25059665aa2a8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 12:02:34 +0200 Subject: [PATCH 068/103] chore: rm unused optimism feature from compat (#11560) --- crates/node/core/Cargo.toml | 3 +-- crates/optimism/node/Cargo.toml | 1 - crates/optimism/payload/Cargo.toml | 1 - crates/rpc/rpc-types-compat/Cargo.toml | 5 +---- crates/rpc/rpc/Cargo.toml | 1 - 5 files changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index f6d686451905..3ac90a888705 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -77,8 +77,7 @@ tempfile.workspace = true [features] optimism = [ - "reth-primitives/optimism", - "reth-rpc-types-compat/optimism", + "reth-primitives/optimism" ] # Features for vergen to generate correct env vars jemalloc = [] diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index cb74bcbc0904..3c298bea9cee 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -81,7 +81,6 @@ optimism = [ "reth-chainspec/optimism", "reth-primitives/optimism", "reth-provider/optimism", - "reth-rpc-types-compat/optimism", "reth-optimism-evm/optimism", "reth-optimism-payload-builder/optimism", "reth-beacon-consensus/optimism", diff --git a/crates/optimism/payload/Cargo.toml b/crates/optimism/payload/Cargo.toml index 117f63201a48..d5f8b520e822 100644 --- a/crates/optimism/payload/Cargo.toml +++ b/crates/optimism/payload/Cargo.toml @@ -52,7 +52,6 @@ optimism = [ "reth-chainspec/optimism", "reth-primitives/optimism", "reth-provider/optimism", - "reth-rpc-types-compat/optimism", "reth-optimism-evm/optimism", "reth-revm/optimism", ] diff --git a/crates/rpc/rpc-types-compat/Cargo.toml b/crates/rpc/rpc-types-compat/Cargo.toml index a9d82d95779c..81b4def204f9 100644 --- a/crates/rpc/rpc-types-compat/Cargo.toml +++ b/crates/rpc/rpc-types-compat/Cargo.toml @@ -26,7 +26,4 @@ alloy-serde.workspace = true alloy-rpc-types-engine.workspace = true [dev-dependencies] -serde_json.workspace = true - -[features] -optimism = ["reth-primitives/optimism"] +serde_json.workspace = true \ No newline at end of file diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index b0994a33e0aa..5399e50ce28a 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -100,7 +100,6 @@ jsonrpsee = { workspace = true, features = ["client"] } js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"] optimism = [ "reth-primitives/optimism", - "reth-rpc-types-compat/optimism", "reth-provider/optimism", "reth-revm/optimism", ] From a2cfa9e0503170ba09d19fcd363db99d40e952cb Mon Sep 17 00:00:00 2001 From: Debjit Bhowal Date: Tue, 8 Oct 2024 15:57:18 +0530 Subject: [PATCH 069/103] Added InternalBlockExecutionError to execute.rs exports (#11525) Co-authored-by: Matthias Seitz --- crates/evm/src/execute.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 3fc2975f0ff7..1bd79378127e 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -1,7 +1,9 @@ //! Traits for execution. // Re-export execution types -pub use reth_execution_errors::{BlockExecutionError, BlockValidationError}; +pub use reth_execution_errors::{ + BlockExecutionError, BlockValidationError, InternalBlockExecutionError, +}; pub use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput, ExecutionOutcome}; pub use reth_storage_errors::provider::ProviderError; From 3765dbe4707ba199f3bc85fc78d4ba7029276fdc Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 8 Oct 2024 11:46:53 +0100 Subject: [PATCH 070/103] docs(exex): include code for ExEx book from real files (#11545) --- .github/workflows/lint.yml | 12 ++- .github/workflows/unit.yml | 5 ++ .gitignore | 5 +- Cargo.toml | 1 + book/developers/exex/hello-world.md | 86 +--------------------- book/sources/Cargo.toml | 9 +++ book/sources/exex/hello-world/Cargo.toml | 13 ++++ book/sources/exex/hello-world/src/bin/1.rs | 9 +++ book/sources/exex/hello-world/src/bin/2.rs | 20 +++++ book/sources/exex/hello-world/src/bin/3.rs | 39 ++++++++++ 10 files changed, 114 insertions(+), 85 deletions(-) create mode 100644 book/sources/Cargo.toml create mode 100644 book/sources/exex/hello-world/Cargo.toml create mode 100644 book/sources/exex/hello-world/src/bin/1.rs create mode 100644 book/sources/exex/hello-world/src/bin/2.rs create mode 100644 book/sources/exex/hello-world/src/bin/3.rs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4bab77b915ee..e8161557fde9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,7 +30,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true - - run: cargo clippy --bin "${{ matrix.binary }}" --workspace --features "${{ matrix.network }} asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + - name: Run clippy on binaries + run: cargo clippy --bin "${{ matrix.binary }}" --workspace --features "${{ matrix.network }} asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + env: + RUSTFLAGS: -D warnings + - name: Run clippy on book binary sources + run: cargo clippy --manifest-path book/sources/Cargo.toml --workspace --bins env: RUSTFLAGS: -D warnings @@ -128,7 +133,10 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt - - run: cargo fmt --all --check + - name: Run fmt + run: cargo fmt --all --check + - name: Run fmt on book sources + run: cargo fmt --manifest-path book/sources/Cargo.toml --all --check udeps: name: udeps diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index a6663aea8843..1056a6fb58c8 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -42,6 +42,11 @@ jobs: --workspace --exclude ef-tests \ --partition hash:${{ matrix.partition }}/2 \ -E "!kind(test)" + - name: Run tests on book sources + run: | + cargo nextest run \ + --manifest-path book/sources/Cargo.toml --workspace \ + -E "!kind(test)" state: name: Ethereum state tests diff --git a/.gitignore b/.gitignore index 2d5d851a5055..00f776542422 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,7 @@ jwttoken/ crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake-build-debug # Rust bug report -rustc-ice-* \ No newline at end of file +rustc-ice-* + +# Book sources should be able to build with the latest version +book/sources/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 8c1ee8c4514c..b6eed127119b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,6 +148,7 @@ members = [ "testing/testing-utils", ] default-members = ["bin/reth"] +exclude = ["book/sources"] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html diff --git a/book/developers/exex/hello-world.md b/book/developers/exex/hello-world.md index facb07e5307f..c1f3e5af9443 100644 --- a/book/developers/exex/hello-world.md +++ b/book/developers/exex/hello-world.md @@ -14,19 +14,7 @@ cd my-exex And add Reth as a dependency in `Cargo.toml` ```toml -[package] -name = "my-exex" -version = "0.1.0" -edition = "2021" - -[dependencies] -reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth -reth-exex = { git = "https://github.com/paradigmxyz/reth.git" } # Execution Extensions -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git" } # Ethereum Node implementation -reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } # Logging - -eyre = "0.6" # Easy error handling -futures-util = "0.3" # Stream utilities for consuming notifications +{{#include ../../sources/exex/hello-world/Cargo.toml}} ``` ### Default Reth node @@ -34,15 +22,7 @@ futures-util = "0.3" # Stream utilities for consuming notifications Now, let's jump to our `main.rs` and start by initializing and launching a default Reth node ```rust,norun,noplayground,ignore -use reth_node_ethereum::EthereumNode; - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder.node(EthereumNode::default()).launch().await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/hello-world/src/bin/1.rs}} ``` You can already test that it works by running the binary and initializing the Holesky node in a custom datadir @@ -63,26 +43,7 @@ $ cargo run -- init --chain holesky --datadir data The simplest ExEx is just an async function that never returns. We need to install it into our node ```rust,norun,noplayground,ignore -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -async fn my_exex(mut _ctx: ExExContext) -> eyre::Result<()> { - loop {} -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/hello-world/src/bin/2.rs}} ``` See that unused `_ctx`? That's the context that we'll use to listen to new notifications coming from the main node, @@ -103,46 +64,7 @@ If you try running a node with an ExEx that exits, the node will exit as well. Now, let's extend our simplest ExEx and start actually listening to new notifications, log them, and send events back to the main node ```rust,norun,noplayground,ignore -use futures_util::StreamExt; -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -async fn my_exex(mut ctx: ExExContext) -> eyre::Result<()> { - while let Some(notification) = ctx.notifications.next().await { - match ¬ification { - ExExNotification::ChainCommitted { new } => { - info!(committed_chain = ?new.range(), "Received commit"); - } - ExExNotification::ChainReorged { old, new } => { - info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); - } - ExExNotification::ChainReverted { old } => { - info!(reverted_chain = ?old.range(), "Received revert"); - } - }; - - if let Some(committed_chain) = notification.committed_chain() { - ctx.events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - } - - Ok(()) -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/hello-world/src/bin/3.rs}} ``` Woah, there's a lot of new stuff here! Let's go through it step by step: diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml new file mode 100644 index 000000000000..c04c8567f94d --- /dev/null +++ b/book/sources/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +members = [ + "exex/hello-world", +] + +# Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 +# https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html +resolver = "2" + diff --git a/book/sources/exex/hello-world/Cargo.toml b/book/sources/exex/hello-world/Cargo.toml new file mode 100644 index 000000000000..e5d32a140549 --- /dev/null +++ b/book/sources/exex/hello-world/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "my-exex" +version = "0.1.0" +edition = "2021" + +[dependencies] +reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth +reth-exex = { git = "https://github.com/paradigmxyz/reth.git" } # Execution Extensions +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git" } # Ethereum Node implementation +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } # Logging + +eyre = "0.6" # Easy error handling +futures-util = "0.3" # Stream utilities for consuming notifications diff --git a/book/sources/exex/hello-world/src/bin/1.rs b/book/sources/exex/hello-world/src/bin/1.rs new file mode 100644 index 000000000000..794609bfd7d1 --- /dev/null +++ b/book/sources/exex/hello-world/src/bin/1.rs @@ -0,0 +1,9 @@ +use reth_node_ethereum::EthereumNode; + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/hello-world/src/bin/2.rs b/book/sources/exex/hello-world/src/bin/2.rs new file mode 100644 index 000000000000..6ab5fc49650e --- /dev/null +++ b/book/sources/exex/hello-world/src/bin/2.rs @@ -0,0 +1,20 @@ +use reth::api::FullNodeComponents; +use reth_exex::ExExContext; +use reth_node_ethereum::EthereumNode; + +async fn my_exex(mut _ctx: ExExContext) -> eyre::Result<()> { + #[allow(clippy::empty_loop)] + loop {} +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/hello-world/src/bin/3.rs b/book/sources/exex/hello-world/src/bin/3.rs new file mode 100644 index 000000000000..21bd25a56dbf --- /dev/null +++ b/book/sources/exex/hello-world/src/bin/3.rs @@ -0,0 +1,39 @@ +use futures_util::TryStreamExt; +use reth::api::FullNodeComponents; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; + +async fn my_exex(mut ctx: ExExContext) -> eyre::Result<()> { + while let Some(notification) = ctx.notifications.try_next().await? { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + info!(committed_chain = ?new.range(), "Received commit"); + } + ExExNotification::ChainReorged { old, new } => { + info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); + } + ExExNotification::ChainReverted { old } => { + info!(reverted_chain = ?old.range(), "Received revert"); + } + }; + + if let Some(committed_chain) = notification.committed_chain() { + ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + } + + Ok(()) +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} From 4163835b932119d9bcd3fa469490aa7391ec86a2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 13:04:48 +0200 Subject: [PATCH 071/103] fix: actually configure the custom gas limit (#11565) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 819aff546d42..c5703bcceef3 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -716,6 +716,7 @@ pub trait Call: LoadState + SpawnBlocking { // Keep a copy of gas related request values let tx_request_gas_limit = request.gas; let tx_request_gas_price = request.gas_price; + // the gas limit of the corresponding block let block_env_gas_limit = block.gas_limit; // Determine the highest possible gas limit, considering both the request's specified limit @@ -1083,6 +1084,8 @@ pub trait Call: LoadState + SpawnBlocking { /// - `disable_eip3607` is set to `true` /// - `disable_base_fee` is set to `true` /// - `nonce` is set to `None` + /// + /// In addition, this changes the block's gas limit to the configured [`Self::call_gas_limit`]. fn prepare_call_env( &self, mut cfg: CfgEnvWithHandlerCfg, @@ -1102,6 +1105,9 @@ pub trait Call: LoadState + SpawnBlocking { ) } + // apply configured gas cap + block.gas_limit = U256::from(self.call_gas_limit()); + // Disabled because eth_call is sometimes used with eoa senders // See cfg.disable_eip3607 = true; From cb431f1f5cc0948d50dba7a5e1330809c6cf0690 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 8 Oct 2024 14:54:22 +0200 Subject: [PATCH 072/103] chore: relax trait bound for `EthTransactions` (#11571) --- crates/rpc/rpc-eth-api/src/helpers/transaction.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index d70ea7e541cb..afcb1b2eeb25 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -22,8 +22,7 @@ use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool} use crate::{FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcReceipt, RpcTransaction}; use super::{ - Call, EthApiSpec, EthSigner, LoadBlock, LoadFee, LoadPendingBlock, LoadReceipt, LoadState, - SpawnBlocking, + Call, EthApiSpec, EthSigner, LoadBlock, LoadPendingBlock, LoadReceipt, LoadState, SpawnBlocking, }; /// Transaction related functions for the [`EthApiServer`](crate::EthApiServer) trait in @@ -344,7 +343,7 @@ pub trait EthTransactions: LoadTransaction { mut request: TransactionRequest, ) -> impl Future> + Send where - Self: EthApiSpec + LoadBlock + LoadPendingBlock + LoadFee + Call, + Self: EthApiSpec + LoadBlock + LoadPendingBlock + Call, { async move { let from = match request.from { From 65e66832959954ef77ca580f2268ee2e0d445a6f Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 8 Oct 2024 21:54:47 +0900 Subject: [PATCH 073/103] fix(provider): fix sub overflow on `tx_range` queries for empty blocks (#11568) --- crates/storage/provider/src/providers/blockchain_provider.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 9d075015397f..c52d2d58b1e0 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -463,8 +463,9 @@ impl BlockchainProvider2 { let block_tx_count = block_state.block_ref().block().body.transactions.len(); let remaining = (tx_range.end() - tx_range.start() + 1) as usize; - // If the transaction range start is higher than this block last transaction, advance - if *tx_range.start() > in_memory_tx_num + block_tx_count as u64 - 1 { + // If the transaction range start is equal or higher than the next block first + // transaction, advance + if *tx_range.start() >= in_memory_tx_num + block_tx_count as u64 { in_memory_tx_num += block_tx_count as u64; continue } From 67db43d0acc8ff9d38fafd37bed8c7d43c840e75 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 8 Oct 2024 22:17:57 +0900 Subject: [PATCH 074/103] chore(provider): add more test coverage on `BlockchainProvider` non-range queries (#11564) --- .../src/providers/blockchain_provider.rs | 1203 +++++------------ 1 file changed, 323 insertions(+), 880 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index c52d2d58b1e0..d25c3f84bd1a 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1714,7 +1714,7 @@ mod tests { StaticFileWriter, }; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; - use alloy_primitives::{BlockNumber, B256}; + use alloy_primitives::{BlockNumber, TxNumber, B256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ @@ -1732,8 +1732,7 @@ mod tests { use reth_errors::ProviderError; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ - Receipt, SealedBlock, StaticFileSegment, TransactionMeta, TransactionSignedNoHash, - Withdrawals, + Receipt, SealedBlock, StaticFileSegment, TransactionSignedNoHash, Withdrawals, }; use reth_storage_api::{ BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, @@ -2258,284 +2257,6 @@ mod tests { Ok(()) } - #[test] - fn test_block_with_senders_by_hash_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_hash = first_in_mem_block.hash(); - - // Get the block with senders by hash and check if it matches the first in-memory block - let block_with_senders = provider - .block_with_senders(BlockHashOrNumber::Hash(block_hash), TransactionVariant::WithHash)? - .unwrap(); - assert_eq!(block_with_senders.block.seal(block_hash), first_in_mem_block.clone()); - assert_eq!(block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_by_number_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_number = first_in_mem_block.number; - - // Get the block with senders by number and check if it matches the first in-memory block - let block_with_senders = provider - .block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!( - block_with_senders.block.seal(first_in_mem_block.hash()), - first_in_mem_block.clone() - ); - assert_eq!(block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_by_hash_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_hash = first_db_block.hash(); - - // Get the block with senders by hash and check if it matches the first database block - let block_with_senders = provider - .block_with_senders(BlockHashOrNumber::Hash(block_hash), TransactionVariant::WithHash)? - .unwrap(); - assert_eq!(block_with_senders.block.seal(block_hash), first_db_block.clone()); - assert_eq!(block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_by_number_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_number = first_db_block.number; - - // Get the block with senders by number and check if it matches the first database block - let block_with_senders = provider - .block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(block_with_senders.block.seal(first_db_block.hash()), first_db_block.clone()); - assert_eq!(block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_non_existent_block() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Generate a random hash (non-existent block) - let non_existent_hash = B256::random(); - let result = provider.block_with_senders( - BlockHashOrNumber::Hash(non_existent_hash), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - // Generate a random number (non-existent block) - let non_existent_number = 9999; - let result = provider.block_with_senders( - BlockHashOrNumber::Number(non_existent_number), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_hash_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_hash = first_in_mem_block.hash(); - - // Get the sealed block with senders by hash and check if it matches the first in-memory - // block - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_in_mem_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_number_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_number = first_in_mem_block.number; - - // Get the sealed block with senders by number and check if it matches the first in-memory - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_in_mem_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_hash_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_hash = first_db_block.hash(); - - // Get the sealed block with senders by hash and check if it matches the first database - // block - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_db_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_number_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_number = first_db_block.number; - - // Get the sealed block with senders by number and check if it matches the first database - // block - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_db_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_non_existent_block() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Generate a random hash (non-existent block) - let non_existent_hash = B256::random(); - let result = provider.sealed_block_with_senders( - BlockHashOrNumber::Hash(non_existent_hash), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - // Generate a random number (non-existent block) - let non_existent_number = 9999; - let result = provider.sealed_block_with_senders( - BlockHashOrNumber::Number(non_existent_number), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - Ok(()) - } - #[test] fn test_block_hash_reader() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -2582,39 +2303,16 @@ mod tests { let blocks = [database_blocks, in_memory_blocks].concat(); - assert_eq!(provider.header(&database_block.hash())?, Some(database_block.header().clone())); - assert_eq!( - provider.header(&in_memory_block.hash())?, - Some(in_memory_block.header().clone()) - ); - - assert_eq!( - provider.header_by_number(database_block.number)?, - Some(database_block.header().clone()) - ); - assert_eq!( - provider.header_by_number(in_memory_block.number)?, - Some(in_memory_block.header().clone()) - ); - assert_eq!( provider.header_td_by_number(database_block.number)?, Some(database_block.difficulty) ); + assert_eq!( provider.header_td_by_number(in_memory_block.number)?, Some(in_memory_block.difficulty) ); - assert_eq!( - provider.sealed_header(database_block.number)?, - Some(database_block.header.clone()) - ); - assert_eq!( - provider.sealed_header(in_memory_block.number)?, - Some(in_memory_block.header.clone()) - ); - assert_eq!( provider.sealed_headers_while(0..=10, |header| header.number <= 8)?, blocks @@ -2936,37 +2634,6 @@ mod tests { Ok(()) } - #[test] - fn test_receipt_provider() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { tx_count: 1..3, ..Default::default() }, - )?; - - let blocks = [database_blocks, in_memory_blocks].concat(); - - for block in blocks { - let block_number = block.number as usize; - for (txn_number, _) in block.body.transactions.iter().enumerate() { - let txn_hash = block.body.transactions.get(txn_number).unwrap().hash(); - let txn_id = provider.transaction_id(txn_hash)?.unwrap(); - assert_eq!( - provider.receipt(txn_id)?.unwrap(), - receipts.get(block_number).unwrap().clone().get(txn_number).unwrap().clone() - ); - assert_eq!( - provider.receipt_by_hash(txn_hash)?.unwrap(), - receipts.get(block_number).unwrap().clone().get(txn_number).unwrap().clone() - ); - } - } - - Ok(()) - } - #[test] fn test_receipt_provider_id_ext_receipts_by_block_id() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -3448,502 +3115,6 @@ mod tests { Ok(()) } - #[test] - fn test_transaction_id() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // Database - // Choose a random transaction from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction ID can be found in the database - let result = provider.transaction_id(tx_hash)?; - assert_eq!(result, Some(0)); - - // In memory - // Choose a random transaction from the in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction ID can be found in the in-memory state - let result = provider.transaction_id(tx_hash)?; - assert!(result.is_some(), "Transaction ID should be found in the in-memory state"); - - // Check that the transaction ID is greater than the last database transaction ID - let last_db_tx_id = provider.database.last_block_number()?; - let last_db_tx_id = - provider.database.block_body_indices(last_db_tx_id)?.unwrap().last_tx_num(); - - assert!( - result.unwrap() > last_db_tx_id, - "In-memory transaction ID should be greater than the last database transaction ID" - ); - assert_eq!(result, Some(last_db_tx_id + 1)); - - // Generate a random hash not present in any transaction - let random_tx_hash = B256::random(); - - // Ensure the transaction ID is not found - let result = provider.transaction_id(random_tx_hash)?; - assert!(result.is_none(), "Transaction ID should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_by_id() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction ID from in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID - let result = provider.transaction_by_id(tx_id)?; - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Database - // Choose a random transaction ID from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID - let result = provider.transaction_by_id(tx_id)?; - assert!(result.is_some(), "Transaction should be found in the database"); - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Generate a random transaction ID not present in any block - let random_tx_id = 999999; - - // Ensure the transaction is not found - let result = provider.transaction_by_id(random_tx_id)?; - assert!(result.is_none(), "Transaction should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_by_id_no_hash() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction ID from in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID without hash - let result = provider.transaction_by_id_no_hash(tx_id)?; - let expected_tx: TransactionSignedNoHash = tx.clone().into(); - assert_eq!( - result.unwrap(), - expected_tx, - "The retrieved transaction without hash should match the expected transaction" - ); - - // Database - // Choose a random transaction ID from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID without hash - let result = provider.transaction_by_id_no_hash(tx_id)?; - let expected_tx: TransactionSignedNoHash = tx.clone().into(); - assert_eq!( - result.unwrap(), - expected_tx, - "The retrieved transaction without hash should match the expected transaction" - ); - - // Generate a random transaction ID not present in any block - let random_tx_id = 7656898; - - // Ensure the transaction is not found without hash - let result = provider.transaction_by_id_no_hash(random_tx_id)?; - assert!(result.is_none(), "Transaction should not be found without hash"); - - Ok(()) - } - - #[test] - fn test_transaction_by_hash() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction hash from the in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction can be retrieved by its hash from the in-memory state - let result = provider.transaction_by_hash(tx_hash)?; - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Database - // Choose a random transaction hash from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction can be retrieved by its hash from the database - let result = provider.transaction_by_hash(tx_hash)?; - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Generate a random hash not present in any transaction - let random_tx_hash = B256::random(); - - // Ensure the transaction is not found by the random hash - let result = provider.transaction_by_hash(random_tx_hash)?; - assert!(result.is_none(), "Transaction should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_by_hash_with_meta() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction from the in-memory block - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Create the expected metadata for this transaction - let meta = TransactionMeta { - tx_hash, - index: 0, - block_hash: in_memory_blocks[0].header.hash(), - block_number: in_memory_blocks[0].header.number, - base_fee: in_memory_blocks[0].header.base_fee_per_gas, - excess_blob_gas: None, - timestamp: in_memory_blocks[0].header.timestamp, - }; - - // Ensure the transaction and its metadata can be retrieved from the in-memory state - let result = provider.transaction_by_hash_with_meta(tx_hash)?; - let (retrieved_tx, retrieved_meta) = result.unwrap(); - assert_eq!( - retrieved_tx, *tx, - "The retrieved transaction should match the expected transaction" - ); - assert_eq!( - retrieved_meta, meta, - "The retrieved metadata should match the expected metadata" - ); - - // Database - // Choose a random transaction from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Create the expected metadata for this transaction - let meta = TransactionMeta { - tx_hash, - index: 0, - block_hash: database_blocks[0].header.hash(), - block_number: database_blocks[0].header.number, - base_fee: database_blocks[0].header.base_fee_per_gas, - excess_blob_gas: None, - timestamp: database_blocks[0].header.timestamp, - }; - - // Ensure the transaction and its metadata can be retrieved from the database - let result = provider.transaction_by_hash_with_meta(tx_hash)?; - let (retrieved_tx, retrieved_meta) = result.unwrap(); - assert_eq!( - retrieved_tx, *tx, - "The retrieved transaction should match the expected transaction" - ); - assert_eq!( - retrieved_meta, meta, - "The retrieved metadata should match the expected metadata" - ); - - // Generate a random hash not present in any transaction - let random_tx_hash = B256::random(); - - // Ensure the transaction with metadata is not found by the random hash - let result = provider.transaction_by_hash_with_meta(random_tx_hash)?; - assert!(result.is_none(), "Transaction with metadata should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_block() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction ID from in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Retrieve the block number for this transaction - let result = provider.transaction_block(tx_id)?; - let block_number = result.unwrap(); - - // Ensure the block number matches the expected block number - assert_eq!( - block_number, in_memory_blocks[0].header.number, - "The block number should match the in-memory block number" - ); - - // Database - // Choose a random transaction from the database block - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Retrieve the block number for this transaction - let result = provider.transaction_block(tx_id)?; - assert_eq!(Some(0), result, "The block number should match the database block number"); - - // Ensure that invalid transaction ID returns None - let result = provider.transaction_block(67675657)?; - - assert!(result.is_none(), "Block number should not be found for an invalid transaction ID"); - - Ok(()) - } - - #[test] - fn transactions_found_by_block_hash() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random block hash from in-memory blocks - let block_hash = in_memory_blocks[0].header.hash(); - - // Retrieve the transactions for this block using the block hash - let result = provider.transactions_by_block(BlockHashOrNumber::Hash(block_hash))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, in_memory_blocks[0].body.transactions, - "The transactions should match the in-memory block transactions" - ); - - // Database - // Choose a random block hash from the database blocks - let block_hash = database_blocks[0].header.hash(); - - // Retrieve the transactions for this block using the block hash - let result = provider.transactions_by_block(BlockHashOrNumber::Hash(block_hash))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, database_blocks[0].body.transactions, - "The transactions should match the database block transactions" - ); - - // Generate a random block hash that does not exist - let random_block_hash = B256::random(); - - // Try to retrieve transactions for a non-existent block hash - let result = provider.transactions_by_block(BlockHashOrNumber::Hash(random_block_hash))?; - - // Ensure no transactions are found - assert!(result.is_none(), "No transactions should be found for a non-existent block hash"); - - Ok(()) - } - - #[test] - fn transactions_found_by_block_number() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random block number from in-memory blocks - let block_number = in_memory_blocks[0].header.number; - - // Retrieve the transactions for this block using the block number - let result = provider.transactions_by_block(BlockHashOrNumber::Number(block_number))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, in_memory_blocks[0].body.transactions, - "The transactions should match the in-memory block transactions" - ); - - // Database - // Choose a random block number from the database blocks - let block_number = database_blocks[0].header.number; - - // Retrieve the transactions for this block using the block number - let result = provider.transactions_by_block(BlockHashOrNumber::Number(block_number))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, database_blocks[0].body.transactions, - "The transactions should match the database block transactions" - ); - - // Generate a block number that is out of range (non-existent) - let non_existent_block_number = u64::MAX; - - // Try to retrieve transactions for a non-existent block number - let result = - provider.transactions_by_block(BlockHashOrNumber::Number(non_existent_block_number))?; - - // Ensure no transactions are found - assert!( - result.is_none(), - "No transactions should be found for a non-existent block number" - ); - - Ok(()) - } - - #[test] - fn transactions_found_entirely_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Define a block range entirely within in-memory blocks - let start_block = in_memory_blocks[0].header.number; - let end_block = in_memory_blocks[1].header.number; - - // Retrieve the transactions for this block range - let result = provider.transactions_by_block_range(start_block..=end_block)?; - - // Ensure the transactions match the expected transactions in the in-memory blocks - assert_eq!(result.len(), 2); - assert_eq!(result[0], in_memory_blocks[0].body.transactions); - assert_eq!(result[1], in_memory_blocks[1].body.transactions); - - // Database - // Define a block range entirely within database blocks - let start_block = database_blocks[0].header.number; - let end_block = database_blocks[1].header.number; - - // Retrieve the transactions for this block range - let result = provider.transactions_by_block_range(start_block..=end_block)?; - - // Ensure the transactions match the expected transactions in the database blocks - assert_eq!(result.len(), 2); - assert_eq!(result[0], database_blocks[0].body.transactions); - assert_eq!(result[1], database_blocks[1].body.transactions); - - Ok(()) - } - macro_rules! test_by_tx_range { ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ let db_tx_count = @@ -4131,65 +3302,337 @@ mod tests { Ok(()) } - #[test] - fn transaction_sender_found_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; + /// Helper macro to call a provider method based on argument count and check its result + macro_rules! call_method { + ($provider:expr, $method:ident, ($($args:expr),*), $expected_item:expr) => {{ + let result = $provider.$method($($args),*)?; + assert_eq!( + result, + $expected_item, + "{}: item does not match the expected item for arguments {:?}", + stringify!($method), + ($($args),*) + ); + }}; - // In memory - // Choose a random transaction from the in-memory block - let tx = &in_memory_blocks[0].body.transactions[0]; + // Handle valid or invalid arguments for one argument + (ONE, $provider:expr, $method:ident, $item_extractor:expr, $txnum:expr, $txhash:expr, $block:expr, $receipts:expr) => {{ + let (arg, expected_item) = $item_extractor($block, $txnum($block), $txhash($block), $receipts); + call_method!($provider, $method, (arg), expected_item); + }}; - // Retrieve the transaction ID - let tx_id = provider.transaction_id(tx.hash())?.unwrap(); + // Handle valid or invalid arguments for two arguments + (TWO, $provider:expr, $method:ident, $item_extractor:expr, $txnum:expr, $txhash:expr, $block:expr, $receipts:expr) => {{ + let ((arg1, arg2), expected_item) = $item_extractor($block, $txnum($block), $txhash($block), $receipts); + call_method!($provider, $method, (arg1, arg2), expected_item); + }}; + } - // Retrieve the sender address for this transaction - let result = provider.transaction_sender(tx_id)?; + /// Macro to test non-range methods. + /// + /// ( `NUMBER_ARGUMENTS`, METHOD, FN -> ((`METHOD_ARGUMENT(s)`,...), `EXPECTED_RESULT`), + /// `INVALID_ARGUMENTS`) + macro_rules! test_non_range { + ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, $receipts:expr, [$(($arg_count:ident, $method:ident, $item_extractor:expr, $invalid_args:expr)),* $(,)?]) => {{ + $( + let tx_hash = |block: &SealedBlock| block.body.transactions[0].hash(); + let tx_num = |block: &SealedBlock| { + $database_blocks + .iter() + .chain($in_memory_blocks.iter()) + .take_while(|b| b.number < block.number) + .map(|b| b.body.transactions.len()) + .sum::() as u64 + }; - // Ensure the sender address matches the expected sender address - let expected_sender = tx.recover_signer().unwrap(); - assert_eq!( - result, - Some(expected_sender), - "The sender address should match the expected sender address" - ); + // Ensure that the first generated in-memory block exists + // In the future this block will be persisted to disk and removed from memory AFTER the firsk database query. + // This ensures that we query the in-memory state before the database avoiding any race condition. + { + call_method!($arg_count, $provider, $method, $item_extractor, tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + } - // Database - // Choose a random transaction from the database block - let tx = &database_blocks[0].body.transactions[0]; + // Invalid/Non-existent argument should return `None` + { + call_method!($arg_count, $provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + } - // Retrieve the transaction ID - let tx_id = provider.transaction_id(tx.hash())?.unwrap(); + // Check that the item is only in memory and not in database + { + let last_mem_block = &$in_memory_blocks[$in_memory_blocks.len() - 1]; - // Retrieve the sender address for this transaction - let result = provider.transaction_sender(tx_id)?; + let (args, expected_item) = $item_extractor(last_mem_block, tx_num(last_mem_block), tx_hash(last_mem_block), &$receipts); + call_method!($arg_count, $provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &$receipts); - // Ensure the sender address matches the expected sender address - let expected_sender = tx.recover_signer().unwrap(); - assert_eq!( - result, - Some(expected_sender), - "The sender address should match the expected sender address" - ); + // Ensure the item is not in storage + call_method!($arg_count, $provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &$receipts); + } + )* + }}; +} - // Generate a random transaction ID that does not exist - let invalid_tx_id = u64::MAX; + #[test] + fn test_non_range_methods() -> eyre::Result<()> { + let test_tx_index = 0; - // Attempt to retrieve the sender address for this invalid transaction ID - let result = provider.transaction_sender(invalid_tx_id)?; + let mut rng = generators::rng(); + let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; - // Ensure no sender address is found - assert!( - result.is_none(), - "No sender address should be found for an invalid transaction ID" + test_non_range!( + provider, + database_blocks, + in_memory_blocks, + receipts, + [ + // TODO: header should use B256 like others instead of &B256 + // ( + // ONE, + // header, + // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), + // (&B256::random()) + // ), + ( + ONE, + header_by_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.header().clone()) + ), + u64::MAX + ), + ( + ONE, + sealed_header, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.clone()) + ), + u64::MAX + ), + ( + ONE, + block_hash, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.hash()) + ), + u64::MAX + ), + ( + ONE, + block_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.hash(), + Some(block.number) + ), + B256::random() + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.clone().unseal()) + ), + BlockHashOrNumber::Hash(B256::random()) + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.clone().unseal()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + block_body_indices, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(StoredBlockBodyIndices { + first_tx_num: tx_num, + tx_count: block.body.transactions.len() as u64 + }) + ), + u64::MAX + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() + ), + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() + ), + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + Some( + block + .clone() + .unseal() + .with_recovered_senders() + .unwrap() + .seal(block.hash()) + ) + ), + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + Some( + block + .clone() + .unseal() + .with_recovered_senders() + .unwrap() + .seal(block.hash()) + ) + ), + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + ONE, + transaction_id, + |_: &SealedBlock, tx_num: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(tx_num) + ), + B256::random() + ), + ( + ONE, + transaction_by_id, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.body.transactions[test_tx_index].clone()) + ), + u64::MAX + ), + ( + ONE, + transaction_by_id_no_hash, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(Into::::into( + block.body.transactions[test_tx_index].clone() + )) + ), + u64::MAX + ), + ( + ONE, + transaction_by_hash, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(block.body.transactions[test_tx_index].clone()) + ), + B256::random() + ), + ( + ONE, + transaction_block, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.number) + ), + u64::MAX + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.body.transactions.clone()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.body.transactions.clone()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transaction_sender, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + block.body.transactions[test_tx_index].recover_signer() + ), + u64::MAX + ), + ( + ONE, + receipt, + |block: &SealedBlock, + tx_num: TxNumber, + _: B256, + receipts: &Vec>| ( + tx_num, + Some(receipts[block.number as usize][test_tx_index].clone()) + ), + u64::MAX + ), + ( + ONE, + receipt_by_hash, + |block: &SealedBlock, + _: TxNumber, + tx_hash: B256, + receipts: &Vec>| ( + tx_hash, + Some(receipts[block.number as usize][test_tx_index].clone()) + ), + B256::random() + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(receipts[block.number as usize].clone()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(receipts[block.number as usize].clone()) + ), + BlockHashOrNumber::Hash(B256::random()) + ), + // TODO: withdrawals, requests, ommers + ] ); Ok(()) From 946f1a4c1814e6c4a9a2e1981952dcdb25661fe2 Mon Sep 17 00:00:00 2001 From: Luca Provini Date: Tue, 8 Oct 2024 15:27:42 +0200 Subject: [PATCH 075/103] feat: adding a new method to network config builder (#11569) --- crates/exex/test-utils/src/lib.rs | 1 + crates/net/network/src/config.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index f4561a6b55f2..e219f55031db 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -271,6 +271,7 @@ pub async fn test_exex_context_with_chain_spec( let network_manager = NetworkManager::new( NetworkConfigBuilder::new(SecretKey::new(&mut rand::thread_rng())) + .with_unused_discovery_port() .build(provider_factory.clone()), ) .await?; diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 60b559b9dbc3..9d9183edcf98 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -347,6 +347,18 @@ impl NetworkConfigBuilder { self } + /// Sets the discovery port to an unused port. + /// This is useful for testing. + pub fn with_unused_discovery_port(self) -> Self { + self.discovery_port(0) + } + + /// Sets the listener port to an unused port. + /// This is useful for testing. + pub fn with_unused_listener_port(self) -> Self { + self.listener_port(0) + } + /// Sets the external ip resolver to use for discovery v4. /// /// If no [`Discv4ConfigBuilder`] is set via [`Self::discovery`], this will create a new one. From c32d4c3f2a2920c7f4ab1afc874fa503974bd223 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 16:23:54 +0200 Subject: [PATCH 076/103] chore: rm unused optimism feature from engine api (#11577) --- crates/rpc/rpc-engine-api/Cargo.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index 4463d375a034..00503f2c1dd7 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -55,7 +55,4 @@ reth-testing-utils.workspace = true alloy-rlp.workspace = true -assert_matches.workspace = true - -[features] -optimism = ["reth-primitives/optimism"] +assert_matches.workspace = true \ No newline at end of file From c13ab7a292ab65ee1a1ed5724de1b2aacfa9b87e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 16:23:56 +0200 Subject: [PATCH 077/103] chore: replace some revm deps (#11579) --- crates/primitives/benches/validate_blob_tx.rs | 7 +++---- crates/primitives/src/alloy_compat.rs | 3 +-- crates/primitives/src/transaction/util.rs | 12 ++++-------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/crates/primitives/benches/validate_blob_tx.rs b/crates/primitives/benches/validate_blob_tx.rs index 61fe161f2f74..50498a9420fd 100644 --- a/crates/primitives/benches/validate_blob_tx.rs +++ b/crates/primitives/benches/validate_blob_tx.rs @@ -1,7 +1,7 @@ #![allow(missing_docs)] use alloy_consensus::TxEip4844; -use alloy_eips::eip4844::env_settings::EnvKzgSettings; +use alloy_eips::eip4844::{env_settings::EnvKzgSettings, MAX_BLOBS_PER_BLOCK}; use alloy_primitives::hex; use criterion::{ criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion, @@ -13,7 +13,6 @@ use proptest::{ }; use proptest_arbitrary_interop::arb; use reth_primitives::BlobTransactionSidecar; -use revm_primitives::MAX_BLOB_NUMBER_PER_BLOCK; // constant seed to use for the rng const SEED: [u8; 32] = hex!("1337133713371337133713371337133713371337133713371337133713371337"); @@ -22,9 +21,9 @@ const SEED: [u8; 32] = hex!("133713371337133713371337133713371337133713371337133 fn blob_validation(c: &mut Criterion) { let mut group = c.benchmark_group("Blob Transaction KZG validation"); - for num_blobs in 1..=MAX_BLOB_NUMBER_PER_BLOCK { + for num_blobs in 1..=MAX_BLOBS_PER_BLOCK { println!("Benchmarking validation for tx with {num_blobs} blobs"); - validate_blob_tx(&mut group, "ValidateBlob", num_blobs, EnvKzgSettings::Default); + validate_blob_tx(&mut group, "ValidateBlob", num_blobs as u64, EnvKzgSettings::Default); } } diff --git a/crates/primitives/src/alloy_compat.rs b/crates/primitives/src/alloy_compat.rs index bf7f557b799a..f43af019e75b 100644 --- a/crates/primitives/src/alloy_compat.rs +++ b/crates/primitives/src/alloy_compat.rs @@ -278,9 +278,8 @@ impl TryFrom> for TransactionSigne #[cfg(feature = "optimism")] mod tests { use super::*; - use alloy_primitives::{B256, U256}; + use alloy_primitives::{address, Address, B256, U256}; use alloy_rpc_types::Transaction as AlloyTransaction; - use revm_primitives::{address, Address}; #[test] fn optimism_deposit_tx_conversion_no_mint() { diff --git a/crates/primitives/src/transaction/util.rs b/crates/primitives/src/transaction/util.rs index 6205ec886ca0..7569400e94b4 100644 --- a/crates/primitives/src/transaction/util.rs +++ b/crates/primitives/src/transaction/util.rs @@ -1,6 +1,5 @@ use crate::Signature; use alloy_primitives::Address; -use revm_primitives::B256; #[cfg(feature = "secp256k1")] pub(crate) mod secp256k1 { @@ -20,8 +19,7 @@ mod impl_secp256k1 { ecdsa::{RecoverableSignature, RecoveryId}, Message, PublicKey, SecretKey, SECP256K1, }; - use alloy_primitives::{keccak256, Parity}; - use revm_primitives::U256; + use alloy_primitives::{keccak256, Parity, B256, U256}; /// Recovers the address of the sender using secp256k1 pubkey recovery. /// @@ -65,10 +63,9 @@ mod impl_secp256k1 { #[cfg_attr(feature = "secp256k1", allow(unused, unreachable_pub))] mod impl_k256 { use super::*; - use alloy_primitives::{keccak256, Parity}; + use alloy_primitives::{keccak256, Parity, B256, U256}; pub(crate) use k256::ecdsa::Error; use k256::ecdsa::{RecoveryId, SigningKey, VerifyingKey}; - use revm_primitives::U256; /// Recovers the address of the sender using secp256k1 pubkey recovery. /// @@ -117,11 +114,12 @@ mod impl_k256 { #[cfg(test)] mod tests { + use alloy_primitives::{keccak256, B256}; + #[cfg(feature = "secp256k1")] #[test] fn sanity_ecrecover_call_secp256k1() { use super::impl_secp256k1::*; - use revm_primitives::{keccak256, B256}; let (secret, public) = secp256k1::generate_keypair(&mut rand::thread_rng()); let signer = public_key_to_address(public); @@ -143,7 +141,6 @@ mod tests { #[test] fn sanity_ecrecover_call_k256() { use super::impl_k256::*; - use revm_primitives::{keccak256, B256}; let secret = k256::ecdsa::SigningKey::random(&mut rand::thread_rng()); let public = *secret.verifying_key(); @@ -165,7 +162,6 @@ mod tests { #[test] fn sanity_secp256k1_k256_compat() { use super::{impl_k256, impl_secp256k1}; - use revm_primitives::{keccak256, B256}; let (secp256k1_secret, secp256k1_public) = secp256k1::generate_keypair(&mut rand::thread_rng()); From c61ae1371b93fc95387a7b893d911bf2cbc2030f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 16:25:00 +0200 Subject: [PATCH 078/103] chore: rm bad cap function (#11562) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 7 ++++--- crates/rpc/rpc-eth-types/src/revm_utils.rs | 16 +++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index c5703bcceef3..5bc7d73b2221 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -28,8 +28,8 @@ use reth_rpc_eth_types::{ cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, error::ensure_success, revm_utils::{ - apply_block_overrides, apply_state_overrides, caller_gas_allowance, - cap_tx_gas_limit_with_caller_allowance, get_precompiles, CallFees, + apply_block_overrides, apply_state_overrides, caller_gas_allowance, get_precompiles, + CallFees, }, simulate::{self, EthSimulateError}, EthApiError, RevertError, RpcInvalidTransactionError, StateCacheDb, @@ -379,8 +379,9 @@ pub trait EthCall: Call + LoadPendingBlock { let mut db = CacheDB::new(StateProviderDatabase::new(state)); if request.gas.is_none() && env.tx.gas_price > U256::ZERO { + let cap = caller_gas_allowance(&mut db, &env.tx)?; // no gas limit was provided in the request, so we need to cap the request's gas limit - cap_tx_gas_limit_with_caller_allowance(&mut db, &mut env.tx)?; + env.tx.gas_limit = cap.min(env.block.gas_limit).saturating_to(); } let from = request.from.unwrap_or_default(); diff --git a/crates/rpc/rpc-eth-types/src/revm_utils.rs b/crates/rpc/rpc-eth-types/src/revm_utils.rs index f6fbdc2f7ac8..25c54fd46777 100644 --- a/crates/rpc/rpc-eth-types/src/revm_utils.rs +++ b/crates/rpc/rpc-eth-types/src/revm_utils.rs @@ -23,25 +23,15 @@ pub fn get_precompiles(spec_id: SpecId) -> impl IntoIterator { Precompiles::new(spec).addresses().copied().map(Address::from) } -/// Caps the configured [`TxEnv`] `gas_limit` with the allowance of the caller. -pub fn cap_tx_gas_limit_with_caller_allowance(db: &mut DB, env: &mut TxEnv) -> EthResult<()> -where - DB: Database, - EthApiError: From<::Error>, -{ - if let Ok(gas_limit) = caller_gas_allowance(db, env)?.try_into() { - env.gas_limit = gas_limit; - } - - Ok(()) -} - /// Calculates the caller gas allowance. /// /// `allowance = (account.balance - tx.value) / tx.gas_price` /// /// Returns an error if the caller has insufficient funds. /// Caution: This assumes non-zero `env.gas_price`. Otherwise, zero allowance will be returned. +/// +/// Note: this takes the mut [Database] trait because the loaded sender can be reused for the +/// following operation like `eth_call`. pub fn caller_gas_allowance(db: &mut DB, env: &TxEnv) -> EthResult where DB: Database, From e18b0bab90e17930c9a9a8413cb1b8b4b9839600 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Tue, 8 Oct 2024 18:33:31 +0400 Subject: [PATCH 079/103] feat: impl `Encodable2718` and `Decodable2718` for `PooledTransactionsElement` (#11482) --- Cargo.lock | 1 + .../src/commands/debug_cmd/build_block.rs | 2 +- crates/net/eth-wire/Cargo.toml | 1 + .../net/eth-wire/tests/pooled_transactions.rs | 3 +- crates/primitives/src/transaction/pooled.rs | 396 ++++++------------ crates/primitives/src/transaction/sidecar.rs | 11 +- .../rpc-eth-api/src/helpers/transaction.rs | 2 +- crates/rpc/rpc-eth-types/src/utils.rs | 5 +- crates/transaction-pool/src/maintain.rs | 3 +- crates/transaction-pool/src/traits.rs | 2 +- crates/transaction-pool/src/validate/eth.rs | 3 +- 11 files changed, 150 insertions(+), 279 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93ae84221192..bbf487026d2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7146,6 +7146,7 @@ dependencies = [ name = "reth-eth-wire" version = "1.0.8" dependencies = [ + "alloy-eips", "alloy-primitives", "alloy-rlp", "arbitrary", diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index a7f75c02a8f8..455d8356aff9 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -194,7 +194,7 @@ impl> Command { ) .expect("should not fail to convert blob tx if it is already eip4844"); let pooled = PooledTransactionsElement::BlobTransaction(tx); - let encoded_length = pooled.length_without_header(); + let encoded_length = pooled.encode_2718_len(); // insert the blob into the store blob_store.insert(transaction.hash, sidecar)?; diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index 13224a37ada1..6eea4bc4ac65 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -62,6 +62,7 @@ proptest.workspace = true proptest-arbitrary-interop.workspace = true async-stream.workspace = true serde.workspace = true +alloy-eips.workspace = true [features] arbitrary = [ diff --git a/crates/net/eth-wire/tests/pooled_transactions.rs b/crates/net/eth-wire/tests/pooled_transactions.rs index 7927384c95b0..6690f42631a6 100644 --- a/crates/net/eth-wire/tests/pooled_transactions.rs +++ b/crates/net/eth-wire/tests/pooled_transactions.rs @@ -1,5 +1,6 @@ //! Decoding tests for [`PooledTransactions`] +use alloy_eips::eip2718::Decodable2718; use alloy_primitives::hex; use alloy_rlp::{Decodable, Encodable}; use reth_eth_wire::{EthVersion, PooledTransactions, ProtocolMessage}; @@ -72,5 +73,5 @@ fn decode_blob_rpc_transaction() { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata/rpc_blob_transaction"); let data = fs::read_to_string(network_data_path).expect("Unable to read file"); let hex_data = hex::decode(data.trim()).unwrap(); - let _txs = PooledTransactionsElement::decode_enveloped(&mut hex_data.as_ref()).unwrap(); + let _txs = PooledTransactionsElement::decode_2718(&mut hex_data.as_ref()).unwrap(); } diff --git a/crates/primitives/src/transaction/pooled.rs b/crates/primitives/src/transaction/pooled.rs index cc2dc5766394..ec49f44a680b 100644 --- a/crates/primitives/src/transaction/pooled.rs +++ b/crates/primitives/src/transaction/pooled.rs @@ -10,14 +10,13 @@ use crate::{ BlobTransaction, BlobTransactionSidecar, Signature, Transaction, TransactionSigned, TransactionSignedEcRecovered, EIP4844_TX_TYPE_ID, }; -use alloc::vec::Vec; use alloy_consensus::{ transaction::{TxEip1559, TxEip2930, TxEip4844, TxLegacy}, SignableTransaction, TxEip4844WithSidecar, }; -use alloy_eips::eip2718::{Decodable2718, Eip2718Error}; -use alloy_primitives::{Address, Bytes, TxHash, B256}; -use alloy_rlp::{Decodable, Encodable, Error as RlpError, Header, EMPTY_LIST_CODE}; +use alloy_eips::eip2718::{Decodable2718, Eip2718Result, Encodable2718}; +use alloy_primitives::{Address, TxHash, B256}; +use alloy_rlp::{Decodable, Encodable, Error as RlpError, Header}; use bytes::Buf; use derive_more::{AsRef, Deref}; use serde::{Deserialize, Serialize}; @@ -183,101 +182,6 @@ impl PooledTransactionsElement { } } - /// Decodes the "raw" format of transaction (e.g. `eth_sendRawTransaction`). - /// - /// This should be used for `eth_sendRawTransaction`, for any transaction type. Blob - /// transactions **must** include the blob sidecar as part of the raw encoding. - /// - /// This method can not be used for decoding the `transactions` field of `engine_newPayload`, - /// because EIP-4844 transactions for that method do not include the blob sidecar. The blobs - /// are supplied in an argument separate from the payload. - /// - /// A raw transaction is either a legacy transaction or EIP-2718 typed transaction, with a - /// special case for EIP-4844 transactions. - /// - /// For legacy transactions, the format is encoded as: `rlp(tx)`. This format will start with a - /// RLP list header. - /// - /// For EIP-2718 typed transactions, the format is encoded as the type of the transaction - /// followed by the rlp of the transaction: `type || rlp(tx)`. - /// - /// For EIP-4844 transactions, the format includes a blob sidecar (the blobs, commitments, and - /// proofs) after the transaction: - /// `type || rlp([tx_payload_body, blobs, commitments, proofs])` - /// - /// Where `tx_payload_body` is encoded as a RLP list: - /// `[chain_id, nonce, max_priority_fee_per_gas, ..., y_parity, r, s]` - pub fn decode_enveloped(data: &mut &[u8]) -> alloy_rlp::Result { - if data.is_empty() { - return Err(RlpError::InputTooShort) - } - - // Check if the tx is a list - tx types are less than EMPTY_LIST_CODE (0xc0) - if data[0] >= EMPTY_LIST_CODE { - // decode as legacy transaction - let (transaction, hash, signature) = - TransactionSigned::decode_rlp_legacy_transaction_tuple(data)?; - - Ok(Self::Legacy { transaction, signature, hash }) - } else { - // decode the type byte, only decode BlobTransaction if it is a 4844 transaction - let tx_type = *data.first().ok_or(RlpError::InputTooShort)?; - - // First, we advance the buffer past the type byte - data.advance(1); - - if tx_type == EIP4844_TX_TYPE_ID { - // Recall that the blob transaction response `TransactionPayload` is encoded like - // this: `rlp([tx_payload_body, blobs, commitments, proofs])` - // - // Note that `tx_payload_body` is a list: - // `[chain_id, nonce, max_priority_fee_per_gas, ..., y_parity, r, s]` - // - // This makes the full encoding: - // `tx_type (0x03) || rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` - - // Now, we decode the inner blob transaction: - // `rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` - let blob_tx = BlobTransaction::decode_inner(data)?; - Ok(Self::BlobTransaction(blob_tx)) - } else { - let typed_tx = - TransactionSigned::typed_decode(tx_type, data).map_err(|err| match err { - Eip2718Error::RlpError(err) => err, - _ => RlpError::Custom("failed to decode EIP-2718 transaction"), - })?; - - // because we checked the tx type, we can be sure that the transaction is not a - // blob transaction or legacy - match typed_tx.transaction { - Transaction::Legacy(_) => Err(RlpError::Custom( - "legacy transactions should not be a result of EIP-2718 decoding", - )), - Transaction::Eip4844(_) => Err(RlpError::Custom( - "EIP-4844 transactions can only be decoded with transaction type 0x03", - )), - Transaction::Eip2930(tx) => Ok(Self::Eip2930 { - transaction: tx, - signature: typed_tx.signature, - hash: typed_tx.hash, - }), - Transaction::Eip1559(tx) => Ok(Self::Eip1559 { - transaction: tx, - signature: typed_tx.signature, - hash: typed_tx.hash, - }), - Transaction::Eip7702(tx) => {Ok(Self::Eip7702 { - transaction: tx, - signature: typed_tx.signature, - hash: typed_tx.hash, - })}, - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => Err(RlpError::Custom("Optimism deposit transaction cannot be decoded to PooledTransactionsElement")) - } - } - } - } - /// Create [`TransactionSignedEcRecovered`] by converting this transaction into /// [`TransactionSigned`] and [`Address`] of the signer. pub fn into_ecrecovered_transaction(self, signer: Address) -> TransactionSignedEcRecovered { @@ -309,83 +213,6 @@ impl PooledTransactionsElement { } } - /// Returns the length without an RLP header - this is used for eth/68 sizes. - pub fn length_without_header(&self) -> usize { - match self { - Self::Legacy { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(&with_eip155_parity( - signature, - transaction.chain_id, - )) - } - Self::Eip2930 { transaction, signature, .. } => { - // method computes the payload len without a RLP header - transaction.encoded_len_with_signature(signature, false) - } - Self::Eip1559 { transaction, signature, .. } => { - // method computes the payload len without a RLP header - transaction.encoded_len_with_signature(signature, false) - } - Self::Eip7702 { transaction, signature, .. } => { - // method computes the payload len without a RLP header - transaction.encoded_len_with_signature(signature, false) - } - Self::BlobTransaction(blob_tx) => { - // the encoding does not use a header, so we set `with_header` to false - blob_tx.payload_len_with_type(false) - } - } - } - - /// Returns the enveloped encoded transactions. - /// - /// See also [`alloy_eips::eip2718::Encodable2718::encoded_2718`] - pub fn envelope_encoded(&self) -> Bytes { - let mut buf = Vec::new(); - self.encode_enveloped(&mut buf); - buf.into() - } - - /// Encodes the transaction into the "raw" format (e.g. `eth_sendRawTransaction`). - /// This format is also referred to as "binary" encoding. - /// - /// For legacy transactions, it encodes the RLP of the transaction into the buffer: - /// `rlp(tx-data)` - /// For EIP-2718 typed it encodes the type of the transaction followed by the rlp of the - /// transaction: `tx-type || rlp(tx-data)` - pub fn encode_enveloped(&self, out: &mut dyn bytes::BufMut) { - // The encoding of `tx-data` depends on the transaction type. Refer to these docs for more - // information on the exact format: - // - Legacy: TxLegacy::encode_with_signature - // - EIP-2930: TxEip2930::encode_with_signature - // - EIP-1559: TxEip1559::encode_with_signature - // - EIP-4844: BlobTransaction::encode_with_type_inner - // - EIP-7702: TxEip7702::encode_with_signature - match self { - Self::Legacy { transaction, signature, .. } => transaction - .encode_with_signature_fields( - &with_eip155_parity(signature, transaction.chain_id), - out, - ), - Self::Eip2930 { transaction, signature, .. } => { - transaction.encode_with_signature(signature, out, false) - } - Self::Eip1559 { transaction, signature, .. } => { - transaction.encode_with_signature(signature, out, false) - } - Self::Eip7702 { transaction, signature, .. } => { - transaction.encode_with_signature(signature, out, false) - } - Self::BlobTransaction(blob_tx) => { - // The inner encoding is used with `with_header` set to true, making the final - // encoding: - // `tx_type || rlp([transaction_payload_body, blobs, commitments, proofs]))` - blob_tx.encode_with_type_inner(out, false); - } - } - } - /// Returns true if the transaction is an EIP-4844 transaction. #[inline] pub const fn is_eip4844(&self) -> bool { @@ -481,73 +308,25 @@ impl PooledTransactionsElement { } impl Encodable for PooledTransactionsElement { - /// Encodes an enveloped post EIP-4844 [`PooledTransactionsElement`]. + /// This encodes the transaction _with_ the signature, and an rlp header. /// - /// For legacy transactions, this encodes the transaction as `rlp(tx-data)`. + /// For legacy transactions, it encodes the transaction data: + /// `rlp(tx-data)` /// - /// For EIP-2718 transactions, this encodes the transaction as `rlp(tx_type || rlp(tx-data)))`, - /// ___including__ the RLP-header for the entire transaction. + /// For EIP-2718 typed transactions, it encodes the transaction type followed by the rlp of the + /// transaction: + /// `rlp(tx-type || rlp(tx-data))` fn encode(&self, out: &mut dyn bytes::BufMut) { - // The encoding of `tx-data` depends on the transaction type. Refer to these docs for more - // information on the exact format: - // - Legacy: TxLegacy::encode_with_signature - // - EIP-2930: TxEip2930::encode_with_signature - // - EIP-1559: TxEip1559::encode_with_signature - // - EIP-4844: BlobTransaction::encode_with_type_inner - // - EIP-7702: TxEip7702::encode_with_signature - match self { - Self::Legacy { transaction, signature, .. } => transaction - .encode_with_signature_fields( - &with_eip155_parity(signature, transaction.chain_id), - out, - ), - Self::Eip2930 { transaction, signature, .. } => { - // encodes with string header - transaction.encode_with_signature(signature, out, true) - } - Self::Eip1559 { transaction, signature, .. } => { - // encodes with string header - transaction.encode_with_signature(signature, out, true) - } - Self::Eip7702 { transaction, signature, .. } => { - // encodes with string header - transaction.encode_with_signature(signature, out, true) - } - Self::BlobTransaction(blob_tx) => { - // The inner encoding is used with `with_header` set to true, making the final - // encoding: - // `rlp(tx_type || rlp([transaction_payload_body, blobs, commitments, proofs]))` - blob_tx.encode_with_type_inner(out, true); - } - } + self.network_encode(out); } fn length(&self) -> usize { - match self { - Self::Legacy { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(&with_eip155_parity( - signature, - transaction.chain_id, - )) - } - Self::Eip2930 { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(signature, true) - } - Self::Eip1559 { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(signature, true) - } - Self::Eip7702 { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(signature, true) - } - Self::BlobTransaction(blob_tx) => { - // the encoding uses a header, so we set `with_header` to true - blob_tx.payload_len_with_type(true) - } + let mut payload_length = self.encode_2718_len(); + if !self.is_legacy() { + payload_length += Header { list: false, payload_length }.length(); } + + payload_length } } @@ -581,23 +360,110 @@ impl Decodable for PooledTransactionsElement { // Check if the tx is a list if header.list { // decode as legacy transaction - let (transaction, hash, signature) = - TransactionSigned::decode_rlp_legacy_transaction_tuple(&mut original_encoding)?; + let tx = Self::fallback_decode(&mut original_encoding)?; // advance the buffer by however long the legacy transaction decoding advanced the // buffer *buf = original_encoding; - Ok(Self::Legacy { transaction, signature, hash }) + Ok(tx) } else { // decode the type byte, only decode BlobTransaction if it is a 4844 transaction let tx_type = *buf.first().ok_or(RlpError::InputTooShort)?; let remaining_len = buf.len(); - // Aadvance the buffer past the type byte + // Advance the buffer past the type byte buf.advance(1); - if tx_type == EIP4844_TX_TYPE_ID { + let tx = Self::typed_decode(tx_type, buf).map_err(RlpError::from)?; + + // check that the bytes consumed match the payload length + let bytes_consumed = remaining_len - buf.len(); + if bytes_consumed != header.payload_length { + return Err(RlpError::UnexpectedLength) + } + + Ok(tx) + } + } +} + +impl Encodable2718 for PooledTransactionsElement { + fn type_flag(&self) -> Option { + match self { + Self::Legacy { .. } => None, + Self::Eip2930 { .. } => Some(0x01), + Self::Eip1559 { .. } => Some(0x02), + Self::BlobTransaction { .. } => Some(0x03), + Self::Eip7702 { .. } => Some(0x04), + } + } + + fn encode_2718_len(&self) -> usize { + match self { + Self::Legacy { transaction, signature, .. } => { + // method computes the payload len with a RLP header + transaction.encoded_len_with_signature(&with_eip155_parity( + signature, + transaction.chain_id, + )) + } + Self::Eip2930 { transaction, signature, .. } => { + // method computes the payload len without a RLP header + transaction.encoded_len_with_signature(signature, false) + } + Self::Eip1559 { transaction, signature, .. } => { + // method computes the payload len without a RLP header + transaction.encoded_len_with_signature(signature, false) + } + Self::Eip7702 { transaction, signature, .. } => { + // method computes the payload len without a RLP header + transaction.encoded_len_with_signature(signature, false) + } + Self::BlobTransaction(blob_tx) => { + // the encoding does not use a header, so we set `with_header` to false + blob_tx.payload_len_with_type(false) + } + } + } + + fn encode_2718(&self, out: &mut dyn alloy_rlp::BufMut) { + // The encoding of `tx-data` depends on the transaction type. Refer to these docs for more + // information on the exact format: + // - Legacy: TxLegacy::encode_with_signature + // - EIP-2930: TxEip2930::encode_with_signature + // - EIP-1559: TxEip1559::encode_with_signature + // - EIP-4844: BlobTransaction::encode_with_type_inner + // - EIP-7702: TxEip7702::encode_with_signature + match self { + Self::Legacy { transaction, signature, .. } => transaction + .encode_with_signature_fields( + &with_eip155_parity(signature, transaction.chain_id), + out, + ), + Self::Eip2930 { transaction, signature, .. } => { + transaction.encode_with_signature(signature, out, false) + } + Self::Eip1559 { transaction, signature, .. } => { + transaction.encode_with_signature(signature, out, false) + } + Self::Eip7702 { transaction, signature, .. } => { + transaction.encode_with_signature(signature, out, false) + } + Self::BlobTransaction(blob_tx) => { + // The inner encoding is used with `with_header` set to true, making the final + // encoding: + // `tx_type || rlp([transaction_payload_body, blobs, commitments, proofs]))` + blob_tx.encode_with_type_inner(out, false); + } + } + } +} + +impl Decodable2718 for PooledTransactionsElement { + fn typed_decode(ty: u8, buf: &mut &[u8]) -> Eip2718Result { + match ty { + EIP4844_TX_TYPE_ID => { // Recall that the blob transaction response `TransactionPayload` is encoded like // this: `rlp([tx_payload_body, blobs, commitments, proofs])` // @@ -607,36 +473,23 @@ impl Decodable for PooledTransactionsElement { // This makes the full encoding: // `tx_type (0x03) || rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` - // Decode the inner blob transaction: + // Now, we decode the inner blob transaction: // `rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` let blob_tx = BlobTransaction::decode_inner(buf)?; - - // check that the bytes consumed match the payload length - let bytes_consumed = remaining_len - buf.len(); - if bytes_consumed != header.payload_length { - return Err(RlpError::UnexpectedLength) - } - Ok(Self::BlobTransaction(blob_tx)) - } else { - let typed_tx = - TransactionSigned::typed_decode(tx_type, buf).map_err(RlpError::from)?; - - // check that the bytes consumed match the payload length - let bytes_consumed = remaining_len - buf.len(); - if bytes_consumed != header.payload_length { - return Err(RlpError::UnexpectedLength) - } + } + tx_type => { + let typed_tx = TransactionSigned::typed_decode(tx_type, buf)?; - // because we checked the tx type, we can be sure that the transaction is not a - // blob transaction or legacy match typed_tx.transaction { Transaction::Legacy(_) => Err(RlpError::Custom( - "legacy transactions should not be a result of EIP-2718 decoding", - )), + "legacy transactions should not be a result of typed decoding", + ).into()), + // because we checked the tx type, we can be sure that the transaction is not a + // blob transaction Transaction::Eip4844(_) => Err(RlpError::Custom( "EIP-4844 transactions can only be decoded with transaction type 0x03", - )), + ).into()), Transaction::Eip2930(tx) => Ok(Self::Eip2930 { transaction: tx, signature: typed_tx.signature, @@ -653,11 +506,19 @@ impl Decodable for PooledTransactionsElement { hash: typed_tx.hash, }), #[cfg(feature = "optimism")] - Transaction::Deposit(_) => Err(RlpError::Custom("Optimism deposit transaction cannot be decoded to PooledTransactionsElement")) + Transaction::Deposit(_) => Err(RlpError::Custom("Optimism deposit transaction cannot be decoded to PooledTransactionsElement").into()) } } } } + + fn fallback_decode(buf: &mut &[u8]) -> Eip2718Result { + // decode as legacy transaction + let (transaction, hash, signature) = + TransactionSigned::decode_rlp_legacy_transaction_tuple(buf)?; + + Ok(Self::Legacy { transaction, signature, hash }) + } } impl TryFrom for PooledTransactionsElement { @@ -773,6 +634,7 @@ mod tests { use super::*; use alloy_primitives::{address, hex}; use assert_matches::assert_matches; + use bytes::Bytes; #[test] fn invalid_legacy_pooled_decoding_input_too_short() { @@ -804,7 +666,7 @@ mod tests { // this is a legacy tx so we can attempt the same test with decode_enveloped let input_rlp = &mut &hex_data[..]; - let res = PooledTransactionsElement::decode_enveloped(input_rlp); + let res = PooledTransactionsElement::decode_2718(input_rlp); assert!( res.is_err(), @@ -820,7 +682,7 @@ mod tests { let data = hex!("02f903d382426882ba09832dc6c0848674742682ed9694714b6a4ea9b94a8a7d9fd362ed72630688c8898c80b90364492d24749189822d8512430d3f3ff7a2ede675ac08265c08e2c56ff6fdaa66dae1cdbe4a5d1d7809f3e99272d067364e597542ac0c369d69e22a6399c3e9bee5da4b07e3f3fdc34c32c3d88aa2268785f3e3f8086df0934b10ef92cfffc2e7f3d90f5e83302e31382e302d64657600000000000000000000000000000000000000000000569e75fc77c1a856f6daaf9e69d8a9566ca34aa47f9133711ce065a571af0cfd000000000000000000000000e1e210594771824dad216568b91c9cb4ceed361c00000000000000000000000000000000000000000000000000000000000546e00000000000000000000000000000000000000000000000000000000000e4e1c00000000000000000000000000000000000000000000000000000000065d6750c00000000000000000000000000000000000000000000000000000000000f288000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002cf600000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000000f1628e56fa6d8c50e5b984a58c0df14de31c7b857ce7ba499945b99252976a93d06dcda6776fc42167fbe71cb59f978f5ef5b12577a90b132d14d9c6efa528076f0161d7bf03643cfc5490ec5084f4a041db7f06c50bd97efa08907ba79ddcac8b890f24d12d8db31abbaaf18985d54f400449ee0559a4452afe53de5853ce090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000028000000000000000000000000000000000000000000000000000000000000003e800000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000064ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00000000000000000000000000000000000000000000000000000000c080a01428023fc54a27544abc421d5d017b9a7c5936ad501cbdecd0d9d12d04c1a033a0753104bbf1c87634d6ff3f0ffa0982710612306003eb022363b57994bdef445a" ); - let res = PooledTransactionsElement::decode_enveloped(&mut &data[..]).unwrap(); + let res = PooledTransactionsElement::decode_2718(&mut &data[..]).unwrap(); assert_eq!( res.into_transaction().to(), Some(address!("714b6a4ea9b94a8a7d9fd362ed72630688c8898c")) @@ -854,7 +716,7 @@ mod tests { assert!(input_rlp.is_empty()); // we can also decode_enveloped - let res = PooledTransactionsElement::decode_enveloped(&mut &data[..]); + let res = PooledTransactionsElement::decode_2718(&mut &data[..]); assert_matches!(res, Ok(_tx)); } } diff --git a/crates/primitives/src/transaction/sidecar.rs b/crates/primitives/src/transaction/sidecar.rs index 52c3c68ef9db..87b8c1fbf3e8 100644 --- a/crates/primitives/src/transaction/sidecar.rs +++ b/crates/primitives/src/transaction/sidecar.rs @@ -283,7 +283,10 @@ pub fn generate_blob_sidecar(blobs: Vec) -> BlobTransactionSidecar mod tests { use super::*; use crate::{kzg::Blob, PooledTransactionsElement}; - use alloy_eips::eip4844::Bytes48; + use alloy_eips::{ + eip2718::{Decodable2718, Encodable2718}, + eip4844::Bytes48, + }; use alloy_primitives::hex; use alloy_rlp::Encodable; use std::{fs, path::PathBuf, str::FromStr}; @@ -435,15 +438,15 @@ mod tests { let entry = entry.unwrap(); let content = fs::read_to_string(entry.path()).unwrap(); let raw = hex::decode(content.trim()).unwrap(); - let tx = PooledTransactionsElement::decode_enveloped(&mut raw.as_ref()) + let tx = PooledTransactionsElement::decode_2718(&mut raw.as_ref()) .map_err(|err| { panic!("Failed to decode transaction: {:?} {:?}", err, entry.path()); }) .unwrap(); // We want to test only EIP-4844 transactions assert!(tx.is_eip4844()); - let encoded = tx.envelope_encoded(); - assert_eq!(encoded.as_ref(), &raw[..], "{:?}", entry.path()); + let encoded = tx.encoded_2718(); + assert_eq!(encoded.as_slice(), &raw[..], "{:?}", entry.path()); } } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index afcb1b2eeb25..5e6aadd778bd 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -97,7 +97,7 @@ pub trait EthTransactions: LoadTransaction { async move { // Note: this is mostly used to fetch pooled transactions so we check the pool first if let Some(tx) = - self.pool().get_pooled_transaction_element(hash).map(|tx| tx.envelope_encoded()) + self.pool().get_pooled_transaction_element(hash).map(|tx| tx.encoded_2718().into()) { return Ok(Some(tx)) } diff --git a/crates/rpc/rpc-eth-types/src/utils.rs b/crates/rpc/rpc-eth-types/src/utils.rs index bb7c3d64846e..596acc74ce1e 100644 --- a/crates/rpc/rpc-eth-types/src/utils.rs +++ b/crates/rpc/rpc-eth-types/src/utils.rs @@ -1,5 +1,6 @@ //! Commonly used code snippets +use alloy_eips::eip2718::Decodable2718; use alloy_primitives::Bytes; use reth_primitives::{PooledTransactionsElement, PooledTransactionsElementEcRecovered}; use std::future::Future; @@ -8,13 +9,13 @@ use super::{EthApiError, EthResult}; /// Recovers a [`PooledTransactionsElementEcRecovered`] from an enveloped encoded byte stream. /// -/// See [`PooledTransactionsElement::decode_enveloped`] +/// See [`Decodable2718::decode_2718`] pub fn recover_raw_transaction(data: Bytes) -> EthResult { if data.is_empty() { return Err(EthApiError::EmptyRawTransactionData) } - let transaction = PooledTransactionsElement::decode_enveloped(&mut data.as_ref()) + let transaction = PooledTransactionsElement::decode_2718(&mut data.as_ref()) .map_err(|_| EthApiError::FailedToDecodeSignedTransaction)?; transaction.try_into_ecrecovered().or(Err(EthApiError::InvalidTransactionSignature)) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 523151956dd1..66b98614737e 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -676,6 +676,7 @@ mod tests { blobstore::InMemoryBlobStore, validate::EthTransactionValidatorBuilder, CoinbaseTipOrdering, EthPooledTransaction, Pool, TransactionOrigin, }; + use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; use reth_chainspec::MAINNET; use reth_fs_util as fs; @@ -699,7 +700,7 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap(); let transactions_path = temp_dir.path().join(FILENAME).with_extension(EXTENSION); let tx_bytes = hex!("02f87201830655c2808505ef61f08482565f94388c818ca8b9251b393131c08a736a67ccb192978801049e39c4b5b1f580c001a01764ace353514e8abdfb92446de356b260e3c1225b73fc4c8876a6258d12a129a04f02294aa61ca7676061cd99f29275491218b4754b46a0248e5e42bc5091f507"); - let tx = PooledTransactionsElement::decode_enveloped(&mut &tx_bytes[..]).unwrap(); + let tx = PooledTransactionsElement::decode_2718(&mut &tx_bytes[..]).unwrap(); let provider = MockEthProvider::default(); let transaction: EthPooledTransaction = tx.try_into_ecrecovered().unwrap().into(); let tx_to_cmp = transaction.clone(); diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 1598877ff3b6..adae238e46b3 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1051,7 +1051,7 @@ impl EthPooledTransaction { /// Conversion from the network transaction type to the pool transaction type. impl From for EthPooledTransaction { fn from(tx: PooledTransactionsElementEcRecovered) -> Self { - let encoded_length = tx.length_without_header(); + let encoded_length = tx.encode_2718_len(); let (tx, signer) = tx.into_components(); match tx { PooledTransactionsElement::BlobTransaction(tx) => { diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index acfe46d6e88a..49165f189a05 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -833,6 +833,7 @@ mod tests { blobstore::InMemoryBlobStore, error::PoolErrorKind, CoinbaseTipOrdering, EthPooledTransaction, Pool, TransactionPool, }; + use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; use reth_chainspec::MAINNET; use reth_primitives::PooledTransactionsElement; @@ -842,7 +843,7 @@ mod tests { let raw = "0x02f914950181ad84b2d05e0085117553845b830f7df88080b9143a6040608081523462000414576200133a803803806200001e8162000419565b9283398101608082820312620004145781516001600160401b03908181116200041457826200004f9185016200043f565b92602092838201519083821162000414576200006d9183016200043f565b8186015190946001600160a01b03821692909183900362000414576060015190805193808511620003145760038054956001938488811c9816801562000409575b89891014620003f3578190601f988981116200039d575b50899089831160011462000336576000926200032a575b505060001982841b1c191690841b1781555b8751918211620003145760049788548481811c9116801562000309575b89821014620002f457878111620002a9575b5087908784116001146200023e5793839491849260009562000232575b50501b92600019911b1c19161785555b6005556007805460ff60a01b19169055600880546001600160a01b0319169190911790553015620001f3575060025469d3c21bcecceda100000092838201809211620001de57506000917fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9160025530835282815284832084815401905584519384523093a351610e889081620004b28239f35b601190634e487b7160e01b6000525260246000fd5b90606493519262461bcd60e51b845283015260248201527f45524332303a206d696e7420746f20746865207a65726f2061646472657373006044820152fd5b0151935038806200013a565b9190601f198416928a600052848a6000209460005b8c8983831062000291575050501062000276575b50505050811b0185556200014a565b01519060f884600019921b161c191690553880808062000267565b86860151895590970196948501948893500162000253565b89600052886000208880860160051c8201928b8710620002ea575b0160051c019085905b828110620002dd5750506200011d565b60008155018590620002cd565b92508192620002c4565b60228a634e487b7160e01b6000525260246000fd5b90607f16906200010b565b634e487b7160e01b600052604160045260246000fd5b015190503880620000dc565b90869350601f19831691856000528b6000209260005b8d8282106200038657505084116200036d575b505050811b018155620000ee565b015160001983861b60f8161c191690553880806200035f565b8385015186558a979095019493840193016200034c565b90915083600052896000208980850160051c8201928c8610620003e9575b918891869594930160051c01915b828110620003d9575050620000c5565b60008155859450889101620003c9565b92508192620003bb565b634e487b7160e01b600052602260045260246000fd5b97607f1697620000ae565b600080fd5b6040519190601f01601f191682016001600160401b038111838210176200031457604052565b919080601f84011215620004145782516001600160401b038111620003145760209062000475601f8201601f1916830162000419565b92818452828287010111620004145760005b8181106200049d57508260009394955001015290565b85810183015184820184015282016200048756fe608060408181526004918236101561001657600080fd5b600092833560e01c91826306fdde0314610a1c57508163095ea7b3146109f257816318160ddd146109d35781631b4c84d2146109ac57816323b872dd14610833578163313ce5671461081757816339509351146107c357816370a082311461078c578163715018a6146107685781638124f7ac146107495781638da5cb5b1461072057816395d89b411461061d578163a457c2d714610575578163a9059cbb146104e4578163c9567bf914610120575063dd62ed3e146100d557600080fd5b3461011c578060031936011261011c57806020926100f1610b5a565b6100f9610b75565b6001600160a01b0391821683526001865283832091168252845220549051908152f35b5080fd5b905082600319360112610338576008546001600160a01b039190821633036104975760079283549160ff8360a01c1661045557737a250d5630b4cf539739df2c5dacb4c659f2488d92836bffffffffffffffffffffffff60a01b8092161786553087526020938785528388205430156104065730895260018652848920828a52865280858a205584519081527f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925863092a38554835163c45a015560e01b815290861685828581845afa9182156103dd57849187918b946103e7575b5086516315ab88c960e31b815292839182905afa9081156103dd576044879289928c916103c0575b508b83895196879586946364e329cb60e11b8652308c870152166024850152165af19081156103b6579086918991610389575b50169060065416176006558385541660604730895288865260c4858a20548860085416928751958694859363f305d71960e01b8552308a86015260248501528d60448501528d606485015260848401524260a48401525af1801561037f579084929161034c575b50604485600654169587541691888551978894859363095ea7b360e01b855284015260001960248401525af1908115610343575061030c575b5050805460ff60a01b1916600160a01b17905580f35b81813d831161033c575b6103208183610b8b565b8101031261033857518015150361011c5738806102f6565b8280fd5b503d610316565b513d86823e3d90fd5b6060809293503d8111610378575b6103648183610b8b565b81010312610374578290386102bd565b8580fd5b503d61035a565b83513d89823e3d90fd5b6103a99150863d88116103af575b6103a18183610b8b565b810190610e33565b38610256565b503d610397565b84513d8a823e3d90fd5b6103d79150843d86116103af576103a18183610b8b565b38610223565b85513d8b823e3d90fd5b6103ff919450823d84116103af576103a18183610b8b565b92386101fb565b845162461bcd60e51b81528085018790526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b6020606492519162461bcd60e51b8352820152601760248201527f74726164696e6720697320616c7265616479206f70656e0000000000000000006044820152fd5b608490602084519162461bcd60e51b8352820152602160248201527f4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6044820152603760f91b6064820152fd5b9050346103385781600319360112610338576104fe610b5a565b9060243593303303610520575b602084610519878633610bc3565b5160018152f35b600594919454808302908382041483151715610562576127109004820391821161054f5750925080602061050b565b634e487b7160e01b815260118552602490fd5b634e487b7160e01b825260118652602482fd5b9050823461061a578260031936011261061a57610590610b5a565b918360243592338152600160205281812060018060a01b03861682526020522054908282106105c9576020856105198585038733610d31565b608490602086519162461bcd60e51b8352820152602560248201527f45524332303a2064656372656173656420616c6c6f77616e63652062656c6f77604482015264207a65726f60d81b6064820152fd5b80fd5b83833461011c578160031936011261011c57805191809380549160019083821c92828516948515610716575b6020958686108114610703578589529081156106df5750600114610687575b6106838787610679828c0383610b8b565b5191829182610b11565b0390f35b81529295507f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19b5b8284106106cc57505050826106839461067992820101948680610668565b80548685018801529286019281016106ae565b60ff19168887015250505050151560051b8301019250610679826106838680610668565b634e487b7160e01b845260228352602484fd5b93607f1693610649565b50503461011c578160031936011261011c5760085490516001600160a01b039091168152602090f35b50503461011c578160031936011261011c576020906005549051908152f35b833461061a578060031936011261061a57600880546001600160a01b031916905580f35b50503461011c57602036600319011261011c5760209181906001600160a01b036107b4610b5a565b16815280845220549051908152f35b82843461061a578160031936011261061a576107dd610b5a565b338252600160209081528383206001600160a01b038316845290528282205460243581019290831061054f57602084610519858533610d31565b50503461011c578160031936011261011c576020905160128152f35b83833461011c57606036600319011261011c5761084e610b5a565b610856610b75565b6044359160018060a01b0381169485815260209560018752858220338352875285822054976000198903610893575b505050906105199291610bc3565b85891061096957811561091a5733156108cc5750948481979861051997845260018a528284203385528a52039120558594938780610885565b865162461bcd60e51b8152908101889052602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b6064820152608490fd5b865162461bcd60e51b81529081018890526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b865162461bcd60e51b8152908101889052601d60248201527f45524332303a20696e73756666696369656e7420616c6c6f77616e63650000006044820152606490fd5b50503461011c578160031936011261011c5760209060ff60075460a01c1690519015158152f35b50503461011c578160031936011261011c576020906002549051908152f35b50503461011c578060031936011261011c57602090610519610a12610b5a565b6024359033610d31565b92915034610b0d5783600319360112610b0d57600354600181811c9186908281168015610b03575b6020958686108214610af05750848852908115610ace5750600114610a75575b6106838686610679828b0383610b8b565b929550600383527fc2575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b5b828410610abb575050508261068394610679928201019438610a64565b8054868501880152928601928101610a9e565b60ff191687860152505050151560051b83010192506106798261068338610a64565b634e487b7160e01b845260229052602483fd5b93607f1693610a44565b8380fd5b6020808252825181830181905290939260005b828110610b4657505060409293506000838284010152601f8019910116010190565b818101860151848201604001528501610b24565b600435906001600160a01b0382168203610b7057565b600080fd5b602435906001600160a01b0382168203610b7057565b90601f8019910116810190811067ffffffffffffffff821117610bad57604052565b634e487b7160e01b600052604160045260246000fd5b6001600160a01b03908116918215610cde5716918215610c8d57600082815280602052604081205491808310610c3957604082827fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef958760209652828652038282205586815220818154019055604051908152a3565b60405162461bcd60e51b815260206004820152602660248201527f45524332303a207472616e7366657220616d6f756e7420657863656564732062604482015265616c616e636560d01b6064820152608490fd5b60405162461bcd60e51b815260206004820152602360248201527f45524332303a207472616e7366657220746f20746865207a65726f206164647260448201526265737360e81b6064820152608490fd5b60405162461bcd60e51b815260206004820152602560248201527f45524332303a207472616e736665722066726f6d20746865207a65726f206164604482015264647265737360d81b6064820152608490fd5b6001600160a01b03908116918215610de25716918215610d925760207f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925918360005260018252604060002085600052825280604060002055604051908152a3565b60405162461bcd60e51b815260206004820152602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b6064820152608490fd5b60405162461bcd60e51b8152602060048201526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b90816020910312610b7057516001600160a01b0381168103610b70579056fea2646970667358221220285c200b3978b10818ff576bb83f2dc4a2a7c98dfb6a36ea01170de792aa652764736f6c63430008140033000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000d3fd4f95820a9aa848ce716d6c200eaefb9a2e4900000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000003543131000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000035431310000000000000000000000000000000000000000000000000000000000c001a04e551c75810ffdfe6caff57da9f5a8732449f42f0f4c57f935b05250a76db3b6a046cd47e6d01914270c1ec0d9ac7fae7dfb240ec9a8b6ec7898c4d6aa174388f2"; let data = hex::decode(raw).unwrap(); - let tx = PooledTransactionsElement::decode_enveloped(&mut data.as_ref()).unwrap(); + let tx = PooledTransactionsElement::decode_2718(&mut data.as_ref()).unwrap(); tx.try_into_ecrecovered().unwrap().into() } From b09734a59c709896cd1ef3a6562d4299fe574ee6 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 8 Oct 2024 16:52:36 +0100 Subject: [PATCH 080/103] fix(exex): exhaust backfill job when using a stream (#11578) --- Cargo.lock | 182 ++++++++++++++---------- crates/exex/exex/src/backfill/job.rs | 10 +- crates/exex/exex/src/backfill/stream.rs | 114 +++++++++------ crates/exex/exex/src/notifications.rs | 6 +- 4 files changed, 190 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbf487026d2b..6c135f294c58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.2" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.36" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c225801d42099570d0674701dddd4142f0ef715282aeb5985042e2ec962df7" +checksum = "8158b4878c67837e5413721cc44298e6a2d88d39203175ea025e51892a16ba4c" dependencies = [ "alloy-rlp", "arbitrary", @@ -296,7 +296,7 @@ dependencies = [ "getrandom 0.2.15", "hashbrown 0.14.5", "hex-literal", - "indexmap 2.6.0", + "indexmap 2.5.0", "itoa", "k256", "keccak-asm", @@ -619,7 +619,7 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.6.0", + "indexmap 2.5.0", "proc-macro-error2", "proc-macro2", "quote", @@ -1016,9 +1016,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.13" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "brotli", "flate2", @@ -1203,6 +1203,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.79", +] + [[package]] name = "bindgen" version = "0.70.1" @@ -1313,7 +1333,7 @@ dependencies = [ "bitflags 2.6.0", "boa_interner", "boa_macros", - "indexmap 2.6.0", + "indexmap 2.5.0", "num-bigint", "rustc-hash 2.0.0", ] @@ -1339,7 +1359,7 @@ dependencies = [ "fast-float", "hashbrown 0.14.5", "icu_normalizer", - "indexmap 2.6.0", + "indexmap 2.5.0", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -1385,7 +1405,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.5", - "indexmap 2.6.0", + "indexmap 2.5.0", "once_cell", "phf", "rustc-hash 2.0.0", @@ -1453,9 +1473,9 @@ dependencies = [ [[package]] name = "brotli" -version = "7.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1599,9 +1619,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.25" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", @@ -3113,9 +3133,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -3128,9 +3148,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -3138,15 +3158,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -3155,9 +3175,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -3176,9 +3196,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -3187,15 +3207,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -3209,9 +3229,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -3291,9 +3311,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -3370,7 +3390,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.6.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -3416,12 +3436,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hashbrown" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" - [[package]] name = "hashlink" version = "0.8.4" @@ -3949,13 +3963,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.14.5", "serde", ] @@ -3972,7 +3986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.6.0", + "indexmap 2.5.0", "is-terminal", "itoa", "log", @@ -4070,9 +4084,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "iri-string" @@ -4110,6 +4124,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -4426,6 +4449,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.159" @@ -4469,11 +4498,11 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.9" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cca3586d5efa98fba425856ba227950fd4287525dd5317b352f476ca7d603b" +checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" dependencies = [ - "bindgen", + "bindgen 0.69.4", "errno", "libc", ] @@ -4694,7 +4723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", - "indexmap 2.6.0", + "indexmap 2.5.0", "metrics", "metrics-util", "quanta", @@ -5105,18 +5134,21 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "oorandom" @@ -5462,18 +5494,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", @@ -6009,9 +6041,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.2.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" dependencies = [ "bitflags 2.6.0", ] @@ -7529,7 +7561,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "derive_more 1.0.0", - "indexmap 2.6.0", + "indexmap 2.5.0", "parking_lot 0.12.3", "pprof", "rand 0.8.5", @@ -7545,7 +7577,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "1.0.8" dependencies = [ - "bindgen", + "bindgen 0.70.1", "cc", ] @@ -9450,9 +9482,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", @@ -9751,7 +9783,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "itoa", "memchr", "ryu", @@ -9803,15 +9835,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.5.0", "serde", "serde_derive", "serde_json", @@ -9821,9 +9853,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" dependencies = [ "darling", "proc-macro2", @@ -10621,7 +10653,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -10970,9 +11002,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 7d66ce23d609..77a7b50477b0 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -18,6 +18,8 @@ use reth_revm::database::StateProviderDatabase; use reth_stages_api::ExecutionStageThresholds; use reth_tracing::tracing::{debug, trace}; +pub(super) type BackfillJobResult = Result; + /// Backfill job started for a specific range. /// /// It implements [`Iterator`] that executes blocks in batches according to the provided thresholds @@ -37,7 +39,7 @@ where E: BlockExecutorProvider, P: HeaderProvider + BlockReader + StateProviderFactory, { - type Item = Result; + type Item = BackfillJobResult; fn next(&mut self) -> Option { if self.range.is_empty() { @@ -63,7 +65,7 @@ where self.into() } - fn execute_range(&mut self) -> Result { + fn execute_range(&mut self) -> BackfillJobResult { debug!( target: "exex::backfill", range = ?self.range, @@ -165,7 +167,7 @@ where E: BlockExecutorProvider, P: HeaderProvider + BlockReader + StateProviderFactory, { - type Item = Result<(BlockWithSenders, BlockExecutionOutput), BlockExecutionError>; + type Item = BackfillJobResult<(BlockWithSenders, BlockExecutionOutput)>; fn next(&mut self) -> Option { self.range.next().map(|block_number| self.execute_block(block_number)) @@ -187,7 +189,7 @@ where pub(crate) fn execute_block( &self, block_number: u64, - ) -> Result<(BlockWithSenders, BlockExecutionOutput), BlockExecutionError> { + ) -> BackfillJobResult<(BlockWithSenders, BlockExecutionOutput)> { let td = self .provider .header_td_by_number(block_number)? diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index ca4bd326daa0..c55b8651daf1 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -18,12 +18,25 @@ use reth_stages_api::ExecutionStageThresholds; use reth_tracing::tracing::debug; use tokio::task::JoinHandle; +use super::job::BackfillJobResult; + /// The default parallelism for active tasks in [`StreamBackfillJob`]. pub(crate) const DEFAULT_PARALLELISM: usize = 4; /// The default batch size for active tasks in [`StreamBackfillJob`]. const DEFAULT_BATCH_SIZE: usize = 100; -type BackfillTasks = FuturesOrdered>>; +/// Boxed thread-safe iterator that yields [`BackfillJobResult`]s. +type BackfillTaskIterator = + Box> + Send + Sync + 'static>; + +/// Backfill task output. +struct BackfillTaskOutput { + job: BackfillTaskIterator, + result: Option>, +} + +/// Ordered queue of [`JoinHandle`]s that yield [`BackfillTaskOutput`]s. +type BackfillTasks = FuturesOrdered>>; type SingleBlockStreamItem = (BlockWithSenders, BlockExecutionOutput); type BatchBlockStreamItem = Chain; @@ -44,7 +57,10 @@ pub struct StreamBackfillJob { thresholds: ExecutionStageThresholds, } -impl StreamBackfillJob { +impl StreamBackfillJob +where + T: Send + Sync + 'static, +{ /// Configures the parallelism of the [`StreamBackfillJob`] to handle active tasks. pub const fn with_parallelism(mut self, parallelism: usize) -> Self { self.parallelism = parallelism; @@ -57,14 +73,30 @@ impl StreamBackfillJob { self } - fn poll_next_task( - &mut self, - cx: &mut Context<'_>, - ) -> Poll>> { - match ready!(self.tasks.poll_next_unpin(cx)) { - Some(res) => Poll::Ready(Some(res.map_err(BlockExecutionError::other)?)), - None => Poll::Ready(None), + /// Spawns a new task calling the [`BackfillTaskIterator::next`] method and pushes it to the + /// [`BackfillTasks`] queue. + fn push_task(&mut self, mut job: BackfillTaskIterator) { + self.tasks.push_back(tokio::task::spawn_blocking(move || BackfillTaskOutput { + result: job.next(), + job, + })); + } + + /// Polls the next task in the [`BackfillTasks`] queue until it returns a non-empty result. + fn poll_next_task(&mut self, cx: &mut Context<'_>) -> Poll>> { + while let Some(res) = ready!(self.tasks.poll_next_unpin(cx)) { + let task_result = res.map_err(BlockExecutionError::other)?; + + if let BackfillTaskOutput { result: Some(job_result), job } = task_result { + // If the task returned a non-empty result, a new task advancing the job is created + // and pushed to the front of the queue. + self.push_task(job); + + return Poll::Ready(Some(job_result)) + }; } + + Poll::Ready(None) } } @@ -73,27 +105,28 @@ where E: BlockExecutorProvider + Clone + Send + 'static, P: HeaderProvider + BlockReader + StateProviderFactory + Clone + Send + Unpin + 'static, { - type Item = Result; + type Item = BackfillJobResult; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); // Spawn new tasks only if we are below the parallelism configured. while this.tasks.len() < this.parallelism { - // If we have a block number, then we can spawn a new task for that block - if let Some(block_number) = this.range.next() { - let mut job = SingleBlockBackfillJob { - executor: this.executor.clone(), - provider: this.provider.clone(), - range: block_number..=block_number, - stream_parallelism: this.parallelism, - }; - let task = - tokio::task::spawn_blocking(move || job.next().expect("non-empty range")); - this.tasks.push_back(task); - } else { + // Get the next block number from the range. If it is empty, we are done. + let Some(block_number) = this.range.next() else { + debug!(target: "exex::backfill", tasks = %this.tasks.len(), range = ?this.range, "No more single blocks to backfill"); break; - } + }; + + // Spawn a new task for that block + debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?block_number, "Spawning new single block backfill task"); + let job = Box::new(SingleBlockBackfillJob { + executor: this.executor.clone(), + provider: this.provider.clone(), + range: block_number..=block_number, + stream_parallelism: this.parallelism, + }) as BackfillTaskIterator<_>; + this.push_task(job); } this.poll_next_task(cx) @@ -105,7 +138,7 @@ where E: BlockExecutorProvider + Clone + Send + 'static, P: HeaderProvider + BlockReader + StateProviderFactory + Clone + Send + Unpin + 'static, { - type Item = Result; + type Item = BackfillJobResult; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -117,24 +150,23 @@ where let start = range.next(); let range_bounds = start.zip(range.last().or(start)); - // If we have range bounds, then we can spawn a new task for that range - if let Some((first, last)) = range_bounds { - let range = first..=last; - debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?range, "Spawning new backfill task"); - let mut job = BackfillJob { - executor: this.executor.clone(), - provider: this.provider.clone(), - prune_modes: this.prune_modes.clone(), - thresholds: this.thresholds.clone(), - range, - stream_parallelism: this.parallelism, - }; - let task = - tokio::task::spawn_blocking(move || job.next().expect("non-empty range")); - this.tasks.push_back(task); - } else { + // Create the range from the range bounds. If it is empty, we are done. + let Some(range) = range_bounds.map(|(first, last)| first..=last) else { + debug!(target: "exex::backfill", tasks = %this.tasks.len(), range = ?this.range, "No more block batches to backfill"); break; - } + }; + + // Spawn a new task for that range + debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?range, "Spawning new block batch backfill task"); + let job = Box::new(BackfillJob { + executor: this.executor.clone(), + provider: this.provider.clone(), + prune_modes: this.prune_modes.clone(), + thresholds: this.thresholds.clone(), + range, + stream_parallelism: this.parallelism, + }) as BackfillTaskIterator<_>; + this.push_task(job); } this.poll_next_task(cx) diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 6efdb1775cf1..d0c94d34f644 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -338,9 +338,11 @@ where } if let Some(backfill_job) = &mut this.backfill_job { - if let Some(chain) = ready!(backfill_job.poll_next_unpin(cx)) { + debug!(target: "exex::notifications", "Polling backfill job"); + if let Some(chain) = ready!(backfill_job.poll_next_unpin(cx)).transpose()? { + debug!(target: "exex::notifications", range = ?chain.range(), "Backfill job returned a chain"); return Poll::Ready(Some(Ok(ExExNotification::ChainCommitted { - new: Arc::new(chain?), + new: Arc::new(chain), }))) } From fda6e353d51d42ec2d9e601dc63b82df892628de Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Tue, 8 Oct 2024 20:18:27 +0400 Subject: [PATCH 081/103] fix: in-memory trie updates pruning (#11580) Co-authored-by: Matthias Seitz --- crates/engine/tree/src/tree/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 47be69a3dee4..fd3f2df5c01b 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -282,7 +282,7 @@ impl TreeState { } // remove trie updates that are below the finalized block - self.persisted_trie_updates.retain(|_, (block_num, _)| *block_num < finalized_num); + self.persisted_trie_updates.retain(|_, (block_num, _)| *block_num > finalized_num); // The only block that should remain at the `finalized` number now, is the finalized // block, if it exists. From 10eb1e8b3fae32c452fc95e51585be7baf15be08 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:10:23 +0900 Subject: [PATCH 082/103] chore(providers): test race condition on all `BlockchainProvider2` macro tests (#11574) --- .../src/providers/blockchain_provider.rs | 734 +++++++++--------- 1 file changed, 379 insertions(+), 355 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index d25c3f84bd1a..54f28b77b912 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -3116,66 +3116,96 @@ mod tests { } macro_rules! test_by_tx_range { - ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ - let db_tx_count = - $database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - let in_mem_tx_count = - $in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + ([$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ - let db_range = 0..=(db_tx_count - 1); - let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); + // Get the number methods being tested. + // Since each method tested will move a block from memory to storage, this ensures we have enough. + let extra_blocks = [$(stringify!($method)),*].len(); + + let mut rng = generators::rng(); + let (provider, mut database_blocks, mut in_memory_blocks, receipts) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT + extra_blocks, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + let provider = Arc::new(provider); $( + // Since data moves for each tried method, need to recalculate everything + let db_tx_count = + database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + let in_mem_tx_count = + in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + + let db_range = 0..=(db_tx_count - 1); + let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); + // Retrieve the expected database data let database_data = - $database_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(db_range.clone())?, database_data); + database_blocks.iter().flat_map(|b| $data_extractor(b, &receipts)).collect::>(); + assert_eq!(provider.$method(db_range.clone())?, database_data, "full db data"); // Retrieve the expected in-memory data let in_memory_data = - $in_memory_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); + in_memory_blocks.iter().flat_map(|b| $data_extractor(b, &receipts)).collect::>(); + assert_eq!(provider.$method(in_mem_range.clone())?, in_memory_data, "full mem data"); // Test partial in-memory range assert_eq!( - &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, - &in_memory_data[1..in_memory_data.len() - 1] + &provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &in_memory_data[1..in_memory_data.len() - 1], + "partial mem data" ); // Test range in in-memory to unbounded end - assert_eq!($provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..]); + assert_eq!(provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..], "unbounded mem data"); // Test last element in-memory - assert_eq!($provider.$method(in_mem_range.end()..)?, &in_memory_data[in_memory_data.len() -1 ..]); - - // Test range that spans database and in-memory - assert_eq!( - $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, - database_data[database_data.len() - 2..] - .iter() - .chain(&in_memory_data[..in_memory_data.len() - 1]) - .cloned() - .collect::>() - ); + assert_eq!(provider.$method(in_mem_range.end()..)?, &in_memory_data[in_memory_data.len() -1 ..], "last mem data"); // Test range that spans database and in-memory with unbounded end assert_eq!( - $provider.$method(in_mem_range.start() - 2..)?, + provider.$method(in_mem_range.start() - 2..)?, database_data[database_data.len() - 2..] .iter() .chain(&in_memory_data[..]) .cloned() - .collect::>() + .collect::>(), + "unbounded span data" ); + // Test range that spans database and in-memory + #[allow(unused_assignments)] + { + // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + + assert_eq!( + provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>(), + "span data" + ); + + // Adjust our blocks accordingly + database_blocks.push(in_memory_blocks.remove(0)); + } + // Test invalid range let start_tx_num = u64::MAX; let end_tx_num = u64::MAX; - let result = $provider.$method(start_tx_num..end_tx_num)?; + let result = provider.$method(start_tx_num..end_tx_num)?; assert!(result.is_empty(), "No data should be found for an invalid transaction range"); // Test empty range - let result = $provider.$method(in_mem_range.end()+10..in_mem_range.end()+20)?; + let result = provider.$method(in_mem_range.end()+10..in_mem_range.end()+20)?; assert!(result.is_empty(), "No data should be found for an empty transaction range"); )* }}; @@ -3183,80 +3213,93 @@ mod tests { #[test] fn test_methods_by_tx_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - test_by_tx_range!( - provider, - database_blocks, - in_memory_blocks, - [ - (senders_by_tx_range, |block: &SealedBlock| block.senders().unwrap()), - (transactions_by_tx_range, |block: &SealedBlock| block - .body - .transactions - .iter() - .map(|tx| Into::::into(tx.clone())) - .collect::>()), - (receipts_by_tx_range, |block: &SealedBlock| receipts[block.number as usize] - .clone()) - ] - ); + test_by_tx_range!([ + (senders_by_tx_range, |block: &SealedBlock, _: &Vec>| block + .senders() + .unwrap()), + (transactions_by_tx_range, |block: &SealedBlock, _: &Vec>| block + .body + .transactions + .iter() + .map(|tx| Into::::into(tx.clone())) + .collect::>()), + (receipts_by_tx_range, |block: &SealedBlock, receipts: &Vec>| receipts + [block.number as usize] + .clone()) + ]); Ok(()) } macro_rules! test_by_block_range { - ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ - let db_block_count = $database_blocks.len() as u64; - let in_mem_block_count = $in_memory_blocks.len() as u64; + ([$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ + // Get the number methods being tested. + // Since each method tested will move a block from memory to storage, this ensures we have enough. + let extra_blocks = [$(stringify!($method)),*].len(); - let db_range = 0..=db_block_count - 1; - let in_mem_range = db_block_count..=(in_mem_block_count + db_range.end()); + let mut rng = generators::rng(); + let (provider, mut database_blocks, mut in_memory_blocks, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT + extra_blocks, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + let provider = Arc::new(provider); $( + // Since data moves for each tried method, need to recalculate everything + let db_block_count = database_blocks.len() as u64; + let in_mem_block_count = in_memory_blocks.len() as u64; + + let db_range = 0..=db_block_count - 1; + let in_mem_range = db_block_count..=(in_mem_block_count + db_range.end()); + // Retrieve the expected database data let database_data = - $database_blocks.iter().map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(db_range.clone())?, database_data); + database_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!(provider.$method(db_range.clone())?, database_data); // Retrieve the expected in-memory data let in_memory_data = - $in_memory_blocks.iter().map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); + in_memory_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!(provider.$method(in_mem_range.clone())?, in_memory_data); // Test partial in-memory range assert_eq!( - &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, &in_memory_data[1..in_memory_data.len() - 1] ); // Test range that spans database and in-memory - assert_eq!( - $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, - database_data[database_data.len() - 2..] - .iter() - .chain(&in_memory_data[..in_memory_data.len() - 1]) - .cloned() - .collect::>() - ); + { + + // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + + assert_eq!( + provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>() + ); + + // Adjust our blocks accordingly + database_blocks.push(in_memory_blocks.remove(0)); + } // Test invalid range let start_block_num = u64::MAX; let end_block_num = u64::MAX; - let result = $provider.$method(start_block_num..=end_block_num-1)?; + let result = provider.$method(start_block_num..=end_block_num-1)?; assert!(result.is_empty(), "No data should be found for an invalid block range"); // Test valid range with empty results - let result = $provider.$method(in_mem_range.end() + 10..=in_mem_range.end() + 20)?; + let result = provider.$method(in_mem_range.end() + 10..=in_mem_range.end() + 20)?; assert!(result.is_empty(), "No data should be found for an empty block range"); )* }}; @@ -3264,40 +3307,21 @@ mod tests { #[test] fn test_methods_by_block_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - // todo(joshie) add canonical_hashes_range below after changing its interface into range // instead start end - test_by_block_range!( - provider, - database_blocks, - in_memory_blocks, - [ - (headers_range, |block: &SealedBlock| block.header().clone()), - (sealed_headers_range, |block: &SealedBlock| block.header.clone()), - (block_range, |block: &SealedBlock| block.clone().unseal()), - (block_with_senders_range, |block: &SealedBlock| block - .clone() - .unseal() - .with_senders_unchecked(vec![])), - (sealed_block_with_senders_range, |block: &SealedBlock| block - .clone() - .with_senders_unchecked(vec![])), - (transactions_by_block_range, |block: &SealedBlock| block - .body - .transactions - .clone()), - ] - ); + test_by_block_range!([ + (headers_range, |block: &SealedBlock| block.header().clone()), + (sealed_headers_range, |block: &SealedBlock| block.header.clone()), + (block_range, |block: &SealedBlock| block.clone().unseal()), + (block_with_senders_range, |block: &SealedBlock| block + .clone() + .unseal() + .with_senders_unchecked(vec![])), + (sealed_block_with_senders_range, |block: &SealedBlock| block + .clone() + .with_senders_unchecked(vec![])), + (transactions_by_block_range, |block: &SealedBlock| block.body.transactions.clone()), + ]); Ok(()) } @@ -3333,39 +3357,72 @@ mod tests { /// ( `NUMBER_ARGUMENTS`, METHOD, FN -> ((`METHOD_ARGUMENT(s)`,...), `EXPECTED_RESULT`), /// `INVALID_ARGUMENTS`) macro_rules! test_non_range { - ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, $receipts:expr, [$(($arg_count:ident, $method:ident, $item_extractor:expr, $invalid_args:expr)),* $(,)?]) => {{ + ([$(($arg_count:ident, $method:ident, $item_extractor:expr, $invalid_args:expr)),* $(,)?]) => {{ + + // Get the number methods being tested. + // Since each method tested will move a block from memory to storage, this ensures we have enough. + let extra_blocks = [$(stringify!($arg_count)),*].len(); + + let mut rng = generators::rng(); + let (provider, mut database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT + extra_blocks, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + let provider = Arc::new(provider); + + let mut in_memory_blocks: std::collections::VecDeque<_> = in_memory_blocks.into(); + $( let tx_hash = |block: &SealedBlock| block.body.transactions[0].hash(); let tx_num = |block: &SealedBlock| { - $database_blocks + database_blocks .iter() - .chain($in_memory_blocks.iter()) + .chain(in_memory_blocks.iter()) .take_while(|b| b.number < block.number) .map(|b| b.body.transactions.len()) .sum::() as u64 }; // Ensure that the first generated in-memory block exists - // In the future this block will be persisted to disk and removed from memory AFTER the firsk database query. - // This ensures that we query the in-memory state before the database avoiding any race condition. { - call_method!($arg_count, $provider, $method, $item_extractor, tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + + call_method!($arg_count, provider, $method, $item_extractor, tx_num, tx_hash, &in_memory_blocks[0], &receipts); + + // Move the block as well in our own structures + database_blocks.push(in_memory_blocks.pop_front().unwrap()); } + // database_blocks is changed above + let tx_num = |block: &SealedBlock| { + database_blocks + .iter() + .chain(in_memory_blocks.iter()) + .take_while(|b| b.number < block.number) + .map(|b| b.body.transactions.len()) + .sum::() as u64 + }; + // Invalid/Non-existent argument should return `None` { - call_method!($arg_count, $provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + call_method!($arg_count, provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &in_memory_blocks[0], &receipts); } // Check that the item is only in memory and not in database { - let last_mem_block = &$in_memory_blocks[$in_memory_blocks.len() - 1]; + let last_mem_block = &in_memory_blocks[in_memory_blocks.len() - 1]; - let (args, expected_item) = $item_extractor(last_mem_block, tx_num(last_mem_block), tx_hash(last_mem_block), &$receipts); - call_method!($arg_count, $provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &$receipts); + let (args, expected_item) = $item_extractor(last_mem_block, tx_num(last_mem_block), tx_hash(last_mem_block), &receipts); + call_method!($arg_count, provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &receipts); // Ensure the item is not in storage - call_method!($arg_count, $provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &$receipts); + call_method!($arg_count, provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &receipts); } )* }}; @@ -3375,265 +3432,232 @@ mod tests { fn test_non_range_methods() -> eyre::Result<()> { let test_tx_index = 0; - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - test_non_range!( - provider, - database_blocks, - in_memory_blocks, - receipts, - [ - // TODO: header should use B256 like others instead of &B256 - // ( - // ONE, - // header, - // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), - // (&B256::random()) - // ), - ( - ONE, - header_by_number, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(block.header.header().clone()) - ), - u64::MAX + test_non_range!([ + // TODO: header should use B256 like others instead of &B256 + // ( + // ONE, + // header, + // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), + // (&B256::random()) + // ), + ( + ONE, + header_by_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.header().clone()) ), - ( - ONE, - sealed_header, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(block.header.clone()) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + sealed_header, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.clone()) ), - ( - ONE, - block_hash, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(block.hash()) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + block_hash, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.hash()) ), - ( - ONE, - block_number, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.hash(), - Some(block.number) - ), - B256::random() + u64::MAX + ), + ( + ONE, + block_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.hash(), + Some(block.number) ), - ( - ONE, - block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Hash(block.hash()), - Some(block.clone().unseal()) - ), - BlockHashOrNumber::Hash(B256::random()) + B256::random() + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.clone().unseal()) ), - ( - ONE, - block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Number(block.number), - Some(block.clone().unseal()) - ), - BlockHashOrNumber::Number(u64::MAX) + BlockHashOrNumber::Hash(B256::random()) + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.clone().unseal()) ), - ( - ONE, - block_body_indices, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(StoredBlockBodyIndices { - first_tx_num: tx_num, - tx_count: block.body.transactions.len() as u64 - }) - ), - u64::MAX + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + block_body_indices, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(StoredBlockBodyIndices { + first_tx_num: tx_num, + tx_count: block.body.transactions.len() as u64 + }) ), - ( - TWO, - block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), - block.clone().unseal().with_recovered_senders() - ), - (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + u64::MAX + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() ), - ( - TWO, - block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), - block.clone().unseal().with_recovered_senders() - ), - (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() ), - ( - TWO, - sealed_block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), - Some( - block - .clone() - .unseal() - .with_recovered_senders() - .unwrap() - .seal(block.hash()) - ) - ), - (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + Some( + block.clone().unseal().with_recovered_senders().unwrap().seal(block.hash()) + ) ), - ( - TWO, - sealed_block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), - Some( - block - .clone() - .unseal() - .with_recovered_senders() - .unwrap() - .seal(block.hash()) - ) - ), - (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + Some( + block.clone().unseal().with_recovered_senders().unwrap().seal(block.hash()) + ) ), - ( - ONE, - transaction_id, - |_: &SealedBlock, tx_num: TxNumber, tx_hash: B256, _: &Vec>| ( - tx_hash, - Some(tx_num) - ), - B256::random() + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + ONE, + transaction_id, + |_: &SealedBlock, tx_num: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(tx_num) ), - ( - ONE, - transaction_by_id, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - Some(block.body.transactions[test_tx_index].clone()) - ), - u64::MAX + B256::random() + ), + ( + ONE, + transaction_by_id, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.body.transactions[test_tx_index].clone()) ), - ( - ONE, - transaction_by_id_no_hash, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - Some(Into::::into( - block.body.transactions[test_tx_index].clone() - )) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + transaction_by_id_no_hash, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(Into::::into( + block.body.transactions[test_tx_index].clone() + )) ), - ( - ONE, - transaction_by_hash, - |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( - tx_hash, - Some(block.body.transactions[test_tx_index].clone()) - ), - B256::random() + u64::MAX + ), + ( + ONE, + transaction_by_hash, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(block.body.transactions[test_tx_index].clone()) ), - ( - ONE, - transaction_block, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - Some(block.number) - ), - u64::MAX + B256::random() + ), + ( + ONE, + transaction_block, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.number) ), - ( - ONE, - transactions_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Number(block.number), - Some(block.body.transactions.clone()) - ), - BlockHashOrNumber::Number(u64::MAX) + u64::MAX + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.body.transactions.clone()) ), - ( - ONE, - transactions_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Hash(block.hash()), - Some(block.body.transactions.clone()) - ), - BlockHashOrNumber::Number(u64::MAX) + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.body.transactions.clone()) ), - ( - ONE, - transaction_sender, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - block.body.transactions[test_tx_index].recover_signer() - ), - u64::MAX + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transaction_sender, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + block.body.transactions[test_tx_index].recover_signer() ), - ( - ONE, - receipt, - |block: &SealedBlock, - tx_num: TxNumber, - _: B256, - receipts: &Vec>| ( - tx_num, - Some(receipts[block.number as usize][test_tx_index].clone()) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + receipt, + |block: &SealedBlock, tx_num: TxNumber, _: B256, receipts: &Vec>| ( + tx_num, + Some(receipts[block.number as usize][test_tx_index].clone()) ), - ( - ONE, - receipt_by_hash, - |block: &SealedBlock, - _: TxNumber, - tx_hash: B256, - receipts: &Vec>| ( - tx_hash, - Some(receipts[block.number as usize][test_tx_index].clone()) - ), - B256::random() + u64::MAX + ), + ( + ONE, + receipt_by_hash, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, receipts: &Vec>| ( + tx_hash, + Some(receipts[block.number as usize][test_tx_index].clone()) ), - ( - ONE, - receipts_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( - BlockHashOrNumber::Number(block.number), - Some(receipts[block.number as usize].clone()) - ), - BlockHashOrNumber::Number(u64::MAX) + B256::random() + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(receipts[block.number as usize].clone()) ), - ( - ONE, - receipts_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( - BlockHashOrNumber::Hash(block.hash()), - Some(receipts[block.number as usize].clone()) - ), - BlockHashOrNumber::Hash(B256::random()) + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(receipts[block.number as usize].clone()) ), - // TODO: withdrawals, requests, ommers - ] - ); + BlockHashOrNumber::Hash(B256::random()) + ), + // TODO: withdrawals, requests, ommers + ]); Ok(()) } From 6917b12d82d3282423b73c0f75aaba4480998880 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 20:31:37 +0200 Subject: [PATCH 083/103] chore: bump alloy primitives 0 8 7 (#11586) --- Cargo.lock | 22 +++++++++++++++++++--- Cargo.toml | 2 +- crates/primitives-traits/src/withdrawal.rs | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c135f294c58..863f0f35c18d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260d3ff3bff0bb84599f032a2f2c6828180b0ea0cd41fdaf44f39cef3ba41861" +checksum = "8ecb848c43f6b06ae3de2e4a67496cbbabd78ae87db0f1248934f15d76192c6a" dependencies = [ "alloy-rlp", "arbitrary", @@ -293,8 +293,9 @@ dependencies = [ "const-hex", "derive_arbitrary", "derive_more 1.0.0", + "foldhash", "getrandom 0.2.15", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "hex-literal", "indexmap 2.5.0", "itoa", @@ -3101,6 +3102,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -3433,6 +3440,15 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "foldhash", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index b6eed127119b..5c772a844429 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -420,7 +420,7 @@ revm-primitives = { version = "10.0.0", features = [ # eth alloy-chains = "0.1.32" alloy-dyn-abi = "0.8.0" -alloy-primitives = { version = "0.8.4", default-features = false } +alloy-primitives = { version = "0.8.7", default-features = false } alloy-rlp = "0.3.4" alloy-sol-types = "0.8.0" alloy-trie = { version = "0.6", default-features = false } diff --git a/crates/primitives-traits/src/withdrawal.rs b/crates/primitives-traits/src/withdrawal.rs index f6b0607e7f03..995e60292c6e 100644 --- a/crates/primitives-traits/src/withdrawal.rs +++ b/crates/primitives-traits/src/withdrawal.rs @@ -133,7 +133,7 @@ mod tests { // #[test] fn test_withdrawal_serde_roundtrip() { - let input = r#"[{"index":"0x0","validatorIndex":"0x0","address":"0x0000000000000000000000000000000000001000","amount":"0x1"},{"index":"0x1","validatorIndex":"0x1","address":"0x0000000000000000000000000000000000001001","amount":"0x1"},{"index":"0x2","validatorIndex":"0x2","address":"0x0000000000000000000000000000000000001002","amount":"0x1"},{"index":"0x3","validatorIndex":"0x3","address":"0x0000000000000000000000000000000000001003","amount":"0x1"},{"index":"0x4","validatorIndex":"0x4","address":"0x0000000000000000000000000000000000001004","amount":"0x1"},{"index":"0x5","validatorIndex":"0x5","address":"0x0000000000000000000000000000000000001005","amount":"0x1"},{"index":"0x6","validatorIndex":"0x6","address":"0x0000000000000000000000000000000000001006","amount":"0x1"},{"index":"0x7","validatorIndex":"0x7","address":"0x0000000000000000000000000000000000001007","amount":"0x1"},{"index":"0x8","validatorIndex":"0x8","address":"0x0000000000000000000000000000000000001008","amount":"0x1"},{"index":"0x9","validatorIndex":"0x9","address":"0x0000000000000000000000000000000000001009","amount":"0x1"},{"index":"0xa","validatorIndex":"0xa","address":"0x000000000000000000000000000000000000100A","amount":"0x1"},{"index":"0xb","validatorIndex":"0xb","address":"0x000000000000000000000000000000000000100b","amount":"0x1"},{"index":"0xc","validatorIndex":"0xc","address":"0x000000000000000000000000000000000000100C","amount":"0x1"},{"index":"0xd","validatorIndex":"0xd","address":"0x000000000000000000000000000000000000100D","amount":"0x1"},{"index":"0xe","validatorIndex":"0xe","address":"0x000000000000000000000000000000000000100e","amount":"0x1"},{"index":"0xf","validatorIndex":"0xf","address":"0x000000000000000000000000000000000000100f","amount":"0x1"}]"#; + let input = r#"[{"index":"0x0","validatorIndex":"0x0","address":"0x0000000000000000000000000000000000001000","amount":"0x1"},{"index":"0x1","validatorIndex":"0x1","address":"0x0000000000000000000000000000000000001001","amount":"0x1"},{"index":"0x2","validatorIndex":"0x2","address":"0x0000000000000000000000000000000000001002","amount":"0x1"},{"index":"0x3","validatorIndex":"0x3","address":"0x0000000000000000000000000000000000001003","amount":"0x1"},{"index":"0x4","validatorIndex":"0x4","address":"0x0000000000000000000000000000000000001004","amount":"0x1"},{"index":"0x5","validatorIndex":"0x5","address":"0x0000000000000000000000000000000000001005","amount":"0x1"},{"index":"0x6","validatorIndex":"0x6","address":"0x0000000000000000000000000000000000001006","amount":"0x1"},{"index":"0x7","validatorIndex":"0x7","address":"0x0000000000000000000000000000000000001007","amount":"0x1"},{"index":"0x8","validatorIndex":"0x8","address":"0x0000000000000000000000000000000000001008","amount":"0x1"},{"index":"0x9","validatorIndex":"0x9","address":"0x0000000000000000000000000000000000001009","amount":"0x1"},{"index":"0xa","validatorIndex":"0xa","address":"0x000000000000000000000000000000000000100a","amount":"0x1"},{"index":"0xb","validatorIndex":"0xb","address":"0x000000000000000000000000000000000000100b","amount":"0x1"},{"index":"0xc","validatorIndex":"0xc","address":"0x000000000000000000000000000000000000100c","amount":"0x1"},{"index":"0xd","validatorIndex":"0xd","address":"0x000000000000000000000000000000000000100d","amount":"0x1"},{"index":"0xe","validatorIndex":"0xe","address":"0x000000000000000000000000000000000000100e","amount":"0x1"},{"index":"0xf","validatorIndex":"0xf","address":"0x000000000000000000000000000000000000100f","amount":"0x1"}]"#; let withdrawals: Vec = serde_json::from_str(input).unwrap(); let s = serde_json::to_string(&withdrawals).unwrap(); From 95913cb9b5845f3dfbdeb12a24eba022868c3781 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Tue, 8 Oct 2024 20:31:55 +0200 Subject: [PATCH 084/103] chore(ci): remove expected failures related to checksummed addresses (#11589) --- .github/assets/hive/expected_failures.yaml | 17 ----------------- .../hive/expected_failures_experimental.yaml | 17 ----------------- 2 files changed, 34 deletions(-) diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index ddf1383ff4d9..7a212a51dd5e 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -20,23 +20,6 @@ rpc-compat: - eth_getBlockByNumber/get-latest (reth) - eth_getBlockByNumber/get-safe (reth) - - eth_createAccessList/create-al-contract-eip1559 (reth) - - eth_createAccessList/create-al-contract (reth) - - eth_getProof/get-account-proof-blockhash (reth) - - eth_getProof/get-account-proof-latest (reth) - - eth_getProof/get-account-proof-with-storage (reth) - - eth_getTransactionByBlockHashAndIndex/get-block-n (reth) - - eth_getTransactionByBlockNumberAndIndex/get-block-n (reth) - - eth_getTransactionByHash/get-access-list (reth) - - eth_getTransactionByHash/get-blob-tx (reth) - - eth_getTransactionByHash/get-dynamic-fee (reth) - - eth_getTransactionByHash/get-legacy-create (reth) - - eth_getTransactionByHash/get-legacy-input (reth) - - eth_getTransactionByHash/get-legacy-tx (reth) - - eth_getTransactionReceipt/get-legacy-contract (reth) - - eth_getTransactionReceipt/get-legacy-input (reth) - - eth_getTransactionReceipt/get-legacy-receipt (reth)' - # https://github.com/paradigmxyz/reth/issues/8732 engine-withdrawals: - Withdrawals Fork On Genesis (Paris) (reth) diff --git a/.github/assets/hive/expected_failures_experimental.yaml b/.github/assets/hive/expected_failures_experimental.yaml index 91fd1a88ab83..d4b3d2bcbd3c 100644 --- a/.github/assets/hive/expected_failures_experimental.yaml +++ b/.github/assets/hive/expected_failures_experimental.yaml @@ -20,23 +20,6 @@ rpc-compat: - eth_getBlockByNumber/get-latest (reth) - eth_getBlockByNumber/get-safe (reth) - - eth_createAccessList/create-al-contract-eip1559 (reth) - - eth_createAccessList/create-al-contract (reth) - - eth_getProof/get-account-proof-blockhash (reth) - - eth_getProof/get-account-proof-latest (reth) - - eth_getProof/get-account-proof-with-storage (reth) - - eth_getTransactionByBlockHashAndIndex/get-block-n (reth) - - eth_getTransactionByBlockNumberAndIndex/get-block-n (reth) - - eth_getTransactionByHash/get-access-list (reth) - - eth_getTransactionByHash/get-blob-tx (reth) - - eth_getTransactionByHash/get-dynamic-fee (reth) - - eth_getTransactionByHash/get-legacy-create (reth) - - eth_getTransactionByHash/get-legacy-input (reth) - - eth_getTransactionByHash/get-legacy-tx (reth) - - eth_getTransactionReceipt/get-legacy-contract (reth) - - eth_getTransactionReceipt/get-legacy-input (reth) - - eth_getTransactionReceipt/get-legacy-receipt (reth)' - # https://github.com/paradigmxyz/reth/issues/8732 engine-withdrawals: - Withdrawals Fork On Genesis (Paris) (reth) From f7c76291a160a1f56bcf47bd6308b593b3afb5d3 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:26:44 +0900 Subject: [PATCH 085/103] chore(rpc): use `block_hash` instead on fetching `debug_trace_block` block (#11587) --- crates/rpc/rpc/src/debug.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index a83ba68b42f5..9fbde05bb400 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -209,7 +209,7 @@ where let ((cfg, block_env, _), block) = futures::try_join!( self.eth_api().evm_env_at(block_hash.into()), - self.eth_api().block_with_senders(block_id), + self.eth_api().block_with_senders(block_hash.into()), )?; let block = block.ok_or(EthApiError::HeaderNotFound(block_id))?; From f332f7156112ad489bcf4aa51b55d2d1d3ad9e2e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 20:30:30 +0200 Subject: [PATCH 086/103] chore: also derive arb for test (#11588) --- crates/primitives-traits/src/storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/primitives-traits/src/storage.rs b/crates/primitives-traits/src/storage.rs index 801edba088f4..39b6155ee284 100644 --- a/crates/primitives-traits/src/storage.rs +++ b/crates/primitives-traits/src/storage.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; /// /// `key` is the subkey when used as a value in the `StorageChangeSets` table. #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct StorageEntry { /// Storage key. From f5f76206fda26b48ee999cc9877cab2e05b04acd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 21:03:47 +0200 Subject: [PATCH 087/103] feat: add mul support for SubPoolLimit (#11591) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- crates/transaction-pool/src/config.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 67328c7f1be1..1b4b010a8e14 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -7,7 +7,8 @@ use reth_primitives::{ constants::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}, EIP4844_TX_TYPE_ID, }; -use std::collections::HashSet; +use std::{collections::HashSet, ops::Mul}; + /// Guarantees max transactions for one sender, compatible with geth/erigon pub const TXPOOL_MAX_ACCOUNT_SLOTS_PER_SENDER: usize = 16; @@ -107,6 +108,15 @@ impl SubPoolLimit { } } +impl Mul for SubPoolLimit { + type Output = Self; + + fn mul(self, rhs: usize) -> Self::Output { + let Self { max_txs, max_size } = self; + Self { max_txs: max_txs * rhs, max_size: max_size * rhs } + } +} + impl Default for SubPoolLimit { fn default() -> Self { // either 10k transactions or 20MB @@ -318,4 +328,14 @@ mod tests { let new_config = config.set_propagate_local_transactions(false); assert!(!new_config.propagate_local_transactions); } + + #[test] + fn scale_pool_limit() { + let limit = SubPoolLimit::default(); + let double = limit * 2; + assert_eq!( + double, + SubPoolLimit { max_txs: limit.max_txs * 2, max_size: limit.max_size * 2 } + ) + } } From e427eb202715d12e09ba99df653e537cf5f5926a Mon Sep 17 00:00:00 2001 From: John Date: Wed, 9 Oct 2024 01:27:08 +0600 Subject: [PATCH 088/103] docs: delete missing part path (#11590) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- docs/repo/layout.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 8eb0782b988c..6ed91e79656f 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -168,7 +168,7 @@ Small utility crates. - [`tasks`](../../crates/tasks): An executor-agnostic task abstraction, used to spawn tasks on different async executors. Supports blocking tasks and handles panics gracefully. A tokio implementation is provided by default. - [`metrics/common`](../../crates/metrics/src/common): Common metrics types (e.g. metered channels) -- [`metrics/metrics-derive`](../../crates/metrics/metrics-derive): A derive-style API for creating metrics +- [`metrics/metrics-derive`](https://github.com/rkrasiuk/metrics-derive): A derive-style API for creating metrics - [`tracing`](../../crates/tracing): A small utility crate to install a uniform [`tracing`][tracing] subscriber [libmdbx-rs]: https://crates.io/crates/libmdbx From 3141f6db26926cdd2a3e65e9d9cb9b4f19d037d7 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 9 Oct 2024 06:04:16 +0400 Subject: [PATCH 089/103] fix: simplify reorg handling (#11592) --- crates/engine/tree/src/tree/mod.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index fd3f2df5c01b..0478c73c90d2 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -899,16 +899,9 @@ where return Ok(None); } - if old_hash == current_hash { - // We've found the fork point - break; - } - if let Some(block) = self.executed_block_by_hash(current_hash)? { - if self.is_fork(block.block.hash())? { - current_hash = block.block.parent_hash; - new_chain.push(block); - } + current_hash = block.block.parent_hash; + new_chain.push(block); } else { // This shouldn't happen as we've already walked this path warn!(target: "engine::tree", invalid_hash=?current_hash, "New chain block not found in TreeState"); From 1e1de3dd77b114977f2461982ca371a971dab005 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 04:48:36 +0200 Subject: [PATCH 090/103] fix: use original bytes for codes (#11593) --- crates/rpc/rpc/src/debug.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 9fbde05bb400..fb0407039776 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -619,7 +619,7 @@ where .cache .contracts .iter() - .map(|(hash, code)| (*hash, code.bytes())) + .map(|(hash, code)| (*hash, code.original_bytes())) .collect(); for (address, account) in &statedb.cache.accounts { From 62954591d25c164b1cc42084c32010e15cd4a75a Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:33:29 +0900 Subject: [PATCH 091/103] perf(rpc): use `Arc>>>; /// The type that can send the response to a requested [`BlockWithSenders`] -type BlockWithSendersResponseSender = oneshot::Sender>>; +type BlockWithSendersResponseSender = + oneshot::Sender>>>; /// The type that can send the response to the requested receipts of a block. type ReceiptsResponseSender = oneshot::Sender>>>>; @@ -48,7 +49,7 @@ type EnvResponseSender = oneshot::Sender = MultiConsumerLruCache< B256, - BlockWithSenders, + Arc, L, Either, >; @@ -151,7 +152,7 @@ impl EthStateCache { rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)?; if let Ok(Some(block_with_senders)) = block_with_senders_res { - Ok(Some(block_with_senders.block)) + Ok(Some(block_with_senders.block.clone())) } else { Ok(None) } @@ -186,7 +187,7 @@ impl EthStateCache { Ok(self .get_block_with_senders(block_hash) .await? - .map(|block| block.into_transactions_ecrecovered().collect())) + .map(|block| (*block).clone().into_transactions_ecrecovered().collect())) } /// Fetches both transactions and receipts for the given block hash. @@ -208,7 +209,7 @@ impl EthStateCache { pub async fn get_block_with_senders( &self, block_hash: B256, - ) -> ProviderResult> { + ) -> ProviderResult>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? @@ -221,7 +222,10 @@ impl EthStateCache { &self, block_hash: B256, ) -> ProviderResult> { - Ok(self.get_block_with_senders(block_hash).await?.map(|block| block.seal(block_hash))) + Ok(self + .get_block_with_senders(block_hash) + .await? + .map(|block| (*block).clone().seal(block_hash))) } /// Requests the [Receipt] for the block hash @@ -288,7 +292,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter, + LimitBlocks: Limiter>, LimitReceipts: Limiter>>, LimitEnvs: Limiter, { @@ -318,7 +322,11 @@ where Tasks: TaskSpawner + Clone + 'static, EvmConfig: ConfigureEvm
, { - fn on_new_block(&mut self, block_hash: B256, res: ProviderResult>) { + fn on_new_block( + &mut self, + block_hash: B256, + res: ProviderResult>>, + ) { if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders for tx in queued { @@ -328,7 +336,7 @@ where } Either::Right(transaction_tx) => { let _ = transaction_tx.send(res.clone().map(|maybe_block| { - maybe_block.map(|block| block.block.body.transactions) + maybe_block.map(|block| block.block.body.transactions.clone()) })); } } @@ -360,6 +368,7 @@ where } fn on_reorg_block(&mut self, block_hash: B256, res: ProviderResult>) { + let res = res.map(|b| b.map(Arc::new)); if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders for tx in queued { @@ -369,7 +378,7 @@ where } Either::Right(transaction_tx) => { let _ = transaction_tx.send(res.clone().map(|maybe_block| { - maybe_block.map(|block| block.block.body.transactions) + maybe_block.map(|block| block.block.body.transactions.clone()) })); } } @@ -431,10 +440,12 @@ where let _permit = rate_limiter.acquire().await; // Only look in the database to prevent situations where we // looking up the tree is blocking - let block_sender = provider.block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ); + let block_sender = provider + .block_with_senders( + BlockHashOrNumber::Hash(block_hash), + TransactionVariant::WithHash, + ) + .map(|maybe_block| maybe_block.map(Arc::new)); let _ = action_tx.send(CacheAction::BlockWithSendersResult { block_hash, res: block_sender, @@ -459,10 +470,12 @@ where let _permit = rate_limiter.acquire().await; // Only look in the database to prevent situations where we // looking up the tree is blocking - let res = provider.block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ); + let res = provider + .block_with_senders( + BlockHashOrNumber::Hash(block_hash), + TransactionVariant::WithHash, + ) + .map(|b| b.map(Arc::new)); let _ = action_tx.send(CacheAction::BlockWithSendersResult { block_hash, res, @@ -561,7 +574,7 @@ where } CacheAction::CacheNewCanonicalChain { chain_change } => { for block in chain_change.blocks { - this.on_new_block(block.hash(), Ok(Some(block.unseal()))); + this.on_new_block(block.hash(), Ok(Some(Arc::new(block.unseal())))); } for block_receipts in chain_change.receipts { @@ -601,7 +614,7 @@ enum CacheAction { GetBlockTransactions { block_hash: B256, response_tx: BlockTransactionsResponseSender }, GetEnv { block_hash: B256, response_tx: EnvResponseSender }, GetReceipts { block_hash: B256, response_tx: ReceiptsResponseSender }, - BlockWithSendersResult { block_hash: B256, res: ProviderResult> }, + BlockWithSendersResult { block_hash: B256, res: ProviderResult>> }, ReceiptsResult { block_hash: B256, res: ProviderResult>>> }, EnvResult { block_hash: B256, res: Box> }, CacheNewCanonicalChain { chain_change: ChainChange }, From d5372a739e9edde259250b8d91cac04103f827dc Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 12:03:25 +0200 Subject: [PATCH 092/103] fix: active inflight count (#11598) --- crates/net/network/src/transactions/fetcher.rs | 11 ++++++++--- crates/net/network/src/transactions/mod.rs | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index e15972df08b2..0b849b659101 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -98,6 +98,11 @@ pub struct TransactionFetcher { // === impl TransactionFetcher === impl TransactionFetcher { + /// Removes the peer from the active set. + pub(crate) fn remove_peer(&mut self, peer_id: &PeerId) { + self.active_peers.remove(peer_id); + } + /// Updates metrics. #[inline] pub fn update_metrics(&self) { @@ -157,7 +162,7 @@ impl TransactionFetcher { fn decrement_inflight_request_count_for(&mut self, peer_id: &PeerId) { let remove = || -> bool { if let Some(inflight_count) = self.active_peers.get(peer_id) { - *inflight_count -= 1; + *inflight_count = inflight_count.saturating_sub(1); if *inflight_count == 0 { return true } @@ -659,8 +664,6 @@ impl TransactionFetcher { return Some(new_announced_hashes) } - *inflight_count += 1; - #[cfg(debug_assertions)] { for hash in &new_announced_hashes { @@ -695,6 +698,8 @@ impl TransactionFetcher { } } } + + *inflight_count += 1; // stores a new request future for the request self.inflight_requests.push(GetPooledTxRequestFut::new(peer_id, new_announced_hashes, rx)); diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 2fa4ccfbb606..24cc8f61407d 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -904,6 +904,7 @@ where NetworkEvent::SessionClosed { peer_id, .. } => { // remove the peer self.peers.remove(&peer_id); + self.transaction_fetcher.remove_peer(&peer_id); } NetworkEvent::SessionEstablished { peer_id, client_version, messages, version, .. From 570c1df7ecb1fffefbc19980a7acb5dbabbba218 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Wed, 9 Oct 2024 12:32:18 +0200 Subject: [PATCH 093/103] fix(net): max inflight tx reqs default (#11602) --- crates/net/network/src/transactions/fetcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index 0b849b659101..cf5c09045f28 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -1325,7 +1325,7 @@ impl TransactionFetcherInfo { impl Default for TransactionFetcherInfo { fn default() -> Self { Self::new( - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize * DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER as usize, + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize, DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, From 8ba7bc0799bf71f8d92d64acb63cbd6f2712709f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 13:00:17 +0200 Subject: [PATCH 094/103] fix: set deposit gasprice correctly (#11603) --- crates/optimism/rpc/src/eth/transaction.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 3ccda419cad8..a4b4c064970d 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -105,7 +105,11 @@ impl TransactionCompat for OpTxBuilder { fn fill(tx: TransactionSignedEcRecovered, tx_info: TransactionInfo) -> Self::Transaction { let signed_tx = tx.clone().into_signed(); - let inner = EthTxBuilder::fill(tx, tx_info).inner; + let mut inner = EthTxBuilder::fill(tx, tx_info).inner; + + if signed_tx.is_deposit() { + inner.gas_price = Some(signed_tx.max_fee_per_gas()) + } Transaction { inner, From 3c4a94fb763d04bc16c3afacc441b3e9bf87a123 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 13:12:28 +0200 Subject: [PATCH 095/103] fix: set system tx correctly (#11601) --- crates/optimism/rpc/src/eth/transaction.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index a4b4c064970d..ab7525016a1a 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -116,7 +116,8 @@ impl TransactionCompat for OpTxBuilder { source_hash: signed_tx.source_hash(), mint: signed_tx.mint(), // only include is_system_tx if true: - is_system_tx: signed_tx.is_deposit().then_some(signed_tx.is_system_transaction()), + is_system_tx: (signed_tx.is_deposit() && signed_tx.is_system_transaction()) + .then_some(true), deposit_receipt_version: None, // todo: how to fill this field? } } From 5ad6484386debb85abc5cca5e777571c79ba3c7c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 9 Oct 2024 13:20:58 +0200 Subject: [PATCH 096/103] fix(trie): prefix set extension (#11605) --- crates/trie/trie/src/prefix_set.rs | 21 +++++++++++++++++---- crates/trie/trie/src/proof.rs | 4 ++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/trie/trie/src/prefix_set.rs b/crates/trie/trie/src/prefix_set.rs index 4997228050a3..af0fb173d98a 100644 --- a/crates/trie/trie/src/prefix_set.rs +++ b/crates/trie/trie/src/prefix_set.rs @@ -20,9 +20,9 @@ pub struct TriePrefixSetsMut { impl TriePrefixSetsMut { /// Extends prefix sets with contents of another prefix set. pub fn extend(&mut self, other: Self) { - self.account_prefix_set.extend(other.account_prefix_set.keys); + self.account_prefix_set.extend(other.account_prefix_set); for (hashed_address, prefix_set) in other.storage_prefix_sets { - self.storage_prefix_sets.entry(hashed_address).or_default().extend(prefix_set.keys); + self.storage_prefix_sets.entry(hashed_address).or_default().extend(prefix_set); } self.destroyed_accounts.extend(other.destroyed_accounts); } @@ -115,12 +115,18 @@ impl PrefixSetMut { self.keys.push(nibbles); } + /// Extend prefix set with contents of another prefix set. + pub fn extend(&mut self, other: Self) { + self.all |= other.all; + self.keys.extend(other.keys); + } + /// Extend prefix set keys with contents of provided iterator. - pub fn extend(&mut self, nibbles_iter: I) + pub fn extend_keys(&mut self, keys: I) where I: IntoIterator, { - self.keys.extend(nibbles_iter); + self.keys.extend(keys); } /// Returns the number of elements in the set. @@ -270,4 +276,11 @@ mod tests { assert_eq!(prefix_set.keys.len(), 3); // Length should be 3 (excluding duplicate) assert_eq!(prefix_set.keys.capacity(), 3); // Capacity should be 3 after shrinking } + + #[test] + fn test_prefix_set_all_extend() { + let mut prefix_set_mut = PrefixSetMut::default(); + prefix_set_mut.extend(PrefixSetMut::all()); + assert!(prefix_set_mut.all); + } } diff --git a/crates/trie/trie/src/proof.rs b/crates/trie/trie/src/proof.rs index bff681a3498d..d31d63fd9a8b 100644 --- a/crates/trie/trie/src/proof.rs +++ b/crates/trie/trie/src/proof.rs @@ -96,7 +96,7 @@ where // Create the walker. let mut prefix_set = self.prefix_sets.account_prefix_set.clone(); - prefix_set.extend(targets.keys().map(Nibbles::unpack)); + prefix_set.extend_keys(targets.keys().map(Nibbles::unpack)); let walker = TrieWalker::new(trie_cursor, prefix_set.freeze()); // Create a hash builder to rebuild the root node since it is not available in the database. @@ -225,7 +225,7 @@ where } let target_nibbles = targets.into_iter().map(Nibbles::unpack).collect::>(); - self.prefix_set.extend(target_nibbles.clone()); + self.prefix_set.extend_keys(target_nibbles.clone()); let trie_cursor = self.trie_cursor_factory.storage_trie_cursor(self.hashed_address)?; let walker = TrieWalker::new(trie_cursor, self.prefix_set.freeze()); From 83962095ecfe9ca8b806fa4edbc01815c5375af9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 13:42:55 +0200 Subject: [PATCH 097/103] feat: add tx propagation mode (#11594) --- crates/net/network/src/transactions/config.rs | 28 +++++++++++++++++++ crates/net/network/src/transactions/mod.rs | 16 +++++------ crates/node/core/src/args/network.rs | 1 + 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index b8023ca7928c..81ec293ea1ff 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -18,6 +18,9 @@ pub struct TransactionsManagerConfig { pub transaction_fetcher_config: TransactionFetcherConfig, /// Max number of seen transactions to store for each peer. pub max_transactions_seen_by_peer_history: u32, + /// How new pending transactions are propagated. + #[cfg_attr(feature = "serde", serde(default))] + pub propagation_mode: TransactionPropagationMode, } impl Default for TransactionsManagerConfig { @@ -25,6 +28,31 @@ impl Default for TransactionsManagerConfig { Self { transaction_fetcher_config: TransactionFetcherConfig::default(), max_transactions_seen_by_peer_history: DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, + propagation_mode: TransactionPropagationMode::default(), + } + } +} + +/// Determines how new pending transactions are propagated to other peers in full. +#[derive(Debug, Clone, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum TransactionPropagationMode { + /// Send full transactions to sqrt of current peers. + #[default] + Sqrt, + /// Always send transactions in full. + All, + /// Send full transactions to a maximum number of peers + Max(usize), +} + +impl TransactionPropagationMode { + /// Returns the number of peers that should + pub(crate) fn full_peer_count(&self, peer_count: usize) -> usize { + match self { + Self::Sqrt => (peer_count as f64).sqrt().round() as usize, + Self::All => peer_count, + Self::Max(max) => peer_count.min(*max), } } } diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 24cc8f61407d..0c488ff919dc 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -12,7 +12,7 @@ pub use self::constants::{ tx_fetcher::DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, }; -pub use config::{TransactionFetcherConfig, TransactionsManagerConfig}; +pub use config::{TransactionFetcherConfig, TransactionPropagationMode, TransactionsManagerConfig}; pub use validation::*; pub(crate) use fetcher::{FetchEvent, TransactionFetcher}; @@ -246,8 +246,8 @@ pub struct TransactionsManager { pending_transactions: ReceiverStream, /// Incoming events from the [`NetworkManager`](crate::NetworkManager). transaction_events: UnboundedMeteredReceiver, - /// Max number of seen transactions to store for each peer. - max_transactions_seen_by_peer_history: u32, + /// How the `TransactionsManager` is configured. + config: TransactionsManagerConfig, /// `TransactionsManager` metrics metrics: TransactionsManagerMetrics, } @@ -298,8 +298,7 @@ impl TransactionsManager { from_network, NETWORK_POOL_TRANSACTIONS_SCOPE, ), - max_transactions_seen_by_peer_history: transactions_manager_config - .max_transactions_seen_by_peer_history, + config: transactions_manager_config, metrics, } } @@ -424,9 +423,8 @@ where return propagated } - // send full transactions to a fraction of the connected peers (square root of the total - // number of connected peers) - let max_num_full = (self.peers.len() as f64).sqrt().round() as usize; + // send full transactions to a set of the connected peers based on the configured mode + let max_num_full = self.config.propagation_mode.full_peer_count(self.peers.len()); // Note: Assuming ~random~ order due to random state of the peers map hasher for (peer_idx, (peer_id, peer)) in self.peers.iter_mut().enumerate() { @@ -914,7 +912,7 @@ where messages, version, client_version, - self.max_transactions_seen_by_peer_history, + self.config.max_transactions_seen_by_peer_history, ); let peer = match self.peers.entry(peer_id) { Entry::Occupied(mut entry) => { diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index 0f1465bc5795..04153b93ecdd 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -226,6 +226,7 @@ impl NetworkArgs { self.max_capacity_cache_txns_pending_fetch, ), max_transactions_seen_by_peer_history: self.max_seen_tx_history, + propagation_mode: Default::default(), }; // Configure basic network stack From 67e83b6685e2a236709f53af043edb3b501112ca Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 14:12:54 +0200 Subject: [PATCH 098/103] feat: add helper function to provde the tx manager config (#11608) --- crates/node/builder/src/builder/mod.rs | 40 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index d2ce2a1d8e2f..8b57781ea97c 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -9,6 +9,13 @@ pub use states::*; use std::sync::Arc; +use crate::{ + common::WithConfigs, + components::NodeComponentsBuilder, + node::FullNode, + rpc::{EthApiBuilderProvider, RethRpcServerHandles, RpcContext}, + DefaultNodeLauncher, LaunchNode, Node, NodeHandle, +}; use futures::Future; use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks}; use reth_cli_util::get_secret_key; @@ -18,7 +25,8 @@ use reth_db_api::{ }; use reth_exex::ExExContext; use reth_network::{ - NetworkBuilder, NetworkConfig, NetworkConfigBuilder, NetworkHandle, NetworkManager, + transactions::TransactionsManagerConfig, NetworkBuilder, NetworkConfig, NetworkConfigBuilder, + NetworkHandle, NetworkManager, }; use reth_node_api::{ FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter, @@ -38,14 +46,6 @@ use reth_transaction_pool::{PoolConfig, TransactionPool}; use secp256k1::SecretKey; use tracing::{info, trace, warn}; -use crate::{ - common::WithConfigs, - components::NodeComponentsBuilder, - node::FullNode, - rpc::{EthApiBuilderProvider, RethRpcServerHandles, RpcContext}, - DefaultNodeLauncher, LaunchNode, Node, NodeHandle, -}; - /// The adapter type for a reth node with the builtin provider type // Note: we need to hardcode this because custom components might depend on it in associated types. pub type RethFullAdapter = FullNodeTypesAdapter< @@ -583,16 +583,34 @@ impl BuilderContext { self.config().builder.clone() } - /// Convenience function to start the network. + /// Convenience function to start the network tasks. /// /// Spawns the configured network and associated tasks and returns the [`NetworkHandle`] /// connected to that network. pub fn start_network(&self, builder: NetworkBuilder<(), ()>, pool: Pool) -> NetworkHandle + where + Pool: TransactionPool + Unpin + 'static, + { + self.start_network_with(builder, pool, Default::default()) + } + + /// Convenience function to start the network tasks. + /// + /// Accepts the config for the transaction task. + /// + /// Spawns the configured network and associated tasks and returns the [`NetworkHandle`] + /// connected to that network. + pub fn start_network_with( + &self, + builder: NetworkBuilder<(), ()>, + pool: Pool, + tx_config: TransactionsManagerConfig, + ) -> NetworkHandle where Pool: TransactionPool + Unpin + 'static, { let (handle, network, txpool, eth) = builder - .transactions(pool, Default::default()) + .transactions(pool, tx_config) .request_handler(self.provider().clone()) .split_with_handle(); From f5d68443a344074b34c371ce30ac7484e7ae3fa5 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 9 Oct 2024 13:12:44 +0100 Subject: [PATCH 099/103] fix(grafana): set instance variable from `reth_info` metric (#11607) --- etc/grafana/dashboards/reth-performance.json | 28 +++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/etc/grafana/dashboards/reth-performance.json b/etc/grafana/dashboards/reth-performance.json index e0ff5865dd59..02d890dceeff 100644 --- a/etc/grafana/dashboards/reth-performance.json +++ b/etc/grafana/dashboards/reth-performance.json @@ -15,7 +15,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.2.0" + "version": "11.1.0" }, { "type": "datasource", @@ -83,7 +83,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -137,9 +136,9 @@ "options": { "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -202,7 +201,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -256,9 +254,9 @@ "options": { "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -315,7 +313,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values($instance)", + "definition": "query_result(reth_info)", "hide": 0, "includeAll": false, "label": "instance", @@ -323,12 +321,12 @@ "name": "instance", "options": [], "query": { - "qryType": 1, - "query": "label_values($instance)", + "qryType": 3, + "query": "query_result(reth_info)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, - "regex": "", + "regex": "/.*instance=\\\"([^\\\"]*).*/", "skipUrlSync": false, "sort": 0, "type": "query" @@ -341,8 +339,8 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Reth Performance", + "title": "Reth - Performance", "uid": "bdywb3xjphfy8a", "version": 2, "weekStart": "" -} \ No newline at end of file +} From fb8bd77df34fe462cc59efd63761037d9cd94cfd Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Wed, 9 Oct 2024 14:32:16 +0200 Subject: [PATCH 100/103] fix(net): add concurrency param from config to `TransactionFetcherInfo` (#11600) Co-authored-by: Matthias Seitz --- .../net/network/src/transactions/fetcher.rs | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index cf5c09045f28..9276219d593b 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -130,20 +130,27 @@ impl TransactionFetcher { /// Sets up transaction fetcher with config pub fn with_transaction_fetcher_config(config: &TransactionFetcherConfig) -> Self { - let mut tx_fetcher = Self::default(); + let TransactionFetcherConfig { + max_inflight_requests, + max_capacity_cache_txns_pending_fetch, + .. + } = *config; - tx_fetcher.info.soft_limit_byte_size_pooled_transactions_response = - config.soft_limit_byte_size_pooled_transactions_response; - tx_fetcher.info.soft_limit_byte_size_pooled_transactions_response_on_pack_request = - config.soft_limit_byte_size_pooled_transactions_response_on_pack_request; - tx_fetcher - .metrics - .capacity_inflight_requests - .increment(tx_fetcher.info.max_inflight_requests as u64); - tx_fetcher.info.max_capacity_cache_txns_pending_fetch = - config.max_capacity_cache_txns_pending_fetch; + let info = config.clone().into(); - tx_fetcher + let metrics = TransactionFetcherMetrics::default(); + metrics.capacity_inflight_requests.increment(max_inflight_requests as u64); + + Self { + active_peers: LruMap::new(max_inflight_requests), + hashes_pending_fetch: LruCache::new(max_capacity_cache_txns_pending_fetch), + hashes_fetch_inflight_and_pending_fetch: LruMap::new( + max_inflight_requests + max_capacity_cache_txns_pending_fetch, + ), + info, + metrics, + ..Default::default() + } } /// Removes the specified hashes from inflight tracking. @@ -178,7 +185,7 @@ impl TransactionFetcher { /// Returns `true` if peer is idle with respect to `self.inflight_requests`. pub fn is_idle(&self, peer_id: &PeerId) -> bool { let Some(inflight_count) = self.active_peers.peek(peer_id) else { return true }; - if *inflight_count < DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER { + if *inflight_count < self.info.max_inflight_requests_per_peer { return true } false @@ -653,12 +660,12 @@ impl TransactionFetcher { return Some(new_announced_hashes) }; - if *inflight_count >= DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER { + if *inflight_count >= self.info.max_inflight_requests_per_peer { trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), hashes=?*new_announced_hashes, %conn_eth_version, - max_concurrent_tx_reqs_per_peer=DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, + max_concurrent_tx_reqs_per_peer=self.info.max_inflight_requests_per_peer, "limit for concurrent `GetPooledTransactions` requests per peer reached" ); return Some(new_announced_hashes) @@ -1288,10 +1295,12 @@ pub enum VerificationOutcome { } /// Tracks stats about the [`TransactionFetcher`]. -#[derive(Debug)] +#[derive(Debug, Constructor)] pub struct TransactionFetcherInfo { /// Max inflight [`GetPooledTransactions`] requests. pub max_inflight_requests: usize, + /// Max inflight [`GetPooledTransactions`] requests per peer. + pub max_inflight_requests_per_peer: u8, /// Soft limit for the byte size of the expected [`PooledTransactions`] response, upon packing /// a [`GetPooledTransactions`] request with hashes (by default less than 2 MiB worth of /// transactions is requested). @@ -1305,27 +1314,11 @@ pub struct TransactionFetcherInfo { pub max_capacity_cache_txns_pending_fetch: u32, } -impl TransactionFetcherInfo { - /// Creates a new max - pub const fn new( - max_inflight_requests: usize, - soft_limit_byte_size_pooled_transactions_response_on_pack_request: usize, - soft_limit_byte_size_pooled_transactions_response: usize, - max_capacity_cache_txns_pending_fetch: u32, - ) -> Self { - Self { - max_inflight_requests, - soft_limit_byte_size_pooled_transactions_response_on_pack_request, - soft_limit_byte_size_pooled_transactions_response, - max_capacity_cache_txns_pending_fetch, - } - } -} - impl Default for TransactionFetcherInfo { fn default() -> Self { Self::new( DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize, + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, @@ -1333,6 +1326,26 @@ impl Default for TransactionFetcherInfo { } } +impl From for TransactionFetcherInfo { + fn from(config: TransactionFetcherConfig) -> Self { + let TransactionFetcherConfig { + max_inflight_requests, + max_inflight_requests_per_peer, + soft_limit_byte_size_pooled_transactions_response, + soft_limit_byte_size_pooled_transactions_response_on_pack_request, + max_capacity_cache_txns_pending_fetch, + } = config; + + Self::new( + max_inflight_requests as usize, + max_inflight_requests_per_peer, + soft_limit_byte_size_pooled_transactions_response_on_pack_request, + soft_limit_byte_size_pooled_transactions_response, + max_capacity_cache_txns_pending_fetch, + ) + } +} + #[derive(Debug, Default)] struct TxFetcherSearchDurations { find_idle_peer: Duration, From b787d9e521fad1cb28a372637474ae4ec4986bf3 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:45:40 +0900 Subject: [PATCH 101/103] perf(rpc): optimistically retrieve block if near the tip on `eth_getLogs` (#11582) --- crates/rpc/rpc-eth-types/src/logs_utils.rs | 67 +++++++++++------- crates/rpc/rpc/src/eth/filter.rs | 81 ++++++++++++---------- 2 files changed, 87 insertions(+), 61 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index f26555bb70da..c64bbe055b79 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -6,7 +6,7 @@ use alloy_primitives::TxHash; use alloy_rpc_types::{FilteredParams, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; -use reth_primitives::{BlockNumHash, Receipt}; +use reth_primitives::{BlockNumHash, Receipt, SealedBlock}; use reth_storage_api::BlockReader; /// Returns all matching of a block's receipts when the transaction hashes are known. @@ -45,11 +45,20 @@ where all_logs } +/// Helper enum to fetch a transaction either from a block or from the provider. +#[derive(Debug)] +pub enum ProviderOrBlock<'a, P: BlockReader> { + /// Provider + Provider(&'a P), + /// [`SealedBlock`] + Block(SealedBlock), +} + /// Appends all matching logs of a block's receipts. /// If the log matches, look up the corresponding transaction hash. -pub fn append_matching_block_logs( +pub fn append_matching_block_logs( all_logs: &mut Vec, - provider: impl BlockReader, + provider_or_block: ProviderOrBlock<'_, P>, filter: &FilteredParams, block_num_hash: BlockNumHash, receipts: &[Receipt], @@ -60,8 +69,8 @@ pub fn append_matching_block_logs( let mut log_index: u64 = 0; // Lazy loaded number of the first transaction in the block. - // This is useful for blocks with multiple matching logs because it prevents - // re-querying the block body indices. + // This is useful for blocks with multiple matching logs because it + // prevents re-querying the block body indices. let mut loaded_first_tx_num = None; // Iterate over receipts and append matching logs. @@ -71,27 +80,37 @@ pub fn append_matching_block_logs( for log in &receipt.logs { if log_matches_filter(block_num_hash, log, filter) { - let first_tx_num = match loaded_first_tx_num { - Some(num) => num, - None => { - let block_body_indices = - provider.block_body_indices(block_num_hash.number)?.ok_or( - ProviderError::BlockBodyIndicesNotFound(block_num_hash.number), - )?; - loaded_first_tx_num = Some(block_body_indices.first_tx_num); - block_body_indices.first_tx_num - } - }; - // if this is the first match in the receipt's logs, look up the transaction hash if transaction_hash.is_none() { - // This is safe because Transactions and Receipts have the same keys. - let transaction_id = first_tx_num + receipt_idx as u64; - let transaction = provider - .transaction_by_id(transaction_id)? - .ok_or_else(|| ProviderError::TransactionNotFound(transaction_id.into()))?; - - transaction_hash = Some(transaction.hash()); + transaction_hash = match &provider_or_block { + ProviderOrBlock::Block(block) => { + block.body.transactions.get(receipt_idx).map(|t| t.hash()) + } + ProviderOrBlock::Provider(provider) => { + let first_tx_num = match loaded_first_tx_num { + Some(num) => num, + None => { + let block_body_indices = provider + .block_body_indices(block_num_hash.number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound( + block_num_hash.number, + ))?; + loaded_first_tx_num = Some(block_body_indices.first_tx_num); + block_body_indices.first_tx_num + } + }; + + // This is safe because Transactions and Receipts have the same + // keys. + let transaction_id = first_tx_num + receipt_idx as u64; + let transaction = + provider.transaction_by_id(transaction_id)?.ok_or_else(|| { + ProviderError::TransactionNotFound(transaction_id.into()) + })?; + + Some(transaction.hash()) + } + }; } let log = Log { diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index c5581f42a06c..9efecf3dae7f 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -19,11 +19,11 @@ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_chainspec::ChainInfo; use reth_node_api::EthApiTypes; -use reth_primitives::TransactionSignedEcRecovered; +use reth_primitives::{Receipt, SealedBlock, TransactionSignedEcRecovered}; use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; use reth_rpc_eth_types::{ - logs_utils::{self, append_matching_block_logs}, + logs_utils::{self, append_matching_block_logs, ProviderOrBlock}, EthApiError, EthFilterConfig, EthStateCache, EthSubscriptionIdProvider, }; use reth_rpc_server_types::{result::rpc_error_with_code, ToRpcResult}; @@ -376,29 +376,34 @@ where FilterBlockOption::AtBlockHash(block_hash) => { // for all matching logs in the block // get the block header with the hash - let block = self + let header = self .provider .header_by_hash_or_number(block_hash.into())? .ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?; + let block_num_hash = BlockNumHash::new(header.number, block_hash); + // we also need to ensure that the receipts are available and return an error if // not, in case the block hash been reorged - let receipts = self - .eth_cache - .get_receipts(block_hash) + let (receipts, maybe_block) = self + .receipts_and_maybe_block( + &block_num_hash, + self.provider.chain_info()?.best_number, + ) .await? .ok_or(EthApiError::HeaderNotFound(block_hash.into()))?; let mut all_logs = Vec::new(); - let filter = FilteredParams::new(Some(filter)); - logs_utils::append_matching_block_logs( + append_matching_block_logs( &mut all_logs, - &self.provider, - &filter, - (block_hash, block.number).into(), + maybe_block + .map(|b| ProviderOrBlock::Block(b)) + .unwrap_or_else(|| ProviderOrBlock::Provider(&self.provider)), + &FilteredParams::new(Some(filter)), + block_num_hash, &receipts, false, - block.timestamp, + header.timestamp, )?; Ok(all_logs) @@ -454,7 +459,6 @@ where chain_info: ChainInfo, ) -> Result, EthFilterError> { trace!(target: "rpc::eth::filter", from=from_block, to=to_block, ?filter, "finding logs in range"); - let best_number = chain_info.best_number; if to_block < from_block { return Err(EthFilterError::InvalidBlockRangeParams) @@ -467,27 +471,6 @@ where let mut all_logs = Vec::new(); let filter_params = FilteredParams::new(Some(filter.clone())); - if (to_block == best_number) && (from_block == best_number) { - // only one block to check and it's the current best block which we can fetch directly - // Note: In case of a reorg, the best block's hash might have changed, hence we only - // return early of we were able to fetch the best block's receipts - // perf: we're fetching the best block here which is expected to be cached - if let Some((block, receipts)) = - self.eth_cache.get_block_and_receipts(chain_info.best_hash).await? - { - logs_utils::append_matching_block_logs( - &mut all_logs, - &self.provider, - &filter_params, - chain_info.into(), - &receipts, - false, - block.header.timestamp, - )?; - } - return Ok(all_logs) - } - // derive bloom filters from filter input, so we can check headers for matching logs let address_filter = FilteredParams::address_filter(&filter.address); let topics_filter = FilteredParams::topics_filter(&filter.topics); @@ -514,12 +497,17 @@ where .ok_or_else(|| ProviderError::HeaderNotFound(header.number.into()))?, }; - if let Some(receipts) = self.eth_cache.get_receipts(block_hash).await? { + let num_hash = BlockNumHash::new(header.number, block_hash); + if let Some((receipts, maybe_block)) = + self.receipts_and_maybe_block(&num_hash, chain_info.best_number).await? + { append_matching_block_logs( &mut all_logs, - &self.provider, + maybe_block + .map(|block| ProviderOrBlock::Block(block)) + .unwrap_or_else(|| ProviderOrBlock::Provider(&self.provider)), &filter_params, - BlockNumHash::new(header.number, block_hash), + num_hash, &receipts, false, header.timestamp, @@ -540,6 +528,25 @@ where Ok(all_logs) } + + /// Retrieves receipts and block from cache if near the tip (4 blocks), otherwise only receipts. + async fn receipts_and_maybe_block( + &self, + block_num_hash: &BlockNumHash, + best_number: u64, + ) -> Result>, Option)>, EthFilterError> { + // The last 4 blocks are most likely cached, so we can just fetch them + let cached_range = best_number.saturating_sub(4)..=best_number; + let receipts_block = if cached_range.contains(&block_num_hash.number) { + self.eth_cache + .get_block_and_receipts(block_num_hash.hash) + .await? + .map(|(b, r)| (r, Some(b))) + } else { + self.eth_cache.get_receipts(block_num_hash.hash).await?.map(|r| (r, None)) + }; + Ok(receipts_block) + } } /// All active filters From 170715365c487fbdb30a9dcf820893c10e057ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Garamv=C3=B6lgyi?= Date: Wed, 9 Oct 2024 16:48:00 +0200 Subject: [PATCH 102/103] update fork base commit --- fork.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fork.yaml b/fork.yaml index 0ae42b86efad..b8c77208f4ab 100644 --- a/fork.yaml +++ b/fork.yaml @@ -4,7 +4,7 @@ footer: | base: name: reth url: https://github.com/paradigmxyz/reth - hash: 4960b927bcf5b1ce1fffd88f76c77929110b9eb0 + hash: b787d9e521fad1cb28a372637474ae4ec4986bf3 fork: name: scroll-reth url: https://github.com/scroll-tech/reth From 68ea30254ab5a42a49154455d23b97fff02a4cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Garamv=C3=B6lgyi?= Date: Thu, 10 Oct 2024 13:25:05 +0200 Subject: [PATCH 103/103] remove new book tests --- .github/workflows/unit.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 0cbecc5f0b47..518e2668ba6a 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -43,11 +43,6 @@ jobs: --workspace --exclude ef-tests \ --partition hash:${{ matrix.partition }}/2 \ -E "!kind(test)" - - name: Run tests on book sources - run: | - cargo nextest run \ - --manifest-path book/sources/Cargo.toml --workspace \ - -E "!kind(test)" state: name: Ethereum state tests