Introduce ThreadBlockBuildingContext, removing old CachedReads.

The idea behind ThreadBlockBuildingContext is to have a place that
is owned by a particular thread (for example, max profit ordering
builder) that can be used as a scratchpad for caching without doing
global caching shared between all threads.

Currently we have only one place for caches BlockBuildingContext. It
is global and shared between all builders, top of block simulation,
and finalization thread. This approach solves many problems with caching. For
example, we use it for eth-sparse-mpt root hash caching.

But sometimes we would like to have a cache that is local to the current
thread to avoid the overhead of mutex and multiple threads.

I've seen the need for caches like this multiple times.
This commit adds support for local caches like this.

The first thing that is moved to this new setup of 2 caches is cached reads. We had a
lot of trouble with passing CachedReads struct around and it
introduced itself into traits where it does not belong such as
BlockBuildingHelper or backtesting code. Here we move CachedReads to this uniform
setup and it simplifies cached reads state handling.

Author: dvush

Diff for: crates/rbuilder/src/backtest/build_block/backtest_build_block.rs

@@ -117,14 +117,13 @@ where
                     builder_name: builder_name.clone(),
                     sim_orders: &sim_orders,
                     provider: provider_factory.clone(),
-                    cached_reads: None,
                 };
                 let build_res = config.build_backtest_block(builder_name, input);
                 if let Err(err) = &build_res {
                     println!("Error building block: {:?}", err);
                     return None;
                 }
-                let (block, _) = build_res.ok()?;
+                let block = build_res.ok()?;
                 println!(
                     "Built block {} with builder: {:?}",
                     ctx.block(),

Diff for: crates/rbuilder/src/backtest/execute.rs

@@ -12,7 +12,6 @@ use crate::{
 };
 use alloy_eips::BlockNumHash;
 use alloy_primitives::{Address, U256};
-use reth::revm::cached::CachedReads;
 use reth_chainspec::ChainSpec;
 use serde::{Deserialize, Serialize};
 use std::{cell::RefCell, rc::Rc, sync::Arc};
@@ -180,19 +179,15 @@ where
     let simulated_total_gas = sim_orders.iter().map(|o| o.sim_value.gas_used).sum();
     let mut builder_outputs = Vec::new();
 
-    let mut cached_reads = Some(CachedReads::default());
     for building_algorithm_name in builders_names {
         let input = BacktestSimulateBlockInput {
             ctx: ctx.clone(),
             builder_name: building_algorithm_name.clone(),
             sim_orders: &sim_orders,
             provider: provider.clone(),
-            cached_reads,
         };
 
-        let (block, new_cached_reads) =
-            config.build_backtest_block(&building_algorithm_name, input)?;
-        cached_reads = Some(new_cached_reads);
+        let block = config.build_backtest_block(&building_algorithm_name, input)?;
         builder_outputs.push(BacktestBuilderOutput {
             orders_included: block.trace.included_orders.len(),
             builder_name: building_algorithm_name,

Diff for: crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs

@@ -1,7 +1,7 @@
 use crate::{
     building::{
         evm_inspector::SlotKey, tracers::AccumulatorSimulationTracer, BlockBuildingContext,
-        BlockState, PartialBlock, PartialBlockFork,
+        BlockState, PartialBlock, PartialBlockFork, ThreadBlockBuildingContext,
     },
     provider::StateProviderFactory,
     utils::{extract_onchain_block_txs, find_suggested_fee_recipient, signed_uint_delta},
@@ -55,27 +55,38 @@ where
         Arc::from(provider.root_hasher(parent_num_hash)?),
     );
 
+    let mut local_ctx = ThreadBlockBuildingContext::default();
+
     let state_provider = provider.history_by_block_hash(ctx.attributes.parent)?;
     let mut partial_block = PartialBlock::new(true);
     let mut state = BlockState::new(state_provider);
 
     partial_block
-        .pre_block_call(&ctx, &mut state)
+        .pre_block_call(&ctx, &mut local_ctx, &mut state)
         .with_context(|| "Failed to pre_block_call")?;
 
     let mut cumulative_gas_used = 0;
     let mut cumulative_blob_gas_used = 0;
     let mut written_slots: HashMap<SlotKey, Vec<B256>> = HashMap::default();
 
     for (idx, tx) in txs.into_iter().enumerate() {
-        let coinbase_balance_before = state.balance(coinbase)?;
+        let coinbase_balance_before = state.balance(
+            coinbase,
+            &ctx.shared_cached_reads,
+            &mut local_ctx.cached_reads,
+        )?;
         let mut accumulator_tracer = AccumulatorSimulationTracer::default();
         let result = {
-            let mut fork = PartialBlockFork::new(&mut state).with_tracer(&mut accumulator_tracer);
-            fork.commit_tx(&tx, &ctx, cumulative_gas_used, 0, cumulative_blob_gas_used)?
+            let mut fork = PartialBlockFork::new(&mut state, &ctx, &mut local_ctx)
+                .with_tracer(&mut accumulator_tracer);
+            fork.commit_tx(&tx, cumulative_gas_used, 0, cumulative_blob_gas_used)?
                 .with_context(|| format!("Failed to commit tx: {} {:?}", idx, tx.hash()))?
         };
-        let coinbase_balance_after = state.balance(coinbase)?;
+        let coinbase_balance_after = state.balance(
+            coinbase,
+            &ctx.shared_cached_reads,
+            &mut local_ctx.cached_reads,
+        )?;
         let coinbase_profit = signed_uint_delta(coinbase_balance_after, coinbase_balance_before);
 
         cumulative_gas_used += result.gas_used;

Diff for: crates/rbuilder/src/bin/debug-bench-machine.rs

@@ -6,12 +6,14 @@ use clap::Parser;
 use eyre::Context;
 use itertools::Itertools;
 use rbuilder::{
-    building::{BlockBuildingContext, BlockState, PartialBlock, PartialBlockFork},
+    building::{
+        BlockBuildingContext, BlockState, PartialBlock, PartialBlockFork,
+        ThreadBlockBuildingContext,
+    },
     live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig, config::Config},
     provider::StateProviderFactory,
     utils::{extract_onchain_block_txs, find_suggested_fee_recipient, http_provider},
 };
-use reth::revm::cached::CachedReads;
 use reth_provider::StateProvider;
 use std::{path::PathBuf, sync::Arc, time::Instant};
 use tracing::{debug, info};
@@ -82,25 +84,24 @@ async fn main() -> eyre::Result<()> {
 
     let mut build_times_ms = Vec::new();
     let mut finalize_time_ms = Vec::new();
-    let mut cached_reads = Some(CachedReads::default());
     for _ in 0..cli.iters {
         let ctx = ctx.clone();
         let txs = txs.clone();
         let state_provider = state_provider.clone();
-        let (new_cached_reads, build_time, finalize_time) =
+        let (build_time, finalize_time) =
             tokio::task::spawn_blocking(move || -> eyre::Result<_> {
                 let partial_block = PartialBlock::new(true);
-                let mut state = BlockState::new_arc(state_provider)
-                    .with_cached_reads(cached_reads.unwrap_or_default());
+                let mut state = BlockState::new_arc(state_provider);
+                let mut local_ctx = ThreadBlockBuildingContext::default();
 
                 let build_time = Instant::now();
 
                 let mut cumulative_gas_used = 0;
                 let mut cumulative_blob_gas_used = 0;
                 for (idx, tx) in txs.into_iter().enumerate() {
                     let result = {
-                        let mut fork = PartialBlockFork::new(&mut state);
-                        fork.commit_tx(&tx, &ctx, cumulative_gas_used, 0, cumulative_blob_gas_used)?
+                        let mut fork = PartialBlockFork::new(&mut state, &ctx, &mut local_ctx);
+                        fork.commit_tx(&tx, cumulative_gas_used, 0, cumulative_blob_gas_used)?
                             .with_context(|| {
                                 format!("Failed to commit tx: {} {:?}", idx, tx.hash())
                             })?
@@ -112,19 +113,18 @@ async fn main() -> eyre::Result<()> {
                 let build_time = build_time.elapsed();
 
                 let finalize_time = Instant::now();
-                let finalized_block = partial_block.finalize(&mut state, &ctx)?;
+                let finalized_block = partial_block.finalize(&mut state, &ctx, &mut local_ctx)?;
                 let finalize_time = finalize_time.elapsed();
 
                 debug!(
                     "Calculated root hash: {:?}",
                     finalized_block.sealed_block.state_root
                 );
 
-                Ok((finalized_block.cached_reads, build_time, finalize_time))
+                Ok((build_time, finalize_time))
             })
             .await??;
 
-        cached_reads = Some(new_cached_reads);
         build_times_ms.push(build_time.as_millis());
         finalize_time_ms.push(finalize_time.as_millis());
     }

Diff for: crates/rbuilder/src/bin/dummy-builder.rs

@@ -16,7 +16,7 @@ use rbuilder::{
             BlockBuildingAlgorithm, BlockBuildingAlgorithmInput, OrderConsumer,
             UnfinishedBlockBuildingSink, UnfinishedBlockBuildingSinkFactory,
         },
-        BlockBuildingContext, SimulatedOrderStore,
+        BlockBuildingContext, SimulatedOrderStore, ThreadBlockBuildingContext,
     },
     live_builder::{
         base_config::{
@@ -206,22 +206,23 @@ impl DummyBuildingAlgorithm {
     where
         P: StateProviderFactory + Clone + 'static,
     {
+        let mut local_ctx = ThreadBlockBuildingContext::default();
         let block_state = provider
             .history_by_block_hash(ctx.attributes.parent)?
             .into();
 
         let mut block_building_helper = BlockBuildingHelperFromProvider::new(
             block_state,
             ctx.clone(),
-            None,
+            &mut local_ctx,
             BUILDER_NAME.to_string(),
             false,
             CancellationToken::new(),
         )?;
 
         for order in orders {
             // don't care about the result
-            let _ = block_building_helper.commit_order(&order, &|_| Ok(()))?;
+            let _ = block_building_helper.commit_order(&mut local_ctx, &order, &|_| Ok(()))?;
         }
         Ok(Box::new(block_building_helper))
     }