add geth_opcodes dataset

crates/freeze/src/datasets/geth_traces.rs → crates/freeze/src/datasets/geth_calls.rs

@@ -3,9 +3,9 @@ use ethers::prelude::*;
 use polars::prelude::*;
 
 /// columns for geth traces
-#[cryo_to_df::to_df(Datatype::GethTraces)]
+#[cryo_to_df::to_df(Datatype::GethCalls)]
 #[derive(Default)]
-pub struct GethTraces {
+pub struct GethCalls {
     n_rows: u64,
     typ: Vec<String>,
     from_address: Vec<Vec<u8>>,
@@ -24,14 +24,14 @@ pub struct GethTraces {
 }
 
 #[async_trait::async_trait]
-impl Dataset for GethTraces {}
+impl Dataset for GethCalls {}
 
 #[async_trait::async_trait]
-impl CollectByBlock for GethTraces {
+impl CollectByBlock for GethCalls {
     type Response = (Option<u32>, Vec<Option<Vec<u8>>>, Vec<CallFrame>);
 
     async fn extract(request: Params, source: Arc<Source>, query: Arc<Query>) -> R<Self::Response> {
-        let schema = query.schemas.get_schema(&Datatype::GethTraces)?;
+        let schema = query.schemas.get_schema(&Datatype::GethCalls)?;
         let include_transaction = schema.has_column("block_number");
         let block_number = request.block_number()? as u32;
         source.fetcher.geth_debug_trace_block_calls(block_number, include_transaction).await
@@ -43,11 +43,11 @@ impl CollectByBlock for GethTraces {
 }
 
 #[async_trait::async_trait]
-impl CollectByTransaction for GethTraces {
+impl CollectByTransaction for GethCalls {
     type Response = (Option<u32>, Vec<Option<Vec<u8>>>, Vec<CallFrame>);
 
     async fn extract(request: Params, source: Arc<Source>, query: Arc<Query>) -> R<Self::Response> {
-        let schema = query.schemas.get_schema(&Datatype::GethTraces)?;
+        let schema = query.schemas.get_schema(&Datatype::GethCalls)?;
         let include_block_number = schema.has_column("block_number");
         source
             .fetcher
@@ -62,11 +62,11 @@ impl CollectByTransaction for GethTraces {
 
 fn process_geth_traces(
     traces: (Option<u32>, Vec<Option<Vec<u8>>>, Vec<CallFrame>),
-    columns: &mut GethTraces,
+    columns: &mut GethCalls,
     schemas: &Schemas,
 ) -> R<()> {
     let (block_number, txs, traces) = traces;
-    let schema = schemas.get(&Datatype::GethTraces).ok_or(err("schema for geth_traces missing"))?;
+    let schema = schemas.get(&Datatype::GethCalls).ok_or(err("schema for geth_traces missing"))?;
     for (tx_index, (tx, trace)) in txs.into_iter().zip(traces).enumerate() {
         process_trace(trace, columns, schema, &block_number, &tx, tx_index as u32, vec![])?
     }
@@ -75,7 +75,7 @@ fn process_geth_traces(
 
 fn process_trace(
     trace: CallFrame,
-    columns: &mut GethTraces,
+    columns: &mut GethCalls,
     schema: &Table,
     block_number: &Option<u32>,
     tx: &Option<Vec<u8>>,

crates/freeze/src/datasets/geth_opcodes.rs

@@ -0,0 +1,160 @@
+use crate::*;
+use ethers::prelude::*;
+use polars::prelude::*;
+
+/// columns for geth traces
+#[cryo_to_df::to_df(Datatype::GethOpcodes)]
+#[derive(Default)]
+pub struct GethOpcodes {
+    n_rows: u64,
+    block_number: Vec<Option<u32>>,
+    transaction_hash: Vec<Option<Vec<u8>>>,
+    transaction_index: Vec<u32>,
+    trace_address: Vec<String>,
+    depth: Vec<u64>,
+    error: Vec<Option<String>>,
+    gas: Vec<u64>,
+    gas_cost: Vec<u64>,
+    op: Vec<String>,
+    pc: Vec<u64>,
+    refund_counter: Vec<Option<u64>>,
+
+    memory: Vec<Option<String>>,
+    stack: Vec<Option<String>>,
+    storage: Vec<Option<String>>,
+    return_data: Vec<Option<Vec<u8>>>,
+    chain_id: Vec<u64>,
+}
+
+#[async_trait::async_trait]
+impl Dataset for GethOpcodes {
+    fn default_columns() -> Option<Vec<&'static str>> {
+        let f = |x: &&str| x != &"memory" && x != &"stack" && x != &"storage";
+        Some(GethOpcodes::column_types().into_keys().filter(f).collect())
+    }
+}
+
+#[async_trait::async_trait]
+impl CollectByBlock for GethOpcodes {
+    type Response = (Option<u32>, Vec<Option<Vec<u8>>>, Vec<DefaultFrame>);
+
+    async fn extract(request: Params, source: Arc<Source>, query: Arc<Query>) -> R<Self::Response> {
+        let schema = query.schemas.get_schema(&Datatype::GethOpcodes)?;
+        let options = GethDebugTracingOptions {
+            disable_storage: Some(!schema.has_column("storage")),
+            disable_stack: Some(!schema.has_column("stack")),
+            enable_memory: Some(schema.has_column("memory")),
+            enable_return_data: Some(schema.has_column("return_data")),
+            ..Default::default()
+        };
+        let include_transaction = schema.has_column("block_number");
+        let block_number = request.block_number()? as u32;
+        source
+            .fetcher
+            .geth_debug_trace_block_opcodes(block_number, include_transaction, options)
+            .await
+    }
+
+    fn transform(response: Self::Response, columns: &mut Self, query: &Arc<Query>) -> R<()> {
+        process_geth_opcodes(response, columns, &query.schemas)
+    }
+}
+
+#[async_trait::async_trait]
+impl CollectByTransaction for GethOpcodes {
+    type Response = (Option<u32>, Vec<Option<Vec<u8>>>, Vec<DefaultFrame>);
+
+    async fn extract(request: Params, source: Arc<Source>, query: Arc<Query>) -> R<Self::Response> {
+        let schema = query.schemas.get_schema(&Datatype::GethOpcodes)?;
+        let options = GethDebugTracingOptions {
+            disable_storage: Some(!schema.has_column("storage")),
+            disable_stack: Some(!schema.has_column("stack")),
+            enable_memory: Some(schema.has_column("memory")),
+            enable_return_data: Some(schema.has_column("return_data")),
+            ..Default::default()
+        };
+        let include_block_number = schema.has_column("block_number");
+        source
+            .fetcher
+            .geth_debug_trace_transaction_opcodes(
+                request.transaction_hash()?,
+                include_block_number,
+                options,
+            )
+            .await
+    }
+
+    fn transform(response: Self::Response, columns: &mut Self, query: &Arc<Query>) -> R<()> {
+        process_geth_opcodes(response, columns, &query.schemas)
+    }
+}
+
+fn process_geth_opcodes(
+    traces: (Option<u32>, Vec<Option<Vec<u8>>>, Vec<DefaultFrame>),
+    columns: &mut GethOpcodes,
+    schemas: &Schemas,
+) -> R<()> {
+    let (block_number, txs, traces) = traces;
+    let schema =
+        schemas.get(&Datatype::GethOpcodes).ok_or(err("schema for geth_traces missing"))?;
+    for (tx_index, (tx, trace)) in txs.into_iter().zip(traces).enumerate() {
+        process_trace(trace, columns, schema, &block_number, &tx, tx_index as u32, vec![])?
+    }
+    Ok(())
+}
+
+fn process_trace(
+    trace: DefaultFrame,
+    columns: &mut GethOpcodes,
+    schema: &Table,
+    block_number: &Option<u32>,
+    tx: &Option<Vec<u8>>,
+    tx_index: u32,
+    trace_address: Vec<u32>,
+) -> R<()> {
+    let n_struct_logs = trace.struct_logs.len();
+    for (i, struct_log) in trace.struct_logs.into_iter().enumerate() {
+        columns.n_rows += 1;
+
+        store!(schema, columns, block_number, *block_number);
+        store!(schema, columns, transaction_hash, tx.clone());
+        store!(schema, columns, transaction_index, tx_index);
+        store!(
+            schema,
+            columns,
+            trace_address,
+            trace_address.iter().map(|&n| n.to_string()).collect::<Vec<_>>().join(" ")
+        );
+
+        store!(schema, columns, depth, struct_log.depth);
+        store!(schema, columns, error, struct_log.error);
+        store!(schema, columns, gas, struct_log.gas);
+        store!(schema, columns, gas_cost, struct_log.gas_cost);
+        store!(schema, columns, pc, struct_log.pc);
+        store!(schema, columns, op, struct_log.op);
+        store!(schema, columns, refund_counter, struct_log.refund_counter);
+
+        if schema.has_column("memory") {
+            let memory_str = serde_json::to_string(&struct_log.memory)
+                .map_err(|_| err("could not encode opcode memory"))?;
+            store!(schema, columns, memory, Some(memory_str));
+        }
+        if schema.has_column("stack") {
+            let stack_str = serde_json::to_string(&struct_log.stack)
+                .map_err(|_| err("could not encode opcode stack"))?;
+            store!(schema, columns, stack, Some(stack_str));
+        }
+        if schema.has_column("storage") {
+            let storage_str = serde_json::to_string(&struct_log.storage)
+                .map_err(|_| err("could not encode opcode storage"))?;
+            store!(schema, columns, storage, Some(storage_str));
+        }
+
+        if i == n_struct_logs - 1 {
+            store!(schema, columns, return_data, Some(trace.return_value.to_vec()));
+        } else {
+            store!(schema, columns, return_data, None);
+        }
+    }
+    Ok(())
+}

crates/freeze/src/datasets/mod.rs

@@ -34,14 +34,16 @@ pub mod eth_calls;
 pub mod four_byte_traces;
 /// geth balance diffs
 pub mod geth_balance_diffs;
+/// geth calls
+pub mod geth_calls;
 /// geth code diffs
 pub mod geth_code_diffs;
 /// geth nonce diffs
 pub mod geth_nonce_diffs;
+/// geth opcodes
+pub mod geth_opcodes;
 /// geth storage diffs
 pub mod geth_storage_diffs;
-/// geth traces
-pub mod geth_traces;
 /// javascript traces
 pub mod javascript_traces;
 /// logs
@@ -87,10 +89,11 @@ pub use erc721_transfers::*;
 pub use eth_calls::*;
 pub use four_byte_traces::*;
 pub use geth_balance_diffs::*;
+pub use geth_calls::*;
 pub use geth_code_diffs::*;
 pub use geth_nonce_diffs::*;
+pub use geth_opcodes::*;
 pub use geth_storage_diffs::*;
-pub use geth_traces::*;
 pub use javascript_traces::*;
 pub use logs::*;
 pub use native_transfers::*;

crates/freeze/src/types/datatypes/scalar.rs

@@ -20,11 +20,12 @@ define_datatypes!(
     Erc721Transfers,
     EthCalls,
     FourByteTraces,
+    GethCalls,
     GethCodeDiffs,
     GethBalanceDiffs,
     GethStorageDiffs,
     GethNonceDiffs,
-    GethTraces,
+    GethOpcodes,
     JavascriptTraces,
     Logs,
     NativeTransfers,

crates/freeze/src/types/sources.rs

@@ -440,7 +440,27 @@ impl<P: JsonRpcClient> Fetcher<P> {
         Ok((block, txs, calls))
     }
 
-    /// get geth debug block call traces
+    /// get geth debug block opcode traces
+    pub async fn geth_debug_trace_block_opcodes(
+        &self,
+        block_number: u32,
+        include_transaction_hashes: bool,
+        options: GethDebugTracingOptions,
+    ) -> Result<(Option<u32>, Vec<Option<Vec<u8>>>, Vec<DefaultFrame>)> {
+        let (block, txs, traces) =
+            self.geth_debug_trace_block(block_number, options, include_transaction_hashes).await?;
+
+        let mut calls = Vec::new();
+        for trace in traces.into_iter() {
+            match trace {
+                GethTrace::Known(GethTraceFrame::Default(frame)) => calls.push(frame),
+                _ => return Err(CollectError::CollectError("invalid trace result".to_string())),
+            }
+        }
+        Ok((block, txs, calls))
+    }
+
+    /// get geth debug block 4byte traces
     pub async fn geth_debug_trace_block_4byte_traces(
         &self,
         block_number: u32,
@@ -608,6 +628,27 @@ impl<P: JsonRpcClient> Fetcher<P> {
         Ok((block, txs, calls))
     }
 
+    /// get geth debug block opcode traces
+    pub async fn geth_debug_trace_transaction_opcodes(
+        &self,
+        transaction_hash: Vec<u8>,
+        include_block_number: bool,
+        options: GethDebugTracingOptions,
+    ) -> Result<(Option<u32>, Vec<Option<Vec<u8>>>, Vec<DefaultFrame>)> {
+        let (block, txs, traces) = self
+            .geth_debug_trace_transaction(transaction_hash, options, include_block_number)
+            .await?;
+
+        let mut calls = Vec::new();
+        for trace in traces.into_iter() {
+            match trace {
+                GethTrace::Known(GethTraceFrame::Default(frame)) => calls.push(frame),
+                _ => return Err(CollectError::CollectError("invalid trace result".to_string())),
+            }
+        }
+        Ok((block, txs, calls))
+    }
+
     /// get geth debug block 4byte traces
     pub async fn geth_debug_trace_transaction_4byte_traces(
         &self,