diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8969e54880..830fbd3417d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,13 +107,13 @@ jobs: # check with root permissions, should be cached from previous build - name: sudo make docker run: sudo DOCKER_TAG=testinprod/op-erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker - - automated-tests: - runs-on: - ubuntu-20.04 - if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }} - steps: - - uses: actions/checkout@v3 - - name: run automated testing - run: BUILD_ERIGON=1 ./tests/automated-testing/run.sh +# automated-tests: +# runs-on: +# ubuntu-20.04 +# if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }} +# steps: +# - uses: actions/checkout@v3 +# +# - name: run automated testing +# run: BUILD_ERIGON=1 ./tests/automated-testing/run.sh diff --git a/DEV_CHAIN.md b/DEV_CHAIN.md index f5484d2b144..d42159f2b64 100644 --- a/DEV_CHAIN.md +++ b/DEV_CHAIN.md @@ -11,7 +11,7 @@ make erigon ``` -## 2. Build RPC deamon +## 2. Build RPC daemon On the same terminal folder you can build the RPC daemon. ```bash @@ -43,7 +43,7 @@ Now save the enode information generated in the logs, we will use this in a minu enode://d30d079163d7b69fcb261c0538c0c3faba4fb4429652970e60fa25deb02a789b4811e98b468726ba0be63b9dc925a019f433177eb6b45c23bb78892f786d8f7a@127.0.0.1:53171 ``` -## 4. Start RPC deamon +## 4. Start RPC daemon Open terminal 2 and navigate to erigon/build/bin folder. Here type the following command @@ -68,7 +68,7 @@ Open terminal 3 and navigate to erigon/build/bin folder. Paste in the following --nodiscover ``` -To check if the nodes are connected, you can go to the log of both the nodes and look for the line +To check if the nodes are connected, you can go to the log of both nodes and look for the line ``` [p2p] GoodPeers eth66=1 ``` diff --git a/Dockerfile b/Dockerfile index f4042e0878e..21ea444e2c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ COPY --from=builder /app/build/bin/state /usr/local/bin/state COPY --from=builder /app/build/bin/txpool /usr/local/bin/txpool COPY --from=builder /app/build/bin/verkle /usr/local/bin/verkle COPY --from=builder /app/build/bin/caplin-phase1 /usr/local/bin/caplin-phase1 - +COPY --from=builder /app/build/bin/caplin-regression /usr/local/bin/caplin-regression EXPOSE 8545 \ diff --git a/Dockerfile.debian b/Dockerfile.debian index 1f57b7c8a9e..08278f855c7 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -75,6 +75,7 @@ COPY --from=builder /app/build/bin/state /usr/local/bin/state COPY --from=builder /app/build/bin/txpool /usr/local/bin/txpool COPY --from=builder /app/build/bin/verkle /usr/local/bin/verkle COPY --from=builder /app/build/bin/caplin-phase1 /usr/local/bin/caplin-phase1 +COPY --from=builder /app/build/bin/caplin-regression /usr/local/bin/caplin-regression EXPOSE 8545 \ 8551 \ diff --git a/Makefile b/Makefile index 41b41c995a5..9ed34609606 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,6 @@ CGO_CFLAGS += -DMDBX_FORCE_ASSERTIONS=0 # Enable MDBX's asserts by default in 'd CGO_CFLAGS += -O CGO_CFLAGS += -D__BLST_PORTABLE__ CGO_CFLAGS += -Wno-error=strict-prototypes # for Clang15, remove it when can https://github.com/ledgerwatch/erigon/issues/6113#issuecomment-1359526277 -CGO_CFLAGS := CGO_CFLAGS="$(CGO_CFLAGS)" -DBG_CGO_CFLAGS += -DMDBX_DEBUG=1 # about netgo see: https://github.com/golang/go/issues/30310#issuecomment-471669125 and https://github.com/golang/go/issues/57757 BUILD_TAGS = nosqlite,noboltdb @@ -35,9 +33,9 @@ PACKAGE = github.com/testinprod-io/op-erigon GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}" -GOBUILD = $(CGO_CFLAGS) $(GO) build $(GO_FLAGS) -GO_DBG_BUILD = $(GO) build $(GO_FLAGS) -tags $(BUILD_TAGS),debug -gcflags=all="-N -l" # see delve docs -GOTEST = $(CGO_CFLAGS) GODEBUG=cgocheck=0 $(GO) test $(GO_FLAGS) ./... -p 2 +GOBUILD = CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GO_FLAGS) +GO_DBG_BUILD = CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l" # see delve docs +GOTEST = CGO_CFLAGS="$(CGO_CFLAGS)" GODEBUG=cgocheck=0 $(GO) test $(GO_FLAGS) ./... -p 2 default: all @@ -87,7 +85,7 @@ setup_xdg_data_home: ## docker-compose: validate build args, setup xdg data home, and run docker-compose up docker-compose: validate_docker_build_args setup_xdg_data_home - docker-compose up + docker compose up ## dbg debug build allows see C stack traces, run it with GOTRACEBACK=crash. You don't need debug build for C pit for profiling. To profile C code use SETCGOTRCKEBACK=1 dbg: @@ -152,14 +150,14 @@ test: $(GOTEST) --timeout 100s test3: - $(GOTEST) --timeout 100s -tags $(BUILD_TAGS),erigon3 + $(GOTEST) --timeout 100s -tags $(BUILD_TAGS),e3 ## test-integration: run integration tests with a 30m timeout test-integration: $(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration test3-integration: - $(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration,erigon3 + $(GOTEST) --timeout 30m -tags $(BUILD_TAGS),integration,e3 ## lint: run golangci-lint with .golangci.yml config file lint: @@ -192,6 +190,7 @@ devtools: $(GOBUILD) -o $(GOBIN)/codecgen github.com/ugorji/go/codec/codecgen PATH=$(GOBIN):$(PATH) go generate ./common # PATH=$(GOBIN):$(PATH) go generate ./core/types + PATH=$(GOBIN):$(PATH) cd ./cmd/rpcdaemon/graphql && go run github.com/99designs/gqlgen . PATH=$(GOBIN):$(PATH) go generate ./consensus/aura/... #PATH=$(GOBIN):$(PATH) go generate ./eth/ethconfig/... @type "npm" 2> /dev/null || echo 'Please install node.js and npm' @@ -205,7 +204,7 @@ bindings: ## prometheus: run prometheus and grafana with docker-compose prometheus: - docker-compose up prometheus grafana + docker compose up prometheus grafana ## escape: run escape path={path} to check for memory leaks e.g. run escape path=cmd/erigon escape: @@ -221,7 +220,7 @@ git-submodules: @git submodule update --quiet --init --recursive --force || true PACKAGE_NAME := github.com/testinprod-io/op-erigon -GOLANG_CROSS_VERSION ?= v1.20.4 +GOLANG_CROSS_VERSION ?= v1.20.5 .PHONY: release-dry-run release-dry-run: git-submodules diff --git a/README.md b/README.md index 1a077bf5652..4b266b4bc0a 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,7 @@ _Flags:_ - `log.json` - `log.console.json` (alias for `log.json`) - `log.dir.path` +- `log.dir.prefix` - `log.dir.verbosity` - `log.dir.json` @@ -304,7 +305,7 @@ int value specifying the highest output log level: LvlTrace = 5 ``` -To set an output dir for logs to be collected on disk, please set `--log.dir.path`. The flag `--log.dir.verbosity` is +To set an output dir for logs to be collected on disk, please set `--log.dir.path` If you want to change the filename prodiced from `erigon` you should also set the `--log.dir.prefix` flag to an alternate name. The flag `--log.dir.verbosity` is also available to control the verbosity of this logging, with the same int value as above, or the string value e.g. ' debug' or 'info'. Default verbosity is 'debug' (4), for disk logging. @@ -451,7 +452,7 @@ http.api : ["eth","debug","net"] Erigon can be used as an Execution Layer (EL) for Consensus Layer clients (CL). Default configuration is OK. If your CL client is on a different device, add `--authrpc.addr 0.0.0.0` ([Engine API] listens on localhost by default) -as well as `--authrpc.vhosts `. +as well as `--authrpc.vhosts ` where `` is your source host or `any`. [Engine API]: https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md @@ -662,7 +663,7 @@ Windows support for docker-compose is not ready yet. Please help us with .ps1 po ### Grafana dashboard -`docker-compose up prometheus grafana`, [detailed docs](./cmd/prometheus/Readme.md). +`docker compose up prometheus grafana`, [detailed docs](./cmd/prometheus/Readme.md). ### diff --git a/RELEASE_INSTRUCTIONS.md b/RELEASE_INSTRUCTIONS.md index 8a68e125b05..f4f064b12ad 100644 --- a/RELEASE_INSTRUCTIONS.md +++ b/RELEASE_INSTRUCTIONS.md @@ -15,7 +15,7 @@ If there are any transactions where code bitmap was useful, warning messages lik ```` WARN [08-01|14:54:27.778] Code Bitmap used for detecting invalid jump tx=0x86e55d1818b5355424975de9633a57c40789ca08552297b726333a9433949c92 block number=6426298 ```` -In such cases (unless there are too many instances), all block numbers need to be excluded in the `SkipAnalysis` function, and comment to it. The constant `MainnetNotCheckedFrom` needs to be update to the first block number we have not checked. The value can be taken from the output of the `checkChangeSets` +In such cases (unless there are too many instances), all block numbers need to be excluded in the `SkipAnalysis` function, and comment to it. The constant `MainnetNotCheckedFrom` needs to be updated to the first block number we have not checked. The value can be taken from the output of the `checkChangeSets` utility before it exits, like this: ```` INFO [08-01|15:36:04.282] Checked blocks=10573804 next time specify --block=10573804 duration=36m54.789025062s @@ -23,11 +23,11 @@ INFO [08-01|15:36:04.282] Checked blocks=105738 ## Update DB Schema version if required -In the file `common/dbutils/bucket.go` there is variable `DBSchemaVersion` that needs to be update if there are any changes in the database schema, leading to data migrations. +In the file `common/dbutils/bucket.go` there is variable `DBSchemaVersion` that needs to be updated if there are any changes in the database schema, leading to data migrations. In most cases, it is enough to bump minor version. ## Update remote KV version if required -In the file `ethdb/remote/remotedbserver/server.go` there is variable `KvServiceAPIVersion` that needs to be update if there are any changes in the remote KV interface, or +In the file `ethdb/remote/remotedbserver/server.go` there is variable `KvServiceAPIVersion` that needs to be updated if there are any changes in the remote KV interface, or database schema, leading to data migrations. In most cases, it is enough to bump minor version. It is best to change both DB schema version and remove KV version together. diff --git a/TESTING.md b/TESTING.md index 76b8a4539cc..e57163f5473 100644 --- a/TESTING.md +++ b/TESTING.md @@ -53,7 +53,7 @@ INFO [03-24|13:41:20.391] Commit cycle in=2.16378229 Here we see that the sync cycle went through all the stages for a single block `12101885`. -After that, it is useful to wait more until an Unwind is encoutered and check that Erigon handled it without errors. +After that, it is useful to wait more until an Unwind is encountered and check that Erigon handled it without errors. Usually, errors occur at the stage `[7/14 IntermediateHashes]` and manifest in the wrong trie root. Here is an example of processing an unwind without errors (look for the word "Unwind" in the log): @@ -148,7 +148,7 @@ ERROR[08-01|14:30:38.299] Demoting invalidated transaction hash="859191 ERROR[08-01|14:30:38.299] Demoting invalidated transaction hash="25ee67…e73153" ``` -this is also likely to disappered after the introduction of new downloader/sentry design +this is also likely to disappear after the introduction of new downloader/sentry design ### Assessing relative performance of sync @@ -172,7 +172,7 @@ INFO [03-24|13:41:20.391] Commit cycle in=2.16378229 The line above shows how long was commit cycle. We saw in the past that after some changes the commit time dramatically increases, and these -regressions need to be investigaged. We expect "commit cycle" on Linux with NVMe drive to usually take less than a +regressions need to be investigated. We expect "commit cycle" on Linux with NVMe drive to usually take less than a second. For other operating systems and devices the typical time may vary, but it should significantly increase from one release to another. Perhaps we need to log some extra information in the log to make it easier for the tester to filter out the log @@ -274,11 +274,11 @@ requests for certain RPC methods, using hits provided by options `--blockFrom` a useful ones are: 1. `bench8` tests `eth_getLogs` RPC method (compatibility with go-ethereum and OpenEthereum) -2. `bench11` tests `trace_call` RPC method (compatiblity with OpenEthereum tracing) +2. `bench11` tests `trace_call` RPC method (compatibility with OpenEthereum tracing) 3. `bench12` tests `debug_traceCall` RPC method (compatibility with go-ethereum tracing) -4. `bench13` tests `trace_callMany` RPC method (compability with OpenEthereum tracing) +4. `bench13` tests `trace_callMany` RPC method (compatibility with OpenEthereum tracing) -Options `--erigonUrl` and `--gethUrl` specify HTTP endpoints that needs to be tested against each other. Despite its +Options `--erigonUrl` and `--gethUrl` specify HTTP endpoints that need to be tested against each other. Despite its name, `--gethUrl` option does not have to point to go-ethereum node, it can point to anything that it supposed to be "correct" for the purpose of the test ( go-ethereum node, OpenEthereum node, @@ -286,7 +286,7 @@ or Erigon RPC daemon & Erigon node built from the previous release code). Option `--needCompare` triggers the comparison of JSON RPC responses. If omitted, requests to `--gethUrl` are not done. When comparison is turned on, -the utility stops at the first occurrence of mistmatch. +the utility stops at the first occurrence of mismatch. ## RPC test recording and replay diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index dbeb7d5689b..567564ba682 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -26,13 +26,16 @@ import ( "time" "github.com/holiman/uint256" - ethereum "github.com/ledgerwatch/erigon" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv" state2 "github.com/ledgerwatch/erigon-lib/state" types2 "github.com/ledgerwatch/erigon-lib/types" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" + + ethereum "github.com/ledgerwatch/erigon" "github.com/ledgerwatch/erigon/accounts/abi" "github.com/ledgerwatch/erigon/accounts/abi/bind" "github.com/ledgerwatch/erigon/common/math" @@ -44,12 +47,9 @@ import ( "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/event" "github.com/ledgerwatch/erigon/params" - "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/stages" - "github.com/ledgerwatch/log/v3" ) // This nil assignment ensures at compile time that SimulatedBackend implements bind.ContractBackend. @@ -93,14 +93,13 @@ func NewSimulatedBackendWithConfig(alloc types.GenesisAlloc, config *chain.Confi genesis := types.Genesis{Config: config, GasLimit: gasLimit, Alloc: alloc} engine := ethash.NewFaker() m := stages.MockWithGenesisEngine(nil, &genesis, engine, false) - br, _ := m.NewBlocksIO() backend := &SimulatedBackend{ m: m, prependBlock: m.Genesis, getHeader: func(hash libcommon.Hash, number uint64) (h *types.Header) { var err error if err = m.DB.View(context.Background(), func(tx kv.Tx) error { - h, err = br.Header(context.Background(), tx, hash, number) + h, err = m.BlockReader.Header(context.Background(), tx, hash, number) return nil }); err != nil { panic(err) @@ -125,14 +124,11 @@ func NewTestSimulatedBackendWithConfig(t *testing.T, alloc types.GenesisAlloc, c t.Cleanup(b.Close) return b } -func (b *SimulatedBackend) DB() kv.RwDB { return b.m.DB } -func (b *SimulatedBackend) Agg() *state2.AggregatorV3 { return b.m.HistoryV3Components() } -func (b *SimulatedBackend) HistoryV3() bool { return b.m.HistoryV3 } -func (b *SimulatedBackend) Engine() consensus.Engine { return b.m.Engine } -func (b *SimulatedBackend) BlockReader() services.FullBlockReader { - br, _ := b.m.NewBlocksIO() - return br -} +func (b *SimulatedBackend) DB() kv.RwDB { return b.m.DB } +func (b *SimulatedBackend) Agg() *state2.AggregatorV3 { return b.m.HistoryV3Components() } +func (b *SimulatedBackend) HistoryV3() bool { return b.m.HistoryV3 } +func (b *SimulatedBackend) Engine() consensus.Engine { return b.m.Engine } +func (b *SimulatedBackend) BlockReader() services.FullBlockReader { return b.m.BlockReader } // Close terminates the underlying blockchain's update loop. func (b *SimulatedBackend) Close() { @@ -151,7 +147,7 @@ func (b *SimulatedBackend) Commit() { Headers: []*types.Header{b.pendingHeader}, Blocks: []*types.Block{b.pendingBlock}, TopBlock: b.pendingBlock, - }); err != nil { + }, nil); err != nil { panic(err) } //nolint:prealloc @@ -186,13 +182,7 @@ func (b *SimulatedBackend) emptyPendingBlock() { panic(err) } b.pendingReaderTx = tx - - if ethconfig.EnableHistoryV4InTest { - panic("implement me") - //b.pendingReader = state.NewReaderV4(b.pendingReaderTx.(kv.TemporalTx)) - } else { - b.pendingReader = state.NewPlainStateReader(b.pendingReaderTx) - } + b.pendingReader = b.m.NewStateReader(b.pendingReaderTx) b.pendingState = state.New(b.pendingReader) } @@ -747,7 +737,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx types.Transac defer b.mu.Unlock() // Check transaction validity. - signer := types.MakeSigner(b.m.ChainConfig, b.pendingBlock.NumberU64()) + signer := types.MakeSigner(b.m.ChainConfig, b.pendingBlock.NumberU64(), b.pendingBlock.Time()) sender, senderErr := tx.Sender(*signer) if senderErr != nil { return fmt.Errorf("invalid transaction: %w", senderErr) diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 046fa9e4264..0d33ca281b9 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -63,7 +63,7 @@ func TestSimulatedBackend(t *testing.T) { // generate a transaction and confirm you can retrieve it code := `6060604052600a8060106000396000f360606040526008565b00` var gas uint64 = 3000000 - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewContractCreation(0, u256.Num0, gas, u256.Num1, common.FromHex(code)) tx, _ = types.SignTx(tx, *signer, key) @@ -175,7 +175,7 @@ func TestNewSimulatedBackend_AdjustTimeFail(t *testing.T) { // Create tx and send amount, _ := uint256.FromBig(big.NewInt(1000)) gasPrice, _ := uint256.FromBig(big.NewInt(1)) - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(0, testAddr, amount, params.TxGas, gasPrice, nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -298,7 +298,7 @@ func TestSimulatedBackend_NonceAt(t *testing.T) { } // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(nonce, testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -339,7 +339,7 @@ func TestSimulatedBackend_SendTransaction(t *testing.T) { bgCtx := context.Background() // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(uint64(0), testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -373,7 +373,7 @@ func TestSimulatedBackend_TransactionByHash(t *testing.T) { bgCtx := context.Background() // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(uint64(0), testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -681,7 +681,7 @@ func TestSimulatedBackend_TransactionCount(t *testing.T) { } // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(uint64(0), testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -736,7 +736,7 @@ func TestSimulatedBackend_TransactionInBlock(t *testing.T) { } // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(uint64(0), testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -791,7 +791,7 @@ func TestSimulatedBackend_PendingNonceAt(t *testing.T) { } // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(uint64(0), testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { @@ -843,7 +843,7 @@ func TestSimulatedBackend_TransactionReceipt(t *testing.T) { bgCtx := context.Background() // create a signed transaction to send - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(uint64(0), testAddr, uint256.NewInt(1000), params.TxGas, uint256.NewInt(1), nil) signedTx, err := types.SignTx(tx, *signer, testKey) if err != nil { diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 91a858d0ef5..3c2632e4d15 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -77,7 +77,7 @@ func TestWaitDeployed(t *testing.T) { // Create the transaction. // Create the transaction. var tx types.Transaction = types.NewContractCreation(0, u256.Num0, test.gas, u256.Num1, common.FromHex(test.code)) - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) tx, _ = types.SignTx(tx, *signer, testKey) // Wait for it to get mined in the background. @@ -127,7 +127,7 @@ func TestWaitDeployedCornerCases(t *testing.T) { // Create a transaction to an account. code := "6060604052600a8060106000396000f360606040526008565b00" - signer := types.MakeSigner(params.TestChainConfig, 1) + signer := types.MakeSigner(params.TestChainConfig, 1, 0) var tx types.Transaction = types.NewTransaction(0, libcommon.HexToAddress("0x01"), u256.Num0, 3000000, u256.Num1, common.FromHex(code)) tx, _ = types.SignTx(tx, *signer, testKey) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cl/cltypes/eth1_block.go b/cl/cltypes/eth1_block.go index 6ff85c320fa..207bc72a541 100644 --- a/cl/cltypes/eth1_block.go +++ b/cl/cltypes/eth1_block.go @@ -72,10 +72,8 @@ func NewEth1BlockFromHeaderAndBody(header *types.Header, body *types.RawBody) *E Withdrawals: solid.NewStaticListSSZFromList(body.Withdrawals, 16, 44), } - if header.DataGasUsed != nil { + if header.DataGasUsed != nil && header.ExcessDataGas != nil { block.DataGasUsed = *header.DataGasUsed - } - if header.ExcessDataGas != nil { block.ExcessDataGas = *header.ExcessDataGas block.version = clparams.DenebVersion } else if header.WithdrawalsHash != nil { @@ -104,6 +102,12 @@ func (b *Eth1Block) PayloadHeader() (*Eth1Header, error) { } } + var dataGasUsed, excessDataGas uint64 + if b.version >= clparams.DenebVersion { + dataGasUsed = b.DataGasUsed + excessDataGas = b.ExcessDataGas + } + return &Eth1Header{ ParentHash: b.ParentHash, FeeRecipient: b.FeeRecipient, @@ -120,8 +124,8 @@ func (b *Eth1Block) PayloadHeader() (*Eth1Header, error) { BlockHash: b.BlockHash, TransactionsRoot: transactionsRoot, WithdrawalsRoot: withdrawalsRoot, - DataGasUsed: b.DataGasUsed, - ExcessDataGas: b.ExcessDataGas, + DataGasUsed: dataGasUsed, + ExcessDataGas: excessDataGas, version: b.version, }, nil } @@ -238,6 +242,10 @@ func (b *Eth1Block) RlpHeader() (*types.Header, error) { return header, nil } +func (b *Eth1Block) Version() clparams.StateVersion { + return b.version +} + // Body returns the equivalent raw body (only eth1 body section). func (b *Eth1Block) Body() *types.RawBody { withdrawals := make([]*types.Withdrawal, b.Withdrawals.Len()) diff --git a/cl/phase1/cache/attestation_indicies_cache.go b/cl/phase1/cache/attestation_indicies_cache.go index 564fc7a8931..36c3b83e366 100644 --- a/cl/phase1/cache/attestation_indicies_cache.go +++ b/cl/phase1/cache/attestation_indicies_cache.go @@ -1,18 +1,30 @@ package cache import ( + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cl/cltypes/solid" "github.com/ledgerwatch/erigon/cl/phase1/core/state/lru" + "github.com/ledgerwatch/erigon/cl/utils" ) -var attestationIndiciesCache *lru.Cache[*solid.AttestationData, []uint64] +var attestationIndiciesCache *lru.Cache[common.Hash, []uint64] const attestationIndiciesCacheSize = 256 -func LoadAttestatingIndicies(attestation *solid.AttestationData) ([]uint64, bool) { - return attestationIndiciesCache.Get(attestation) +func LoadAttestatingIndicies(attestation *solid.AttestationData, aggregationBits []byte) ([]uint64, bool) { + bitsHash := utils.Keccak256(aggregationBits) + hash, err := attestation.HashSSZ() + if err != nil { + return nil, false + } + return attestationIndiciesCache.Get(utils.Keccak256(hash[:], bitsHash[:])) } -func StoreAttestation(attestation *solid.AttestationData, indicies []uint64) { - attestationIndiciesCache.Add(attestation, indicies) +func StoreAttestation(attestation *solid.AttestationData, aggregationBits []byte, indicies []uint64) { + bitsHash := utils.Keccak256(aggregationBits) + hash, err := attestation.HashSSZ() + if err != nil { + return + } + attestationIndiciesCache.Add(utils.Keccak256(hash[:], bitsHash[:]), indicies) } diff --git a/cl/phase1/cache/cache_test.go b/cl/phase1/cache/cache_test.go index 7f0010d7cf9..bedade366bb 100644 --- a/cl/phase1/cache/cache_test.go +++ b/cl/phase1/cache/cache_test.go @@ -10,9 +10,9 @@ import ( func TestAttestationsCache(t *testing.T) { input := []uint64{1} - a := &solid.AttestationData{} - cache.StoreAttestation(a, []uint64{1}) - output, valid := cache.LoadAttestatingIndicies(a) + a := solid.NewAttestationData() + cache.StoreAttestation(&a, []byte{2}, []uint64{1}) + output, valid := cache.LoadAttestatingIndicies(&a, []byte{2}) require.True(t, valid) require.Equal(t, input, output) } diff --git a/cl/phase1/cache/init.go b/cl/phase1/cache/init.go index f1675de1ab7..1f349b2b402 100644 --- a/cl/phase1/cache/init.go +++ b/cl/phase1/cache/init.go @@ -1,13 +1,13 @@ package cache import ( - "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cl/phase1/core/state/lru" ) func init() { var err error - if attestationIndiciesCache, err = lru.New[*solid.AttestationData, []uint64]("attestationIndiciesCacheSize", attestationIndiciesCacheSize); err != nil { + if attestationIndiciesCache, err = lru.New[common.Hash, []uint64]("attestationIndiciesCacheSize", attestationIndiciesCacheSize); err != nil { panic(err) } } diff --git a/cl/phase1/core/state/cache_accessors.go b/cl/phase1/core/state/cache_accessors.go index 552c6f2a166..06da54a180f 100644 --- a/cl/phase1/core/state/cache_accessors.go +++ b/cl/phase1/core/state/cache_accessors.go @@ -244,7 +244,7 @@ func (b *BeaconState) ComputeNextSyncCommittee() (*solid.SyncCommittee, error) { // GetAttestingIndicies retrieves attesting indicies for a specific attestation. however some tests will not expect the aggregation bits check. // thus, it is a flag now. func (b *BeaconState) GetAttestingIndicies(attestation solid.AttestationData, aggregationBits []byte, checkBitsLength bool) ([]uint64, error) { - if cached, ok := cache.LoadAttestatingIndicies(&attestation); ok { + if cached, ok := cache.LoadAttestatingIndicies(&attestation, aggregationBits); ok { return cached, nil } committee, err := b.GetBeaconCommitee(attestation.Slot(), attestation.ValidatorIndex()) @@ -267,7 +267,7 @@ func (b *BeaconState) GetAttestingIndicies(attestation solid.AttestationData, ag attestingIndices = append(attestingIndices, member) } } - cache.StoreAttestation(&attestation, attestingIndices) + cache.StoreAttestation(&attestation, aggregationBits, attestingIndices) return attestingIndices, nil } diff --git a/cl/phase1/core/transition/block_transition.go b/cl/phase1/core/transition/block_transition.go deleted file mode 100644 index 400f27128b6..00000000000 --- a/cl/phase1/core/transition/block_transition.go +++ /dev/null @@ -1,211 +0,0 @@ -package transition - -import ( - "errors" - "fmt" - - "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon/cl/clparams" - "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/metrics/methelp" -) - -// processBlock takes a block and transitions the state to the next slot, using the provided execution payload if enabled. -func processBlock(state *state2.BeaconState, signedBlock *cltypes.SignedBeaconBlock, fullValidation bool) error { - block := signedBlock.Block - version := state.Version() - // Check the state version is correct. - if signedBlock.Version() != version { - return fmt.Errorf("processBlock: wrong state version for block at slot %d", block.Slot) - } - - h := methelp.NewHistTimer("beacon_process_block") - - c := h.Tag("process_step", "block_header") - // Process the block header. - if err := ProcessBlockHeader(state, block, fullValidation); err != nil { - return fmt.Errorf("processBlock: failed to process block header: %v", err) - } - c.PutSince() - - // Process execution payload if enabled. - if version >= clparams.BellatrixVersion && executionEnabled(state, block.Body.ExecutionPayload) { - if state.Version() >= clparams.CapellaVersion { - // Process withdrawals in the execution payload. - c = h.Tag("process_step", "withdrawals") - if err := ProcessWithdrawals(state, block.Body.ExecutionPayload.Withdrawals, fullValidation); err != nil { - return fmt.Errorf("processBlock: failed to process withdrawals: %v", err) - } - c.PutSince() - } - - // Process the execution payload. - c = h.Tag("process_step", "execution_payload") - if err := ProcessExecutionPayload(state, block.Body.ExecutionPayload); err != nil { - return fmt.Errorf("processBlock: failed to process execution payload: %v", err) - } - c.PutSince() - } - - // Process RANDAO reveal. - c = h.Tag("process_step", "randao_reveal") - if err := ProcessRandao(state, block.Body.RandaoReveal, block.ProposerIndex, fullValidation); err != nil { - return fmt.Errorf("processBlock: failed to process RANDAO reveal: %v", err) - } - c.PutSince() - - // Process Eth1 data. - c = h.Tag("process_step", "eth1_data") - if err := ProcessEth1Data(state, block.Body.Eth1Data); err != nil { - return fmt.Errorf("processBlock: failed to process Eth1 data: %v", err) - } - c.PutSince() - - // Process block body operations. - c = h.Tag("process_step", "operations") - if err := processOperations(state, block.Body, fullValidation); err != nil { - return fmt.Errorf("processBlock: failed to process block body operations: %v", err) - } - c.PutSince() - - // Process sync aggregate in case of Altair version. - if version >= clparams.AltairVersion { - c = h.Tag("process_step", "sync_aggregate") - if err := ProcessSyncAggregate(state, block.Body.SyncAggregate, fullValidation); err != nil { - return fmt.Errorf("processBlock: failed to process sync aggregate: %v", err) - } - c.PutSince() - } - - if version >= clparams.DenebVersion && fullValidation { - c = h.Tag("process_step", "blob_kzg_commitments") - verified, err := VerifyKzgCommitmentsAgainstTransactions(block.Body.ExecutionPayload.Transactions, block.Body.BlobKzgCommitments) - if err != nil { - return fmt.Errorf("processBlock: failed to process blob kzg commitments: %w", err) - } - if !verified { - return fmt.Errorf("processBlock: failed to process blob kzg commitments: commitments are not equal") - } - c.PutSince() - } - - h.PutSince() - return nil -} - -func processOperations(state *state2.BeaconState, blockBody *cltypes.BeaconBody, fullValidation bool) error { - if blockBody.Deposits.Len() != int(maximumDeposits(state)) { - return errors.New("outstanding deposits do not match maximum deposits") - } - h := methelp.NewHistTimer("beacon_process_block_operations") - - // Process each proposer slashing - var err error - c := h.Tag("operation", "proposer_slashings") - if err := solid.RangeErr[*cltypes.ProposerSlashing](blockBody.ProposerSlashings, func(index int, slashing *cltypes.ProposerSlashing, length int) error { - if err = ProcessProposerSlashing(state, slashing); err != nil { - return fmt.Errorf("ProcessProposerSlashing: %s", err) - } - return nil - }); err != nil { - return err - } - c.PutSince() - - c = h.Tag("operation", "attester_slashings") - if err := solid.RangeErr[*cltypes.AttesterSlashing](blockBody.AttesterSlashings, func(index int, slashing *cltypes.AttesterSlashing, length int) error { - if err = ProcessAttesterSlashing(state, slashing); err != nil { - return fmt.Errorf("ProcessAttesterSlashing: %s", err) - } - return nil - }); err != nil { - return err - } - - c.PutSince() - - // Process each attestations - c = h.Tag("operation", "attestations", "validation", "false") - if fullValidation { - c = h.Tag("operation", "attestations", "validation", "true") - } - if err := ProcessAttestations(state, blockBody.Attestations, fullValidation); err != nil { - return fmt.Errorf("ProcessAttestation: %s", err) - } - c.PutSince() - - // Process each deposit - c = h.Tag("operation", "deposit") - if err := solid.RangeErr[*cltypes.Deposit](blockBody.Deposits, func(index int, deposit *cltypes.Deposit, length int) error { - if err = ProcessDeposit(state, deposit, fullValidation); err != nil { - return fmt.Errorf("ProcessDeposit: %s", err) - } - return nil - }); err != nil { - return err - } - c.PutSince() - - // Process each voluntary exit. - c = h.Tag("operation", "voluntary_exit") - if err := solid.RangeErr[*cltypes.SignedVoluntaryExit](blockBody.VoluntaryExits, func(index int, exit *cltypes.SignedVoluntaryExit, length int) error { - if err = ProcessVoluntaryExit(state, exit, fullValidation); err != nil { - return fmt.Errorf("ProcessVoluntaryExit: %s", err) - } - return nil - }); err != nil { - return err - } - c.PutSince() - if state.Version() < clparams.CapellaVersion { - return nil - } - // Process each execution change. this will only have entries after the capella fork. - c = h.Tag("operation", "execution_change") - if err := solid.RangeErr[*cltypes.SignedBLSToExecutionChange](blockBody.ExecutionChanges, func(index int, addressChange *cltypes.SignedBLSToExecutionChange, length int) error { - if err := ProcessBlsToExecutionChange(state, addressChange, fullValidation); err != nil { - return fmt.Errorf("ProcessBlsToExecutionChange: %s", err) - } - return nil - }); err != nil { - return err - } - c.PutSince() - return nil -} - -func maximumDeposits(state *state2.BeaconState) (maxDeposits uint64) { - maxDeposits = state.Eth1Data().DepositCount - state.Eth1DepositIndex() - if maxDeposits > state.BeaconConfig().MaxDeposits { - maxDeposits = state.BeaconConfig().MaxDeposits - } - return -} - -// ProcessExecutionPayload sets the latest payload header accordinly. -func ProcessExecutionPayload(s *state2.BeaconState, payload *cltypes.Eth1Block) error { - if state2.IsMergeTransitionComplete(s.BeaconState) { - if payload.ParentHash != s.LatestExecutionPayloadHeader().BlockHash { - return fmt.Errorf("ProcessExecutionPayload: invalid eth1 chain. mismatching parent") - } - } - if payload.PrevRandao != s.GetRandaoMixes(state2.Epoch(s.BeaconState)) { - return fmt.Errorf("ProcessExecutionPayload: randao mix mismatches with mix digest") - } - if payload.Time != state2.ComputeTimestampAtSlot(s.BeaconState, s.Slot()) { - return fmt.Errorf("ProcessExecutionPayload: invalid Eth1 timestamp") - } - payloadHeader, err := payload.PayloadHeader() - if err != nil { - return err - } - s.SetLatestExecutionPayloadHeader(payloadHeader) - return nil -} - -func executionEnabled(s *state2.BeaconState, payload *cltypes.Eth1Block) bool { - return (!state2.IsMergeTransitionComplete(s.BeaconState) && payload.BlockHash != libcommon.Hash{}) || state2.IsMergeTransitionComplete(s.BeaconState) -} diff --git a/cl/phase1/core/transition/operations.go b/cl/phase1/core/transition/operations.go deleted file mode 100644 index c79d0546670..00000000000 --- a/cl/phase1/core/transition/operations.go +++ /dev/null @@ -1,281 +0,0 @@ -package transition - -import ( - "errors" - "fmt" - - "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - "github.com/Giulio2002/bls" - "github.com/ledgerwatch/log/v3" - - "github.com/ledgerwatch/erigon/cl/clparams" - "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/cl/fork" - "github.com/ledgerwatch/erigon/cl/utils" - "github.com/ledgerwatch/erigon/core/types" -) - -func ProcessProposerSlashing(s *state2.BeaconState, propSlashing *cltypes.ProposerSlashing) error { - h1 := propSlashing.Header1.Header - h2 := propSlashing.Header2.Header - - if h1.Slot != h2.Slot { - return fmt.Errorf("non-matching slots on proposer slashing: %d != %d", h1.Slot, h2.Slot) - } - - if h1.ProposerIndex != h2.ProposerIndex { - return fmt.Errorf("non-matching proposer indices proposer slashing: %d != %d", h1.ProposerIndex, h2.ProposerIndex) - } - - h1Root, err := h1.HashSSZ() - if err != nil { - return fmt.Errorf("unable to hash header1: %v", err) - } - h2Root, err := h2.HashSSZ() - if err != nil { - return fmt.Errorf("unable to hash header2: %v", err) - } - if h1Root == h2Root { - return fmt.Errorf("propose slashing headers are the same: %v == %v", h1Root, h2Root) - } - - proposer, err := s.ValidatorForValidatorIndex(int(h1.ProposerIndex)) - if err != nil { - return err - } - if !proposer.IsSlashable(state2.Epoch(s.BeaconState)) { - return fmt.Errorf("proposer is not slashable: %v", proposer) - } - - for _, signedHeader := range []*cltypes.SignedBeaconBlockHeader{propSlashing.Header1, propSlashing.Header2} { - domain, err := s.GetDomain(s.BeaconConfig().DomainBeaconProposer, state2.GetEpochAtSlot(s.BeaconConfig(), signedHeader.Header.Slot)) - if err != nil { - return fmt.Errorf("unable to get domain: %v", err) - } - signingRoot, err := fork.ComputeSigningRoot(signedHeader.Header, domain) - if err != nil { - return fmt.Errorf("unable to compute signing root: %v", err) - } - pk := proposer.PublicKey() - valid, err := bls.Verify(signedHeader.Signature[:], signingRoot[:], pk[:]) - if err != nil { - return fmt.Errorf("unable to verify signature: %v", err) - } - if !valid { - return fmt.Errorf("invalid signature: signature %v, root %v, pubkey %v", signedHeader.Signature[:], signingRoot[:], pk) - } - } - - // Set whistleblower index to 0 so current proposer gets reward. - s.SlashValidator(h1.ProposerIndex, nil) - return nil -} - -func ProcessAttesterSlashing(s *state2.BeaconState, attSlashing *cltypes.AttesterSlashing) error { - att1 := attSlashing.Attestation_1 - att2 := attSlashing.Attestation_2 - - if !cltypes.IsSlashableAttestationData(att1.Data, att2.Data) { - return fmt.Errorf("attestation data not slashable: %+v; %+v", att1.Data, att2.Data) - } - - valid, err := state2.IsValidIndexedAttestation(s.BeaconState, att1) - if err != nil { - return fmt.Errorf("error calculating indexed attestation 1 validity: %v", err) - } - if !valid { - return fmt.Errorf("invalid indexed attestation 1") - } - - valid, err = state2.IsValidIndexedAttestation(s.BeaconState, att2) - if err != nil { - return fmt.Errorf("error calculating indexed attestation 2 validity: %v", err) - } - if !valid { - return fmt.Errorf("invalid indexed attestation 2") - } - - slashedAny := false - currentEpoch := state2.GetEpochAtSlot(s.BeaconConfig(), s.Slot()) - for _, ind := range solid.IntersectionOfSortedSets(att1.AttestingIndices, att2.AttestingIndices) { - validator, err := s.ValidatorForValidatorIndex(int(ind)) - if err != nil { - return err - } - if validator.IsSlashable(currentEpoch) { - err := s.SlashValidator(ind, nil) - if err != nil { - return fmt.Errorf("unable to slash validator: %d", ind) - } - slashedAny = true - } - } - - if !slashedAny { - return fmt.Errorf("no validators slashed") - } - return nil -} - -func ProcessDeposit(s *state2.BeaconState, deposit *cltypes.Deposit, fullValidation bool) error { - if deposit == nil { - return nil - } - depositLeaf, err := deposit.Data.HashSSZ() - if err != nil { - return err - } - depositIndex := s.Eth1DepositIndex() - eth1Data := s.Eth1Data() - rawProof := []common.Hash{} - deposit.Proof.Range(func(_ int, l common.Hash, _ int) bool { - rawProof = append(rawProof, l) - return true - }) - // Validate merkle proof for deposit leaf. - if fullValidation && !utils.IsValidMerkleBranch( - depositLeaf, - rawProof, - s.BeaconConfig().DepositContractTreeDepth+1, - depositIndex, - eth1Data.Root, - ) { - return fmt.Errorf("processDepositForAltair: Could not validate deposit root") - } - - // Increment index - s.SetEth1DepositIndex(depositIndex + 1) - publicKey := deposit.Data.PubKey - amount := deposit.Data.Amount - // Check if pub key is in validator set - validatorIndex, has := s.ValidatorIndexByPubkey(publicKey) - if !has { - // Agnostic domain. - domain, err := fork.ComputeDomain(s.BeaconConfig().DomainDeposit[:], utils.Uint32ToBytes4(s.BeaconConfig().GenesisForkVersion), [32]byte{}) - if err != nil { - return err - } - depositMessageRoot, err := deposit.Data.MessageHash() - if err != nil { - return err - } - signedRoot := utils.Keccak256(depositMessageRoot[:], domain) - // Perform BLS verification and if successful noice. - valid, err := bls.Verify(deposit.Data.Signature[:], signedRoot[:], publicKey[:]) - // Literally you can input it trash. - if !valid || err != nil { - log.Debug("Validator BLS verification failed", "valid", valid, "err", err) - return nil - } - // Append validator - s.AddValidator(state2.ValidatorFromDeposit(s.BeaconConfig(), deposit), amount) - // Altair forward - if s.Version() >= clparams.AltairVersion { - s.AddCurrentEpochParticipationFlags(cltypes.ParticipationFlags(0)) - s.AddPreviousEpochParticipationFlags(cltypes.ParticipationFlags(0)) - s.AddInactivityScore(0) - } - return nil - } - // Increase the balance if exists already - return state2.IncreaseBalance(s.BeaconState, validatorIndex, amount) -} - -// ProcessVoluntaryExit takes a voluntary exit and applies state transition. -func ProcessVoluntaryExit(s *state2.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit, fullValidation bool) error { - // Sanity checks so that we know it is good. - voluntaryExit := signedVoluntaryExit.VolunaryExit - currentEpoch := state2.Epoch(s.BeaconState) - validator, err := s.ValidatorForValidatorIndex(int(voluntaryExit.ValidatorIndex)) - if err != nil { - return err - } - if !validator.Active(currentEpoch) { - return errors.New("ProcessVoluntaryExit: validator is not active") - } - if validator.ExitEpoch() != s.BeaconConfig().FarFutureEpoch { - return errors.New("ProcessVoluntaryExit: another exit for the same validator is already getting processed") - } - if currentEpoch < voluntaryExit.Epoch { - return errors.New("ProcessVoluntaryExit: exit is happening in the future") - } - if currentEpoch < validator.ActivationEpoch()+s.BeaconConfig().ShardCommitteePeriod { - return errors.New("ProcessVoluntaryExit: exit is happening too fast") - } - - // We can skip it in some instances if we want to optimistically sync up. - if fullValidation { - domain, err := s.GetDomain(s.BeaconConfig().DomainVoluntaryExit, voluntaryExit.Epoch) - if err != nil { - return err - } - signingRoot, err := fork.ComputeSigningRoot(voluntaryExit, domain) - if err != nil { - return err - } - pk := validator.PublicKey() - valid, err := bls.Verify(signedVoluntaryExit.Signature[:], signingRoot[:], pk[:]) - if err != nil { - return err - } - if !valid { - return errors.New("ProcessVoluntaryExit: BLS verification failed") - } - } - // Do the exit (same process in slashing). - return s.InitiateValidatorExit(voluntaryExit.ValidatorIndex) -} - -// ProcessWithdrawals processes withdrawals by decreasing the balance of each validator -// and updating the next withdrawal index and validator index. -func ProcessWithdrawals(s *state2.BeaconState, withdrawals *solid.ListSSZ[*types.Withdrawal], fullValidation bool) error { - // Get the list of withdrawals, the expected withdrawals (if performing full validation), - // and the beacon configuration. - beaconConfig := s.BeaconConfig() - numValidators := uint64(s.ValidatorLength()) - - // Check if full validation is required and verify expected withdrawals. - if fullValidation { - expectedWithdrawals := state2.ExpectedWithdrawals(s.BeaconState) - if len(expectedWithdrawals) != withdrawals.Len() { - return fmt.Errorf("ProcessWithdrawals: expected %d withdrawals, but got %d", len(expectedWithdrawals), withdrawals.Len()) - } - if err := solid.RangeErr[*types.Withdrawal](withdrawals, func(i int, w *types.Withdrawal, _ int) error { - if !expectedWithdrawals[i].Equal(w) { - return fmt.Errorf("ProcessWithdrawals: withdrawal %d does not match expected withdrawal", i) - } - return nil - }); err != nil { - return err - } - } - - if err := solid.RangeErr[*types.Withdrawal](withdrawals, func(_ int, w *types.Withdrawal, _ int) error { - if err := state2.DecreaseBalance(s.BeaconState, w.Validator, w.Amount); err != nil { - return err - } - return nil - }); err != nil { - return err - } - - // Update next withdrawal index based on number of withdrawals. - if withdrawals.Len() > 0 { - lastWithdrawalIndex := withdrawals.Get(withdrawals.Len() - 1).Index - s.SetNextWithdrawalIndex(lastWithdrawalIndex + 1) - } - - // Update next withdrawal validator index based on number of withdrawals. - if withdrawals.Len() == int(beaconConfig.MaxWithdrawalsPerPayload) { - lastWithdrawalValidatorIndex := withdrawals.Get(withdrawals.Len()-1).Validator + 1 - s.SetNextWithdrawalValidatorIndex(lastWithdrawalValidatorIndex % numValidators) - } else { - nextIndex := s.NextWithdrawalValidatorIndex() + beaconConfig.MaxValidatorsPerWithdrawalsSweep - s.SetNextWithdrawalValidatorIndex(nextIndex % numValidators) - } - - return nil -} diff --git a/cl/phase1/core/transition/process_attestations.go b/cl/phase1/core/transition/process_attestations.go deleted file mode 100644 index 55491897c69..00000000000 --- a/cl/phase1/core/transition/process_attestations.go +++ /dev/null @@ -1,260 +0,0 @@ -package transition - -import ( - "errors" - "fmt" - - "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - "github.com/ledgerwatch/erigon/cl/clparams" - "github.com/ledgerwatch/erigon/cl/utils" - "github.com/ledgerwatch/erigon/metrics/methelp" - "golang.org/x/exp/slices" -) - -func ProcessAttestations(s *state2.BeaconState, attestations *solid.ListSSZ[*solid.Attestation], fullValidation bool) error { - attestingIndiciesSet := make([][]uint64, attestations.Len()) - h := methelp.NewHistTimer("beacon_process_attestations") - baseRewardPerIncrement := s.BaseRewardPerIncrement() - - c := h.Tag("attestation_step", "process") - var err error - if err := solid.RangeErr[*solid.Attestation](attestations, func(i int, a *solid.Attestation, _ int) error { - if attestingIndiciesSet[i], err = processAttestation(s, a, baseRewardPerIncrement); err != nil { - return err - } - return nil - }); err != nil { - return err - } - if err != nil { - return err - } - var valid bool - c.PutSince() - if fullValidation { - c = h.Tag("attestation_step", "validate") - valid, err = verifyAttestations(s, attestations, attestingIndiciesSet) - if err != nil { - return err - } - if !valid { - return errors.New("ProcessAttestation: wrong bls data") - } - c.PutSince() - } - - return nil -} - -func processAttestationPostAltair(s *state2.BeaconState, attestation *solid.Attestation, baseRewardPerIncrement uint64) ([]uint64, error) { - data := attestation.AttestantionData() - currentEpoch := state2.Epoch(s.BeaconState) - stateSlot := s.Slot() - beaconConfig := s.BeaconConfig() - - h := methelp.NewHistTimer("beacon_process_attestation_post_altair") - - c := h.Tag("step", "get_participation_flag") - participationFlagsIndicies, err := s.GetAttestationParticipationFlagIndicies(attestation.AttestantionData(), stateSlot-data.Slot()) - if err != nil { - return nil, err - } - c.PutSince() - - c = h.Tag("step", "get_attesting_indices") - - attestingIndicies, err := s.GetAttestingIndicies(attestation.AttestantionData(), attestation.AggregationBits(), true) - if err != nil { - return nil, err - } - - c.PutSince() - - var proposerRewardNumerator uint64 - - isCurrentEpoch := data.Target().Epoch() == currentEpoch - - c = h.Tag("step", "update_attestation") - for _, attesterIndex := range attestingIndicies { - val, err := s.ValidatorEffectiveBalance(int(attesterIndex)) - if err != nil { - return nil, err - } - - baseReward := (val / beaconConfig.EffectiveBalanceIncrement) * baseRewardPerIncrement - for flagIndex, weight := range beaconConfig.ParticipationWeights() { - flagParticipation := s.EpochParticipationForValidatorIndex(isCurrentEpoch, int(attesterIndex)) - if !slices.Contains(participationFlagsIndicies, uint8(flagIndex)) || flagParticipation.HasFlag(flagIndex) { - continue - } - s.SetEpochParticipationForValidatorIndex(isCurrentEpoch, int(attesterIndex), flagParticipation.Add(flagIndex)) - proposerRewardNumerator += baseReward * weight - } - } - c.PutSince() - // Reward proposer - c = h.Tag("step", "get_proposer_index") - proposer, err := s.GetBeaconProposerIndex() - if err != nil { - return nil, err - } - c.PutSince() - proposerRewardDenominator := (beaconConfig.WeightDenominator - beaconConfig.ProposerWeight) * beaconConfig.WeightDenominator / beaconConfig.ProposerWeight - reward := proposerRewardNumerator / proposerRewardDenominator - return attestingIndicies, state2.IncreaseBalance(s.BeaconState, proposer, reward) -} - -// processAttestationsPhase0 implements the rules for phase0 processing. -func processAttestationPhase0(s *state2.BeaconState, attestation *solid.Attestation) ([]uint64, error) { - data := attestation.AttestantionData() - committee, err := s.GetBeaconCommitee(data.Slot(), data.ValidatorIndex()) - if err != nil { - return nil, err - } - - if len(committee) != utils.GetBitlistLength(attestation.AggregationBits()) { - return nil, fmt.Errorf("processAttestationPhase0: mismatching aggregation bits size") - } - // Cached so it is performant. - proposerIndex, err := s.GetBeaconProposerIndex() - if err != nil { - return nil, err - } - // Create the attestation to add to pending attestations - pendingAttestation := solid.NewPendingAttestionFromParameters( - attestation.AggregationBits(), - data, - s.Slot()-data.Slot(), - proposerIndex, - ) - - isCurrentAttestation := data.Target().Epoch() == state2.Epoch(s.BeaconState) - // Depending of what slot we are on we put in either the current justified or previous justified. - if isCurrentAttestation { - if !data.Source().Equal(s.CurrentJustifiedCheckpoint()) { - return nil, fmt.Errorf("processAttestationPhase0: mismatching sources") - } - s.AddCurrentEpochAtteastation(pendingAttestation) - } else { - if !data.Source().Equal(s.PreviousJustifiedCheckpoint()) { - return nil, fmt.Errorf("processAttestationPhase0: mismatching sources") - } - s.AddPreviousEpochAttestation(pendingAttestation) - } - // Not required by specs but needed if we want performant epoch transition. - indicies, err := s.GetAttestingIndicies(attestation.AttestantionData(), attestation.AggregationBits(), true) - if err != nil { - return nil, err - } - epochRoot, err := state2.GetBlockRoot(s.BeaconState, attestation.AttestantionData().Target().Epoch()) - if err != nil { - return nil, err - } - slotRoot, err := s.GetBlockRootAtSlot(attestation.AttestantionData().Slot()) - if err != nil { - return nil, err - } - // Basically we flag all validators we are currently attesting. will be important for rewards/finalization processing. - for _, index := range indicies { - minCurrentInclusionDelayAttestation, err := s.ValidatorMinCurrentInclusionDelayAttestation(int(index)) - if err != nil { - return nil, err - } - - minPreviousInclusionDelayAttestation, err := s.ValidatorMinPreviousInclusionDelayAttestation(int(index)) - if err != nil { - return nil, err - } - // NOTE: does not affect state root. - // We need to set it to currents or previouses depending on which attestation we process. - if isCurrentAttestation { - if minCurrentInclusionDelayAttestation == nil || - minCurrentInclusionDelayAttestation.InclusionDelay() > pendingAttestation.InclusionDelay() { - if err := s.SetValidatorMinCurrentInclusionDelayAttestation(int(index), pendingAttestation); err != nil { - return nil, err - } - } - if err := s.SetValidatorIsCurrentMatchingSourceAttester(int(index), true); err != nil { - return nil, err - } - if attestation.AttestantionData().Target().BlockRoot() == epochRoot { - if err := s.SetValidatorIsCurrentMatchingTargetAttester(int(index), true); err != nil { - return nil, err - } - } else { - continue - } - if attestation.AttestantionData().BeaconBlockRoot() == slotRoot { - if err := s.SetValidatorIsCurrentMatchingHeadAttester(int(index), true); err != nil { - return nil, err - } - } - } else { - if minPreviousInclusionDelayAttestation == nil || - minPreviousInclusionDelayAttestation.InclusionDelay() > pendingAttestation.InclusionDelay() { - if err := s.SetValidatorMinPreviousInclusionDelayAttestation(int(index), pendingAttestation); err != nil { - return nil, err - } - } - if err := s.SetValidatorIsPreviousMatchingSourceAttester(int(index), true); err != nil { - return nil, err - } - if attestation.AttestantionData().Target().BlockRoot() != epochRoot { - continue - } - if err := s.SetValidatorIsPreviousMatchingTargetAttester(int(index), true); err != nil { - return nil, err - } - if attestation.AttestantionData().BeaconBlockRoot() == slotRoot { - if err := s.SetValidatorIsPreviousMatchingHeadAttester(int(index), true); err != nil { - return nil, err - } - } - } - } - return indicies, nil -} - -// ProcessAttestation takes an attestation and process it. -func processAttestation(s *state2.BeaconState, attestation *solid.Attestation, baseRewardPerIncrement uint64) ([]uint64, error) { - data := attestation.AttestantionData() - currentEpoch := state2.Epoch(s.BeaconState) - previousEpoch := state2.PreviousEpoch(s.BeaconState) - stateSlot := s.Slot() - beaconConfig := s.BeaconConfig() - // Prelimary checks. - if (data.Target().Epoch() != currentEpoch && data.Target().Epoch() != previousEpoch) || data.Target().Epoch() != state2.GetEpochAtSlot(s.BeaconConfig(), data.Slot()) { - return nil, errors.New("ProcessAttestation: attestation with invalid epoch") - } - if data.Slot()+beaconConfig.MinAttestationInclusionDelay > stateSlot || stateSlot > data.Slot()+beaconConfig.SlotsPerEpoch { - return nil, errors.New("ProcessAttestation: attestation slot not in range") - } - if data.ValidatorIndex() >= s.CommitteeCount(data.Target().Epoch()) { - return nil, errors.New("ProcessAttestation: attester index out of range") - } - // check if we need to use rules for phase0 or post-altair. - if s.Version() == clparams.Phase0Version { - return processAttestationPhase0(s, attestation) - } - return processAttestationPostAltair(s, attestation, baseRewardPerIncrement) -} - -func verifyAttestations(s *state2.BeaconState, attestations *solid.ListSSZ[*solid.Attestation], attestingIndicies [][]uint64) (bool, error) { - var err error - valid := true - attestations.Range(func(idx int, a *solid.Attestation, _ int) bool { - indexedAttestation := state2.GetIndexedAttestation(a, attestingIndicies[idx]) - valid, err = state2.IsValidIndexedAttestation(s.BeaconState, indexedAttestation) - if err != nil { - return false - } - if !valid { - return false - } - return true - }) - - return valid, err -} diff --git a/cl/phase1/core/transition/process_bls_to_execution_change.go b/cl/phase1/core/transition/process_bls_to_execution_change.go deleted file mode 100644 index 8ac006b9660..00000000000 --- a/cl/phase1/core/transition/process_bls_to_execution_change.go +++ /dev/null @@ -1,65 +0,0 @@ -package transition - -import ( - "bytes" - "fmt" - - "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - "github.com/Giulio2002/bls" - "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/cl/fork" - "github.com/ledgerwatch/erigon/cl/utils" -) - -// ProcessBlsToExecutionChange processes a BLSToExecutionChange message by updating a validator's withdrawal credentials. -func ProcessBlsToExecutionChange(state *state.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange, fullValidation bool) error { - change := signedChange.Message - - beaconConfig := state.BeaconConfig() - validator, err := state.ValidatorForValidatorIndex(int(change.ValidatorIndex)) - if err != nil { - return err - } - - // Perform full validation if requested. - wc := validator.WithdrawalCredentials() - if fullValidation { - // Check the validator's withdrawal credentials prefix. - if wc[0] != beaconConfig.BLSWithdrawalPrefixByte { - return fmt.Errorf("invalid withdrawal credentials prefix") - } - - // Check the validator's withdrawal credentials against the provided message. - hashedFrom := utils.Keccak256(change.From[:]) - if !bytes.Equal(hashedFrom[1:], wc[1:]) { - return fmt.Errorf("invalid withdrawal credentials") - } - - // Compute the signing domain and verify the message signature. - domain, err := fork.ComputeDomain(beaconConfig.DomainBLSToExecutionChange[:], utils.Uint32ToBytes4(beaconConfig.GenesisForkVersion), state.GenesisValidatorsRoot()) - if err != nil { - return err - } - signedRoot, err := fork.ComputeSigningRoot(change, domain) - if err != nil { - return err - } - valid, err := bls.Verify(signedChange.Signature[:], signedRoot[:], change.From[:]) - if err != nil { - return err - } - if !valid { - return fmt.Errorf("invalid signature") - } - } - credentials := wc - // Reset the validator's withdrawal credentials. - credentials[0] = beaconConfig.ETH1AddressWithdrawalPrefixByte - copy(credentials[1:], make([]byte, 11)) - copy(credentials[12:], change.To[:]) - - // Update the state with the modified validator. - state.SetWithdrawalCredentialForValidatorAtIndex(int(change.ValidatorIndex), credentials) - return nil -} diff --git a/cl/phase1/core/transition/process_epoch.go b/cl/phase1/core/transition/process_epoch.go deleted file mode 100644 index ce0cae47f69..00000000000 --- a/cl/phase1/core/transition/process_epoch.go +++ /dev/null @@ -1,103 +0,0 @@ -package transition - -import ( - "github.com/ledgerwatch/erigon/cl/clparams" - "github.com/ledgerwatch/erigon/cl/cltypes/solid" - "github.com/ledgerwatch/erigon/cl/phase1/core/state" -) - -// ProcessEpoch process epoch transition. -func ProcessEpoch(state *state.BeaconState) error { - if err := ProcessJustificationBitsAndFinality(state); err != nil { - return err - } - if state.Version() >= clparams.AltairVersion { - if err := ProcessInactivityScores(state); err != nil { - return err - } - } - if err := ProcessRewardsAndPenalties(state); err != nil { - return err - } - if err := ProcessRegistryUpdates(state); err != nil { - return err - } - if err := ProcessSlashings(state); err != nil { - return err - } - ProcessEth1DataReset(state) - if err := ProcessEffectiveBalanceUpdates(state); err != nil { - return err - } - ProcessSlashingsReset(state) - ProcessRandaoMixesReset(state) - if err := ProcessHistoricalRootsUpdate(state); err != nil { - return err - } - if state.Version() == clparams.Phase0Version { - if err := ProcessParticipationRecordUpdates(state); err != nil { - return err - } - } - - if state.Version() >= clparams.AltairVersion { - ProcessParticipationFlagUpdates(state) - if err := ProcessSyncCommitteeUpdate(state); err != nil { - return err - } - } - return nil -} - -func ProcessParticipationRecordUpdates(state *state.BeaconState) error { - state.SetPreviousEpochAttestations(state.CurrentEpochAttestations()) - state.ResetCurrentEpochAttestations() - var err error - // Also mark all current attesters as previous - state.ForEachValidator(func(_ solid.Validator, idx, total int) bool { - - var oldCurrentMatchingSourceAttester, oldCurrentMatchingTargetAttester, oldCurrentMatchingHeadAttester bool - var oldMinCurrentInclusionDelayAttestation *solid.PendingAttestation - - if oldCurrentMatchingSourceAttester, err = state.ValidatorIsCurrentMatchingSourceAttester(idx); err != nil { - return false - } - if oldCurrentMatchingTargetAttester, err = state.ValidatorIsCurrentMatchingTargetAttester(idx); err != nil { - return false - } - if oldCurrentMatchingHeadAttester, err = state.ValidatorIsCurrentMatchingHeadAttester(idx); err != nil { - return false - } - if oldMinCurrentInclusionDelayAttestation, err = state.ValidatorMinCurrentInclusionDelayAttestation(idx); err != nil { - return false - } - // Previous sources/target/head - if err = state.SetValidatorIsPreviousMatchingSourceAttester(idx, oldCurrentMatchingSourceAttester); err != nil { - return false - } - if err = state.SetValidatorIsPreviousMatchingTargetAttester(idx, oldCurrentMatchingTargetAttester); err != nil { - return false - } - if err = state.SetValidatorIsPreviousMatchingHeadAttester(idx, oldCurrentMatchingHeadAttester); err != nil { - return false - } - if err = state.SetValidatorMinPreviousInclusionDelayAttestation(idx, oldMinCurrentInclusionDelayAttestation); err != nil { - return false - } - // Current sources/target/head - if err = state.SetValidatorIsCurrentMatchingSourceAttester(idx, false); err != nil { - return false - } - if err = state.SetValidatorIsCurrentMatchingTargetAttester(idx, false); err != nil { - return false - } - if err = state.SetValidatorIsCurrentMatchingHeadAttester(idx, false); err != nil { - return false - } - if err = state.SetValidatorMinCurrentInclusionDelayAttestation(idx, nil); err != nil { - return false - } - return true - }) - return err -} diff --git a/cl/phase1/core/transition/process_slashings.go b/cl/phase1/core/transition/process_slashings.go deleted file mode 100644 index ec2f5788dd8..00000000000 --- a/cl/phase1/core/transition/process_slashings.go +++ /dev/null @@ -1,56 +0,0 @@ -package transition - -import ( - "github.com/ledgerwatch/erigon/cl/clparams" - "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" -) - -func processSlashings(s *state2.BeaconState, slashingMultiplier uint64) error { - // Get the current epoch - epoch := state2.Epoch(s.BeaconState) - // Get the total active balance - totalBalance := s.GetTotalActiveBalance() - // Calculate the total slashing amount - // by summing all slashings and multiplying by the provided multiplier - slashing := state2.GetTotalSlashingAmount(s.BeaconState) * slashingMultiplier - // Adjust the total slashing amount to be no greater than the total active balance - if totalBalance < slashing { - slashing = totalBalance - } - beaconConfig := s.BeaconConfig() - // Apply penalties to validators who have been slashed and reached the withdrawable epoch - var err error - s.ForEachValidator(func(validator solid.Validator, i, total int) bool { - if !validator.Slashed() || epoch+beaconConfig.EpochsPerSlashingsVector/2 != validator.WithdrawableEpoch() { - return true - } - // Get the effective balance increment - increment := beaconConfig.EffectiveBalanceIncrement - // Calculate the penalty numerator by multiplying the validator's effective balance by the total slashing amount - penaltyNumerator := validator.EffectiveBalance() / increment * slashing - // Calculate the penalty by dividing the penalty numerator by the total balance and multiplying by the increment - penalty := penaltyNumerator / totalBalance * increment - // Decrease the validator's balance by the calculated penalty - if err = state2.DecreaseBalance(s.BeaconState, uint64(i), penalty); err != nil { - return false - } - return true - }) - if err != nil { - return err - } - return nil -} - -func ProcessSlashings(state *state2.BeaconState) error { - // Depending on the version of the state, use different multipliers - switch state.Version() { - case clparams.Phase0Version: - return processSlashings(state, state.BeaconConfig().ProportionalSlashingMultiplier) - case clparams.AltairVersion: - return processSlashings(state, state.BeaconConfig().ProportionalSlashingMultiplierAltair) - default: - return processSlashings(state, state.BeaconConfig().ProportionalSlashingMultiplierBellatrix) - } -} diff --git a/cl/phase1/core/transition/process_slots.go b/cl/phase1/core/transition/process_slots.go deleted file mode 100644 index 8632bc68fd5..00000000000 --- a/cl/phase1/core/transition/process_slots.go +++ /dev/null @@ -1,183 +0,0 @@ -package transition - -import ( - "fmt" - "time" - - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - "github.com/Giulio2002/bls" - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon/cl/clparams" - "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/cl/fork" - "github.com/ledgerwatch/erigon/cl/utils" - "github.com/ledgerwatch/log/v3" -) - -func TransitionState(s *state2.BeaconState, block *cltypes.SignedBeaconBlock, fullValidation bool) error { - currentBlock := block.Block - if err := ProcessSlots(s, currentBlock.Slot); err != nil { - return err - } - - if fullValidation { - valid, err := verifyBlockSignature(s, block) - if err != nil { - return fmt.Errorf("error validating block signature: %v", err) - } - if !valid { - return fmt.Errorf("block not valid") - } - } - // Transition block - if err := processBlock(s, block, fullValidation); err != nil { - return err - } - if fullValidation { - expectedStateRoot, err := s.HashSSZ() - if err != nil { - return fmt.Errorf("unable to generate state root: %v", err) - } - if expectedStateRoot != currentBlock.StateRoot { - return fmt.Errorf("expected state root differs from received state root") - } - } - - s.SetPreviousStateRoot(currentBlock.StateRoot) - return nil -} - -// transitionSlot is called each time there is a new slot to process -func transitionSlot(s *state2.BeaconState) error { - slot := s.Slot() - previousStateRoot := s.PreviousStateRoot() - var err error - if previousStateRoot == (libcommon.Hash{}) { - previousStateRoot, err = s.HashSSZ() - if err != nil { - return err - } - } - - beaconConfig := s.BeaconConfig() - - s.SetStateRootAt(int(slot%beaconConfig.SlotsPerHistoricalRoot), previousStateRoot) - - latestBlockHeader := s.LatestBlockHeader() - if latestBlockHeader.Root == [32]byte{} { - latestBlockHeader.Root = previousStateRoot - s.SetLatestBlockHeader(&latestBlockHeader) - } - blockHeader := s.LatestBlockHeader() - - previousBlockRoot, err := (&blockHeader).HashSSZ() - if err != nil { - return err - } - s.SetBlockRootAt(int(slot%beaconConfig.SlotsPerHistoricalRoot), previousBlockRoot) - return nil -} - -func ProcessSlots(s *state2.BeaconState, slot uint64) error { - beaconConfig := s.BeaconConfig() - sSlot := s.Slot() - if slot <= sSlot { - return fmt.Errorf("new slot: %d not greater than s slot: %d", slot, sSlot) - } - // Process each slot. - for i := sSlot; i < slot; i++ { - err := transitionSlot(s) - if err != nil { - return fmt.Errorf("unable to process slot transition: %v", err) - } - // TODO(Someone): Add epoch transition. - if (sSlot+1)%beaconConfig.SlotsPerEpoch == 0 { - start := time.Now() - if err := ProcessEpoch(s); err != nil { - return err - } - log.Debug("Processed new epoch successfully", "epoch", state2.Epoch(s.BeaconState), "process_epoch_elpsed", time.Since(start)) - } - // TODO: add logic to process epoch updates. - sSlot += 1 - s.SetSlot(sSlot) - if sSlot%beaconConfig.SlotsPerEpoch != 0 { - continue - } - if state2.Epoch(s.BeaconState) == beaconConfig.AltairForkEpoch { - if err := s.UpgradeToAltair(); err != nil { - return err - } - } - if state2.Epoch(s.BeaconState) == beaconConfig.BellatrixForkEpoch { - if err := s.UpgradeToBellatrix(); err != nil { - return err - } - } - if state2.Epoch(s.BeaconState) == beaconConfig.CapellaForkEpoch { - if err := s.UpgradeToCapella(); err != nil { - return err - } - } - if state2.Epoch(s.BeaconState) == beaconConfig.DenebForkEpoch { - if err := s.UpgradeToDeneb(); err != nil { - return err - } - } - } - return nil -} - -func verifyBlockSignature(s *state2.BeaconState, block *cltypes.SignedBeaconBlock) (bool, error) { - proposer, err := s.ValidatorForValidatorIndex(int(block.Block.ProposerIndex)) - if err != nil { - return false, err - } - domain, err := s.GetDomain(s.BeaconConfig().DomainBeaconProposer, state2.Epoch(s.BeaconState)) - if err != nil { - return false, err - } - sigRoot, err := fork.ComputeSigningRoot(block.Block, domain) - if err != nil { - return false, err - } - pk := proposer.PublicKey() - return bls.Verify(block.Signature[:], sigRoot[:], pk[:]) -} - -// ProcessHistoricalRootsUpdate updates the historical root data structure by computing a new historical root batch when it is time to do so. -func ProcessHistoricalRootsUpdate(s *state2.BeaconState) error { - nextEpoch := state2.Epoch(s.BeaconState) + 1 - beaconConfig := s.BeaconConfig() - blockRoots := s.BlockRoots() - stateRoots := s.StateRoots() - - // Check if it's time to compute the historical root batch. - if nextEpoch%(beaconConfig.SlotsPerHistoricalRoot/beaconConfig.SlotsPerEpoch) != 0 { - return nil - } - - // Compute historical root batch. - blockRootsLeaf, err := blockRoots.HashSSZ() - if err != nil { - return err - } - stateRootsLeaf, err := stateRoots.HashSSZ() - if err != nil { - return err - } - - // Add the historical summary or root to the s. - if s.Version() >= clparams.CapellaVersion { - s.AddHistoricalSummary(&cltypes.HistoricalSummary{ - BlockSummaryRoot: blockRootsLeaf, - StateSummaryRoot: stateRootsLeaf, - }) - } else { - historicalRoot := utils.Keccak256(blockRootsLeaf[:], stateRootsLeaf[:]) - s.AddHistoricalRoot(historicalRoot) - } - - return nil -} diff --git a/cl/phase1/core/transition/process_sync_aggregate.go b/cl/phase1/core/transition/process_sync_aggregate.go deleted file mode 100644 index bfafc00ab63..00000000000 --- a/cl/phase1/core/transition/process_sync_aggregate.go +++ /dev/null @@ -1,93 +0,0 @@ -package transition - -import ( - "errors" - - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - "github.com/Giulio2002/bls" - "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/cl/fork" - "github.com/ledgerwatch/erigon/cl/utils" -) - -// processSyncAggregate applies all the logic in the spec function `process_sync_aggregate` except -// verifying the BLS signatures. It returns the modified beacons state and the list of validators' -// public keys that voted, for future signature verification. -func processSyncAggregate(s *state2.BeaconState, sync *cltypes.SyncAggregate) ([][]byte, error) { - currentSyncCommittee := s.CurrentSyncCommittee() - - if currentSyncCommittee == nil { - return nil, errors.New("nil current sync committee in s") - } - committeeKeys := currentSyncCommittee.GetCommittee() - if len(sync.SyncCommiteeBits)*8 > len(committeeKeys) { - return nil, errors.New("bits length exceeds committee length") - } - var votedKeys [][]byte - - proposerReward, participantReward, err := s.SyncRewards() - if err != nil { - return nil, err - } - - proposerIndex, err := s.GetBeaconProposerIndex() - if err != nil { - return nil, err - } - - syncAggregateBits := sync.SyncCommiteeBits - earnedProposerReward := uint64(0) - currPubKeyIndex := 0 - for i := range syncAggregateBits { - for bit := 1; bit <= 128; bit *= 2 { - vIdx, exists := s.ValidatorIndexByPubkey(committeeKeys[currPubKeyIndex]) - // Impossible scenario. - if !exists { - return nil, errors.New("validator public key does not exist in state") - } - if syncAggregateBits[i]&byte(bit) > 0 { - votedKeys = append(votedKeys, committeeKeys[currPubKeyIndex][:]) - if err := state2.IncreaseBalance(s.BeaconState, vIdx, participantReward); err != nil { - return nil, err - } - earnedProposerReward += proposerReward - } else { - if err := state2.DecreaseBalance(s.BeaconState, vIdx, participantReward); err != nil { - return nil, err - } - } - currPubKeyIndex++ - } - } - - return votedKeys, state2.IncreaseBalance(s.BeaconState, proposerIndex, earnedProposerReward) -} - -func ProcessSyncAggregate(s *state2.BeaconState, sync *cltypes.SyncAggregate, fullValidation bool) error { - votedKeys, err := processSyncAggregate(s, sync) - if err != nil { - return err - } - if fullValidation { - previousSlot := s.PreviousSlot() - - domain, err := fork.Domain(s.Fork(), state2.GetEpochAtSlot(s.BeaconConfig(), previousSlot), s.BeaconConfig().DomainSyncCommittee, s.GenesisValidatorsRoot()) - if err != nil { - return nil - } - blockRoot, err := s.GetBlockRootAtSlot(previousSlot) - if err != nil { - return err - } - msg := utils.Keccak256(blockRoot[:], domain) - isValid, err := bls.VerifyAggregate(sync.SyncCommiteeSignature[:], msg[:], votedKeys) - if err != nil { - return err - } - if !isValid { - return errors.New("ProcessSyncAggregate: cannot validate sync committee signature") - } - } - return nil -} diff --git a/cl/phase1/core/transition/processing.go b/cl/phase1/core/transition/processing.go deleted file mode 100644 index 669f9cc03f9..00000000000 --- a/cl/phase1/core/transition/processing.go +++ /dev/null @@ -1,120 +0,0 @@ -package transition - -import ( - "encoding/binary" - "fmt" - - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - - libcommon "github.com/ledgerwatch/erigon-lib/common" - - "github.com/Giulio2002/bls" - "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/cl/utils" -) - -func computeSigningRootEpoch(epoch uint64, domain []byte) (libcommon.Hash, error) { - b := make([]byte, 32) - binary.LittleEndian.PutUint64(b, epoch) - return utils.Keccak256(b, domain), nil -} - -func ProcessBlockHeader(state *state2.BeaconState, block *cltypes.BeaconBlock, fullValidation bool) error { - if fullValidation { - if block.Slot != state.Slot() { - return fmt.Errorf("state slot: %d, not equal to block slot: %d", state.Slot(), block.Slot) - } - if block.Slot <= state.LatestBlockHeader().Slot { - return fmt.Errorf("slock slot: %d, not greater than latest block slot: %d", block.Slot, state.LatestBlockHeader().Slot) - } - propInd, err := state.GetBeaconProposerIndex() - if err != nil { - return fmt.Errorf("error in GetBeaconProposerIndex: %v", err) - } - if block.ProposerIndex != propInd { - return fmt.Errorf("block proposer index: %d, does not match beacon proposer index: %d", block.ProposerIndex, propInd) - } - blockHeader := state.LatestBlockHeader() - latestRoot, err := (&blockHeader).HashSSZ() - if err != nil { - return fmt.Errorf("unable to hash tree root of latest block header: %v", err) - } - if block.ParentRoot != latestRoot { - return fmt.Errorf("block parent root: %x, does not match latest block root: %x", block.ParentRoot, latestRoot) - } - } - - bodyRoot, err := block.Body.HashSSZ() - if err != nil { - return fmt.Errorf("unable to hash tree root of block body: %v", err) - } - state.SetLatestBlockHeader(&cltypes.BeaconBlockHeader{ - Slot: block.Slot, - ProposerIndex: block.ProposerIndex, - ParentRoot: block.ParentRoot, - BodyRoot: bodyRoot, - }) - - proposer, err := state.ValidatorForValidatorIndex(int(block.ProposerIndex)) - if err != nil { - return err - } - if proposer.Slashed() { - return fmt.Errorf("proposer: %d is slashed", block.ProposerIndex) - } - return nil -} - -func ProcessRandao(s *state2.BeaconState, randao [96]byte, proposerIndex uint64, fullValidation bool) error { - epoch := state2.Epoch(s.BeaconState) - proposer, err := s.ValidatorForValidatorIndex(int(proposerIndex)) - if err != nil { - return err - } - if fullValidation { - domain, err := s.GetDomain(s.BeaconConfig().DomainRandao, epoch) - if err != nil { - return fmt.Errorf("ProcessRandao: unable to get domain: %v", err) - } - signingRoot, err := computeSigningRootEpoch(epoch, domain) - if err != nil { - return fmt.Errorf("ProcessRandao: unable to compute signing root: %v", err) - } - pk := proposer.PublicKey() - valid, err := bls.Verify(randao[:], signingRoot[:], pk[:]) - if err != nil { - return fmt.Errorf("ProcessRandao: unable to verify public key: %x, with signing root: %x, and signature: %x, %v", pk[:], signingRoot[:], randao[:], err) - } - if !valid { - return fmt.Errorf("ProcessRandao: invalid signature: public key: %x, signing root: %x, signature: %x", pk[:], signingRoot[:], randao[:]) - } - } - - randaoMixes := s.GetRandaoMixes(epoch) - randaoHash := utils.Keccak256(randao[:]) - mix := [32]byte{} - for i := range mix { - mix[i] = randaoMixes[i] ^ randaoHash[i] - } - s.SetRandaoMixAt(int(epoch%s.BeaconConfig().EpochsPerHistoricalVector), mix) - return nil -} - -func ProcessEth1Data(state *state2.BeaconState, eth1Data *cltypes.Eth1Data) error { - state.AddEth1DataVote(eth1Data) - newVotes := state.Eth1DataVotes() - - // Count how many times body.Eth1Data appears in the votes. - numVotes := 0 - newVotes.Range(func(index int, value *cltypes.Eth1Data, length int) bool { - if eth1Data.Equal(value) { - numVotes += 1 - } - return true - }) - - if uint64(numVotes*2) > state.BeaconConfig().EpochsPerEth1VotingPeriod*state.BeaconConfig().SlotsPerEpoch { - state.SetEth1Data(eth1Data) - } - return nil -} diff --git a/cl/phase1/execution_client/execution_engine.go b/cl/phase1/execution_client/execution_engine.go index 293d93832a1..8d073338bc2 100644 --- a/cl/phase1/execution_client/execution_engine.go +++ b/cl/phase1/execution_client/execution_engine.go @@ -12,6 +12,7 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces/types" types2 "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/ethdb/privateapi" ) @@ -146,10 +147,16 @@ func convertPayloadToGrpc(e *cltypes.Eth1Block) (*types.ExecutionPayload, error) Transactions: e.Transactions.UnderlyngReference(), Withdrawals: privateapi.ConvertWithdrawalsToRpc(withdrawals), } - if e.Withdrawals != nil { + if e.Version() >= clparams.CapellaVersion { res.Version = 2 res.Withdrawals = privateapi.ConvertWithdrawalsToRpc(withdrawals) } + if e.Version() >= clparams.DenebVersion { + res.DataGasUsed = &e.DataGasUsed + res.ExcessDataGas = &e.ExcessDataGas + res.Version = 3 + } + return res, nil } diff --git a/cl/phase1/forkchoice/fork_graph/fork_graph.go b/cl/phase1/forkchoice/fork_graph/fork_graph.go index 061cc921596..8221ab2c79b 100644 --- a/cl/phase1/forkchoice/fork_graph/fork_graph.go +++ b/cl/phase1/forkchoice/fork_graph/fork_graph.go @@ -6,7 +6,7 @@ import ( "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/cl/cltypes/solid" "github.com/ledgerwatch/erigon/cl/phase1/core/state" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" + "github.com/ledgerwatch/erigon/cl/transition" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" ) @@ -153,8 +153,8 @@ func (f *ForkGraph) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, full // Add block to list of invalid blocks log.Debug("Invalid beacon block", "reason", invalidBlockErr) f.badBlocks[blockRoot] = struct{}{} - f.currentReferenceState.CopyInto(f.currentState) - f.currentStateBlockRoot, err = f.currentReferenceState.BlockRoot() + f.nextReferenceState.CopyInto(f.currentState) + f.currentStateBlockRoot, err = f.nextReferenceState.BlockRoot() if err != nil { log.Error("[Caplin] Could not recover from invalid block") } diff --git a/cl/phase1/forkchoice/on_attestation.go b/cl/phase1/forkchoice/on_attestation.go index b58e004c4a3..c5fc7ac3583 100644 --- a/cl/phase1/forkchoice/on_attestation.go +++ b/cl/phase1/forkchoice/on_attestation.go @@ -19,7 +19,7 @@ func (f *ForkChoiceStore) OnAttestation(attestation *solid.Attestation, fromBloc return err } target := data.Target() - if cachedIndicies, ok := cache.LoadAttestatingIndicies(&data); ok { + if cachedIndicies, ok := cache.LoadAttestatingIndicies(&data, attestation.AggregationBits()); ok { f.processAttestingIndicies(attestation, cachedIndicies) return nil } diff --git a/cl/phase1/forkchoice/on_block.go b/cl/phase1/forkchoice/on_block.go index cd6860e6add..920e268ef1b 100644 --- a/cl/phase1/forkchoice/on_block.go +++ b/cl/phase1/forkchoice/on_block.go @@ -3,12 +3,12 @@ package forkchoice import ( "fmt" - "github.com/ledgerwatch/erigon/cl/freezer" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - "github.com/ledgerwatch/erigon/cl/phase1/forkchoice/fork_graph" + "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/cl/freezer" + "github.com/ledgerwatch/erigon/cl/phase1/forkchoice/fork_graph" + "github.com/ledgerwatch/erigon/cl/transition/impl/eth2/statechange" ) func (f *ForkChoiceStore) OnBlock(block *cltypes.SignedBeaconBlock, newPayload, fullValidation bool) error { @@ -74,7 +74,7 @@ func (f *ForkChoiceStore) OnBlock(block *cltypes.SignedBeaconBlock, newPayload, justificationBits = lastProcessedState.JustificationBits().Copy() ) // Eagerly compute unrealized justification and finality - if err := transition.ProcessJustificationBitsAndFinality(lastProcessedState); err != nil { + if err := statechange.ProcessJustificationBitsAndFinality(lastProcessedState); err != nil { return err } f.updateUnrealizedCheckpoints(lastProcessedState.CurrentJustifiedCheckpoint().Copy(), lastProcessedState.FinalizedCheckpoint().Copy()) diff --git a/cl/phase1/forkchoice/utils.go b/cl/phase1/forkchoice/utils.go index 2627341f99f..ada018f16f7 100644 --- a/cl/phase1/forkchoice/utils.go +++ b/cl/phase1/forkchoice/utils.go @@ -2,12 +2,11 @@ package forkchoice import ( "fmt" + "github.com/ledgerwatch/erigon/cl/transition" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cl/cltypes/solid" "github.com/ledgerwatch/erigon/cl/phase1/core/state" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - - libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/log/v3" ) @@ -85,7 +84,7 @@ func (f *ForkChoiceStore) getCheckpointState(checkpoint solid.Checkpoint) (*chec if baseState.Slot() < f.computeStartSlotAtEpoch(checkpoint.Epoch()) { log.Debug("Long checkpoint detected") // If we require to change it then process the future epoch - if err := transition.ProcessSlots(baseState, f.computeStartSlotAtEpoch(checkpoint.Epoch())); err != nil { + if err := transition.DefaultMachine.ProcessSlots(baseState, f.computeStartSlotAtEpoch(checkpoint.Epoch())); err != nil { return nil, err } } diff --git a/cl/phase1/stages/stages_beacon_state.go b/cl/phase1/stages/stages_beacon_state.go index cee0fa059ca..eeb043d7281 100644 --- a/cl/phase1/stages/stages_beacon_state.go +++ b/cl/phase1/stages/stages_beacon_state.go @@ -2,10 +2,10 @@ package stages import ( "context" + "github.com/ledgerwatch/erigon/cl/transition" "github.com/ledgerwatch/erigon/cl/phase1/core/rawdb" state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" "github.com/ledgerwatch/erigon/cl/phase1/execution_client" libcommon "github.com/ledgerwatch/erigon-lib/common" diff --git a/cl/spectest/consensus_tests/epoch_processing.go b/cl/spectest/consensus_tests/epoch_processing.go index 7544baee8bc..eb3b0e4c527 100644 --- a/cl/spectest/consensus_tests/epoch_processing.go +++ b/cl/spectest/consensus_tests/epoch_processing.go @@ -1,13 +1,12 @@ package consensus_tests import ( + "github.com/ledgerwatch/erigon/cl/transition/impl/eth2/statechange" "io/fs" "os" "testing" "github.com/ledgerwatch/erigon/cl/phase1/core/state" - transition2 "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - "github.com/ledgerwatch/erigon/spectest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,58 +51,57 @@ func (b *EpochProcessing) Run(t *testing.T, root fs.FS, c spectest.TestCase) (er } var effectiveBalancesUpdateTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessEffectiveBalanceUpdates(s) + return statechange.ProcessEffectiveBalanceUpdates(s) }) var eth1DataResetTest = NewEpochProcessing(func(s *state.BeaconState) error { - transition2.ProcessEth1DataReset(s) + statechange.ProcessEth1DataReset(s) return nil }) var historicalRootsUpdateTest = NewEpochProcessing(func(s *state.BeaconState) error { - transition2.ProcessHistoricalRootsUpdate(s) - return nil + return statechange.ProcessHistoricalRootsUpdate(s) }) var inactivityUpdateTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessInactivityScores(s) + return statechange.ProcessInactivityScores(s) }) var justificationFinalizationTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessJustificationBitsAndFinality(s) + return statechange.ProcessJustificationBitsAndFinality(s) }) var participationFlagUpdatesTest = NewEpochProcessing(func(s *state.BeaconState) error { - transition2.ProcessParticipationFlagUpdates(s) + statechange.ProcessParticipationFlagUpdates(s) return nil }) var participationRecordUpdatesTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessParticipationRecordUpdates(s) + return statechange.ProcessParticipationRecordUpdates(s) }) var randaoMixesTest = NewEpochProcessing(func(s *state.BeaconState) error { - transition2.ProcessRandaoMixesReset(s) + statechange.ProcessRandaoMixesReset(s) return nil }) var registryUpdatesTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessRegistryUpdates(s) + return statechange.ProcessRegistryUpdates(s) }) var rewardsAndPenaltiesTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessRewardsAndPenalties(s) + return statechange.ProcessRewardsAndPenalties(s) }) var slashingsTest = NewEpochProcessing(func(s *state.BeaconState) error { - return transition2.ProcessSlashings(s) + return statechange.ProcessSlashings(s) }) var slashingsResetTest = NewEpochProcessing(func(s *state.BeaconState) error { - transition2.ProcessSlashingsReset(s) + statechange.ProcessSlashingsReset(s) return nil }) var recordsResetTest = NewEpochProcessing(func(s *state.BeaconState) error { - transition2.ProcessParticipationRecordUpdates(s) + statechange.ProcessParticipationRecordUpdates(s) return nil }) diff --git a/cl/spectest/consensus_tests/finality.go b/cl/spectest/consensus_tests/finality.go index 0faf1995cd2..1c72fcfb059 100644 --- a/cl/spectest/consensus_tests/finality.go +++ b/cl/spectest/consensus_tests/finality.go @@ -2,11 +2,10 @@ package consensus_tests import ( "fmt" + "github.com/ledgerwatch/erigon/cl/transition/machine" "io/fs" "testing" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - "github.com/ledgerwatch/erigon/spectest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,7 +25,7 @@ var FinalityFinality = spectest.HandlerFunc(func(t *testing.T, root fs.FS, c spe } startSlot := testState.Slot() for _, block := range blocks { - if err := transition.TransitionState(testState, block, true); err != nil { + if err := machine.TransitionState(c.Machine, testState, block); err != nil { require.NoError(t, fmt.Errorf("cannot transition state: %w. slot=%d. start_slot=%d", err, block.Block.Slot, startSlot)) } } diff --git a/cl/spectest/consensus_tests/operations.go b/cl/spectest/consensus_tests/operations.go index c4c1c755dcc..1aaca37cc04 100644 --- a/cl/spectest/consensus_tests/operations.go +++ b/cl/spectest/consensus_tests/operations.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/ledgerwatch/erigon/cl/cltypes/solid" - transition2 "github.com/ledgerwatch/erigon/cl/phase1/core/transition" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/spectest" @@ -39,7 +38,7 @@ func operationAttestationHandler(t *testing.T, root fs.FS, c spectest.TestCase) if err := spectest.ReadSszOld(root, att, c.Version(), attestationFileName); err != nil { return err } - if err := transition2.ProcessAttestations(preState, solid.NewDynamicListSSZFromList([]*solid.Attestation{att}, 128), true); err != nil { + if err := c.Machine.ProcessAttestations(preState, solid.NewDynamicListSSZFromList([]*solid.Attestation{att}, 128)); err != nil { if expectedError { return nil } @@ -69,7 +68,7 @@ func operationAttesterSlashingHandler(t *testing.T, root fs.FS, c spectest.TestC if err := spectest.ReadSszOld(root, att, c.Version(), attesterSlashingFileName); err != nil { return err } - if err := transition2.ProcessAttesterSlashing(preState, att); err != nil { + if err := c.Machine.ProcessAttesterSlashing(preState, att); err != nil { if expectedError { return nil } @@ -99,7 +98,7 @@ func operationProposerSlashingHandler(t *testing.T, root fs.FS, c spectest.TestC if err := spectest.ReadSszOld(root, att, c.Version(), proposerSlashingFileName); err != nil { return err } - if err := transition2.ProcessProposerSlashing(preState, att); err != nil { + if err := c.Machine.ProcessProposerSlashing(preState, att); err != nil { if expectedError { return nil } @@ -129,7 +128,7 @@ func operationBlockHeaderHandler(t *testing.T, root fs.FS, c spectest.TestCase) if err := spectest.ReadSszOld(root, block, c.Version(), blockFileName); err != nil { return err } - if err := transition2.ProcessBlockHeader(preState, block, true); err != nil { + if err := c.Machine.ProcessBlockHeader(preState, block); err != nil { if expectedError { return nil } @@ -159,7 +158,7 @@ func operationDepositHandler(t *testing.T, root fs.FS, c spectest.TestCase) erro if err := spectest.ReadSszOld(root, deposit, c.Version(), depositFileName); err != nil { return err } - if err := transition2.ProcessDeposit(preState, deposit, true); err != nil { + if err := c.Machine.ProcessDeposit(preState, deposit); err != nil { if expectedError { return nil } @@ -189,7 +188,7 @@ func operationSyncAggregateHandler(t *testing.T, root fs.FS, c spectest.TestCase if err := spectest.ReadSszOld(root, agg, c.Version(), syncAggregateFileName); err != nil { return err } - if err := transition2.ProcessSyncAggregate(preState, agg, true); err != nil { + if err := c.Machine.ProcessSyncAggregate(preState, agg); err != nil { if expectedError { return nil } @@ -219,7 +218,7 @@ func operationVoluntaryExitHandler(t *testing.T, root fs.FS, c spectest.TestCase if err := spectest.ReadSszOld(root, vo, c.Version(), voluntaryExitFileName); err != nil { return err } - if err := transition2.ProcessVoluntaryExit(preState, vo, true); err != nil { + if err := c.Machine.ProcessVoluntaryExit(preState, vo); err != nil { if expectedError { return nil } @@ -249,7 +248,7 @@ func operationWithdrawalHandler(t *testing.T, root fs.FS, c spectest.TestCase) e if err := spectest.ReadSszOld(root, executionPayload, c.Version(), executionPayloadFileName); err != nil { return err } - if err := transition2.ProcessWithdrawals(preState, executionPayload.Withdrawals, true); err != nil { + if err := c.Machine.ProcessWithdrawals(preState, executionPayload.Withdrawals); err != nil { if expectedError { return nil } @@ -279,7 +278,7 @@ func operationSignedBlsChangeHandler(t *testing.T, root fs.FS, c spectest.TestCa if err := spectest.ReadSszOld(root, change, c.Version(), addressChangeFileName); err != nil { return err } - if err := transition2.ProcessBlsToExecutionChange(preState, change, true); err != nil { + if err := c.Machine.ProcessBlsToExecutionChange(preState, change); err != nil { if expectedError { return nil } diff --git a/cl/spectest/consensus_tests/sanity.go b/cl/spectest/consensus_tests/sanity.go index ddc7182227b..493f3ddcf4e 100644 --- a/cl/spectest/consensus_tests/sanity.go +++ b/cl/spectest/consensus_tests/sanity.go @@ -1,12 +1,11 @@ package consensus_tests import ( + "github.com/ledgerwatch/erigon/cl/transition/machine" "io/fs" "os" "testing" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/spectest" "github.com/stretchr/testify/assert" @@ -25,7 +24,7 @@ var SanitySlots = spectest.HandlerFunc(func(t *testing.T, root fs.FS, c spectest expectedState, err := spectest.ReadBeaconState(root, c.Version(), spectest.PostSsz) require.NoError(t, err) - err = transition.ProcessSlots(testState, expectedState.Slot()) + err = c.Machine.ProcessSlots(testState, expectedState.Slot()) require.NoError(t, err) expectedRoot, err := expectedState.HashSSZ() @@ -65,7 +64,7 @@ var SanityBlocks = spectest.HandlerFunc(func(t *testing.T, root fs.FS, c spectes var block *cltypes.SignedBeaconBlock for _, block = range blocks { - err = transition.TransitionState(testState, block, true) + err = machine.TransitionState(c.Machine, testState, block) if err != nil { break } diff --git a/cl/spectest/consensus_tests/transition.go b/cl/spectest/consensus_tests/transition.go index db31b851159..105ab477fb9 100644 --- a/cl/spectest/consensus_tests/transition.go +++ b/cl/spectest/consensus_tests/transition.go @@ -2,11 +2,10 @@ package consensus_tests import ( "fmt" + "github.com/ledgerwatch/erigon/cl/transition/machine" "io/fs" "testing" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/spectest" @@ -60,7 +59,7 @@ func (b *TransitionCore) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err break } blockIndex++ - if err := transition.TransitionState(startState, block, true); err != nil { + if err := machine.TransitionState(c.Machine, startState, block); err != nil { return fmt.Errorf("cannot transition state: %s. slot=%d. start_slot=%d", err, block.Block.Slot, startSlot) } } diff --git a/cl/spectest/tests_test.go b/cl/spectest/tests_test.go index 0f89aa7cdfa..db4fe28c987 100644 --- a/cl/spectest/tests_test.go +++ b/cl/spectest/tests_test.go @@ -1,18 +1,16 @@ -//go:build spectest - -// once all tests are implemented, we can allow this test in the ci build path - package spectest import ( "os" "testing" + "github.com/ledgerwatch/erigon/cl/transition" + "github.com/ledgerwatch/erigon/cl/spectest/consensus_tests" "github.com/ledgerwatch/erigon/spectest" ) func Test(t *testing.T) { - spectest.RunCases(t, consensus_tests.TestFormats, os.DirFS("./tests")) + spectest.RunCases(t, consensus_tests.TestFormats, transition.ValidatingMachine, os.DirFS("./tests")) } diff --git a/cl/transition/compat.go b/cl/transition/compat.go new file mode 100644 index 00000000000..dea4d8d05f7 --- /dev/null +++ b/cl/transition/compat.go @@ -0,0 +1,19 @@ +package transition + +import ( + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/transition/impl/eth2" + machine2 "github.com/ledgerwatch/erigon/cl/transition/machine" + + "github.com/ledgerwatch/erigon/cl/cltypes" +) + +var _ machine2.Interface = (*eth2.Impl)(nil) + +var DefaultMachine = ð2.Impl{} +var ValidatingMachine = ð2.Impl{FullValidation: true} + +func TransitionState(s *state.BeaconState, block *cltypes.SignedBeaconBlock, fullValidation bool) error { + cvm := ð2.Impl{FullValidation: fullValidation} + return machine2.TransitionState(cvm, s, block) +} diff --git a/cl/phase1/core/transition/block_processing_test.go b/cl/transition/impl/eth2/block_processing_test.go similarity index 51% rename from cl/phase1/core/transition/block_processing_test.go rename to cl/transition/impl/eth2/block_processing_test.go index ae4a67c1997..bfae09874c9 100644 --- a/cl/phase1/core/transition/block_processing_test.go +++ b/cl/transition/impl/eth2/block_processing_test.go @@ -1,26 +1,28 @@ -package transition +package eth2_test import ( _ "embed" "testing" + "github.com/stretchr/testify/require" + "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/transition" "github.com/ledgerwatch/erigon/cl/utils" - "github.com/stretchr/testify/require" ) -//go:embed test_data/block_processing/capella_block.ssz_snappy +//go:embed statechange/test_data/block_processing/capella_block.ssz_snappy var capellaBlock []byte -//go:embed test_data/block_processing/capella_state.ssz_snappy +//go:embed statechange/test_data/block_processing/capella_state.ssz_snappy var capellaState []byte func TestBlockProcessing(t *testing.T) { - state := state.New(&clparams.MainnetBeaconConfig) - require.NoError(t, utils.DecodeSSZSnappy(state, capellaState, int(clparams.CapellaVersion))) + s := state.New(&clparams.MainnetBeaconConfig) + require.NoError(t, utils.DecodeSSZSnappy(s, capellaState, int(clparams.CapellaVersion))) block := &cltypes.SignedBeaconBlock{} require.NoError(t, utils.DecodeSSZSnappy(block, capellaBlock, int(clparams.CapellaVersion))) - require.NoError(t, TransitionState(state, block, true)) // All checks already made in transition state + require.NoError(t, transition.TransitionState(s, block, true)) // All checks already made in transition state } diff --git a/cl/transition/impl/eth2/impl.go b/cl/transition/impl/eth2/impl.go new file mode 100644 index 00000000000..dcd233bca0c --- /dev/null +++ b/cl/transition/impl/eth2/impl.go @@ -0,0 +1,11 @@ +package eth2 + +import "github.com/ledgerwatch/erigon/cl/transition/machine" + +type Impl = impl + +var _ machine.Interface = (*impl)(nil) + +type impl struct { + FullValidation bool +} diff --git a/cl/transition/impl/eth2/operations.go b/cl/transition/impl/eth2/operations.go new file mode 100644 index 00000000000..bc70a7e36ce --- /dev/null +++ b/cl/transition/impl/eth2/operations.go @@ -0,0 +1,895 @@ +package eth2 + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "time" + + "github.com/ledgerwatch/erigon/cl/transition/impl/eth2/statechange" + "github.com/ledgerwatch/erigon/metrics/methelp" + "golang.org/x/exp/slices" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + + "github.com/Giulio2002/bls" + "github.com/ledgerwatch/log/v3" + + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/fork" + "github.com/ledgerwatch/erigon/cl/utils" + "github.com/ledgerwatch/erigon/core/types" +) + +func (I *impl) ProcessProposerSlashing(s *state.BeaconState, propSlashing *cltypes.ProposerSlashing) error { + h1 := propSlashing.Header1.Header + h2 := propSlashing.Header2.Header + + if h1.Slot != h2.Slot { + return fmt.Errorf("non-matching slots on proposer slashing: %d != %d", h1.Slot, h2.Slot) + } + + if h1.ProposerIndex != h2.ProposerIndex { + return fmt.Errorf("non-matching proposer indices proposer slashing: %d != %d", h1.ProposerIndex, h2.ProposerIndex) + } + + h1Root, err := h1.HashSSZ() + if err != nil { + return fmt.Errorf("unable to hash header1: %v", err) + } + h2Root, err := h2.HashSSZ() + if err != nil { + return fmt.Errorf("unable to hash header2: %v", err) + } + if h1Root == h2Root { + return fmt.Errorf("propose slashing headers are the same: %v == %v", h1Root, h2Root) + } + + proposer, err := s.ValidatorForValidatorIndex(int(h1.ProposerIndex)) + if err != nil { + return err + } + if !proposer.IsSlashable(state.Epoch(s.BeaconState)) { + return fmt.Errorf("proposer is not slashable: %v", proposer) + } + + for _, signedHeader := range []*cltypes.SignedBeaconBlockHeader{propSlashing.Header1, propSlashing.Header2} { + domain, err := s.GetDomain(s.BeaconConfig().DomainBeaconProposer, state.GetEpochAtSlot(s.BeaconConfig(), signedHeader.Header.Slot)) + if err != nil { + return fmt.Errorf("unable to get domain: %v", err) + } + signingRoot, err := fork.ComputeSigningRoot(signedHeader.Header, domain) + if err != nil { + return fmt.Errorf("unable to compute signing root: %v", err) + } + pk := proposer.PublicKey() + valid, err := bls.Verify(signedHeader.Signature[:], signingRoot[:], pk[:]) + if err != nil { + return fmt.Errorf("unable to verify signature: %v", err) + } + if !valid { + return fmt.Errorf("invalid signature: signature %v, root %v, pubkey %v", signedHeader.Signature[:], signingRoot[:], pk) + } + } + + // Set whistleblower index to 0 so current proposer gets reward. + s.SlashValidator(h1.ProposerIndex, nil) + return nil +} + +func (I *impl) ProcessAttesterSlashing(s *state.BeaconState, attSlashing *cltypes.AttesterSlashing) error { + att1 := attSlashing.Attestation_1 + att2 := attSlashing.Attestation_2 + + if !cltypes.IsSlashableAttestationData(att1.Data, att2.Data) { + return fmt.Errorf("attestation data not slashable: %+v; %+v", att1.Data, att2.Data) + } + + valid, err := state.IsValidIndexedAttestation(s.BeaconState, att1) + if err != nil { + return fmt.Errorf("error calculating indexed attestation 1 validity: %v", err) + } + if !valid { + return fmt.Errorf("invalid indexed attestation 1") + } + + valid, err = state.IsValidIndexedAttestation(s.BeaconState, att2) + if err != nil { + return fmt.Errorf("error calculating indexed attestation 2 validity: %v", err) + } + if !valid { + return fmt.Errorf("invalid indexed attestation 2") + } + + slashedAny := false + currentEpoch := state.GetEpochAtSlot(s.BeaconConfig(), s.Slot()) + for _, ind := range solid.IntersectionOfSortedSets( + solid.IterableSSZ[uint64](att1.AttestingIndices), + solid.IterableSSZ[uint64](att2.AttestingIndices)) { + validator, err := s.ValidatorForValidatorIndex(int(ind)) + if err != nil { + return err + } + if validator.IsSlashable(currentEpoch) { + err := s.SlashValidator(ind, nil) + if err != nil { + return fmt.Errorf("unable to slash validator: %d", ind) + } + slashedAny = true + } + } + + if !slashedAny { + return fmt.Errorf("no validators slashed") + } + return nil +} + +func (I *impl) ProcessDeposit(s *state.BeaconState, deposit *cltypes.Deposit) error { + if deposit == nil { + return nil + } + depositLeaf, err := deposit.Data.HashSSZ() + if err != nil { + return err + } + depositIndex := s.Eth1DepositIndex() + eth1Data := s.Eth1Data() + rawProof := []common.Hash{} + deposit.Proof.Range(func(_ int, l common.Hash, _ int) bool { + rawProof = append(rawProof, l) + return true + }) + // Validate merkle proof for deposit leaf. + if I.FullValidation && !utils.IsValidMerkleBranch( + depositLeaf, + rawProof, + s.BeaconConfig().DepositContractTreeDepth+1, + depositIndex, + eth1Data.Root, + ) { + return fmt.Errorf("processDepositForAltair: Could not validate deposit root") + } + + // Increment index + s.SetEth1DepositIndex(depositIndex + 1) + publicKey := deposit.Data.PubKey + amount := deposit.Data.Amount + // Check if pub key is in validator set + validatorIndex, has := s.ValidatorIndexByPubkey(publicKey) + if !has { + // Agnostic domain. + domain, err := fork.ComputeDomain(s.BeaconConfig().DomainDeposit[:], utils.Uint32ToBytes4(s.BeaconConfig().GenesisForkVersion), [32]byte{}) + if err != nil { + return err + } + depositMessageRoot, err := deposit.Data.MessageHash() + if err != nil { + return err + } + signedRoot := utils.Keccak256(depositMessageRoot[:], domain) + // Perform BLS verification and if successful noice. + valid, err := bls.Verify(deposit.Data.Signature[:], signedRoot[:], publicKey[:]) + // Literally you can input it trash. + if !valid || err != nil { + log.Debug("Validator BLS verification failed", "valid", valid, "err", err) + return nil + } + // Append validator + s.AddValidator(state.ValidatorFromDeposit(s.BeaconConfig(), deposit), amount) + // Altair forward + if s.Version() >= clparams.AltairVersion { + s.AddCurrentEpochParticipationFlags(cltypes.ParticipationFlags(0)) + s.AddPreviousEpochParticipationFlags(cltypes.ParticipationFlags(0)) + s.AddInactivityScore(0) + } + return nil + } + // Increase the balance if exists already + return state.IncreaseBalance(s.BeaconState, validatorIndex, amount) +} + +// ProcessVoluntaryExit takes a voluntary exit and applies state transition. +func (I *impl) ProcessVoluntaryExit(s *state.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit) error { + // Sanity checks so that we know it is good. + voluntaryExit := signedVoluntaryExit.VolunaryExit + currentEpoch := state.Epoch(s.BeaconState) + validator, err := s.ValidatorForValidatorIndex(int(voluntaryExit.ValidatorIndex)) + if err != nil { + return err + } + if !validator.Active(currentEpoch) { + return errors.New("ProcessVoluntaryExit: validator is not active") + } + if validator.ExitEpoch() != s.BeaconConfig().FarFutureEpoch { + return errors.New("ProcessVoluntaryExit: another exit for the same validator is already getting processed") + } + if currentEpoch < voluntaryExit.Epoch { + return errors.New("ProcessVoluntaryExit: exit is happening in the future") + } + if currentEpoch < validator.ActivationEpoch()+s.BeaconConfig().ShardCommitteePeriod { + return errors.New("ProcessVoluntaryExit: exit is happening too fast") + } + + // We can skip it in some instances if we want to optimistically sync up. + if I.FullValidation { + domain, err := s.GetDomain(s.BeaconConfig().DomainVoluntaryExit, voluntaryExit.Epoch) + if err != nil { + return err + } + signingRoot, err := fork.ComputeSigningRoot(voluntaryExit, domain) + if err != nil { + return err + } + pk := validator.PublicKey() + valid, err := bls.Verify(signedVoluntaryExit.Signature[:], signingRoot[:], pk[:]) + if err != nil { + return err + } + if !valid { + return errors.New("ProcessVoluntaryExit: BLS verification failed") + } + } + // Do the exit (same process in slashing). + return s.InitiateValidatorExit(voluntaryExit.ValidatorIndex) +} + +// ProcessWithdrawals processes withdrawals by decreasing the balance of each validator +// and updating the next withdrawal index and validator index. +func (I *impl) ProcessWithdrawals(s *state.BeaconState, withdrawals *solid.ListSSZ[*types.Withdrawal]) error { + // Get the list of withdrawals, the expected withdrawals (if performing full validation), + // and the beacon configuration. + beaconConfig := s.BeaconConfig() + numValidators := uint64(s.ValidatorLength()) + + // Check if full validation is required and verify expected withdrawals. + if I.FullValidation { + expectedWithdrawals := state.ExpectedWithdrawals(s.BeaconState) + if len(expectedWithdrawals) != withdrawals.Len() { + return fmt.Errorf("ProcessWithdrawals: expected %d withdrawals, but got %d", len(expectedWithdrawals), withdrawals.Len()) + } + if err := solid.RangeErr[*types.Withdrawal](withdrawals, func(i int, w *types.Withdrawal, _ int) error { + if !expectedWithdrawals[i].Equal(w) { + return fmt.Errorf("ProcessWithdrawals: withdrawal %d does not match expected withdrawal", i) + } + return nil + }); err != nil { + return err + } + } + + if err := solid.RangeErr[*types.Withdrawal](withdrawals, func(_ int, w *types.Withdrawal, _ int) error { + if err := state.DecreaseBalance(s.BeaconState, w.Validator, w.Amount); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + // Update next withdrawal index based on number of withdrawals. + if withdrawals.Len() > 0 { + lastWithdrawalIndex := withdrawals.Get(withdrawals.Len() - 1).Index + s.SetNextWithdrawalIndex(lastWithdrawalIndex + 1) + } + + // Update next withdrawal validator index based on number of withdrawals. + if withdrawals.Len() == int(beaconConfig.MaxWithdrawalsPerPayload) { + lastWithdrawalValidatorIndex := withdrawals.Get(withdrawals.Len()-1).Validator + 1 + s.SetNextWithdrawalValidatorIndex(lastWithdrawalValidatorIndex % numValidators) + } else { + nextIndex := s.NextWithdrawalValidatorIndex() + beaconConfig.MaxValidatorsPerWithdrawalsSweep + s.SetNextWithdrawalValidatorIndex(nextIndex % numValidators) + } + + return nil +} + +// ProcessExecutionPayload sets the latest payload header accordinly. +func (I *impl) ProcessExecutionPayload(s *state.BeaconState, payload *cltypes.Eth1Block) error { + if state.IsMergeTransitionComplete(s.BeaconState) { + if payload.ParentHash != s.LatestExecutionPayloadHeader().BlockHash { + return fmt.Errorf("ProcessExecutionPayload: invalid eth1 chain. mismatching parent") + } + } + if payload.PrevRandao != s.GetRandaoMixes(state.Epoch(s.BeaconState)) { + return fmt.Errorf("ProcessExecutionPayload: randao mix mismatches with mix digest") + } + if payload.Time != state.ComputeTimestampAtSlot(s.BeaconState, s.Slot()) { + return fmt.Errorf("ProcessExecutionPayload: invalid Eth1 timestamp") + } + payloadHeader, err := payload.PayloadHeader() + if err != nil { + return err + } + s.SetLatestExecutionPayloadHeader(payloadHeader) + return nil +} + +func (I *impl) ProcessSyncAggregate(s *state.BeaconState, sync *cltypes.SyncAggregate) error { + votedKeys, err := processSyncAggregate(s, sync) + if err != nil { + return err + } + if I.FullValidation { + previousSlot := s.PreviousSlot() + + domain, err := fork.Domain(s.Fork(), state.GetEpochAtSlot(s.BeaconConfig(), previousSlot), s.BeaconConfig().DomainSyncCommittee, s.GenesisValidatorsRoot()) + if err != nil { + return nil + } + blockRoot, err := s.GetBlockRootAtSlot(previousSlot) + if err != nil { + return err + } + msg := utils.Keccak256(blockRoot[:], domain) + isValid, err := bls.VerifyAggregate(sync.SyncCommiteeSignature[:], msg[:], votedKeys) + if err != nil { + return err + } + if !isValid { + return errors.New("ProcessSyncAggregate: cannot validate sync committee signature") + } + } + return nil +} + +// processSyncAggregate applies all the logic in the spec function `process_sync_aggregate` except +// verifying the BLS signatures. It returns the modified beacons state and the list of validators' +// public keys that voted, for future signature verification. +func processSyncAggregate(s *state.BeaconState, sync *cltypes.SyncAggregate) ([][]byte, error) { + currentSyncCommittee := s.CurrentSyncCommittee() + + if currentSyncCommittee == nil { + return nil, errors.New("nil current sync committee in s") + } + committeeKeys := currentSyncCommittee.GetCommittee() + if len(sync.SyncCommiteeBits)*8 > len(committeeKeys) { + return nil, errors.New("bits length exceeds committee length") + } + var votedKeys [][]byte + + proposerReward, participantReward, err := s.SyncRewards() + if err != nil { + return nil, err + } + + proposerIndex, err := s.GetBeaconProposerIndex() + if err != nil { + return nil, err + } + + syncAggregateBits := sync.SyncCommiteeBits + earnedProposerReward := uint64(0) + currPubKeyIndex := 0 + for i := range syncAggregateBits { + for bit := 1; bit <= 128; bit *= 2 { + vIdx, exists := s.ValidatorIndexByPubkey(committeeKeys[currPubKeyIndex]) + // Impossible scenario. + if !exists { + return nil, errors.New("validator public key does not exist in state") + } + if syncAggregateBits[i]&byte(bit) > 0 { + votedKeys = append(votedKeys, committeeKeys[currPubKeyIndex][:]) + if err := state.IncreaseBalance(s.BeaconState, vIdx, participantReward); err != nil { + return nil, err + } + earnedProposerReward += proposerReward + } else { + if err := state.DecreaseBalance(s.BeaconState, vIdx, participantReward); err != nil { + return nil, err + } + } + currPubKeyIndex++ + } + } + + return votedKeys, state.IncreaseBalance(s.BeaconState, proposerIndex, earnedProposerReward) +} + +// ProcessBlsToExecutionChange processes a BLSToExecutionChange message by updating a validator's withdrawal credentials. +func (I *impl) ProcessBlsToExecutionChange(s *state.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange) error { + change := signedChange.Message + + beaconConfig := s.BeaconConfig() + validator, err := s.ValidatorForValidatorIndex(int(change.ValidatorIndex)) + if err != nil { + return err + } + + // Perform full validation if requested. + wc := validator.WithdrawalCredentials() + if I.FullValidation { + // Check the validator's withdrawal credentials prefix. + if wc[0] != beaconConfig.BLSWithdrawalPrefixByte { + return fmt.Errorf("invalid withdrawal credentials prefix") + } + + // Check the validator's withdrawal credentials against the provided message. + hashedFrom := utils.Keccak256(change.From[:]) + if !bytes.Equal(hashedFrom[1:], wc[1:]) { + return fmt.Errorf("invalid withdrawal credentials") + } + + // Compute the signing domain and verify the message signature. + domain, err := fork.ComputeDomain(beaconConfig.DomainBLSToExecutionChange[:], utils.Uint32ToBytes4(beaconConfig.GenesisForkVersion), s.GenesisValidatorsRoot()) + if err != nil { + return err + } + signedRoot, err := fork.ComputeSigningRoot(change, domain) + if err != nil { + return err + } + valid, err := bls.Verify(signedChange.Signature[:], signedRoot[:], change.From[:]) + if err != nil { + return err + } + if !valid { + return fmt.Errorf("invalid signature") + } + } + credentials := wc + // Reset the validator's withdrawal credentials. + credentials[0] = beaconConfig.ETH1AddressWithdrawalPrefixByte + copy(credentials[1:], make([]byte, 11)) + copy(credentials[12:], change.To[:]) + + // Update the state with the modified validator. + s.SetWithdrawalCredentialForValidatorAtIndex(int(change.ValidatorIndex), credentials) + return nil +} + +func (I *impl) VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) { + if I.FullValidation { + return true, nil + } + allVersionedHashes := []common.Hash{} + transactions.ForEach(func(tx []byte, idx, total int) bool { + if tx[0] != types.BlobTxType { + return true + } + + allVersionedHashes = append(allVersionedHashes, txPeekBlobVersionedHashes(tx)...) + return true + }) + + commitmentVersionedHash := []common.Hash{} + var err error + var versionedHash common.Hash + kzgCommitments.Range(func(index int, value *cltypes.KZGCommitment, length int) bool { + versionedHash, err = kzgCommitmentToVersionedHash(value) + if err != nil { + return false + } + + commitmentVersionedHash = append(commitmentVersionedHash, versionedHash) + return true + }) + if err != nil { + return false, err + } + + return reflect.DeepEqual(allVersionedHashes, commitmentVersionedHash), nil +} + +func (I *impl) ProcessAttestations(s *state.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error { + attestingIndiciesSet := make([][]uint64, attestations.Len()) + h := methelp.NewHistTimer("beacon_process_attestations") + baseRewardPerIncrement := s.BaseRewardPerIncrement() + + c := h.Tag("attestation_step", "process") + var err error + if err := solid.RangeErr[*solid.Attestation](attestations, func(i int, a *solid.Attestation, _ int) error { + if attestingIndiciesSet[i], err = processAttestation(s, a, baseRewardPerIncrement); err != nil { + return err + } + return nil + }); err != nil { + return err + } + if err != nil { + return err + } + var valid bool + c.PutSince() + if I.FullValidation { + c = h.Tag("attestation_step", "validate") + valid, err = verifyAttestations(s, attestations, attestingIndiciesSet) + if err != nil { + return err + } + if !valid { + return errors.New("ProcessAttestation: wrong bls data") + } + c.PutSince() + } + + return nil +} + +func processAttestationPostAltair(s *state.BeaconState, attestation *solid.Attestation, baseRewardPerIncrement uint64) ([]uint64, error) { + data := attestation.AttestantionData() + currentEpoch := state.Epoch(s.BeaconState) + stateSlot := s.Slot() + beaconConfig := s.BeaconConfig() + + h := methelp.NewHistTimer("beacon_process_attestation_post_altair") + + c := h.Tag("step", "get_participation_flag") + participationFlagsIndicies, err := s.GetAttestationParticipationFlagIndicies(attestation.AttestantionData(), stateSlot-data.Slot()) + if err != nil { + return nil, err + } + c.PutSince() + + c = h.Tag("step", "get_attesting_indices") + + attestingIndicies, err := s.GetAttestingIndicies(attestation.AttestantionData(), attestation.AggregationBits(), true) + if err != nil { + return nil, err + } + + c.PutSince() + + var proposerRewardNumerator uint64 + + isCurrentEpoch := data.Target().Epoch() == currentEpoch + + c = h.Tag("step", "update_attestation") + for _, attesterIndex := range attestingIndicies { + val, err := s.ValidatorEffectiveBalance(int(attesterIndex)) + if err != nil { + return nil, err + } + + baseReward := (val / beaconConfig.EffectiveBalanceIncrement) * baseRewardPerIncrement + for flagIndex, weight := range beaconConfig.ParticipationWeights() { + flagParticipation := s.EpochParticipationForValidatorIndex(isCurrentEpoch, int(attesterIndex)) + if !slices.Contains(participationFlagsIndicies, uint8(flagIndex)) || flagParticipation.HasFlag(flagIndex) { + continue + } + s.SetEpochParticipationForValidatorIndex(isCurrentEpoch, int(attesterIndex), flagParticipation.Add(flagIndex)) + proposerRewardNumerator += baseReward * weight + } + } + c.PutSince() + // Reward proposer + c = h.Tag("step", "get_proposer_index") + proposer, err := s.GetBeaconProposerIndex() + if err != nil { + return nil, err + } + c.PutSince() + proposerRewardDenominator := (beaconConfig.WeightDenominator - beaconConfig.ProposerWeight) * beaconConfig.WeightDenominator / beaconConfig.ProposerWeight + reward := proposerRewardNumerator / proposerRewardDenominator + return attestingIndicies, state.IncreaseBalance(s.BeaconState, proposer, reward) +} + +// processAttestationsPhase0 implements the rules for phase0 processing. +func processAttestationPhase0(s *state.BeaconState, attestation *solid.Attestation) ([]uint64, error) { + data := attestation.AttestantionData() + committee, err := s.GetBeaconCommitee(data.Slot(), data.ValidatorIndex()) + if err != nil { + return nil, err + } + + if len(committee) != utils.GetBitlistLength(attestation.AggregationBits()) { + return nil, fmt.Errorf("processAttestationPhase0: mismatching aggregation bits size") + } + // Cached so it is performant. + proposerIndex, err := s.GetBeaconProposerIndex() + if err != nil { + return nil, err + } + // Create the attestation to add to pending attestations + pendingAttestation := solid.NewPendingAttestionFromParameters( + attestation.AggregationBits(), + data, + s.Slot()-data.Slot(), + proposerIndex, + ) + + isCurrentAttestation := data.Target().Epoch() == state.Epoch(s.BeaconState) + // Depending of what slot we are on we put in either the current justified or previous justified. + if isCurrentAttestation { + if !data.Source().Equal(s.CurrentJustifiedCheckpoint()) { + return nil, fmt.Errorf("processAttestationPhase0: mismatching sources") + } + s.AddCurrentEpochAtteastation(pendingAttestation) + } else { + if !data.Source().Equal(s.PreviousJustifiedCheckpoint()) { + return nil, fmt.Errorf("processAttestationPhase0: mismatching sources") + } + s.AddPreviousEpochAttestation(pendingAttestation) + } + // Not required by specs but needed if we want performant epoch transition. + indicies, err := s.GetAttestingIndicies(attestation.AttestantionData(), attestation.AggregationBits(), true) + if err != nil { + return nil, err + } + epochRoot, err := state.GetBlockRoot(s.BeaconState, attestation.AttestantionData().Target().Epoch()) + if err != nil { + return nil, err + } + slotRoot, err := s.GetBlockRootAtSlot(attestation.AttestantionData().Slot()) + if err != nil { + return nil, err + } + // Basically we flag all validators we are currently attesting. will be important for rewards/finalization processing. + for _, index := range indicies { + minCurrentInclusionDelayAttestation, err := s.ValidatorMinCurrentInclusionDelayAttestation(int(index)) + if err != nil { + return nil, err + } + + minPreviousInclusionDelayAttestation, err := s.ValidatorMinPreviousInclusionDelayAttestation(int(index)) + if err != nil { + return nil, err + } + // NOTE: does not affect state root. + // We need to set it to currents or previouses depending on which attestation we process. + if isCurrentAttestation { + if minCurrentInclusionDelayAttestation == nil || + minCurrentInclusionDelayAttestation.InclusionDelay() > pendingAttestation.InclusionDelay() { + if err := s.SetValidatorMinCurrentInclusionDelayAttestation(int(index), pendingAttestation); err != nil { + return nil, err + } + } + if err := s.SetValidatorIsCurrentMatchingSourceAttester(int(index), true); err != nil { + return nil, err + } + if attestation.AttestantionData().Target().BlockRoot() == epochRoot { + if err := s.SetValidatorIsCurrentMatchingTargetAttester(int(index), true); err != nil { + return nil, err + } + } else { + continue + } + if attestation.AttestantionData().BeaconBlockRoot() == slotRoot { + if err := s.SetValidatorIsCurrentMatchingHeadAttester(int(index), true); err != nil { + return nil, err + } + } + } else { + if minPreviousInclusionDelayAttestation == nil || + minPreviousInclusionDelayAttestation.InclusionDelay() > pendingAttestation.InclusionDelay() { + if err := s.SetValidatorMinPreviousInclusionDelayAttestation(int(index), pendingAttestation); err != nil { + return nil, err + } + } + if err := s.SetValidatorIsPreviousMatchingSourceAttester(int(index), true); err != nil { + return nil, err + } + if attestation.AttestantionData().Target().BlockRoot() != epochRoot { + continue + } + if err := s.SetValidatorIsPreviousMatchingTargetAttester(int(index), true); err != nil { + return nil, err + } + if attestation.AttestantionData().BeaconBlockRoot() == slotRoot { + if err := s.SetValidatorIsPreviousMatchingHeadAttester(int(index), true); err != nil { + return nil, err + } + } + } + } + return indicies, nil +} + +// ProcessAttestation takes an attestation and process it. +func processAttestation(s *state.BeaconState, attestation *solid.Attestation, baseRewardPerIncrement uint64) ([]uint64, error) { + data := attestation.AttestantionData() + currentEpoch := state.Epoch(s.BeaconState) + previousEpoch := state.PreviousEpoch(s.BeaconState) + stateSlot := s.Slot() + beaconConfig := s.BeaconConfig() + // Prelimary checks. + if (data.Target().Epoch() != currentEpoch && data.Target().Epoch() != previousEpoch) || data.Target().Epoch() != state.GetEpochAtSlot(s.BeaconConfig(), data.Slot()) { + return nil, errors.New("ProcessAttestation: attestation with invalid epoch") + } + if data.Slot()+beaconConfig.MinAttestationInclusionDelay > stateSlot || stateSlot > data.Slot()+beaconConfig.SlotsPerEpoch { + return nil, errors.New("ProcessAttestation: attestation slot not in range") + } + if data.ValidatorIndex() >= s.CommitteeCount(data.Target().Epoch()) { + return nil, errors.New("ProcessAttestation: attester index out of range") + } + // check if we need to use rules for phase0 or post-altair. + if s.Version() == clparams.Phase0Version { + return processAttestationPhase0(s, attestation) + } + return processAttestationPostAltair(s, attestation, baseRewardPerIncrement) +} + +func verifyAttestations(s *state.BeaconState, attestations *solid.ListSSZ[*solid.Attestation], attestingIndicies [][]uint64) (bool, error) { + indexedAttestations := make([]*cltypes.IndexedAttestation, 0, attestations.Len()) + attestations.Range(func(idx int, a *solid.Attestation, _ int) bool { + indexedAttestations = append(indexedAttestations, state.GetIndexedAttestation(a, attestingIndicies[idx])) + return true + }) + + return batchVerifyAttestations(s, indexedAttestations) +} + +type indexedAttestationVerificationResult struct { + valid bool + err error +} + +// Concurrent verification of BLS. +func batchVerifyAttestations(s *state.BeaconState, indexedAttestations []*cltypes.IndexedAttestation) (valid bool, err error) { + c := make(chan indexedAttestationVerificationResult, 1) + + for idx := range indexedAttestations { + go func(idx int) { + valid, err := state.IsValidIndexedAttestation(s.BeaconState, indexedAttestations[idx]) + c <- indexedAttestationVerificationResult{ + valid: valid, + err: err, + } + }(idx) + } + for i := 0; i < len(indexedAttestations); i++ { + result := <-c + if result.err != nil { + return false, err + } + if !result.valid { + return false, nil + } + } + return true, nil +} + +func (I *impl) ProcessBlockHeader(s *state.BeaconState, block *cltypes.BeaconBlock) error { + if I.FullValidation { + if block.Slot != s.Slot() { + return fmt.Errorf("state slot: %d, not equal to block slot: %d", s.Slot(), block.Slot) + } + if block.Slot <= s.LatestBlockHeader().Slot { + return fmt.Errorf("slock slot: %d, not greater than latest block slot: %d", block.Slot, s.LatestBlockHeader().Slot) + } + propInd, err := s.GetBeaconProposerIndex() + if err != nil { + return fmt.Errorf("error in GetBeaconProposerIndex: %v", err) + } + if block.ProposerIndex != propInd { + return fmt.Errorf("block proposer index: %d, does not match beacon proposer index: %d", block.ProposerIndex, propInd) + } + blockHeader := s.LatestBlockHeader() + latestRoot, err := (&blockHeader).HashSSZ() + if err != nil { + return fmt.Errorf("unable to hash tree root of latest block header: %v", err) + } + if block.ParentRoot != latestRoot { + return fmt.Errorf("block parent root: %x, does not match latest block root: %x", block.ParentRoot, latestRoot) + } + } + + bodyRoot, err := block.Body.HashSSZ() + if err != nil { + return fmt.Errorf("unable to hash tree root of block body: %v", err) + } + s.SetLatestBlockHeader(&cltypes.BeaconBlockHeader{ + Slot: block.Slot, + ProposerIndex: block.ProposerIndex, + ParentRoot: block.ParentRoot, + BodyRoot: bodyRoot, + }) + + proposer, err := s.ValidatorForValidatorIndex(int(block.ProposerIndex)) + if err != nil { + return err + } + if proposer.Slashed() { + return fmt.Errorf("proposer: %d is slashed", block.ProposerIndex) + } + return nil +} + +func (I *impl) ProcessRandao(s *state.BeaconState, randao [96]byte, proposerIndex uint64) error { + epoch := state.Epoch(s.BeaconState) + proposer, err := s.ValidatorForValidatorIndex(int(proposerIndex)) + if err != nil { + return err + } + if I.FullValidation { + domain, err := s.GetDomain(s.BeaconConfig().DomainRandao, epoch) + if err != nil { + return fmt.Errorf("ProcessRandao: unable to get domain: %v", err) + } + signingRoot, err := computeSigningRootEpoch(epoch, domain) + if err != nil { + return fmt.Errorf("ProcessRandao: unable to compute signing root: %v", err) + } + pk := proposer.PublicKey() + valid, err := bls.Verify(randao[:], signingRoot[:], pk[:]) + if err != nil { + return fmt.Errorf("ProcessRandao: unable to verify public key: %x, with signing root: %x, and signature: %x, %v", pk[:], signingRoot[:], randao[:], err) + } + if !valid { + return fmt.Errorf("ProcessRandao: invalid signature: public key: %x, signing root: %x, signature: %x", pk[:], signingRoot[:], randao[:]) + } + } + + randaoMixes := s.GetRandaoMixes(epoch) + randaoHash := utils.Keccak256(randao[:]) + mix := [32]byte{} + for i := range mix { + mix[i] = randaoMixes[i] ^ randaoHash[i] + } + s.SetRandaoMixAt(int(epoch%s.BeaconConfig().EpochsPerHistoricalVector), mix) + return nil +} + +func (I *impl) ProcessEth1Data(state *state.BeaconState, eth1Data *cltypes.Eth1Data) error { + state.AddEth1DataVote(eth1Data) + newVotes := state.Eth1DataVotes() + + // Count how many times body.Eth1Data appears in the votes. + numVotes := 0 + newVotes.Range(func(index int, value *cltypes.Eth1Data, length int) bool { + if eth1Data.Equal(value) { + numVotes += 1 + } + return true + }) + + if uint64(numVotes*2) > state.BeaconConfig().EpochsPerEth1VotingPeriod*state.BeaconConfig().SlotsPerEpoch { + state.SetEth1Data(eth1Data) + } + return nil +} + +func (I *impl) ProcessSlots(s *state.BeaconState, slot uint64) error { + beaconConfig := s.BeaconConfig() + sSlot := s.Slot() + if slot <= sSlot { + return fmt.Errorf("new slot: %d not greater than s slot: %d", slot, sSlot) + } + // Process each slot. + for i := sSlot; i < slot; i++ { + err := transitionSlot(s) + if err != nil { + return fmt.Errorf("unable to process slot transition: %v", err) + } + // TODO(Someone): Add epoch transition. + if (sSlot+1)%beaconConfig.SlotsPerEpoch == 0 { + start := time.Now() + if err := statechange.ProcessEpoch(s); err != nil { + return err + } + log.Debug("Processed new epoch successfully", "epoch", state.Epoch(s.BeaconState), "process_epoch_elpsed", time.Since(start)) + } + // TODO: add logic to process epoch updates. + sSlot += 1 + s.SetSlot(sSlot) + if sSlot%beaconConfig.SlotsPerEpoch != 0 { + continue + } + if state.Epoch(s.BeaconState) == beaconConfig.AltairForkEpoch { + if err := s.UpgradeToAltair(); err != nil { + return err + } + } + if state.Epoch(s.BeaconState) == beaconConfig.BellatrixForkEpoch { + if err := s.UpgradeToBellatrix(); err != nil { + return err + } + } + if state.Epoch(s.BeaconState) == beaconConfig.CapellaForkEpoch { + if err := s.UpgradeToCapella(); err != nil { + return err + } + } + if state.Epoch(s.BeaconState) == beaconConfig.DenebForkEpoch { + if err := s.UpgradeToDeneb(); err != nil { + return err + } + } + } + return nil +} diff --git a/cl/phase1/core/transition/finalization_and_justification.go b/cl/transition/impl/eth2/statechange/finalization_and_justification.go similarity index 86% rename from cl/phase1/core/transition/finalization_and_justification.go rename to cl/transition/impl/eth2/statechange/finalization_and_justification.go index df905e8485e..d4f625d1518 100644 --- a/cl/phase1/core/transition/finalization_and_justification.go +++ b/cl/transition/impl/eth2/statechange/finalization_and_justification.go @@ -1,17 +1,17 @@ -package transition +package statechange import ( "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" ) // weighJustificationAndFinalization checks justification and finality of epochs and adds records to the state as needed. -func weighJustificationAndFinalization(s *state2.BeaconState, previousEpochTargetBalance, currentEpochTargetBalance uint64) error { +func weighJustificationAndFinalization(s *state.BeaconState, previousEpochTargetBalance, currentEpochTargetBalance uint64) error { totalActiveBalance := s.GetTotalActiveBalance() - currentEpoch := state2.Epoch(s.BeaconState) - previousEpoch := state2.PreviousEpoch(s.BeaconState) + currentEpoch := state.Epoch(s.BeaconState) + previousEpoch := state.PreviousEpoch(s.BeaconState) oldPreviousJustifiedCheckpoint := s.PreviousJustifiedCheckpoint() oldCurrentJustifiedCheckpoint := s.CurrentJustifiedCheckpoint() justificationBits := s.JustificationBits() @@ -23,7 +23,7 @@ func weighJustificationAndFinalization(s *state2.BeaconState, previousEpochTarge justificationBits[0] = false // Update justified checkpoint if super majority is reached on previous epoch if previousEpochTargetBalance*3 >= totalActiveBalance*2 { - checkPointRoot, err := state2.GetBlockRoot(s.BeaconState, previousEpoch) + checkPointRoot, err := state.GetBlockRoot(s.BeaconState, previousEpoch) if err != nil { return err } @@ -32,7 +32,7 @@ func weighJustificationAndFinalization(s *state2.BeaconState, previousEpochTarge justificationBits[1] = true } if currentEpochTargetBalance*3 >= totalActiveBalance*2 { - checkPointRoot, err := state2.GetBlockRoot(s.BeaconState, currentEpoch) + checkPointRoot, err := state.GetBlockRoot(s.BeaconState, currentEpoch) if err != nil { return err } @@ -58,9 +58,9 @@ func weighJustificationAndFinalization(s *state2.BeaconState, previousEpochTarge return nil } -func ProcessJustificationBitsAndFinality(s *state2.BeaconState) error { - currentEpoch := state2.Epoch(s.BeaconState) - previousEpoch := state2.PreviousEpoch(s.BeaconState) +func ProcessJustificationBitsAndFinality(s *state.BeaconState) error { + currentEpoch := state.Epoch(s.BeaconState) + previousEpoch := state.PreviousEpoch(s.BeaconState) beaconConfig := s.BeaconConfig() // Skip for first 2 epochs if currentEpoch <= beaconConfig.GenesisEpoch+1 { diff --git a/cl/phase1/core/transition/process_effective_balance_update.go b/cl/transition/impl/eth2/statechange/process_effective_balance_update.go similarity index 98% rename from cl/phase1/core/transition/process_effective_balance_update.go rename to cl/transition/impl/eth2/statechange/process_effective_balance_update.go index 7430bd66a1e..ce42a2cceff 100644 --- a/cl/phase1/core/transition/process_effective_balance_update.go +++ b/cl/transition/impl/eth2/statechange/process_effective_balance_update.go @@ -1,4 +1,4 @@ -package transition +package statechange import ( "github.com/ledgerwatch/erigon/cl/cltypes/solid" diff --git a/cl/transition/impl/eth2/statechange/process_epoch.go b/cl/transition/impl/eth2/statechange/process_epoch.go new file mode 100644 index 00000000000..fe103ea3075 --- /dev/null +++ b/cl/transition/impl/eth2/statechange/process_epoch.go @@ -0,0 +1,49 @@ +package statechange + +import ( + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" +) + +// ProcessEpoch process epoch transition. +func ProcessEpoch(state *state.BeaconState) error { + if err := ProcessJustificationBitsAndFinality(state); err != nil { + return err + } + if state.Version() >= clparams.AltairVersion { + if err := ProcessInactivityScores(state); err != nil { + return err + } + } + if err := ProcessRewardsAndPenalties(state); err != nil { + return err + } + if err := ProcessRegistryUpdates(state); err != nil { + return err + } + if err := ProcessSlashings(state); err != nil { + return err + } + ProcessEth1DataReset(state) + if err := ProcessEffectiveBalanceUpdates(state); err != nil { + return err + } + ProcessSlashingsReset(state) + ProcessRandaoMixesReset(state) + if err := ProcessHistoricalRootsUpdate(state); err != nil { + return err + } + if state.Version() == clparams.Phase0Version { + if err := ProcessParticipationRecordUpdates(state); err != nil { + return err + } + } + + if state.Version() >= clparams.AltairVersion { + ProcessParticipationFlagUpdates(state) + if err := ProcessSyncCommitteeUpdate(state); err != nil { + return err + } + } + return nil +} diff --git a/cl/phase1/core/transition/process_epoch_test.go b/cl/transition/impl/eth2/statechange/process_epoch_test.go similarity index 90% rename from cl/phase1/core/transition/process_epoch_test.go rename to cl/transition/impl/eth2/statechange/process_epoch_test.go index 4fc13b4dc1b..19a13e3ef2e 100644 --- a/cl/phase1/core/transition/process_epoch_test.go +++ b/cl/transition/impl/eth2/statechange/process_epoch_test.go @@ -1,13 +1,11 @@ -package transition_test +package statechange import ( _ "embed" "testing" - "github.com/ledgerwatch/erigon/cl/phase1/core/state" - transition2 "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/utils" "github.com/stretchr/testify/require" ) @@ -91,64 +89,64 @@ var startingSlashingsResetState []byte func TestProcessRewardsAndPenalties(t *testing.T) { runEpochTransitionConsensusTest(t, startingRewardsPenaltyState, expectedRewardsPenaltyState, func(s *state.BeaconState) error { - return transition2.ProcessRewardsAndPenalties(s) + return ProcessRewardsAndPenalties(s) }) } func TestProcessRegistryUpdates(t *testing.T) { runEpochTransitionConsensusTest(t, startingRegistryUpdatesState, expectedRegistryUpdatesState, func(s *state.BeaconState) error { - return transition2.ProcessRegistryUpdates(s) + return ProcessRegistryUpdates(s) }) } func TestProcessEffectiveBalances(t *testing.T) { runEpochTransitionConsensusTest(t, startingEffectiveBalancesState, expectedEffectiveBalancesState, func(s *state.BeaconState) error { - return transition2.ProcessEffectiveBalanceUpdates(s) + return ProcessEffectiveBalanceUpdates(s) }) } func TestProcessHistoricalRoots(t *testing.T) { runEpochTransitionConsensusTest(t, startingHistoricalRootsState, expectedHistoricalRootsState, func(s *state.BeaconState) error { - return transition2.ProcessHistoricalRootsUpdate(s) + return ProcessHistoricalRootsUpdate(s) }) } func TestProcessParticipationFlagUpdates(t *testing.T) { runEpochTransitionConsensusTest(t, startingParticipationFlagState, expectedParticipationFlagState, func(s *state.BeaconState) error { - transition2.ProcessParticipationFlagUpdates(s) + ProcessParticipationFlagUpdates(s) return nil }) } func TestProcessSlashings(t *testing.T) { runEpochTransitionConsensusTest(t, startingSlashingsState, expectedSlashingsState, func(s *state.BeaconState) error { - return transition2.ProcessSlashings(s) + return ProcessSlashings(s) }) } func TestProcessJustificationAndFinality(t *testing.T) { runEpochTransitionConsensusTest(t, startingJustificationAndFinalityState, expectedJustificationAndFinalityState, func(s *state.BeaconState) error { - return transition2.ProcessJustificationBitsAndFinality(s) + return ProcessJustificationBitsAndFinality(s) }) } func TestEth1DataReset(t *testing.T) { runEpochTransitionConsensusTest(t, startingEth1DataResetState, expectedEth1DataResetState, func(s *state.BeaconState) error { - transition2.ProcessEth1DataReset(s) + ProcessEth1DataReset(s) return nil }) } func TestRandaoMixesReset(t *testing.T) { runEpochTransitionConsensusTest(t, startingRandaoMixesResetState, expectedRandaoMixesResetState, func(s *state.BeaconState) error { - transition2.ProcessRandaoMixesReset(s) + ProcessRandaoMixesReset(s) return nil }) } func TestSlashingsReset(t *testing.T) { runEpochTransitionConsensusTest(t, startingSlashingsResetState, expectedSlashingsResetState, func(s *state.BeaconState) error { - transition2.ProcessSlashingsReset(s) + ProcessSlashingsReset(s) return nil }) } @@ -161,6 +159,6 @@ var startingInactivityScoresState []byte func TestInactivityScores(t *testing.T) { runEpochTransitionConsensusTest(t, startingInactivityScoresState, expectedInactivityScoresState, func(s *state.BeaconState) error { - return transition2.ProcessInactivityScores(s) + return ProcessInactivityScores(s) }) } diff --git a/cl/transition/impl/eth2/statechange/process_historical_roots.go b/cl/transition/impl/eth2/statechange/process_historical_roots.go new file mode 100644 index 00000000000..9bb26f505c5 --- /dev/null +++ b/cl/transition/impl/eth2/statechange/process_historical_roots.go @@ -0,0 +1,59 @@ +package statechange + +import ( + "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" +) + +func ProcessParticipationRecordUpdates(s *state.BeaconState) error { + s.SetPreviousEpochAttestations(s.CurrentEpochAttestations()) + s.ResetCurrentEpochAttestations() + var err error + // Also mark all current attesters as previous + s.ForEachValidator(func(_ solid.Validator, idx, total int) bool { + + var oldCurrentMatchingSourceAttester, oldCurrentMatchingTargetAttester, oldCurrentMatchingHeadAttester bool + var oldMinCurrentInclusionDelayAttestation *solid.PendingAttestation + + if oldCurrentMatchingSourceAttester, err = s.ValidatorIsCurrentMatchingSourceAttester(idx); err != nil { + return false + } + if oldCurrentMatchingTargetAttester, err = s.ValidatorIsCurrentMatchingTargetAttester(idx); err != nil { + return false + } + if oldCurrentMatchingHeadAttester, err = s.ValidatorIsCurrentMatchingHeadAttester(idx); err != nil { + return false + } + if oldMinCurrentInclusionDelayAttestation, err = s.ValidatorMinCurrentInclusionDelayAttestation(idx); err != nil { + return false + } + // Previous sources/target/head + if err = s.SetValidatorIsPreviousMatchingSourceAttester(idx, oldCurrentMatchingSourceAttester); err != nil { + return false + } + if err = s.SetValidatorIsPreviousMatchingTargetAttester(idx, oldCurrentMatchingTargetAttester); err != nil { + return false + } + if err = s.SetValidatorIsPreviousMatchingHeadAttester(idx, oldCurrentMatchingHeadAttester); err != nil { + return false + } + if err = s.SetValidatorMinPreviousInclusionDelayAttestation(idx, oldMinCurrentInclusionDelayAttestation); err != nil { + return false + } + // Current sources/target/head + if err = s.SetValidatorIsCurrentMatchingSourceAttester(idx, false); err != nil { + return false + } + if err = s.SetValidatorIsCurrentMatchingTargetAttester(idx, false); err != nil { + return false + } + if err = s.SetValidatorIsCurrentMatchingHeadAttester(idx, false); err != nil { + return false + } + if err = s.SetValidatorMinCurrentInclusionDelayAttestation(idx, nil); err != nil { + return false + } + return true + }) + return err +} diff --git a/cl/transition/impl/eth2/statechange/process_historical_roots_update.go b/cl/transition/impl/eth2/statechange/process_historical_roots_update.go new file mode 100644 index 00000000000..9490a1c91c2 --- /dev/null +++ b/cl/transition/impl/eth2/statechange/process_historical_roots_update.go @@ -0,0 +1,44 @@ +package statechange + +import ( + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/utils" +) + +// ProcessHistoricalRootsUpdate updates the historical root data structure by computing a new historical root batch when it is time to do so. +func ProcessHistoricalRootsUpdate(s *state.BeaconState) error { + nextEpoch := state.Epoch(s.BeaconState) + 1 + beaconConfig := s.BeaconConfig() + blockRoots := s.BlockRoots() + stateRoots := s.StateRoots() + + // Check if it's time to compute the historical root batch. + if nextEpoch%(beaconConfig.SlotsPerHistoricalRoot/beaconConfig.SlotsPerEpoch) != 0 { + return nil + } + + // Compute historical root batch. + blockRootsLeaf, err := blockRoots.HashSSZ() + if err != nil { + return err + } + stateRootsLeaf, err := stateRoots.HashSSZ() + if err != nil { + return err + } + + // Add the historical summary or root to the s. + if s.Version() >= clparams.CapellaVersion { + s.AddHistoricalSummary(&cltypes.HistoricalSummary{ + BlockSummaryRoot: blockRootsLeaf, + StateSummaryRoot: stateRootsLeaf, + }) + } else { + historicalRoot := utils.Keccak256(blockRootsLeaf[:], stateRootsLeaf[:]) + s.AddHistoricalRoot(historicalRoot) + } + + return nil +} diff --git a/cl/phase1/core/transition/process_inactivity_scores.go b/cl/transition/impl/eth2/statechange/process_inactivity_scores.go similarity index 53% rename from cl/phase1/core/transition/process_inactivity_scores.go rename to cl/transition/impl/eth2/statechange/process_inactivity_scores.go index 0dbb8f4388d..70b5a5a3d81 100644 --- a/cl/phase1/core/transition/process_inactivity_scores.go +++ b/cl/transition/impl/eth2/statechange/process_inactivity_scores.go @@ -1,28 +1,28 @@ -package transition +package statechange import ( - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/utils" ) // ProcessInactivityScores will updates the inactivity registry of each validator. -func ProcessInactivityScores(s *state2.BeaconState) error { - if state2.Epoch(s.BeaconState) == s.BeaconConfig().GenesisEpoch { +func ProcessInactivityScores(s *state.BeaconState) error { + if state.Epoch(s.BeaconState) == s.BeaconConfig().GenesisEpoch { return nil } - previousEpoch := state2.PreviousEpoch(s.BeaconState) - for _, validatorIndex := range state2.EligibleValidatorsIndicies(s.BeaconState) { + previousEpoch := state.PreviousEpoch(s.BeaconState) + for _, validatorIndex := range state.EligibleValidatorsIndicies(s.BeaconState) { // retrieve validator inactivity score index. score, err := s.ValidatorInactivityScore(int(validatorIndex)) if err != nil { return err } - if state2.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, validatorIndex, int(s.BeaconConfig().TimelyTargetFlagIndex)) { + if state.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, validatorIndex, int(s.BeaconConfig().TimelyTargetFlagIndex)) { score -= utils.Min64(1, score) } else { score += s.BeaconConfig().InactivityScoreBias } - if !state2.InactivityLeaking(s.BeaconState) { + if !state.InactivityLeaking(s.BeaconState) { score -= utils.Min64(s.BeaconConfig().InactivityScoreRecoveryRate, score) } if err := s.SetValidatorInactivityScore(int(validatorIndex), score); err != nil { diff --git a/cl/phase1/core/transition/process_registry_updates.go b/cl/transition/impl/eth2/statechange/process_registry_updates.go similarity index 87% rename from cl/phase1/core/transition/process_registry_updates.go rename to cl/transition/impl/eth2/statechange/process_registry_updates.go index ae33d154da5..8a3eda87d99 100644 --- a/cl/phase1/core/transition/process_registry_updates.go +++ b/cl/transition/impl/eth2/statechange/process_registry_updates.go @@ -1,10 +1,10 @@ -package transition +package statechange import ( "sort" "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/clparams" ) @@ -15,15 +15,15 @@ func computeActivationExitEpoch(beaconConfig *clparams.BeaconChainConfig, epoch } // ProcessRegistyUpdates updates every epoch the activation status of validators. Specs at: https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#registry-updates. -func ProcessRegistryUpdates(s *state2.BeaconState) error { +func ProcessRegistryUpdates(s *state.BeaconState) error { beaconConfig := s.BeaconConfig() - currentEpoch := state2.Epoch(s.BeaconState) + currentEpoch := state.Epoch(s.BeaconState) // start also initializing the activation queue. activationQueue := make([]uint64, 0) // Process activation eligibility and ejections. var err error s.ForEachValidator(func(validator solid.Validator, validatorIndex, total int) bool { - if state2.IsValidatorEligibleForActivationQueue(s.BeaconState, validator) { + if state.IsValidatorEligibleForActivationQueue(s.BeaconState, validator) { s.SetActivationEligibilityEpochForValidatorAtIndex(validatorIndex, currentEpoch+1) } if validator.Active(currentEpoch) && validator.EffectiveBalance() <= beaconConfig.EjectionBalance { @@ -32,7 +32,7 @@ func ProcessRegistryUpdates(s *state2.BeaconState) error { } } // Insert in the activation queue in case. - if state2.IsValidatorEligibleForActivation(s.BeaconState, validator) { + if state.IsValidatorEligibleForActivation(s.BeaconState, validator) { activationQueue = append(activationQueue, uint64(validatorIndex)) } return true diff --git a/cl/phase1/core/transition/process_rewards_and_penalties.go b/cl/transition/impl/eth2/statechange/process_rewards_and_penalties.go similarity index 74% rename from cl/phase1/core/transition/process_rewards_and_penalties.go rename to cl/transition/impl/eth2/statechange/process_rewards_and_penalties.go index 4369717a633..2adca23fb28 100644 --- a/cl/phase1/core/transition/process_rewards_and_penalties.go +++ b/cl/transition/impl/eth2/statechange/process_rewards_and_penalties.go @@ -1,18 +1,18 @@ -package transition +package statechange import ( "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes/solid" - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" ) -func processRewardsAndPenaltiesPostAltair(s *state2.BeaconState) (err error) { +func processRewardsAndPenaltiesPostAltair(s *state.BeaconState) (err error) { beaconConfig := s.BeaconConfig() weights := beaconConfig.ParticipationWeights() - eligibleValidators := state2.EligibleValidatorsIndicies(s.BeaconState) + eligibleValidators := state.EligibleValidatorsIndicies(s.BeaconState) // Initialize variables totalActiveBalance := s.GetTotalActiveBalance() - previousEpoch := state2.PreviousEpoch(s.BeaconState) + previousEpoch := state.PreviousEpoch(s.BeaconState) // Inactivity penalties denominator. inactivityPenaltyDenominator := beaconConfig.InactivityScoreBias * beaconConfig.GetPenaltyQuotient(s.Version()) // Make buffer for flag indexes total balances. @@ -20,7 +20,7 @@ func processRewardsAndPenaltiesPostAltair(s *state2.BeaconState) (err error) { // Compute all total balances for each enable unslashed validator indicies with all flags on. s.ForEachValidator(func(validator solid.Validator, validatorIndex, total int) bool { for i := range weights { - if state2.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, uint64(validatorIndex), i) { + if state.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, uint64(validatorIndex), i) { flagsTotalBalances[i] += validator.EffectiveBalance() } } @@ -40,20 +40,20 @@ func processRewardsAndPenaltiesPostAltair(s *state2.BeaconState) (err error) { return } for flagIdx := range weights { - if state2.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, index, flagIdx) { - if !state2.InactivityLeaking(s.BeaconState) { + if state.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, index, flagIdx) { + if !state.InactivityLeaking(s.BeaconState) { rewardNumerator := baseReward * rewardMultipliers[flagIdx] - if err := state2.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { + if err := state.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { return err } } } else if flagIdx != int(beaconConfig.TimelyHeadFlagIndex) { - if err := state2.DecreaseBalance(s.BeaconState, index, baseReward*weights[flagIdx]/beaconConfig.WeightDenominator); err != nil { + if err := state.DecreaseBalance(s.BeaconState, index, baseReward*weights[flagIdx]/beaconConfig.WeightDenominator); err != nil { return err } } } - if !state2.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, index, int(beaconConfig.TimelyTargetFlagIndex)) { + if !state.IsUnslashedParticipatingIndex(s.BeaconState, previousEpoch, index, int(beaconConfig.TimelyTargetFlagIndex)) { inactivityScore, err := s.ValidatorInactivityScore(int(index)) if err != nil { return err @@ -63,19 +63,19 @@ func processRewardsAndPenaltiesPostAltair(s *state2.BeaconState) (err error) { if err != nil { return err } - state2.DecreaseBalance(s.BeaconState, index, (effectiveBalance*inactivityScore)/inactivityPenaltyDenominator) + state.DecreaseBalance(s.BeaconState, index, (effectiveBalance*inactivityScore)/inactivityPenaltyDenominator) } } return } // processRewardsAndPenaltiesPhase0 process rewards and penalties for phase0 state. -func processRewardsAndPenaltiesPhase0(s *state2.BeaconState) (err error) { +func processRewardsAndPenaltiesPhase0(s *state.BeaconState) (err error) { beaconConfig := s.BeaconConfig() - if state2.Epoch(s.BeaconState) == beaconConfig.GenesisEpoch { + if state.Epoch(s.BeaconState) == beaconConfig.GenesisEpoch { return nil } - eligibleValidators := state2.EligibleValidatorsIndicies(s.BeaconState) + eligibleValidators := state.EligibleValidatorsIndicies(s.BeaconState) // Initialize variables rewardDenominator := s.GetTotalActiveBalance() / beaconConfig.EffectiveBalanceIncrement // Make buffer for flag indexes totTargetal balances. @@ -151,47 +151,47 @@ func processRewardsAndPenaltiesPhase0(s *state2.BeaconState) (err error) { } // If we attested then we reward the validator. - if state2.InactivityLeaking(s.BeaconState) { - if err := state2.IncreaseBalance(s.BeaconState, index, baseReward*attested); err != nil { + if state.InactivityLeaking(s.BeaconState) { + if err := state.IncreaseBalance(s.BeaconState, index, baseReward*attested); err != nil { return err } } else { if !currentValidator.Slashed() && previousMatchingSourceAttester { rewardNumerator := baseReward * unslashedMatchingSourceBalanceIncrements - if err := state2.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { + if err := state.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { return err } } if !currentValidator.Slashed() && previousMatchingTargetAttester { rewardNumerator := baseReward * unslashedMatchingTargetBalanceIncrements - if err := state2.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { + if err := state.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { return err } } if !currentValidator.Slashed() && previousMatchingHeadAttester { rewardNumerator := baseReward * unslashedMatchingHeadBalanceIncrements - if err := state2.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { + if err := state.IncreaseBalance(s.BeaconState, index, rewardNumerator/rewardDenominator); err != nil { return err } } } // Process inactivity of the network as a whole finalities. - if state2.InactivityLeaking(s.BeaconState) { + if state.InactivityLeaking(s.BeaconState) { proposerReward := baseReward / beaconConfig.ProposerRewardQuotient // Neutralize rewards. - if state2.DecreaseBalance(s.BeaconState, index, beaconConfig.BaseRewardsPerEpoch*baseReward-proposerReward); err != nil { + if state.DecreaseBalance(s.BeaconState, index, beaconConfig.BaseRewardsPerEpoch*baseReward-proposerReward); err != nil { return err } if currentValidator.Slashed() || !previousMatchingTargetAttester { // Increase penalities linearly if network is leaking. - if state2.DecreaseBalance(s.BeaconState, index, currentValidator.EffectiveBalance()*state2.FinalityDelay(s.BeaconState)/beaconConfig.InactivityPenaltyQuotient); err != nil { + if state.DecreaseBalance(s.BeaconState, index, currentValidator.EffectiveBalance()*state.FinalityDelay(s.BeaconState)/beaconConfig.InactivityPenaltyQuotient); err != nil { return err } } } // For each missed duty we penalize the validator. - if state2.DecreaseBalance(s.BeaconState, index, baseReward*missed); err != nil { + if state.DecreaseBalance(s.BeaconState, index, baseReward*missed); err != nil { return err } @@ -217,11 +217,11 @@ func processRewardsAndPenaltiesPhase0(s *state2.BeaconState) (err error) { } // Compute proposer reward. proposerReward := (baseReward / beaconConfig.ProposerRewardQuotient) - if err = state2.IncreaseBalance(s.BeaconState, attestation.ProposerIndex(), proposerReward); err != nil { + if err = state.IncreaseBalance(s.BeaconState, attestation.ProposerIndex(), proposerReward); err != nil { return false } maxAttesterReward := baseReward - proposerReward - if err = state2.IncreaseBalance(s.BeaconState, uint64(index), maxAttesterReward/attestation.InclusionDelay()); err != nil { + if err = state.IncreaseBalance(s.BeaconState, uint64(index), maxAttesterReward/attestation.InclusionDelay()); err != nil { return false } return true @@ -233,8 +233,8 @@ func processRewardsAndPenaltiesPhase0(s *state2.BeaconState) (err error) { } // ProcessRewardsAndPenalties applies rewards/penalties accumulated during previous epoch. -func ProcessRewardsAndPenalties(s *state2.BeaconState) error { - if state2.Epoch(s.BeaconState) == s.BeaconConfig().GenesisEpoch { +func ProcessRewardsAndPenalties(s *state.BeaconState) error { + if state.Epoch(s.BeaconState) == s.BeaconConfig().GenesisEpoch { return nil } if s.Version() == clparams.Phase0Version { diff --git a/cl/transition/impl/eth2/statechange/process_slashings.go b/cl/transition/impl/eth2/statechange/process_slashings.go new file mode 100644 index 00000000000..7833a053794 --- /dev/null +++ b/cl/transition/impl/eth2/statechange/process_slashings.go @@ -0,0 +1,93 @@ +package statechange + +import ( + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" +) + +func processSlashings(s *state.BeaconState, slashingMultiplier uint64) error { + // Get the current epoch + epoch := state.Epoch(s.BeaconState) + // Get the total active balance + totalBalance := s.GetTotalActiveBalance() + // Calculate the total slashing amount + // by summing all slashings and multiplying by the provided multiplier + slashing := state.GetTotalSlashingAmount(s.BeaconState) * slashingMultiplier + // Adjust the total slashing amount to be no greater than the total active balance + if totalBalance < slashing { + slashing = totalBalance + } + beaconConfig := s.BeaconConfig() + // Apply penalties to validators who have been slashed and reached the withdrawable epoch + var err error + s.ForEachValidator(func(validator solid.Validator, i, total int) bool { + if !validator.Slashed() || epoch+beaconConfig.EpochsPerSlashingsVector/2 != validator.WithdrawableEpoch() { + return true + } + // Get the effective balance increment + increment := beaconConfig.EffectiveBalanceIncrement + // Calculate the penalty numerator by multiplying the validator's effective balance by the total slashing amount + penaltyNumerator := validator.EffectiveBalance() / increment * slashing + // Calculate the penalty by dividing the penalty numerator by the total balance and multiplying by the increment + penalty := penaltyNumerator / totalBalance * increment + // Decrease the validator's balance by the calculated penalty + if err = state.DecreaseBalance(s.BeaconState, uint64(i), penalty); err != nil { + return false + } + return true + }) + if err != nil { + return err + } + return nil +} + +func processSlashings2(s *state.BeaconState, slashingMultiplier uint64) error { + // Get the current epoch + epoch := state.Epoch(s.BeaconState) + // Get the total active balance + totalBalance := s.GetTotalActiveBalance() + // Calculate the total slashing amount + // by summing all slashings and multiplying by the provided multiplier + slashing := state.GetTotalSlashingAmount(s.BeaconState) * slashingMultiplier + // Adjust the total slashing amount to be no greater than the total active balance + if totalBalance < slashing { + slashing = totalBalance + } + beaconConfig := s.BeaconConfig() + // Apply penalties to validators who have been slashed and reached the withdrawable epoch + var err error + s.ForEachValidator(func(validator solid.Validator, i, total int) bool { + if !validator.Slashed() || epoch+beaconConfig.EpochsPerSlashingsVector/2 != validator.WithdrawableEpoch() { + return true + } + // Get the effective balance increment + increment := beaconConfig.EffectiveBalanceIncrement + // Calculate the penalty numerator by multiplying the validator's effective balance by the total slashing amount + penaltyNumerator := validator.EffectiveBalance() / increment * slashing + // Calculate the penalty by dividing the penalty numerator by the total balance and multiplying by the increment + penalty := penaltyNumerator / totalBalance * increment + // Decrease the validator's balance by the calculated penalty + if err = state.DecreaseBalance(s.BeaconState, uint64(i), penalty); err != nil { + return false + } + return true + }) + if err != nil { + return err + } + return nil +} + +func ProcessSlashings(state *state.BeaconState) error { + // Depending on the version of the state, use different multipliers + switch state.Version() { + case clparams.Phase0Version: + return processSlashings(state, state.BeaconConfig().ProportionalSlashingMultiplier) + case clparams.AltairVersion: + return processSlashings(state, state.BeaconConfig().ProportionalSlashingMultiplierAltair) + default: + return processSlashings(state, state.BeaconConfig().ProportionalSlashingMultiplierBellatrix) + } +} diff --git a/cl/phase1/core/transition/process_sync_committee_update.go b/cl/transition/impl/eth2/statechange/process_sync_committee_update.go similarity index 63% rename from cl/phase1/core/transition/process_sync_committee_update.go rename to cl/transition/impl/eth2/statechange/process_sync_committee_update.go index ec043b5a10a..58bd155059e 100644 --- a/cl/phase1/core/transition/process_sync_committee_update.go +++ b/cl/transition/impl/eth2/statechange/process_sync_committee_update.go @@ -1,12 +1,12 @@ -package transition +package statechange import ( - state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" ) // ProcessSyncCommitteeUpdate implements processing for the sync committee update. unfortunately there is no easy way to test it. -func ProcessSyncCommitteeUpdate(s *state2.BeaconState) error { - if (state2.Epoch(s.BeaconState)+1)%s.BeaconConfig().EpochsPerSyncCommitteePeriod != 0 { +func ProcessSyncCommitteeUpdate(s *state.BeaconState) error { + if (state.Epoch(s.BeaconState)+1)%s.BeaconConfig().EpochsPerSyncCommitteePeriod != 0 { return nil } // Set new current sync committee. diff --git a/cl/phase1/core/transition/process_sync_committee_update_test.go b/cl/transition/impl/eth2/statechange/process_sync_committee_update_test.go similarity index 61% rename from cl/phase1/core/transition/process_sync_committee_update_test.go rename to cl/transition/impl/eth2/statechange/process_sync_committee_update_test.go index 5d7d8d35e4e..a7f4b62fe5f 100644 --- a/cl/phase1/core/transition/process_sync_committee_update_test.go +++ b/cl/transition/impl/eth2/statechange/process_sync_committee_update_test.go @@ -1,14 +1,13 @@ -package transition_test +package statechange_test import ( "encoding/binary" + "github.com/ledgerwatch/erigon/cl/transition/impl/eth2/statechange" "testing" + "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes/solid" "github.com/ledgerwatch/erigon/cl/phase1/core/state" - "github.com/ledgerwatch/erigon/cl/phase1/core/transition" - - "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/common" "github.com/stretchr/testify/require" ) @@ -18,7 +17,7 @@ func TestProcessSyncCommittee(t *testing.T) { var pk [48]byte copy(pk[:], pkBytes) validatorNum := 10_000 - state := state.New(&clparams.MainnetBeaconConfig) + s := state.New(&clparams.MainnetBeaconConfig) currentCommittee := &solid.SyncCommittee{} nextCommittee := &solid.SyncCommittee{} for i := 0; i < validatorNum; i++ { @@ -28,13 +27,13 @@ func TestProcessSyncCommittee(t *testing.T) { v.SetExitEpoch(clparams.MainnetBeaconConfig.FarFutureEpoch) v.SetPublicKey(pk) v.SetEffectiveBalance(2000000000) - state.AddValidator(v, 2000000000) + s.AddValidator(v, 2000000000) } - state.SetCurrentSyncCommittee(currentCommittee) - state.SetNextSyncCommittee(nextCommittee) - prevNextSyncCommittee := state.NextSyncCommittee() - state.SetSlot(8160) - require.NoError(t, transition.ProcessSyncCommitteeUpdate(state)) - require.Equal(t, state.CurrentSyncCommittee(), prevNextSyncCommittee) - require.NotEqual(t, state.NextSyncCommittee(), prevNextSyncCommittee) + s.SetCurrentSyncCommittee(currentCommittee) + s.SetNextSyncCommittee(nextCommittee) + prevNextSyncCommittee := s.NextSyncCommittee() + s.SetSlot(8160) + require.NoError(t, statechange.ProcessSyncCommitteeUpdate(s)) + require.Equal(t, s.CurrentSyncCommittee(), prevNextSyncCommittee) + require.NotEqual(t, s.NextSyncCommittee(), prevNextSyncCommittee) } diff --git a/cl/phase1/core/transition/resets.go b/cl/transition/impl/eth2/statechange/resets.go similarity index 97% rename from cl/phase1/core/transition/resets.go rename to cl/transition/impl/eth2/statechange/resets.go index 28f470b7143..8e560946cfc 100644 --- a/cl/phase1/core/transition/resets.go +++ b/cl/transition/impl/eth2/statechange/resets.go @@ -1,4 +1,4 @@ -package transition +package statechange import ( state2 "github.com/ledgerwatch/erigon/cl/phase1/core/state" diff --git a/cl/phase1/core/transition/test_data/block_processing/capella_block.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/block_processing/capella_block.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/block_processing/capella_block.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/block_processing/capella_block.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/block_processing/capella_state.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/block_processing/capella_state.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/block_processing/capella_state.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/block_processing/capella_state.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/effective_balances_expected.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/effective_balances_expected.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/effective_balances_expected.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/effective_balances_expected.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/effective_balances_test_state.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/effective_balances_test_state.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/effective_balances_test_state.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/effective_balances_test_state.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/eth1_data_reset_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/eth1_data_reset_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/eth1_data_reset_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/eth1_data_reset_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/eth1_data_reset_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/eth1_data_reset_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/eth1_data_reset_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/eth1_data_reset_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/historical_roots_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/historical_roots_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/historical_roots_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/historical_roots_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/historical_roots_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/historical_roots_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/historical_roots_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/historical_roots_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/inactivity_scores_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/inactivity_scores_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/inactivity_scores_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/inactivity_scores_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/inactivity_scores_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/inactivity_scores_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/inactivity_scores_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/inactivity_scores_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/justification_and_finality_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/justification_and_finality_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/justification_and_finality_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/justification_and_finality_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/justification_and_finality_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/justification_and_finality_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/justification_and_finality_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/justification_and_finality_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/participation_flag_updates_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/participation_flag_updates_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/participation_flag_updates_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/participation_flag_updates_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/participation_flag_updates_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/participation_flag_updates_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/participation_flag_updates_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/participation_flag_updates_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/randao_mixes_reset_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/randao_mixes_reset_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/randao_mixes_reset_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/randao_mixes_reset_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/randao_mixes_reset_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/randao_mixes_reset_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/randao_mixes_reset_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/randao_mixes_reset_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/registry_updates_test_expected.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/registry_updates_test_expected.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/registry_updates_test_expected.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/registry_updates_test_expected.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/registry_updates_test_state.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/registry_updates_test_state.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/registry_updates_test_state.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/registry_updates_test_state.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/rewards_penalty_test_expected.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/rewards_penalty_test_expected.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/rewards_penalty_test_expected.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/rewards_penalty_test_expected.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/rewards_penalty_test_state.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/rewards_penalty_test_state.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/rewards_penalty_test_state.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/rewards_penalty_test_state.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/slashings_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/slashings_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/slashings_reset_expected_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_reset_expected_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/slashings_reset_expected_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_reset_expected_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/slashings_reset_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_reset_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/slashings_reset_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_reset_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/test_data/epoch_processing/slashings_state_test.ssz_snappy b/cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_state_test.ssz_snappy similarity index 100% rename from cl/phase1/core/transition/test_data/epoch_processing/slashings_state_test.ssz_snappy rename to cl/transition/impl/eth2/statechange/test_data/epoch_processing/slashings_state_test.ssz_snappy diff --git a/cl/phase1/core/transition/process_blob_kzg_commitments.go b/cl/transition/impl/eth2/utils.go similarity index 63% rename from cl/phase1/core/transition/process_blob_kzg_commitments.go rename to cl/transition/impl/eth2/utils.go index 5f2840a6769..cb0843c0390 100644 --- a/cl/phase1/core/transition/process_blob_kzg_commitments.go +++ b/cl/transition/impl/eth2/utils.go @@ -1,12 +1,10 @@ -package transition +package eth2 import ( "encoding/binary" - "reflect" - libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cl/cltypes" - "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/utils" "github.com/ledgerwatch/erigon/core/types" ) @@ -58,32 +56,39 @@ func txPeekBlobVersionedHashes(txBytes []byte) []libcommon.Hash { return versionedHashes } -func VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) { - allVersionedHashes := []libcommon.Hash{} - transactions.ForEach(func(tx []byte, idx, total int) bool { - if tx[0] != types.BlobTxType { - return true - } - - allVersionedHashes = append(allVersionedHashes, txPeekBlobVersionedHashes(tx)...) - return true - }) +func computeSigningRootEpoch(epoch uint64, domain []byte) (libcommon.Hash, error) { + b := make([]byte, 32) + binary.LittleEndian.PutUint64(b, epoch) + return utils.Keccak256(b, domain), nil +} - commitmentVersionedHash := []libcommon.Hash{} +// transitionSlot is called each time there is a new slot to process +func transitionSlot(s *state.BeaconState) error { + slot := s.Slot() + previousStateRoot := s.PreviousStateRoot() var err error - var versionedHash libcommon.Hash - kzgCommitments.Range(func(index int, value *cltypes.KZGCommitment, length int) bool { - versionedHash, err = kzgCommitmentToVersionedHash(value) + if previousStateRoot == (libcommon.Hash{}) { + previousStateRoot, err = s.HashSSZ() if err != nil { - return false + return err } + } - commitmentVersionedHash = append(commitmentVersionedHash, versionedHash) - return true - }) - if err != nil { - return false, err + beaconConfig := s.BeaconConfig() + + s.SetStateRootAt(int(slot%beaconConfig.SlotsPerHistoricalRoot), previousStateRoot) + + latestBlockHeader := s.LatestBlockHeader() + if latestBlockHeader.Root == [32]byte{} { + latestBlockHeader.Root = previousStateRoot + s.SetLatestBlockHeader(&latestBlockHeader) } + blockHeader := s.LatestBlockHeader() - return reflect.DeepEqual(allVersionedHashes, commitmentVersionedHash), nil + previousBlockRoot, err := (&blockHeader).HashSSZ() + if err != nil { + return err + } + s.SetBlockRootAt(int(slot%beaconConfig.SlotsPerHistoricalRoot), previousBlockRoot) + return nil } diff --git a/cl/transition/impl/eth2/validation.go b/cl/transition/impl/eth2/validation.go new file mode 100644 index 00000000000..baa54db1e1f --- /dev/null +++ b/cl/transition/impl/eth2/validation.go @@ -0,0 +1,54 @@ +package eth2 + +import ( + "fmt" + "github.com/Giulio2002/bls" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/fork" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" +) + +func (I *impl) VerifyTransition(s *state.BeaconState, currentBlock *cltypes.BeaconBlock) error { + if !I.FullValidation { + return nil + } + expectedStateRoot, err := s.HashSSZ() + if err != nil { + return fmt.Errorf("unable to generate state root: %v", err) + } + if expectedStateRoot != currentBlock.StateRoot { + return fmt.Errorf("expected state root differs from received state root") + } + return nil +} + +func (I *impl) VerifyBlockSignature(s *state.BeaconState, block *cltypes.SignedBeaconBlock) error { + if !I.FullValidation { + return nil + } + valid, err := verifyBlockSignature(s, block) + if err != nil { + return fmt.Errorf("error validating block signature: %v", err) + } + if !valid { + return fmt.Errorf("block not valid") + } + return nil +} + +func verifyBlockSignature(s *state.BeaconState, block *cltypes.SignedBeaconBlock) (bool, error) { + proposer, err := s.ValidatorForValidatorIndex(int(block.Block.ProposerIndex)) + if err != nil { + return false, err + } + domain, err := s.GetDomain(s.BeaconConfig().DomainBeaconProposer, state.Epoch(s.BeaconState)) + if err != nil { + return false, err + } + sigRoot, err := fork.ComputeSigningRoot(block.Block, domain) + if err != nil { + return false, err + } + pk := proposer.PublicKey() + return bls.Verify(block.Signature[:], sigRoot[:], pk[:]) +} diff --git a/cl/transition/impl/funcmap/impl.go b/cl/transition/impl/funcmap/impl.go new file mode 100644 index 00000000000..e92d926f925 --- /dev/null +++ b/cl/transition/impl/funcmap/impl.go @@ -0,0 +1,94 @@ +package funcmap + +import ( + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/cl/transition/machine" + "github.com/ledgerwatch/erigon/core/types" +) + +var _ machine.Interface = (*Impl)(nil) + +type Impl struct { + FnVerifyBlockSignature func(s *state.BeaconState, block *cltypes.SignedBeaconBlock) error + FnVerifyTransition func(s *state.BeaconState, block *cltypes.BeaconBlock) error + FnProcessSlots func(s *state.BeaconState, slot uint64) error + FnProcessBlockHeader func(s *state.BeaconState, block *cltypes.BeaconBlock) error + FnProcessWithdrawals func(s *state.BeaconState, withdrawals *solid.ListSSZ[*types.Withdrawal]) error + FnProcessExecutionPayload func(s *state.BeaconState, payload *cltypes.Eth1Block) error + FnProcessRandao func(s *state.BeaconState, randao [96]byte, proposerIndex uint64) error + FnProcessEth1Data func(state *state.BeaconState, eth1Data *cltypes.Eth1Data) error + FnProcessSyncAggregate func(s *state.BeaconState, sync *cltypes.SyncAggregate) error + FnVerifyKzgCommitmentsAgainstTransactions func(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) + FnProcessProposerSlashing func(s *state.BeaconState, propSlashing *cltypes.ProposerSlashing) error + FnProcessAttesterSlashing func(s *state.BeaconState, attSlashing *cltypes.AttesterSlashing) error + FnProcessAttestations func(s *state.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error + FnProcessDeposit func(s *state.BeaconState, deposit *cltypes.Deposit) error + FnProcessVoluntaryExit func(s *state.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit) error + FnProcessBlsToExecutionChange func(state *state.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange) error +} + +func (i Impl) VerifyBlockSignature(s *state.BeaconState, block *cltypes.SignedBeaconBlock) error { + return i.FnVerifyBlockSignature(s, block) +} + +func (i Impl) VerifyTransition(s *state.BeaconState, block *cltypes.BeaconBlock) error { + return i.FnVerifyTransition(s, block) +} + +func (i Impl) ProcessBlockHeader(s *state.BeaconState, block *cltypes.BeaconBlock) error { + return i.FnProcessBlockHeader(s, block) +} + +func (i Impl) ProcessWithdrawals(s *state.BeaconState, withdrawals *solid.ListSSZ[*types.Withdrawal]) error { + return i.FnProcessWithdrawals(s, withdrawals) +} + +func (i Impl) ProcessExecutionPayload(s *state.BeaconState, payload *cltypes.Eth1Block) error { + return i.FnProcessExecutionPayload(s, payload) +} + +func (i Impl) ProcessRandao(s *state.BeaconState, randao [96]byte, proposerIndex uint64) error { + return i.FnProcessRandao(s, randao, proposerIndex) +} + +func (i Impl) ProcessEth1Data(state *state.BeaconState, eth1Data *cltypes.Eth1Data) error { + return i.FnProcessEth1Data(state, eth1Data) +} + +func (i Impl) ProcessSyncAggregate(s *state.BeaconState, sync *cltypes.SyncAggregate) error { + return i.FnProcessSyncAggregate(s, sync) +} + +func (i Impl) VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) { + return i.FnVerifyKzgCommitmentsAgainstTransactions(transactions, kzgCommitments) +} + +func (i Impl) ProcessProposerSlashing(s *state.BeaconState, propSlashing *cltypes.ProposerSlashing) error { + return i.FnProcessProposerSlashing(s, propSlashing) +} + +func (i Impl) ProcessAttesterSlashing(s *state.BeaconState, attSlashing *cltypes.AttesterSlashing) error { + return i.FnProcessAttesterSlashing(s, attSlashing) +} + +func (i Impl) ProcessAttestations(s *state.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error { + return i.FnProcessAttestations(s, attestations) +} + +func (i Impl) ProcessDeposit(s *state.BeaconState, deposit *cltypes.Deposit) error { + return i.FnProcessDeposit(s, deposit) +} + +func (i Impl) ProcessVoluntaryExit(s *state.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit) error { + return i.FnProcessVoluntaryExit(s, signedVoluntaryExit) +} + +func (i Impl) ProcessBlsToExecutionChange(state *state.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange) error { + return i.FnProcessBlsToExecutionChange(state, signedChange) +} + +func (i Impl) ProcessSlots(s *state.BeaconState, slot uint64) error { + return i.FnProcessSlots(s, slot) +} diff --git a/cl/transition/machine/block.go b/cl/transition/machine/block.go new file mode 100644 index 00000000000..f3dce1fe9be --- /dev/null +++ b/cl/transition/machine/block.go @@ -0,0 +1,141 @@ +package machine + +import ( + "errors" + "fmt" + + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/metrics/methelp" +) + +// ProcessBlock processes a block with the block processor +func ProcessBlock(impl BlockProcessor, s *state.BeaconState, signedBlock *cltypes.SignedBeaconBlock) error { + block := signedBlock.Block + version := s.Version() + // Check the state version is correct. + if signedBlock.Version() != version { + return fmt.Errorf("processBlock: wrong state version for block at slot %d", block.Slot) + } + h := methelp.NewHistTimer("beacon_process_block") + // Process the block header. + if err := impl.ProcessBlockHeader(s, block); err != nil { + return fmt.Errorf("processBlock: failed to process block header: %v", err) + } + // Process execution payload if enabled. + if version >= clparams.BellatrixVersion && executionEnabled(s, block.Body.ExecutionPayload) { + if s.Version() >= clparams.CapellaVersion { + // Process withdrawals in the execution payload. + if err := impl.ProcessWithdrawals(s, block.Body.ExecutionPayload.Withdrawals); err != nil { + return fmt.Errorf("processBlock: failed to process withdrawals: %v", err) + } + } + // Process the execution payload. + if err := impl.ProcessExecutionPayload(s, block.Body.ExecutionPayload); err != nil { + return fmt.Errorf("processBlock: failed to process execution payload: %v", err) + } + } + // Process RANDAO reveal. + if err := impl.ProcessRandao(s, block.Body.RandaoReveal, block.ProposerIndex); err != nil { + return fmt.Errorf("processBlock: failed to process RANDAO reveal: %v", err) + } + // Process Eth1 data. + if err := impl.ProcessEth1Data(s, block.Body.Eth1Data); err != nil { + return fmt.Errorf("processBlock: failed to process Eth1 data: %v", err) + } + // Process block body operations. + if err := ProcessOperations(impl, s, block.Body); err != nil { + return fmt.Errorf("processBlock: failed to process block body operations: %v", err) + } + // Process sync aggregate in case of Altair version. + if version >= clparams.AltairVersion { + if err := impl.ProcessSyncAggregate(s, block.Body.SyncAggregate); err != nil { + return fmt.Errorf("processBlock: failed to process sync aggregate: %v", err) + } + } + if version >= clparams.DenebVersion { + verified, err := impl.VerifyKzgCommitmentsAgainstTransactions(block.Body.ExecutionPayload.Transactions, block.Body.BlobKzgCommitments) + if err != nil { + return fmt.Errorf("processBlock: failed to process blob kzg commitments: %w", err) + } + if !verified { + return fmt.Errorf("processBlock: failed to process blob kzg commitments: commitments are not equal") + } + } + h.PutSince() + return nil +} + +// ProcessOperations is called by ProcessBlock and prcesses the block body operations +func ProcessOperations(impl BlockOperationProcessor, s *state.BeaconState, blockBody *cltypes.BeaconBody) error { + if blockBody.Deposits.Len() != int(maximumDeposits(s)) { + return errors.New("outstanding deposits do not match maximum deposits") + } + // Process each proposer slashing + var err error + if err := solid.RangeErr[*cltypes.ProposerSlashing](blockBody.ProposerSlashings, func(index int, slashing *cltypes.ProposerSlashing, length int) error { + if err = impl.ProcessProposerSlashing(s, slashing); err != nil { + return fmt.Errorf("ProcessProposerSlashing: %s", err) + } + return nil + }); err != nil { + return err + } + + if err := solid.RangeErr[*cltypes.AttesterSlashing](blockBody.AttesterSlashings, func(index int, slashing *cltypes.AttesterSlashing, length int) error { + if err = impl.ProcessAttesterSlashing(s, slashing); err != nil { + return fmt.Errorf("ProcessAttesterSlashing: %s", err) + } + return nil + }); err != nil { + return err + } + + // Process each attestations + if err := impl.ProcessAttestations(s, blockBody.Attestations); err != nil { + return fmt.Errorf("ProcessAttestation: %s", err) + } + + // Process each deposit + if err := solid.RangeErr[*cltypes.Deposit](blockBody.Deposits, func(index int, deposit *cltypes.Deposit, length int) error { + if err = impl.ProcessDeposit(s, deposit); err != nil { + return fmt.Errorf("ProcessDeposit: %s", err) + } + return nil + }); err != nil { + return err + } + + // Process each voluntary exit. + if err := solid.RangeErr[*cltypes.SignedVoluntaryExit](blockBody.VoluntaryExits, func(index int, exit *cltypes.SignedVoluntaryExit, length int) error { + if err = impl.ProcessVoluntaryExit(s, exit); err != nil { + return fmt.Errorf("ProcessVoluntaryExit: %s", err) + } + return nil + }); err != nil { + return err + } + if s.Version() < clparams.CapellaVersion { + return nil + } + // Process each execution change. this will only have entries after the capella fork. + if err := solid.RangeErr[*cltypes.SignedBLSToExecutionChange](blockBody.ExecutionChanges, func(index int, addressChange *cltypes.SignedBLSToExecutionChange, length int) error { + if err := impl.ProcessBlsToExecutionChange(s, addressChange); err != nil { + return fmt.Errorf("ProcessBlsToExecutionChange: %s", err) + } + return nil + }); err != nil { + return err + } + return nil +} + +func maximumDeposits(s *state.BeaconState) (maxDeposits uint64) { + maxDeposits = s.Eth1Data().DepositCount - s.Eth1DepositIndex() + if maxDeposits > s.BeaconConfig().MaxDeposits { + maxDeposits = s.BeaconConfig().MaxDeposits + } + return +} diff --git a/cl/transition/machine/helpers.go b/cl/transition/machine/helpers.go new file mode 100644 index 00000000000..176efaa06cd --- /dev/null +++ b/cl/transition/machine/helpers.go @@ -0,0 +1,11 @@ +package machine + +import ( + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" +) + +func executionEnabled(s *state.BeaconState, payload *cltypes.Eth1Block) bool { + return (!state.IsMergeTransitionComplete(s.BeaconState) && payload.BlockHash != common.Hash{}) || state.IsMergeTransitionComplete(s.BeaconState) +} diff --git a/cl/transition/machine/machine.go b/cl/transition/machine/machine.go new file mode 100644 index 00000000000..5a6f07581df --- /dev/null +++ b/cl/transition/machine/machine.go @@ -0,0 +1,48 @@ +// Package machine is the interface for eth2 state transition +package machine + +import ( + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/cltypes/solid" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/core/types" +) + +type Interface interface { + BlockValidator + BlockProcessor + SlotProcessor +} + +type BlockProcessor interface { + BlockHeaderProcessor + BlockOperationProcessor +} + +type BlockValidator interface { + VerifyBlockSignature(s *state.BeaconState, block *cltypes.SignedBeaconBlock) error + VerifyTransition(s *state.BeaconState, block *cltypes.BeaconBlock) error +} + +type SlotProcessor interface { + ProcessSlots(s *state.BeaconState, slot uint64) error +} + +type BlockHeaderProcessor interface { + ProcessBlockHeader(s *state.BeaconState, block *cltypes.BeaconBlock) error + ProcessWithdrawals(s *state.BeaconState, withdrawals *solid.ListSSZ[*types.Withdrawal]) error + ProcessExecutionPayload(s *state.BeaconState, payload *cltypes.Eth1Block) error + ProcessRandao(s *state.BeaconState, randao [96]byte, proposerIndex uint64) error + ProcessEth1Data(state *state.BeaconState, eth1Data *cltypes.Eth1Data) error + ProcessSyncAggregate(s *state.BeaconState, sync *cltypes.SyncAggregate) error + VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) +} + +type BlockOperationProcessor interface { + ProcessProposerSlashing(s *state.BeaconState, propSlashing *cltypes.ProposerSlashing) error + ProcessAttesterSlashing(s *state.BeaconState, attSlashing *cltypes.AttesterSlashing) error + ProcessAttestations(s *state.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error + ProcessDeposit(s *state.BeaconState, deposit *cltypes.Deposit) error + ProcessVoluntaryExit(s *state.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit) error + ProcessBlsToExecutionChange(state *state.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange) error +} diff --git a/cl/transition/machine/transition.go b/cl/transition/machine/transition.go new file mode 100644 index 00000000000..2a0672029a6 --- /dev/null +++ b/cl/transition/machine/transition.go @@ -0,0 +1,32 @@ +package machine + +import ( + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" +) + +// TransitionState will call impl..ProcessSlots, then impl.VerifyBlockSignature, then ProcessBlock, then impl.VerifyTransition +func TransitionState(impl Interface, s *state.BeaconState, block *cltypes.SignedBeaconBlock) error { + currentBlock := block.Block + if err := impl.ProcessSlots(s, currentBlock.Slot); err != nil { + return err + } + + if err := impl.VerifyBlockSignature(s, block); err != nil { + return err + } + + // Transition block + if err := ProcessBlock(impl, s, block); err != nil { + return err + } + + // perform validation + if err := impl.VerifyTransition(s, currentBlock); err != nil { + return err + } + + // if validation is successful, transition + s.SetPreviousStateRoot(currentBlock.StateRoot) + return nil +} diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 33342510a40..6d361b48388 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -119,8 +119,6 @@ func init() { } func abigen(c *cli.Context) error { - //fmt.Printf("a? %s\n", c.FlagNames()) - utils.CheckExclusive(c, &abiFlag, &jsonFlag, &solFlag, &vyFlag) // Only one source can be selected. if c.String(pkgFlag.Name) == "" { utils.Fatalf("No destination package specified (--pkg)") diff --git a/cmd/caplin-phase1/main.go b/cmd/caplin-phase1/main.go index 88be1c4f38f..ce45304c44e 100644 --- a/cmd/caplin-phase1/main.go +++ b/cmd/caplin-phase1/main.go @@ -115,13 +115,17 @@ func runCaplinNode(cliCtx *cli.Context) error { engine = execution_client.NewExecutionEnginePhase1FromClient(ctx, remote.NewETHBACKENDClient(cc)) } - apiHandler := handler.NewApiHandler(cfg.GenesisCfg, cfg.BeaconCfg) - go beacon.ListenAndServe(apiHandler, &beacon.RouterConfiguration{ - Protocol: cfg.BeaconProtocol, - Address: cfg.BeaconAddr, - // TODO(enriavil1): Make timeouts configurable via flags - }) - log.Info("Beacon API started", "addr", cfg.BeaconAddr) + if !cfg.NoBeaconApi { + apiHandler := handler.NewApiHandler(cfg.GenesisCfg, cfg.BeaconCfg) + go beacon.ListenAndServe(apiHandler, &beacon.RouterConfiguration{ + Protocol: cfg.BeaconProtocol, + Address: cfg.BeaconAddr, + ReadTimeTimeout: cfg.BeaconApiReadTimeout, + WriteTimeout: cfg.BeaconApiWriteTimeout, + IdleTimeout: cfg.BeaconApiWriteTimeout, + }) + log.Info("Beacon API started", "addr", cfg.BeaconAddr) + } var caplinFreezer freezer.Freezer if cfg.RecordMode { diff --git a/cmd/caplin-regression/main.go b/cmd/caplin-regression/main.go index 6e6ea14e437..7a3abb9c9ca 100644 --- a/cmd/caplin-regression/main.go +++ b/cmd/caplin-regression/main.go @@ -2,12 +2,14 @@ package main import ( "flag" - "net/http" + + "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/cl/phase1/forkchoice" "github.com/ledgerwatch/erigon/cmd/caplin-regression/regression" "github.com/ledgerwatch/log/v3" + "golang.org/x/exp/slices" _ "net/http/pprof" //nolint:gosec ) @@ -18,28 +20,29 @@ var nameTestsMap = map[string]func(*forkchoice.ForkChoiceStore, *cltypes.SignedB "TestRegressionBadBlocks": regression.TestRegressionBadBlocks, } +var excludeTests = []string{"TestRegressionBadBlocks"} + func main() { log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StderrHandler)) test := flag.String("test", "TestRegressionWithValidation", "select test to run. can be TestRegressionWithValidation, TestRegressionWithoutValidation and TestRegressionBadBlocks") - step := flag.Int("step", 1, "how often to log performance") + step := flag.Int("step", 32, "how often to log performance") pprof := flag.Bool("pprof", true, "turn on profiling") + loop := flag.Bool("loop", true, "loop the test in an infinite loop") + testsDir := flag.String("testsDir", "cmd/caplin-regression/caplin-tests", "directory to the tests") + + all := flag.Bool("all", true, "loop trhough all the test") + flag.Parse() if _, ok := nameTestsMap[*test]; !ok { log.Error("Could not start regression tests", "err", "test not found") return } r, err := regression.NewRegressionTester( - "cmd/caplin-regression/caplin-tests", + *testsDir, ) if *pprof { // Server for pprof - go func() { - log.Info("Serving pprof on localhost:6060") - if err := http.ListenAndServe("localhost:6060", nil); err != nil { //nolint:gosec - log.Error("Could not serve pprof", "err", err) - } - - }() + debug.StartPProf("localhost:6060", true) } if err != nil { @@ -47,7 +50,20 @@ func main() { return } - if err := r.Run(*test, nameTestsMap[*test], *step); err != nil { - log.Error("Could not do regression tests", "err", err) + for val := true; val; val = *loop { + if *all { + for name, t := range nameTestsMap { + if slices.Contains(excludeTests, name) { + continue + } + if err := r.Run(name, t, *step); err != nil { + log.Error("Could not do regression tests", "err", err) + } + } + continue + } + if err := r.Run(*test, nameTestsMap[*test], *step); err != nil { + log.Error("Could not do regression tests", "err", err) + } } } diff --git a/cmd/caplin-regression/regression/tester.go b/cmd/caplin-regression/regression/tester.go index 8c7eb06ce8d..4003e04dd40 100644 --- a/cmd/caplin-regression/regression/tester.go +++ b/cmd/caplin-regression/regression/tester.go @@ -34,44 +34,43 @@ func NewRegressionTester(testDirectory string) (*RegressionTester, error) { } func (r *RegressionTester) Run(name string, fn func(*forkchoice.ForkChoiceStore, *cltypes.SignedBeaconBlock) error, step int) error { - for true { - state, err := r.readStartingState() - if err != nil { - return err + state, err := r.readStartingState() + if err != nil { + return err + } + store, err := forkchoice.NewForkChoiceStore(state, nil, nil, true) + if err != nil { + return err + } + log.Info("Loading public keys into memory") + bls.SetEnabledCaching(true) + state.ForEachValidator(func(v solid.Validator, idx, total int) bool { + pk := v.PublicKey() + if err := bls.LoadPublicKeyIntoCache(pk[:], false); err != nil { + panic(err) } - store, err := forkchoice.NewForkChoiceStore(state, nil, nil, true) - if err != nil { + return true + }) + store.OnTick(uint64(time.Now().Unix())) + begin := time.Now() + beginStep := time.Now() + log.Info("Starting test, CTRL+C to stop.", "name", name) + for _, block := range r.blockList { + if err := fn(store, block); err != nil { return err } - log.Info("Loading public keys into memory") - bls.SetEnabledCaching(true) - state.ForEachValidator(func(v solid.Validator, idx, total int) bool { - pk := v.PublicKey() - if err := bls.LoadPublicKeyIntoCache(pk[:], false); err != nil { - panic(err) - } - return true - }) - store.OnTick(uint64(time.Now().Unix())) - begin := time.Now() - beginStep := time.Now() - log.Info("Starting test, CTRL+C to stop.", "name", name) - for _, block := range r.blockList { - if err := fn(store, block); err != nil { - return err - } - if block.Block.Slot%uint64(step) == 0 { - elapsed := time.Since(beginStep) - log.Info("Processed", "slot", block.Block.Slot, "elapsed", elapsed, "sec/blk", elapsed/time.Duration(step)) - beginStep = time.Now() - } + if block.Block.Slot%uint64(step) == 0 { + elapsed := time.Since(beginStep) + log.Info("Processed", "slot", block.Block.Slot, "elapsed", elapsed, "sec/blk", elapsed/time.Duration(step)) + beginStep = time.Now() } - - var m runtime.MemStats - dbg.ReadMemStats(&m) - sum := time.Since(begin) - log.Info("Finished/Restarting test", "name", name, "averageBlockTime", sum/time.Duration(len(r.blockList)), "sys", common.ByteCount(m.Sys)) } + + var m runtime.MemStats + dbg.ReadMemStats(&m) + sum := time.Since(begin) + log.Info("Finished/Restarting test", "name", name, "averageBlockTime", sum/time.Duration(len(r.blockList)), "sys", common.ByteCount(m.Sys)) + return nil } diff --git a/cmd/devnet/args/args.go b/cmd/devnet/args/args.go new file mode 100644 index 00000000000..59208ff41ef --- /dev/null +++ b/cmd/devnet/args/args.go @@ -0,0 +1,154 @@ +package args + +import ( + "fmt" + "reflect" + "strings" + "unicode" + "unicode/utf8" +) + +type Args []string + +func AsArgs(args interface{}) (Args, error) { + + argsValue := reflect.ValueOf(args) + + if argsValue.Kind() == reflect.Ptr { + argsValue = argsValue.Elem() + } + + if argsValue.Kind() != reflect.Struct { + return nil, fmt.Errorf("Args type must be struct or struc pointer, got %T", args) + } + + return gatherArgs(argsValue, func(v reflect.Value, field reflect.StructField) (string, error) { + tag := field.Tag.Get("arg") + + if tag == "-" { + return "", nil + } + + // only process public fields (reflection won't return values of unsafe fields without unsafe operations) + if r, _ := utf8.DecodeRuneInString(field.Name); !(unicode.IsLetter(r) && unicode.IsUpper(r)) { + return "", nil + } + + var key string + var positional bool + + for _, key = range strings.Split(tag, ",") { + if key == "" { + continue + } + + key = strings.TrimLeft(key, " ") + + if pos := strings.Index(key, ":"); pos != -1 { + key = key[:pos] + } + + switch { + case strings.HasPrefix(key, "---"): + return "", fmt.Errorf("%s.%s: too many hyphens", v.Type().Name(), field.Name) + case strings.HasPrefix(key, "--"): + + case strings.HasPrefix(key, "-"): + if len(key) != 2 { + return "", fmt.Errorf("%s.%s: short arguments must be one character only", v.Type().Name(), field.Name) + } + case key == "positional": + key = "" + positional = true + default: + return "", fmt.Errorf("unrecognized tag '%s' on field %s", key, tag) + } + } + + if len(key) == 0 && !positional { + key = "--" + strings.ToLower(field.Name) + } + + var value string + + switch fv := v.FieldByIndex(field.Index); fv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if fv.Int() == 0 { + break + } + fallthrough + default: + value = fmt.Sprintf("%v", fv.Interface()) + } + + flagValue, isFlag := field.Tag.Lookup("flag") + + if isFlag { + if value != "true" { + if flagValue == "true" { + value = flagValue + } + } + } + + if len(value) == 0 { + if defaultString, hasDefault := field.Tag.Lookup("default"); hasDefault { + value = defaultString + } + + if len(value) == 0 { + return "", nil + } + } + + if len(key) == 0 { + return value, nil + } + + if isFlag { + if value == "true" { + return key, nil + } + + return "", nil + } + + if len(value) == 0 { + return key, nil + } + + return fmt.Sprintf("%s=%s", key, value), nil + }) +} + +func gatherArgs(v reflect.Value, visit func(v reflect.Value, field reflect.StructField) (string, error)) (args Args, err error) { + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) + + var gathered Args + + fieldType := field.Type + + if fieldType.Kind() == reflect.Ptr { + fieldType.Elem() + } + + if fieldType.Kind() == reflect.Struct { + gathered, err = gatherArgs(v.FieldByIndex(field.Index), visit) + } else { + var value string + + if value, err = visit(v, field); len(value) > 0 { + gathered = Args{value} + } + } + + if err != nil { + return nil, err + } + + args = append(args, gathered...) + } + + return args, nil +} diff --git a/cmd/devnet/args/node.go b/cmd/devnet/args/node.go new file mode 100644 index 00000000000..a7a9defb5b7 --- /dev/null +++ b/cmd/devnet/args/node.go @@ -0,0 +1,137 @@ +package args + +import ( + "fmt" + "net" + "path/filepath" + "strconv" + + "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/params/networkname" +) + +type Node struct { + requests.RequestGenerator `arg:"-"` + BuildDir string `arg:"positional" default:"./build/bin/devnet"` + DataDir string `arg:"--datadir" default:"./dev"` + Chain string `arg:"--chain" default:"dev"` + Port int `arg:"--port"` + AllowedPorts string `arg:"--p2p.allowed-ports"` + NAT string `arg:"--nat" default:"none"` + ConsoleVerbosity string `arg:"--log.console.verbosity" default:"0"` + DirVerbosity string `arg:"--log.dir.verbosity"` + LogDirPath string `arg:"--log.dir.path"` + LogDirPrefix string `arg:"--log.dir.prefix"` + P2PProtocol string `arg:"--p2p.protocol" default:"68"` + Downloader string `arg:"--no-downloader" default:"true"` + WS string `arg:"--ws" flag:"" default:"true"` + PrivateApiAddr string `arg:"--private.api.addr" default:"localhost:9090"` + HttpPort int `arg:"--http.port" default:"8545"` + HttpVHosts string `arg:"--http.vhosts"` + AuthRpcPort int `arg:"--authrpc.port" default:"8551"` + AuthRpcVHosts string `arg:"--authrpc.vhosts"` + WSPort int `arg:"-" default:"8546"` // flag not defined + GRPCPort int `arg:"-" default:"8547"` // flag not defined + TCPPort int `arg:"-" default:"8548"` // flag not defined + StaticPeers string `arg:"--staticpeers"` + WithoutHeimdall bool `arg:"--bor.withoutheimdall" flag:"" default:"false"` +} + +func (node *Node) configure(base Node, nodeNumber int) error { + node.DataDir = filepath.Join(base.DataDir, fmt.Sprintf("%d", nodeNumber)) + + node.LogDirPath = filepath.Join(base.DataDir, "logs") + node.LogDirPrefix = fmt.Sprintf("node-%d", nodeNumber) + + node.Chain = base.Chain + + node.StaticPeers = base.StaticPeers + + var err error + + node.PrivateApiAddr, _, err = portFromBase(base.PrivateApiAddr, nodeNumber, 1) + + if err != nil { + return err + } + + apiPort := base.HttpPort + (nodeNumber * 5) + + node.HttpPort = apiPort + node.WSPort = apiPort + 1 + node.GRPCPort = apiPort + 2 + node.TCPPort = apiPort + 3 + node.AuthRpcPort = apiPort + 4 + + return nil +} + +type Miner struct { + Node + Mine bool `arg:"--mine" flag:"true"` + DevPeriod int `arg:"--dev.period"` + BorPeriod int `arg:"--bor.period"` + BorMinBlockSize int `arg:"--bor.minblocksize"` + HttpApi string `arg:"--http.api" default:"admin,eth,erigon,web3,net,debug,trace,txpool,parity,ots"` + AccountSlots int `arg:"--txpool.accountslots" default:"16"` +} + +func (m Miner) Configure(baseNode Node, nodeNumber int) (int, interface{}, error) { + err := m.configure(baseNode, nodeNumber) + + if err != nil { + return -1, nil, err + } + + switch m.Chain { + case networkname.DevChainName: + if m.DevPeriod == 0 { + m.DevPeriod = 30 + } + } + + return m.HttpPort, m, nil +} + +func (n Miner) IsMiner() bool { + return true +} + +type NonMiner struct { + Node + HttpApi string `arg:"--http.api" default:"admin,eth,debug,net,trace,web3,erigon,txpool"` + TorrentPort string `arg:"--torrent.port" default:"42070"` + NoDiscover string `arg:"--nodiscover" flag:"" default:"true"` +} + +func (n NonMiner) Configure(baseNode Node, nodeNumber int) (int, interface{}, error) { + err := n.configure(baseNode, nodeNumber) + + if err != nil { + return -1, nil, err + } + + return n.HttpPort, n, nil +} + +func (n NonMiner) IsMiner() bool { + return false +} + +func portFromBase(baseAddr string, increment int, portCount int) (string, int, error) { + apiHost, apiPort, err := net.SplitHostPort(baseAddr) + + if err != nil { + return "", -1, err + } + + portNo, err := strconv.Atoi(apiPort) + + if err != nil { + return "", -1, err + } + + portNo += (increment * portCount) + + return fmt.Sprintf("%s:%d", apiHost, portNo), portNo, nil +} diff --git a/cmd/devnet/node/node_test.go b/cmd/devnet/args/node_test.go similarity index 87% rename from cmd/devnet/node/node_test.go rename to cmd/devnet/args/node_test.go index 8c241c03c7a..707c4da40eb 100644 --- a/cmd/devnet/node/node_test.go +++ b/cmd/devnet/args/node_test.go @@ -1,4 +1,4 @@ -package node_test +package args_test import ( "errors" @@ -6,21 +6,21 @@ import ( "path/filepath" "testing" - "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" - "github.com/ledgerwatch/erigon/cmd/devnet/node" + "github.com/ledgerwatch/erigon/cmd/devnet/args" ) func TestNodeArgs(t *testing.T) { asMap := map[string]struct{}{} - args, _ := devnetutils.AsArgs(node.Miner{ - Node: node.Node{ + nodeArgs, _ := args.AsArgs(args.Miner{ + Node: args.Node{ DataDir: filepath.Join("data", fmt.Sprintf("%d", 1)), PrivateApiAddr: "localhost:9092", }, + DevPeriod: 30, }) - for _, arg := range args { + for _, arg := range nodeArgs { asMap[arg] = struct{}{} } @@ -36,15 +36,15 @@ func TestNodeArgs(t *testing.T) { t.Fatal(asMap, "not found") } - args, _ = devnetutils.AsArgs(node.NonMiner{ - Node: node.Node{ + nodeArgs, _ = args.AsArgs(args.NonMiner{ + Node: args.Node{ DataDir: filepath.Join("data", fmt.Sprintf("%d", 2)), StaticPeers: "enode", PrivateApiAddr: "localhost:9091", }, }) - for _, arg := range args { + for _, arg := range nodeArgs { asMap[arg] = struct{}{} } @@ -160,8 +160,10 @@ func miningNodeArgs(dataDir string, nodeNumber int) []string { downloaderArg, _ := parameterFromArgument("--no-downloader", "true") httpPortArg, _ := parameterFromArgument("--http.port", "8545") authrpcPortArg, _ := parameterFromArgument("--authrpc.port", "8551") + natArg, _ := parameterFromArgument("--nat", "none") + accountSlotsArg, _ := parameterFromArgument("--txpool.accountslots", "16") - return []string{buildDirArg, dataDirArg, chainType, privateApiAddr, httpPortArg, authrpcPortArg, mine, httpApi, ws, devPeriod, consoleVerbosity, p2pProtocol, downloaderArg} + return []string{buildDirArg, dataDirArg, chainType, privateApiAddr, httpPortArg, authrpcPortArg, mine, httpApi, ws, natArg, devPeriod, consoleVerbosity, p2pProtocol, downloaderArg, accountSlotsArg} } // nonMiningNodeArgs returns custom args for starting a non-mining node @@ -176,8 +178,10 @@ func nonMiningNodeArgs(dataDir string, nodeNumber int, enode string) []string { p2pProtocol, _ := parameterFromArgument("--p2p.protocol", "68") downloaderArg, _ := parameterFromArgument("--no-downloader", "true") httpPortArg, _ := parameterFromArgument("--http.port", "8545") - httpApi, _ := parameterFromArgument(httpApiArg, "eth,debug,net,trace,web3,erigon") + httpApi, _ := parameterFromArgument(httpApiArg, "admin,eth,debug,net,trace,web3,erigon,txpool") authrpcPortArg, _ := parameterFromArgument("--authrpc.port", "8551") + natArg, _ := parameterFromArgument("--nat", "none") + ws := wsArg - return []string{buildDirArg, dataDirArg, chainType, privateApiAddr, httpPortArg, authrpcPortArg, httpApi, staticPeers, noDiscover, consoleVerbosity, torrentPort, p2pProtocol, downloaderArg} + return []string{buildDirArg, dataDirArg, chainType, privateApiAddr, httpPortArg, authrpcPortArg, httpApi, ws, natArg, staticPeers, noDiscover, consoleVerbosity, torrentPort, p2pProtocol, downloaderArg} } diff --git a/cmd/devnet/commands/account.go b/cmd/devnet/commands/account.go index 24a41169ac4..b90d4b93b88 100644 --- a/cmd/devnet/commands/account.go +++ b/cmd/devnet/commands/account.go @@ -1,21 +1,33 @@ package commands import ( + "context" + libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon/cmd/devnet/node" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/requests" - "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" ) -const ( - addr = "0x71562b71999873DB5b286dF957af199Ec94617F7" -) +func init() { + scenarios.MustRegisterStepHandlers( + scenarios.StepHandler(GetBalance), + ) +} + +//const ( +// addr = "0x71562b71999873DB5b286dF957af199Ec94617F7" +//) + +func GetBalance(ctx context.Context, addr string, blockNum requests.BlockNumber, checkBal uint64) { + logger := devnet.Logger(ctx) -func callGetBalance(node *node.Node, addr string, blockNum requests.BlockNumber, checkBal uint64, logger log.Logger) { logger.Info("Getting balance", "addeess", addr) + address := libcommon.HexToAddress(addr) - bal, err := node.GetBalance(address, blockNum) + bal, err := devnet.SelectMiner(ctx).GetBalance(address, blockNum) + if err != nil { logger.Error("FAILURE", "error", err) return diff --git a/cmd/devnet/commands/all.go b/cmd/devnet/commands/all.go index b2a9ebbe618..af3e674a449 100644 --- a/cmd/devnet/commands/all.go +++ b/cmd/devnet/commands/all.go @@ -1,31 +1,21 @@ package commands import ( - "github.com/ledgerwatch/erigon/cmd/devnet/models" - "github.com/ledgerwatch/erigon/cmd/devnet/node" - "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" + "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" "github.com/ledgerwatch/erigon/cmd/devnet/services" "github.com/ledgerwatch/log/v3" ) -// ExecuteAllMethods runs all the simulation tests for erigon devnet -func ExecuteAllMethods(nw *node.Network, logger log.Logger) { - // test connection to JSON RPC - logger.Info("PINGING JSON RPC...") - if err := pingErigonRpc(nw.Node(0), logger); err != nil { - return - } - logger.Info("") - - // get balance of the receiver's account - callGetBalance(nw.Node(0), addr, requests.BlockNumbers.Latest, 0, logger) - logger.Info("") - - // confirm that the txpool is empty - logger.Info("CONFIRMING TXPOOL IS EMPTY BEFORE SENDING TRANSACTION...") - services.CheckTxPoolContent(nw.Node(0), 0, 0, 0, logger) - logger.Info("") +func init() { + scenarios.RegisterStepHandlers( + scenarios.StepHandler(services.CheckTxPoolContent), + scenarios.StepHandler(services.InitSubscriptions), + ) +} +// ExecuteAllMethods runs all the simulation tests for erigon devnet +func ExecuteAllMethods(nw *devnet.Network, logger log.Logger) { /* * Cannot run contract tx after running regular tx because contract tx simulates a new backend * and it expects the nonce to be 0. @@ -40,13 +30,6 @@ func ExecuteAllMethods(nw *node.Network, logger log.Logger) { //} //fmt.Println() - _, err := callSendTxWithDynamicFee(nw.Node(0), recipientAddress, models.DevAddress, logger) - if err != nil { - logger.Error("callSendTxWithDynamicFee", "error", err) - return - } - logger.Info("") - // initiate a contract transaction //fmt.Println("INITIATING A CONTRACT TRANSACTION...") //_, err := callContractTx() @@ -55,7 +38,4 @@ func ExecuteAllMethods(nw *node.Network, logger log.Logger) { // return //} //fmt.Println() - - logger.Info("SEND SIGNAL TO QUIT ALL RUNNING NODES") - models.QuitNodeChan <- true } diff --git a/cmd/devnet/commands/block.go b/cmd/devnet/commands/block.go index 9ced304ce2b..b9580e3eb56 100644 --- a/cmd/devnet/commands/block.go +++ b/cmd/devnet/commands/block.go @@ -1,6 +1,7 @@ package commands import ( + "context" "fmt" "time" @@ -8,20 +9,22 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" - "github.com/ledgerwatch/erigon/cmd/devnet/models" - "github.com/ledgerwatch/erigon/cmd/devnet/node" "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" "github.com/ledgerwatch/erigon/cmd/devnet/services" "github.com/ledgerwatch/erigon/common/hexutil" ) -const ( - recipientAddress = "0x71562b71999873DB5b286dF957af199Ec94617F7" - sendValue uint64 = 10000 -) +func init() { + scenarios.MustRegisterStepHandlers( + scenarios.StepHandler(SendTxWithDynamicFee), + scenarios.StepHandler(AwaitBlocks), + ) +} -func callSendTx(node *node.Node, value uint64, toAddr, fromAddr string, logger log.Logger) (*libcommon.Hash, error) { +func callSendTx(node devnet.Node, value uint64, toAddr, fromAddr string, logger log.Logger) (*libcommon.Hash, error) { logger.Info("Sending tx", "value", value, "to", toAddr, "from", fromAddr) // get the latest nonce for the next transaction @@ -32,7 +35,7 @@ func callSendTx(node *node.Node, value uint64, toAddr, fromAddr string, logger l } // create a non-contract transaction and sign it - signedTx, _, _, _, err := services.CreateTransaction(models.NonContractTx, toAddr, value, nonce) + signedTx, _, err := services.CreateTransaction(toAddr, value, nonce) if err != nil { logger.Error("failed to create a transaction", "error", err) return nil, err @@ -53,33 +56,40 @@ func callSendTx(node *node.Node, value uint64, toAddr, fromAddr string, logger l return hash, nil } -func callSendTxWithDynamicFee(node *node.Node, toAddr, fromAddr string, logger log.Logger) ([]*libcommon.Hash, error) { +func SendTxWithDynamicFee(ctx context.Context, toAddr, fromAddr string, amount uint64) ([]*libcommon.Hash, error) { // get the latest nonce for the next transaction + node := devnet.SelectNode(ctx) + logger := devnet.Logger(ctx) + nonce, err := services.GetNonce(node, libcommon.HexToAddress(fromAddr)) + if err != nil { logger.Error("failed to get latest nonce", "error", err) return nil, err } - lowerThanBaseFeeTxs, higherThanBaseFeeTxs, err := services.CreateManyEIP1559TransactionsRefWithBaseFee2(node, toAddr, &nonce, logger) + lowerThanBaseFeeTxs, higherThanBaseFeeTxs, err := services.CreateManyEIP1559TransactionsRefWithBaseFee2(ctx, toAddr, &nonce) if err != nil { logger.Error("failed CreateManyEIP1559TransactionsRefWithBaseFee", "error", err) return nil, err } - higherThanBaseFeeHashlist, err := services.SendManyTransactions(node, higherThanBaseFeeTxs, logger) + higherThanBaseFeeHashlist, err := services.SendManyTransactions(ctx, higherThanBaseFeeTxs) if err != nil { logger.Error("failed SendManyTransactions(higherThanBaseFeeTxs)", "error", err) return nil, err } - lowerThanBaseFeeHashlist, err := services.SendManyTransactions(node, lowerThanBaseFeeTxs, logger) + lowerThanBaseFeeHashlist, err := services.SendManyTransactions(ctx, lowerThanBaseFeeTxs) + if err != nil { logger.Error("failed SendManyTransactions(lowerThanBaseFeeTxs)", "error", err) return nil, err } - services.CheckTxPoolContent(node, 100, 0, 100, logger) + services.CheckTxPoolContent(ctx, 100, 0, 100) + + services.CheckTxPoolContent(ctx, -1, -1, -1) hashmap := make(map[libcommon.Hash]bool) for _, hash := range higherThanBaseFeeHashlist { @@ -92,37 +102,51 @@ func callSendTxWithDynamicFee(node *node.Node, toAddr, fromAddr string, logger l logger.Info("SUCCESS: All transactions in pending pool included in blocks") + return append(lowerThanBaseFeeHashlist, higherThanBaseFeeHashlist...), nil +} + +func AwaitBlocks(ctx context.Context, sleepTime time.Duration) error { + logger := devnet.Logger(ctx) + for i := 1; i <= 20; i++ { + node := devnet.SelectNode(ctx) + blockNumber, err := node.BlockNumber() + if err != nil { logger.Error("FAILURE => error getting block number", "error", err) } else { logger.Info("Got block number", "blockNum", blockNumber) } + pendingSize, queuedSize, baseFeeSize, err := node.TxpoolContent() + if err != nil { logger.Error("FAILURE getting txpool content", "error", err) } else { logger.Info("Txpool subpool sizes", "pending", pendingSize, "queued", queuedSize, "basefee", baseFeeSize) } - time.Sleep(5 * time.Second) + + time.Sleep(sleepTime) } - return append(lowerThanBaseFeeHashlist, higherThanBaseFeeHashlist...), nil + return nil } -func callContractTx(node *node.Node, logger log.Logger) (*libcommon.Hash, error) { +func callContractTx(node devnet.Node, logger log.Logger) (*libcommon.Hash, error) { // hashset to hold hashes for search after mining hashes := make(map[libcommon.Hash]bool) // get the latest nonce for the next transaction - nonce, err := services.GetNonce(node, libcommon.HexToAddress(models.DevAddress)) + nonce, err := services.GetNonce(node, libcommon.HexToAddress(services.DevAddress)) + if err != nil { logger.Error("failed to get latest nonce", "error", err) return nil, err } // subscriptionContract is the handler to the contract for further operations - signedTx, address, subscriptionContract, transactOpts, err := services.CreateTransaction(models.ContractTx, "", 0, nonce) + signedTx, address, subscriptionContract, transactOpts, err := services.DeploySubsriptionContract(nonce) + if err != nil { logger.Error("failed to create transaction", "error", err) return nil, err @@ -130,18 +154,22 @@ func callContractTx(node *node.Node, logger log.Logger) (*libcommon.Hash, error) // send the contract transaction to the node hash, err := node.SendTransaction(signedTx) + if err != nil { logger.Error("failed to send transaction", "error", err) return nil, err } + hashes[*hash] = true logger.Info("") eventHash, err := services.EmitFallbackEvent(node, subscriptionContract, transactOpts, logger) + if err != nil { logger.Error("failed to emit events", "error", err) return nil, err } + hashes[*eventHash] = true txToBlockMap, err := services.SearchReservesForTransactionHash(hashes, logger) @@ -151,13 +179,13 @@ func callContractTx(node *node.Node, logger log.Logger) (*libcommon.Hash, error) blockNum := (*txToBlockMap)[*eventHash] - block, err := node.GetBlockByNumber(requests.HexToInt(blockNum), true) + block, err := node.GetBlockByNumber(devnetutils.HexToInt(blockNum), true) if err != nil { return nil, err } expectedLog := requests.BuildLog(*eventHash, blockNum, address, - devnetutils.GenerateTopic(models.SolContractMethodSignature), hexutility.Bytes{}, hexutil.Uint(1), + devnetutils.GenerateTopic(services.SolContractMethodSignature), hexutility.Bytes{}, hexutil.Uint(1), block.Result.Hash, hexutil.Uint(0), false) if err = node.GetAndCompareLogs(0, 20, expectedLog); err != nil { @@ -166,8 +194,3 @@ func callContractTx(node *node.Node, logger log.Logger) (*libcommon.Hash, error) return hash, nil } - -func makeEIP1559Checks() { - // run the check for baseFee effect twice - -} diff --git a/cmd/devnet/commands/ping.go b/cmd/devnet/commands/ping.go new file mode 100644 index 00000000000..9298f29e952 --- /dev/null +++ b/cmd/devnet/commands/ping.go @@ -0,0 +1,22 @@ +package commands + +import ( + "context" + + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" + "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" +) + +func init() { + scenarios.MustRegisterStepHandlers( + scenarios.StepHandler(PingErigonRpc), + ) +} + +func PingErigonRpc(ctx context.Context) error { + err := devnet.SelectNode(ctx).PingErigonRpc().Err + if err != nil { + devnet.Logger(ctx).Error("FAILURE", "error", err) + } + return err +} diff --git a/cmd/devnet/commands/request.go b/cmd/devnet/commands/request.go deleted file mode 100644 index 693c61183c4..00000000000 --- a/cmd/devnet/commands/request.go +++ /dev/null @@ -1,14 +0,0 @@ -package commands - -import ( - "github.com/ledgerwatch/erigon/cmd/devnet/node" - "github.com/ledgerwatch/log/v3" -) - -func pingErigonRpc(node *node.Node, logger log.Logger) error { - err := node.PingErigonRpc().Err - if err != nil { - logger.Error("FAILURE", "error", err) - } - return err -} diff --git a/cmd/devnet/devnet/context.go b/cmd/devnet/devnet/context.go new file mode 100644 index 00000000000..bb103b03f5f --- /dev/null +++ b/cmd/devnet/devnet/context.go @@ -0,0 +1,154 @@ +package devnet + +import ( + go_context "context" + + "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" + "github.com/ledgerwatch/log/v3" + "github.com/urfave/cli/v2" +) + +type ctxKey int + +const ( + ckLogger ctxKey = iota + ckNetwork + ckNode + ckCliContext +) + +type Context interface { + go_context.Context + WithValue(key, value interface{}) Context +} + +type context struct { + go_context.Context +} + +func (c *context) WithValue(key, value interface{}) Context { + return &context{go_context.WithValue(c, key, value)} +} + +func AsContext(ctx go_context.Context) Context { + if ctx, ok := ctx.(Context); ok { + return ctx + } + + return &context{ctx} +} + +func WithNetwork(ctx go_context.Context, nw *Network) Context { + return &context{go_context.WithValue(go_context.WithValue(ctx, ckNetwork, nw), ckLogger, nw.Logger)} +} + +func Logger(ctx go_context.Context) log.Logger { + if logger, ok := ctx.Value(ckLogger).(log.Logger); ok { + return logger + } + + return log.Root() +} + +type cnode struct { + selector interface{} + node Node +} + +func WithCurrentNode(ctx go_context.Context, selector interface{}) Context { + return &context{go_context.WithValue(ctx, ckNode, &cnode{selector: selector})} +} + +func WithCliContext(ctx go_context.Context, cliCtx *cli.Context) Context { + return &context{go_context.WithValue(ctx, ckCliContext, cliCtx)} +} + +func CurrentNode(ctx go_context.Context) Node { + if cn, ok := ctx.Value(ckNode).(*cnode); ok { + if cn.node == nil { + if network, ok := ctx.Value(ckNetwork).(*Network); ok { + cn.node = network.SelectNode(ctx, cn.selector) + } + } + + return cn.node + } + + return nil +} + +func SelectNode(ctx go_context.Context, selector ...interface{}) Node { + if network, ok := ctx.Value(ckNetwork).(*Network); ok { + if len(selector) > 0 { + return network.SelectNode(ctx, selector[0]) + } + + if current := CurrentNode(ctx); current != nil { + return current + } + + return network.AnyNode(ctx) + } + + return nil +} + +func SelectMiner(ctx go_context.Context, selector ...interface{}) Node { + if network, ok := ctx.Value(ckNetwork).(*Network); ok { + if len(selector) > 0 { + miners := network.Miners() + switch selector := selector[0].(type) { + case int: + if selector < len(miners) { + return miners[selector] + } + case NodeSelector: + for _, node := range miners { + if selector.Test(ctx, node) { + return node + } + } + } + } + + if current := CurrentNode(ctx); current != nil && current.IsMiner() { + return current + } + + if miners := network.Miners(); len(miners) > 0 { + return miners[devnetutils.RandomInt(len(miners)-1)] + } + } + + return nil +} + +func SelectNonMiner(ctx go_context.Context, selector ...interface{}) Node { + if network, ok := ctx.Value(ckNetwork).(*Network); ok { + if len(selector) > 0 { + nonMiners := network.NonMiners() + switch selector := selector[0].(type) { + case int: + if selector < len(nonMiners) { + return nonMiners[selector] + } + case NodeSelector: + for _, node := range nonMiners { + if selector.Test(ctx, node) { + return node + } + } + } + } + + if current := CurrentNode(ctx); current != nil && !current.IsMiner() { + return current + } + + if nonMiners := network.NonMiners(); len(nonMiners) > 0 { + return nonMiners[devnetutils.RandomInt(len(nonMiners)-1)] + } + } + + return nil +} diff --git a/cmd/devnet/devnet/network.go b/cmd/devnet/devnet/network.go new file mode 100644 index 00000000000..8a12d602a30 --- /dev/null +++ b/cmd/devnet/devnet/network.go @@ -0,0 +1,238 @@ +package devnet + +import ( + go_context "context" + "errors" + "fmt" + "net" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/erigon/cmd/devnet/args" + "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" + "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" + erigonapp "github.com/ledgerwatch/erigon/turbo/app" + erigoncli "github.com/ledgerwatch/erigon/turbo/cli" + "github.com/ledgerwatch/log/v3" +) + +type Network struct { + DataDir string + Chain string + Logger log.Logger + BasePrivateApiAddr string + BaseRPCAddr string + Nodes []Node + wg sync.WaitGroup + peers []string +} + +// Start starts the process for two erigon nodes running on the dev chain +func (nw *Network) Start() error { + + type configurable interface { + Configure(baseNode args.Node, nodeNumber int) (int, interface{}, error) + } + + apiHost, apiPort, err := net.SplitHostPort(nw.BaseRPCAddr) + + if err != nil { + return err + } + + apiPortNo, err := strconv.Atoi(apiPort) + + if err != nil { + return err + } + + baseNode := args.Node{ + DataDir: nw.DataDir, + Chain: nw.Chain, + HttpPort: apiPortNo, + PrivateApiAddr: nw.BasePrivateApiAddr, + } + + for i, node := range nw.Nodes { + if configurable, ok := node.(configurable); ok { + nodePort, args, err := configurable.Configure(baseNode, i) + + if err == nil { + node, err = nw.startNode(fmt.Sprintf("http://%s:%d", apiHost, nodePort), args, i) + } + + if err != nil { + nw.Stop() + return err + } + + nw.Nodes[i] = node + + // get the enode of the node + // - note this has the side effect of waiting for the node to start + if enode, err := getEnode(node); err == nil { + nw.peers = append(nw.peers, enode) + baseNode.StaticPeers = strings.Join(nw.peers, ",") + + // TODO do we need to call AddPeer to the nodes to make them aware of this one + // the current model only works for an appending node network where the peers gossip + // connections - not sure if this is the case ? + } + } + } + + return nil +} + +// startNode starts an erigon node on the dev chain +func (nw *Network) startNode(nodeAddr string, cfg interface{}, nodeNumber int) (Node, error) { + + args, err := devnetutils.AsArgs(cfg) + + if err != nil { + return nil, err + } + + nw.wg.Add(1) + + node := node{ + requests.NewRequestGenerator(nodeAddr, nw.Logger), + cfg, + &nw.wg, + nil, + } + + go func() { + nw.Logger.Info("Running node", "number", nodeNumber, "args", args) + + // catch any errors and avoid panics if an error occurs + defer func() { + panicResult := recover() + if panicResult == nil { + return + } + + nw.Logger.Error("catch panic", "err", panicResult, "stack", dbg.Stack()) + nw.Stop() + os.Exit(1) + }() + + app := erigonapp.MakeApp(fmt.Sprintf("node-%d", nodeNumber), node.run, erigoncli.DefaultFlags) + + if err := app.Run(args); err != nil { + _, printErr := fmt.Fprintln(os.Stderr, err) + if printErr != nil { + nw.Logger.Warn("Error writing app run error to stderr", "err", printErr) + } + } + }() + + return node, nil +} + +// getEnode returns the enode of the mining node +func getEnode(n Node) (string, error) { + reqCount := 0 + + for { + nodeInfo, err := n.AdminNodeInfo() + + if err != nil { + if reqCount < 10 { + var urlErr *url.Error + if errors.As(err, &urlErr) { + var opErr *net.OpError + if errors.As(urlErr.Err, &opErr) { + var callErr *os.SyscallError + if errors.As(opErr.Err, &callErr) { + if callErr.Syscall == "connectex" { + reqCount++ + time.Sleep(time.Duration(devnetutils.RandomInt(5)) * time.Second) + continue + } + } + } + } + } + + return "", err + } + + enode, err := devnetutils.UniqueIDFromEnode(nodeInfo.Enode) + + if err != nil { + return "", err + } + + return enode, nil + } +} + +func (nw *Network) Run(ctx go_context.Context, scenario scenarios.Scenario) error { + return scenarios.Run(WithNetwork(ctx, nw), &scenario) +} + +func (nw *Network) Stop() { + type stoppable interface { + Stop() + } + + for _, n := range nw.Nodes { + if stoppable, ok := n.(stoppable); ok { + stoppable.Stop() + } + } + + nw.wg.Wait() +} + +func (nw *Network) AnyNode(ctx go_context.Context) Node { + return nw.SelectNode(ctx, devnetutils.RandomInt(len(nw.Nodes)-1)) +} + +func (nw *Network) SelectNode(ctx go_context.Context, selector interface{}) Node { + switch selector := selector.(type) { + case int: + if selector < len(nw.Nodes) { + return nw.Nodes[selector] + } + case NodeSelector: + for _, node := range nw.Nodes { + if selector.Test(ctx, node) { + return node + } + } + } + + return nil +} + +func (nw *Network) Miners() []Node { + var miners []Node + + for _, node := range nw.Nodes { + if node.IsMiner() { + miners = append(miners, node) + } + } + + return miners +} + +func (nw *Network) NonMiners() []Node { + var nonMiners []Node + + for _, node := range nw.Nodes { + if !node.IsMiner() { + nonMiners = append(nonMiners, node) + } + } + + return nonMiners +} diff --git a/cmd/devnet/devnet/node.go b/cmd/devnet/devnet/node.go new file mode 100644 index 00000000000..d14761fdc3a --- /dev/null +++ b/cmd/devnet/devnet/node.go @@ -0,0 +1,85 @@ +package devnet + +import ( + go_context "context" + "sync" + + "github.com/ledgerwatch/erigon/cmd/devnet/args" + "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/turbo/debug" + enode "github.com/ledgerwatch/erigon/turbo/node" + "github.com/ledgerwatch/log/v3" + "github.com/urfave/cli/v2" +) + +type Node interface { + requests.RequestGenerator + IsMiner() bool +} + +type NodeSelector interface { + Test(ctx go_context.Context, node Node) bool +} + +type NodeSelectorFunc func(ctx go_context.Context, node Node) bool + +func (f NodeSelectorFunc) Test(ctx go_context.Context, node Node) bool { + return f(ctx, node) +} + +type node struct { + requests.RequestGenerator + args interface{} + wg *sync.WaitGroup + ethNode *enode.ErigonNode +} + +func (n *node) Stop() { + if n.ethNode != nil { + toClose := n.ethNode + n.ethNode = nil + toClose.Close() + } + + if n.wg != nil { + wg := n.wg + n.wg = nil + wg.Done() + } +} + +func (n node) IsMiner() bool { + _, isMiner := n.args.(args.Miner) + return isMiner +} + +// run configures, creates and serves an erigon node +func (n *node) run(ctx *cli.Context) error { + var logger log.Logger + var err error + + if logger, err = debug.Setup(ctx, false /* rootLogger */); err != nil { + return err + } + + logger.Info("Build info", "git_branch", params.GitBranch, "git_tag", params.GitTag, "git_commit", params.GitCommit) + + nodeCfg := enode.NewNodConfigUrfave(ctx, logger) + ethCfg := enode.NewEthConfigUrfave(ctx, nodeCfg, logger) + + n.ethNode, err = enode.New(nodeCfg, ethCfg, logger) + + if err != nil { + logger.Error("Node startup", "err", err) + return err + } + + err = n.ethNode.Serve() + + if err != nil { + logger.Error("error while serving Devnet node", "err", err) + } + + return err +} diff --git a/cmd/devnet/devnetutils/utils.go b/cmd/devnet/devnetutils/utils.go index c66d7eefb04..960e0b5abaf 100644 --- a/cmd/devnet/devnetutils/utils.go +++ b/cmd/devnet/devnetutils/utils.go @@ -2,8 +2,8 @@ package devnetutils import ( "crypto/rand" + "encoding/binary" "fmt" - "math/big" "net" "os" "path/filepath" @@ -46,6 +46,13 @@ func ClearDevDB(dataDir string, logger log.Logger) error { return nil } +// HexToInt converts a hexadecimal string to uint64 +func HexToInt(hexStr string) uint64 { + cleaned := strings.ReplaceAll(hexStr, "0x", "") // remove the 0x prefix + result, _ := strconv.ParseUint(cleaned, 16, 64) + return result +} + // UniqueIDFromEnode returns the unique ID from a node's enode, removing the `?discport=0` part func UniqueIDFromEnode(enode string) (string, error) { if len(enode) == 0 { @@ -85,6 +92,16 @@ func UniqueIDFromEnode(enode string) (string, error) { return enode[:i], nil } +func RandomInt(max int) int { + if max == 0 { + return 0 + } + + var n uint16 + binary.Read(rand.Reader, binary.LittleEndian, &n) + return int(n) % (max + 1) +} + // NamespaceAndSubMethodFromMethod splits a parent method into namespace and the actual method func NamespaceAndSubMethodFromMethod(method string) (string, string, error) { parts := strings.SplitN(method, "_", 2) @@ -105,14 +122,7 @@ func RandomNumberInRange(min, max uint64) (uint64, error) { return 0, fmt.Errorf("Invalid range: upper bound %d less or equal than lower bound %d", max, min) } - diff := int64(max - min) - - n, err := rand.Int(rand.Reader, big.NewInt(diff)) - if err != nil { - return 0, err - } - - return uint64(n.Int64() + int64(min)), nil + return uint64(RandomInt(int(max-min)) + int(min)), nil } type Args []string @@ -176,7 +186,17 @@ func AsArgs(args interface{}) (Args, error) { key = "--" + strings.ToLower(field.Name) } - value := fmt.Sprintf("%v", v.FieldByIndex(field.Index).Interface()) + var value string + + switch fv := v.FieldByIndex(field.Index); fv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if fv.Int() == 0 { + break + } + fallthrough + default: + value = fmt.Sprintf("%v", fv.Interface()) + } flagValue, isFlag := field.Tag.Lookup("flag") @@ -188,7 +208,7 @@ func AsArgs(args interface{}) (Args, error) { } } - if len(value) == 0 || value == "0" { + if len(value) == 0 { if defaultString, hasDefault := field.Tag.Lookup("default"); hasDefault { value = defaultString } diff --git a/cmd/devnet/devnetutils/utils_test.go b/cmd/devnet/devnetutils/utils_test.go index a756e4c9dbe..94f22d7155a 100644 --- a/cmd/devnet/devnetutils/utils_test.go +++ b/cmd/devnet/devnetutils/utils_test.go @@ -7,6 +7,23 @@ import ( "github.com/stretchr/testify/require" ) +func TestHexToInt(t *testing.T) { + testCases := []struct { + hexStr string + expected uint64 + }{ + {"0x0", 0}, + {"0x32424", 205860}, + {"0x200", 512}, + {"0x39", 57}, + } + + for _, testCase := range testCases { + got := HexToInt(testCase.hexStr) + require.EqualValues(t, testCase.expected, got) + } +} + func TestUniqueIDFromEnode(t *testing.T) { testCases := []struct { input string diff --git a/cmd/devnet/main.go b/cmd/devnet/main.go index 2a405526449..28db249ac8c 100644 --- a/cmd/devnet/main.go +++ b/cmd/devnet/main.go @@ -1,15 +1,20 @@ package main import ( + "context" "fmt" "os" "path/filepath" + dbg "runtime/debug" "time" - "github.com/ledgerwatch/erigon/cmd/devnet/commands" + _ "github.com/ledgerwatch/erigon/cmd/devnet/commands" + + "github.com/ledgerwatch/erigon/cmd/devnet/args" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" - "github.com/ledgerwatch/erigon/cmd/devnet/node" "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/cmd/devnet/scenarios" "github.com/ledgerwatch/erigon/cmd/devnet/services" "github.com/ledgerwatch/erigon/params/networkname" "github.com/ledgerwatch/log/v3" @@ -21,20 +26,32 @@ import ( "github.com/urfave/cli/v2" ) -var DataDirFlag = flags.DirectoryFlag{ - Name: "datadir", - Usage: "Data directory for the devnet", - Value: flags.DirectoryString(""), - Required: true, -} +var ( + DataDirFlag = flags.DirectoryFlag{ + Name: "datadir", + Usage: "Data directory for the devnet", + Value: flags.DirectoryString(""), + Required: true, + } + + ChainFlag = cli.StringFlag{ + Name: "chain", + Usage: "The devnet chain to run (dev,bor-devnet)", + Value: networkname.DevChainName, + } + + WithoutHeimdallFlag = cli.BoolFlag{ + Name: "bor.withoutheimdall", + Usage: "Run without Heimdall service", + } +) type PanicHandler struct { } func (ph PanicHandler) Log(r *log.Record) error { - fmt.Println("LEGACY LOG:", r.Msg) - //fmt.Printf("Stack: %s\n", dbg.Stack()) - //os.Exit(1) + fmt.Printf("Msg: %s\nStack: %s\n", r.Msg, dbg.Stack()) + os.Exit(1) return nil } @@ -49,6 +66,8 @@ func main() { } app.Flags = []cli.Flag{ &DataDirFlag, + &ChainFlag, + &WithoutHeimdallFlag, } app.After = func(ctx *cli.Context) error { @@ -61,12 +80,19 @@ func main() { } } +const ( + recipientAddress = "0x71562b71999873DB5b286dF957af199Ec94617F7" + sendValue uint64 = 10000 +) + func action(ctx *cli.Context) error { dataDir := ctx.String("datadir") logsDir := filepath.Join(dataDir, "logs") + if err := os.MkdirAll(logsDir, 0755); err != nil { return err } + logger := logging.SetupLoggerCtx("devnet", ctx, false /* rootLogger */) // Make root logger fail @@ -77,33 +103,133 @@ func action(ctx *cli.Context) error { return err } - network := &node.Network{ - DataDir: dataDir, - Chain: networkname.DevChainName, - //Chain: networkname.BorDevnetChainName, - Logger: logger, - BasePrivateApiAddr: "localhost:9090", - BaseRPCAddr: "localhost:8545", - Nodes: []node.NetworkNode{ - &node.Miner{}, - &node.NonMiner{}, - }, + network, err := selectNetwork(ctx, logger) + + if err != nil { + return err } // start the network with each node in a go routine - network.Start() - - // sleep for seconds to allow the nodes fully start up - time.Sleep(time.Second * 10) + logger.Info("Starting Network") + if err := network.Start(); err != nil { + return fmt.Errorf("Network start failed: %w", err) + } - // start up the subscription services for the different sub methods - services.InitSubscriptions([]requests.SubMethod{requests.Methods.ETHNewHeads}, logger) + runCtx := devnet.WithCliContext(context.Background(), ctx) - // execute all rpc methods amongst the two nodes - commands.ExecuteAllMethods(network, logger) + if ctx.String(ChainFlag.Name) == networkname.DevChainName { + // the dev network currently inserts blocks very slowly when run in multi node mode - needs investigaton + // this effectively makes it a ingle node network by routing all traffic to node 0 + // devnet.WithCurrentNode(devnet.WithCliContext(context.Background(), ctx), 0) + services.MaxNumberOfEmptyBlockChecks = 30 + } - // wait for all goroutines to complete before exiting - network.Wait() + network.Run( + runCtx, + scenarios.Scenario{ + Name: "all", + Steps: []*scenarios.Step{ + {Text: "InitSubscriptions", Args: []any{[]requests.SubMethod{requests.Methods.ETHNewHeads}}}, + {Text: "PingErigonRpc"}, + {Text: "CheckTxPoolContent", Args: []any{0, 0, 0}}, + {Text: "SendTxWithDynamicFee", Args: []any{recipientAddress, services.DevAddress, sendValue}}, + {Text: "AwaitBlocks", Args: []any{2 * time.Second}}, + }, + }) + + logger.Info("Stopping Network") + network.Stop() return nil } + +func selectNetwork(ctx *cli.Context, logger log.Logger) (*devnet.Network, error) { + dataDir := ctx.String(DataDirFlag.Name) + chain := ctx.String(ChainFlag.Name) + + switch chain { + case networkname.BorDevnetChainName: + if ctx.Bool(WithoutHeimdallFlag.Name) { + return &devnet.Network{ + DataDir: dataDir, + Chain: networkname.BorDevnetChainName, + Logger: logger, + BasePrivateApiAddr: "localhost:10090", + BaseRPCAddr: "localhost:8545", + Nodes: []devnet.Node{ + args.Miner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + WithoutHeimdall: true, + }, + AccountSlots: 200, + }, + args.NonMiner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + WithoutHeimdall: true, + }, + }, + }, + }, nil + } else { + return &devnet.Network{ + DataDir: dataDir, + Chain: networkname.BorDevnetChainName, + Logger: logger, + BasePrivateApiAddr: "localhost:10090", + BaseRPCAddr: "localhost:8545", + Nodes: []devnet.Node{ + args.Miner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + AccountSlots: 200, + }, + args.Miner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + AccountSlots: 200, + }, + args.NonMiner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + }, + }, + }, nil + } + + case networkname.DevChainName: + return &devnet.Network{ + DataDir: dataDir, + Chain: networkname.DevChainName, + Logger: logger, + BasePrivateApiAddr: "localhost:10090", + BaseRPCAddr: "localhost:8545", + Nodes: []devnet.Node{ + args.Miner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + AccountSlots: 200, + }, + args.NonMiner{ + Node: args.Node{ + ConsoleVerbosity: "0", + DirVerbosity: "5", + }, + }, + }, + }, nil + } + + return nil, fmt.Errorf(`Unknown network: "%s"`, chain) +} diff --git a/cmd/devnet/models/errors.go b/cmd/devnet/models/errors.go deleted file mode 100644 index eb385c5128e..00000000000 --- a/cmd/devnet/models/errors.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -import "errors" - -var ( - // ErrInvalidTransactionType for invalid transaction types - ErrInvalidTransactionType = errors.New("invalid transaction type") -) diff --git a/cmd/devnet/models/model.go b/cmd/devnet/models/model.go deleted file mode 100644 index 9d832977b4f..00000000000 --- a/cmd/devnet/models/model.go +++ /dev/null @@ -1,77 +0,0 @@ -package models - -import ( - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon/accounts/abi/bind/backends" - "github.com/ledgerwatch/erigon/cmd/devnet/requests" - "github.com/ledgerwatch/erigon/common/hexutil" - "github.com/ledgerwatch/erigon/core" - "github.com/ledgerwatch/erigon/crypto" - "github.com/ledgerwatch/erigon/rpc" -) - -type ( - // TransactionType is the type of transaction attempted to be made, can be regular or contract - TransactionType string -) - -const ( - // MaxNumberOfBlockChecks is the max number of blocks to look for a transaction in - MaxNumberOfBlockChecks = 3 - - // hexPrivateKey is the hex value for the private key - hexPrivateKey = "26e86e45f6fc45ec6e2ecd128cec80fa1d1505e5507dcd2ae58c3130a7a97b48" - // DevAddress is the developer address for sending - DevAddress = "0x67b1d87101671b127f5f8714789C7192f7ad340e" - - // NonContractTx is the transaction type for sending ether - NonContractTx TransactionType = "non-contract" - // ContractTx is the transaction type for sending ether - ContractTx TransactionType = "contract" - // DynamicFee is the transaction type for dynamic fee - DynamicFee TransactionType = "dynamic-fee" - - // SolContractMethodSignature is the function signature for the event in the solidity contract definition - SolContractMethodSignature = "SubscriptionEvent()" -) - -var ( - // DevSignedPrivateKey is the signed private key for signing transactions - DevSignedPrivateKey, _ = crypto.HexToECDSA(hexPrivateKey) - // gspec is the geth dev genesis block - gspec = core.DeveloperGenesisBlock(uint64(0), libcommon.HexToAddress(DevAddress)) - // ContractBackend is a simulated backend created using a simulated blockchain - ContractBackend = backends.NewSimulatedBackendWithConfig(gspec.Alloc, gspec.Config, 1_000_000) - - // MethodSubscriptionMap is a container for all the subscription methods - MethodSubscriptionMap *map[requests.SubMethod]*MethodSubscription - - // NewHeadsChan is the block cache the eth_NewHeads - NewHeadsChan chan interface{} - - //QuitNodeChan is the channel for receiving a quit signal on all nodes - QuitNodeChan chan bool -) - -// MethodSubscription houses the client subscription, name and channel for its delivery -type MethodSubscription struct { - Client *rpc.Client - ClientSub *rpc.ClientSubscription - Name requests.SubMethod - SubChan chan interface{} -} - -// NewMethodSubscription returns a new MethodSubscription instance -func NewMethodSubscription(name requests.SubMethod) *MethodSubscription { - return &MethodSubscription{ - Name: name, - SubChan: make(chan interface{}), - } -} - -// Block represents a simple block for queries -type Block struct { - Number *hexutil.Big - Transactions []libcommon.Hash - BlockHash libcommon.Hash -} diff --git a/cmd/devnet/node/network.go b/cmd/devnet/node/network.go deleted file mode 100644 index 74cfecbfe44..00000000000 --- a/cmd/devnet/node/network.go +++ /dev/null @@ -1,58 +0,0 @@ -package node - -import ( - "sync" - - "github.com/ledgerwatch/erigon/cmd/devnet/models" - "github.com/ledgerwatch/log/v3" -) - -type Network struct { - DataDir string - Chain string - Logger log.Logger - BasePrivateApiAddr string - BaseRPCAddr string - Nodes []NetworkNode - wg sync.WaitGroup - peers []string -} - -// Start starts the process for two erigon nodes running on the dev chain -func (nw *Network) Start() { - for i, node := range nw.Nodes { - node.Start(nw, i) - - // get the enode of the node - // - note this has the side effect of waiting for the node to start - if enode, err := node.getEnode(); err != nil { - nw.peers = append(nw.peers, enode) - - // TODO we need to call AddPeer to the nodes to make them aware of this one - // the current model only works for a 2 node network - } - - } - - quitOnSignal(&nw.wg) -} - -func (nw *Network) Wait() { - nw.wg.Wait() -} - -func (nw *Network) Node(nodeNumber int) *Node { - return nw.Nodes[nodeNumber].node() -} - -// QuitOnSignal stops the node goroutines after all checks have been made on the devnet -func quitOnSignal(wg *sync.WaitGroup) { - models.QuitNodeChan = make(chan bool) - go func() { - for <-models.QuitNodeChan { - // TODO this assumes 2 nodes and it should be node.Stop() - wg.Done() - wg.Done() - } - }() -} diff --git a/cmd/devnet/node/node.go b/cmd/devnet/node/node.go deleted file mode 100644 index dc3d95a5a29..00000000000 --- a/cmd/devnet/node/node.go +++ /dev/null @@ -1,271 +0,0 @@ -package node - -import ( - "crypto/rand" - "errors" - "fmt" - "math/big" - "net" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/urfave/cli/v2" - - "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" - "github.com/ledgerwatch/erigon/cmd/devnet/requests" - "github.com/ledgerwatch/erigon/params" - "github.com/ledgerwatch/erigon/params/networkname" - erigonapp "github.com/ledgerwatch/erigon/turbo/app" - erigoncli "github.com/ledgerwatch/erigon/turbo/cli" - "github.com/ledgerwatch/erigon/turbo/debug" - "github.com/ledgerwatch/erigon/turbo/node" - "github.com/ledgerwatch/log/v3" -) - -type NetworkNode interface { - Start(nw *Network, nodeNumber int) error - getEnode() (string, error) - node() *Node -} - -type Node struct { - *requests.RequestGenerator `arg:"-"` - BuildDir string `arg:"positional" default:"./build/bin/devnet"` - DataDir string `arg:"--datadir" default:"./dev"` - Chain string `arg:"--chain" default:"dev"` - ConsoleVerbosity string `arg:"--log.console.verbosity" default:"0"` - DirVerbosity string `arg:"--log.dir.verbosity"` - LogDirPath string `arg:"--log.dir.path"` //default:"./cmd/devnet/debug_logs" - P2PProtocol string `arg:"--p2p.protocol" default:"68"` - Downloader string `arg:"--no-downloader" default:"true"` - PrivateApiAddr string `arg:"--private.api.addr" default:"localhost:9090"` - HttpPort int `arg:"--http.port" default:"8545"` - AuthRpcPort int `arg:"--authrpc.port" default:"8551"` - WSPort int `arg:"-" default:"8546"` // flag not defined - GRPCPort int `arg:"-" default:"8547"` // flag not defined - TCPPort int `arg:"-" default:"8548"` // flag not defined - StaticPeers string `arg:"--staticpeers"` - WithoutHeimdall bool `arg:"--bor.withoutheimdall" flag:"" default:"false"` -} - -// getEnode returns the enode of the mining node -func (node Node) getEnode() (string, error) { - reqCount := 0 - - for { - nodeInfo, err := node.AdminNodeInfo() - - if err != nil { - if reqCount < 10 { - var urlErr *url.Error - if errors.As(err, &urlErr) { - var opErr *net.OpError - if errors.As(urlErr.Err, &opErr) { - var callErr *os.SyscallError - if errors.As(opErr.Err, &callErr) { - if callErr.Syscall == "connectex" { - reqCount++ - delay, _ := rand.Int(rand.Reader, big.NewInt(4)) - time.Sleep(time.Duration(delay.Int64()+1) * time.Second) - continue - } - } - } - } - } - - return "", err - } - - enode, err := devnetutils.UniqueIDFromEnode(nodeInfo.Enode) - - if err != nil { - return "", err - } - - return enode, nil - } -} - -func (node *Node) configure(nw *Network, nodeNumber int) (err error) { - node.DataDir = filepath.Join(nw.DataDir, fmt.Sprintf("%d", nodeNumber)) - - //TODO add a log.dir.prefix arg and set it to the node name (node-%d) - //node.LogDirPath = filepath.Join(nw.DataDir, "logs") - - node.Chain = nw.Chain - - node.StaticPeers = strings.Join(nw.peers, ",") - - node.PrivateApiAddr, _, err = portFromBase(nw.BasePrivateApiAddr, nodeNumber, 1) - - if err != nil { - return err - } - - httpApiAddr, apiPort, err := portFromBase(nw.BaseRPCAddr, nodeNumber, 5) - - if err != nil { - return err - } - - node.HttpPort = apiPort - node.WSPort = apiPort + 1 - node.GRPCPort = apiPort + 2 - node.TCPPort = apiPort + 3 - node.AuthRpcPort = apiPort + 4 - - node.RequestGenerator = requests.NewRequestGenerator("http://"+httpApiAddr, nw.Logger) - - return nil -} - -type Miner struct { - Node - Mine bool `arg:"--mine" flag:"true"` - DevPeriod string `arg:"--dev.period" default:"30"` - HttpApi string `arg:"--http.api" default:"admin,eth,erigon,web3,net,debug,trace,txpool,parity,ots"` - WS string `arg:"--ws" flag:"" default:"true"` -} - -func (node *Miner) node() *Node { - return &node.Node -} - -func (node *Miner) Start(nw *Network, nodeNumber int) (err error) { - err = node.configure(nw, nodeNumber) - - if err != nil { - return err - } - - switch node.Chain { - case networkname.BorDevnetChainName: - node.WithoutHeimdall = true - } - - args, err := devnetutils.AsArgs(node) - - if err != nil { - return err - } - - nw.wg.Add(1) - - go startNode(&nw.wg, args, nodeNumber, nw.Logger) - - return nil -} - -type NonMiner struct { - Node - HttpApi string `arg:"--http.api" default:"eth,debug,net,trace,web3,erigon"` - TorrentPort string `arg:"--torrent.port" default:"42070"` - NoDiscover string `arg:"--nodiscover" flag:"" default:"true"` -} - -func (node *NonMiner) node() *Node { - return &node.Node -} - -func (node *NonMiner) Start(nw *Network, nodeNumber int) (err error) { - err = node.configure(nw, nodeNumber) - - if err != nil { - return err - } - - args, err := devnetutils.AsArgs(node) - - if err != nil { - return err - } - - nw.wg.Add(1) - - go startNode(&nw.wg, args, nodeNumber, nw.Logger) - - return nil -} - -func portFromBase(baseAddr string, increment int, portCount int) (string, int, error) { - apiHost, apiPort, err := net.SplitHostPort(baseAddr) - - if err != nil { - return "", -1, err - } - - portNo, err := strconv.Atoi(apiPort) - - if err != nil { - return "", -1, err - } - - portNo += (increment * portCount) - - return fmt.Sprintf("%s:%d", apiHost, portNo), portNo, nil -} - -// startNode starts an erigon node on the dev chain -func startNode(wg *sync.WaitGroup, args []string, nodeNumber int, logger log.Logger) { - logger.Info("Running node", "number", nodeNumber, "args", args) - - // catch any errors and avoid panics if an error occurs - defer func() { - panicResult := recover() - if panicResult == nil { - wg.Done() - return - } - - logger.Error("catch panic", "err", panicResult, "stack", dbg.Stack()) - wg.Done() - //TODO - this should be at process end not here wg should be enough - os.Exit(1) - }() - - app := erigonapp.MakeApp(fmt.Sprintf("node-%d", nodeNumber), runNode, erigoncli.DefaultFlags) - - if err := app.Run(args); err != nil { - _, printErr := fmt.Fprintln(os.Stderr, err) - if printErr != nil { - logger.Warn("Error writing app run error to stderr", "err", printErr) - } - wg.Done() - //TODO - this should be at process end not here wg should be enough - os.Exit(1) - } -} - -// runNode configures, creates and serves an erigon node -func runNode(ctx *cli.Context) error { - // Initializing the node and providing the current git commit there - - var logger log.Logger - var err error - if logger, err = debug.Setup(ctx, false /* rootLogger */); err != nil { - return err - } - logger.Info("Build info", "git_branch", params.GitBranch, "git_tag", params.GitTag, "git_commit", params.GitCommit) - - nodeCfg := node.NewNodConfigUrfave(ctx, logger) - ethCfg := node.NewEthConfigUrfave(ctx, nodeCfg, logger) - - ethNode, err := node.New(nodeCfg, ethCfg, logger) - if err != nil { - logger.Error("Devnet startup", "err", err) - return err - } - - err = ethNode.Serve() - if err != nil { - logger.Error("error while serving Devnet node", "err", err) - } - return err -} diff --git a/cmd/devnet/requests/account.go b/cmd/devnet/requests/account.go index 150cb71f1e7..3c31bed43a7 100644 --- a/cmd/devnet/requests/account.go +++ b/cmd/devnet/requests/account.go @@ -23,7 +23,7 @@ type EthTransaction struct { Value hexutil.Big `json:"value"` } -func (reqGen *RequestGenerator) GetBalance(address libcommon.Address, blockNum BlockNumber) (uint64, error) { +func (reqGen *requestGenerator) GetBalance(address libcommon.Address, blockNum BlockNumber) (uint64, error) { var b EthBalance method, body := reqGen.getBalance(address, blockNum) @@ -38,7 +38,7 @@ func (reqGen *RequestGenerator) GetBalance(address libcommon.Address, blockNum B return b.Balance.ToInt().Uint64(), nil } -func (req *RequestGenerator) getBalance(address libcommon.Address, blockNum BlockNumber) (RPCMethod, string) { +func (req *requestGenerator) getBalance(address libcommon.Address, blockNum BlockNumber) (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":["0x%x","%v"],"id":%d}` return Methods.ETHGetBalance, fmt.Sprintf(template, Methods.ETHGetBalance, address, blockNum, req.reqID) } diff --git a/cmd/devnet/requests/admin.go b/cmd/devnet/requests/admin.go index 662bdd16a99..cba12a5720f 100644 --- a/cmd/devnet/requests/admin.go +++ b/cmd/devnet/requests/admin.go @@ -12,7 +12,7 @@ type AdminNodeInfoResponse struct { Result p2p.NodeInfo `json:"result"` } -func (reqGen *RequestGenerator) AdminNodeInfo() (p2p.NodeInfo, error) { +func (reqGen *requestGenerator) AdminNodeInfo() (p2p.NodeInfo, error) { var b AdminNodeInfoResponse method, body := reqGen.adminNodeInfo() @@ -23,7 +23,7 @@ func (reqGen *RequestGenerator) AdminNodeInfo() (p2p.NodeInfo, error) { return b.Result, nil } -func (req *RequestGenerator) adminNodeInfo() (RPCMethod, string) { +func (req *requestGenerator) adminNodeInfo() (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"id":%d}` return Methods.AdminNodeInfo, fmt.Sprintf(template, Methods.AdminNodeInfo, req.reqID) } diff --git a/cmd/devnet/requests/block.go b/cmd/devnet/requests/block.go index 3012e6fc8da..992b33b7b0f 100644 --- a/cmd/devnet/requests/block.go +++ b/cmd/devnet/requests/block.go @@ -38,7 +38,7 @@ type EthSendRawTransaction struct { TxnHash libcommon.Hash `json:"result"` } -func (reqGen *RequestGenerator) BlockNumber() (uint64, error) { +func (reqGen *requestGenerator) BlockNumber() (uint64, error) { var b EthBlockNumber method, body := reqGen.blockNumber() @@ -52,12 +52,12 @@ func (reqGen *RequestGenerator) BlockNumber() (uint64, error) { return number, nil } -func (req *RequestGenerator) blockNumber() (RPCMethod, string) { +func (req *requestGenerator) blockNumber() (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"id":%d}` return Methods.ETHBlockNumber, fmt.Sprintf(template, Methods.ETHBlockNumber, req.reqID) } -func (reqGen *RequestGenerator) GetBlockByNumber(blockNum uint64, withTxs bool) (EthBlockByNumber, error) { +func (reqGen *requestGenerator) GetBlockByNumber(blockNum uint64, withTxs bool) (EthBlockByNumber, error) { var b EthBlockByNumber method, body := reqGen.getBlockByNumber(blockNum, withTxs) @@ -73,17 +73,17 @@ func (reqGen *RequestGenerator) GetBlockByNumber(blockNum uint64, withTxs bool) return b, nil } -func (req *RequestGenerator) getBlockByNumber(blockNum uint64, withTxs bool) (RPCMethod, string) { +func (req *requestGenerator) getBlockByNumber(blockNum uint64, withTxs bool) (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":["0x%x",%t],"id":%d}` return Methods.ETHGetBlockByNumber, fmt.Sprintf(template, Methods.ETHGetBlockByNumber, blockNum, withTxs, req.reqID) } -func (req *RequestGenerator) getBlockByNumberI(blockNum string, withTxs bool) (RPCMethod, string) { +func (req *requestGenerator) getBlockByNumberI(blockNum string, withTxs bool) (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":["%s",%t],"id":%d}` return Methods.ETHGetBlockByNumber, fmt.Sprintf(template, Methods.ETHGetBlockByNumber, blockNum, withTxs, req.reqID) } -func (reqGen *RequestGenerator) GetBlockByNumberDetails(blockNum string, withTxs bool) (map[string]interface{}, error) { +func (reqGen *requestGenerator) GetBlockByNumberDetails(blockNum string, withTxs bool) (map[string]interface{}, error) { var b struct { CommonResponse Result interface{} `json:"result"` @@ -107,7 +107,7 @@ func (reqGen *RequestGenerator) GetBlockByNumberDetails(blockNum string, withTxs return m, nil } -func (reqGen *RequestGenerator) GetTransactionCount(address libcommon.Address, blockNum BlockNumber) (EthGetTransactionCount, error) { +func (reqGen *requestGenerator) GetTransactionCount(address libcommon.Address, blockNum BlockNumber) (EthGetTransactionCount, error) { var b EthGetTransactionCount method, body := reqGen.getTransactionCount(address, blockNum) @@ -122,16 +122,16 @@ func (reqGen *RequestGenerator) GetTransactionCount(address libcommon.Address, b return b, nil } -func (req *RequestGenerator) getTransactionCount(address libcommon.Address, blockNum BlockNumber) (RPCMethod, string) { +func (req *requestGenerator) getTransactionCount(address libcommon.Address, blockNum BlockNumber) (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":["0x%x","%v"],"id":%d}` return Methods.ETHGetTransactionCount, fmt.Sprintf(template, Methods.ETHGetTransactionCount, address, blockNum, req.reqID) } -func (reqGen *RequestGenerator) SendTransaction(signedTx *types.Transaction) (*libcommon.Hash, error) { +func (reqGen *requestGenerator) SendTransaction(signedTx types.Transaction) (*libcommon.Hash, error) { var b EthSendRawTransaction var buf bytes.Buffer - if err := (*signedTx).MarshalBinary(&buf); err != nil { + if err := signedTx.MarshalBinary(&buf); err != nil { return nil, fmt.Errorf("failed to marshal binary: %v", err) } @@ -140,10 +140,23 @@ func (reqGen *RequestGenerator) SendTransaction(signedTx *types.Transaction) (*l return nil, fmt.Errorf("could not make to request to eth_sendRawTransaction: %v", res.Err) } + zeroHash := true + + for _, hb := range b.TxnHash { + if hb != 0 { + zeroHash = false + break + } + } + + if zeroHash { + return nil, fmt.Errorf("Request: %d, hash: %s, nonce %d: returned a zero transaction hash", b.RequestId, signedTx.Hash().Hex(), signedTx.GetNonce()) + } + return &b.TxnHash, nil } -func (req *RequestGenerator) sendRawTransaction(signedTx []byte) (RPCMethod, string) { +func (req *requestGenerator) sendRawTransaction(signedTx []byte) (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":["0x%x"],"id":%d}` return Methods.ETHSendRawTransaction, fmt.Sprintf(template, Methods.ETHSendRawTransaction, signedTx, req.reqID) } diff --git a/cmd/devnet/requests/event.go b/cmd/devnet/requests/event.go index 9823bd84eba..cfe380eccfa 100644 --- a/cmd/devnet/requests/event.go +++ b/cmd/devnet/requests/event.go @@ -4,10 +4,10 @@ import ( "encoding/json" "fmt" "strconv" - "strings" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/log/v3" ) @@ -51,7 +51,7 @@ func BuildLog(hash libcommon.Hash, blockNum string, address libcommon.Address, t Address: address, Topics: topics, Data: data, - BlockNumber: hexutil.Uint64(HexToInt(blockNum)), + BlockNumber: hexutil.Uint64(devnetutils.HexToInt(blockNum)), TxHash: hash, TxIndex: txIndex, BlockHash: blockHash, @@ -60,14 +60,7 @@ func BuildLog(hash libcommon.Hash, blockNum string, address libcommon.Address, t } } -// HexToInt converts a hexadecimal string to uint64 -func HexToInt(hexStr string) uint64 { - cleaned := strings.ReplaceAll(hexStr, "0x", "") // remove the 0x prefix - result, _ := strconv.ParseUint(cleaned, 16, 64) - return result -} - -func (reqGen *RequestGenerator) GetAndCompareLogs(fromBlock uint64, toBlock uint64, expected Log) error { +func (reqGen *requestGenerator) GetAndCompareLogs(fromBlock uint64, toBlock uint64, expected Log) error { reqGen.logger.Info("GETTING AND COMPARING LOGS") var b EthGetLogs @@ -148,7 +141,7 @@ func hashSlicesAreEqual(s1, s2 []libcommon.Hash) bool { return true } -func (req *RequestGenerator) getLogs(fromBlock, toBlock uint64, address libcommon.Address) (RPCMethod, string) { +func (req *requestGenerator) getLogs(fromBlock, toBlock uint64, address libcommon.Address) (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":[{"fromBlock":"0x%x","toBlock":"0x%x","address":"0x%x"}],"id":%d}` return Methods.ETHGetLogs, fmt.Sprintf(template, Methods.ETHGetLogs, fromBlock, toBlock, address, req.reqID) } diff --git a/cmd/devnet/requests/mock_request.go b/cmd/devnet/requests/mock_request.go deleted file mode 100644 index efcbbd31609..00000000000 --- a/cmd/devnet/requests/mock_request.go +++ /dev/null @@ -1,16 +0,0 @@ -package requests - -import ( - "fmt" - - "github.com/ledgerwatch/log/v3" -) - -func PingErigonRpc(reqGen *RequestGenerator, logger log.Logger) error { - res := reqGen.PingErigonRpc() - if res.Err != nil { - return fmt.Errorf("failed to ping erigon rpc url: %v", res.Err) - } - logger.Info("SUCCESS => OK") - return nil -} diff --git a/cmd/devnet/requests/request_generator.go b/cmd/devnet/requests/request_generator.go index 60d15d60795..c51cb49bf36 100644 --- a/cmd/devnet/requests/request_generator.go +++ b/cmd/devnet/requests/request_generator.go @@ -8,6 +8,9 @@ import ( "strings" "time" + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/p2p" "github.com/ledgerwatch/log/v3" "github.com/valyala/fastjson" ) @@ -34,7 +37,20 @@ type EthError struct { Message string `json:"message"` } -type RequestGenerator struct { +type RequestGenerator interface { + PingErigonRpc() CallResult + GetBalance(address libcommon.Address, blockNum BlockNumber) (uint64, error) + AdminNodeInfo() (p2p.NodeInfo, error) + GetBlockByNumberDetails(blockNum string, withTxs bool) (map[string]interface{}, error) + GetBlockByNumber(blockNum uint64, withTxs bool) (EthBlockByNumber, error) + BlockNumber() (uint64, error) + GetTransactionCount(address libcommon.Address, blockNum BlockNumber) (EthGetTransactionCount, error) + SendTransaction(signedTx types.Transaction) (*libcommon.Hash, error) + GetAndCompareLogs(fromBlock uint64, toBlock uint64, expected Log) error + TxpoolContent() (int, int, int, error) +} + +type requestGenerator struct { reqID int client *http.Client logger log.Logger @@ -100,7 +116,7 @@ var Methods = struct { ETHNewHeads: "eth_newHeads", } -func (req *RequestGenerator) call(method RPCMethod, body string, response interface{}) CallResult { +func (req *requestGenerator) call(method RPCMethod, body string, response interface{}) CallResult { start := time.Now() err := post(req.client, req.target, string(method), body, response, req.logger) req.reqID++ @@ -114,7 +130,7 @@ func (req *RequestGenerator) call(method RPCMethod, body string, response interf } } -func (req *RequestGenerator) PingErigonRpc() CallResult { +func (req *requestGenerator) PingErigonRpc() CallResult { start := time.Now() res := CallResult{ RequestID: req.reqID, @@ -156,17 +172,15 @@ func (req *RequestGenerator) PingErigonRpc() CallResult { return res } -func NewRequestGenerator(target string, logger log.Logger) *RequestGenerator { - var client = &http.Client{ - Timeout: time.Second * 600, - } - reqGen := RequestGenerator{ - client: client, +func NewRequestGenerator(target string, logger log.Logger) RequestGenerator { + return &requestGenerator{ + client: &http.Client{ + Timeout: time.Second * 600, + }, reqID: 1, logger: logger, target: target, } - return &reqGen } func post(client *http.Client, url, method, request string, response interface{}, logger log.Logger) error { diff --git a/cmd/devnet/requests/request_generator_test.go b/cmd/devnet/requests/request_generator_test.go index f6188c6adf1..8cc16e0f654 100644 --- a/cmd/devnet/requests/request_generator_test.go +++ b/cmd/devnet/requests/request_generator_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" ) -func MockRequestGenerator(reqId int) *RequestGenerator { - return &RequestGenerator{ +func MockRequestGenerator(reqId int) *requestGenerator { + return &requestGenerator{ reqID: reqId, client: nil, } @@ -258,20 +258,3 @@ func TestParseResponse(t *testing.T) { require.EqualValues(t, testCase.expected, got) } } - -func TestHexToInt(t *testing.T) { - testCases := []struct { - hexStr string - expected uint64 - }{ - {"0x0", 0}, - {"0x32424", 205860}, - {"0x200", 512}, - {"0x39", 57}, - } - - for _, testCase := range testCases { - got := HexToInt(testCase.hexStr) - require.EqualValues(t, testCase.expected, got) - } -} diff --git a/cmd/devnet/requests/tx.go b/cmd/devnet/requests/tx.go index 06e33fa1b9f..2d32a776221 100644 --- a/cmd/devnet/requests/tx.go +++ b/cmd/devnet/requests/tx.go @@ -9,7 +9,7 @@ type EthTxPool struct { Result interface{} `json:"result"` } -func (reqGen *RequestGenerator) TxpoolContent() (int, int, int, error) { +func (reqGen *requestGenerator) TxpoolContent() (int, int, int, error) { var ( b EthTxPool pending map[string]interface{} @@ -22,7 +22,11 @@ func (reqGen *RequestGenerator) TxpoolContent() (int, int, int, error) { return len(pending), len(queued), len(baseFee), fmt.Errorf("failed to fetch txpool content: %v", res.Err) } - resp := b.Result.(map[string]interface{}) + resp, ok := b.Result.(map[string]interface{}) + + if !ok { + return 0, 0, 0, fmt.Errorf("Unexpected result type: %T", b.Result) + } pendingLen := 0 queuedLen := 0 @@ -52,7 +56,7 @@ func (reqGen *RequestGenerator) TxpoolContent() (int, int, int, error) { return pendingLen, queuedLen, baseFeeLen, nil } -func (req *RequestGenerator) txpoolContent() (RPCMethod, string) { +func (req *requestGenerator) txpoolContent() (RPCMethod, string) { const template = `{"jsonrpc":"2.0","method":%q,"params":[],"id":%d}` return Methods.TxpoolContent, fmt.Sprintf(template, Methods.TxpoolContent, req.reqID) } diff --git a/cmd/devnet/scenarios/context.go b/cmd/devnet/scenarios/context.go new file mode 100644 index 00000000000..622f0a66281 --- /dev/null +++ b/cmd/devnet/scenarios/context.go @@ -0,0 +1,7 @@ +package scenarios + +import "context" + +func stepRunners(ctx context.Context) []*stepRunner { + return nil +} diff --git a/cmd/devnet/scenarios/errors.go b/cmd/devnet/scenarios/errors.go new file mode 100644 index 00000000000..5b7941608a6 --- /dev/null +++ b/cmd/devnet/scenarios/errors.go @@ -0,0 +1,24 @@ +package scenarios + +import "fmt" + +// ErrUndefined is returned in case if step definition was not found +var ErrUndefined = fmt.Errorf("step is undefined") + +type ScenarioError struct { + error + Result ScenarioResult + Cause error +} + +func (e *ScenarioError) Unwrap() error { + return e.error +} + +func NewScenarioError(err error, result ScenarioResult, cause error) *ScenarioError { + return &ScenarioError{err, result, cause} +} + +func (e *ScenarioError) Error() string { + return fmt.Sprintf("%s: Cause: %s", e.error, e.Cause) +} diff --git a/cmd/devnet/scenarios/results.go b/cmd/devnet/scenarios/results.go new file mode 100644 index 00000000000..2f2c414e9dd --- /dev/null +++ b/cmd/devnet/scenarios/results.go @@ -0,0 +1,54 @@ +package scenarios + +import ( + "time" +) + +type ScenarioResult struct { + ScenarioId string + StartedAt time.Time + + StepResults []StepResult +} + +type StepResult struct { + Status StepStatus + FinishedAt time.Time + Err error + + ScenarioId string + + Step *Step +} + +func NewStepResult(scenarioId string, step *Step) StepResult { + return StepResult{FinishedAt: TimeNowFunc(), ScenarioId: scenarioId, Step: step} +} + +type StepStatus int + +const ( + Passed StepStatus = iota + Failed + Skipped + Undefined + Pending +) + +// String ... +func (st StepStatus) String() string { + switch st { + case Passed: + return "passed" + case Failed: + return "failed" + case Skipped: + return "skipped" + case Undefined: + return "undefined" + case Pending: + return "pending" + default: + return "unknown" + } +} diff --git a/cmd/devnet/scenarios/run.go b/cmd/devnet/scenarios/run.go new file mode 100644 index 00000000000..26729163711 --- /dev/null +++ b/cmd/devnet/scenarios/run.go @@ -0,0 +1,120 @@ +package scenarios + +import ( + "context" + "sync" + + "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" +) + +type SimulationInitializer func(*SimulationContext) + +func Run(ctx context.Context, scenarios ...*Scenario) error { + return runner{scenarios: scenarios}.runWithOptions(ctx, getDefaultOptions()) +} + +type runner struct { + randomize bool + stopOnFailure bool + + scenarios []*Scenario + + simulationInitializer SimulationInitializer +} + +func (r *runner) concurrent(ctx context.Context, rate int) (err error) { + var copyLock sync.Mutex + + queue := make(chan int, rate) + scenarios := make([]*Scenario, len(r.scenarios)) + + if r.randomize { + for i := range r.scenarios { + j := devnetutils.RandomInt(i + 1) + scenarios[i] = r.scenarios[j] + } + + } else { + copy(scenarios, r.scenarios) + } + + simulationContext := SimulationContext{ + suite: &suite{ + randomize: r.randomize, + defaultContext: ctx, + stepRunners: stepRunners(ctx), + }, + } + + for i, s := range scenarios { + scenario := *s + + queue <- i // reserve space in queue + + runScenario := func(err *error, Scenario *Scenario) { + defer func() { + <-queue // free a space in queue + }() + + if r.stopOnFailure && *err != nil { + return + } + + // Copy base suite. + suite := *simulationContext.suite + + if r.simulationInitializer != nil { + sc := SimulationContext{suite: &suite} + r.simulationInitializer(&sc) + } + + _, serr := suite.runScenario(&scenario) + if suite.shouldFail(serr) { + copyLock.Lock() + *err = serr + copyLock.Unlock() + } + } + + if rate == 1 { + // Running within the same goroutine for concurrency 1 + // to preserve original stacks and simplify debugging. + runScenario(&err, &scenario) + } else { + go runScenario(&err, &scenario) + } + } + + // wait until last are processed + for i := 0; i < rate; i++ { + queue <- i + } + + close(queue) + + return err +} + +func (runner runner) runWithOptions(ctx context.Context, opt *Options) error { + //var output io.Writer = os.Stdout + //if nil != opt.Output { + // output = opt.Output + //} + + if opt.Concurrency < 1 { + opt.Concurrency = 1 + } + + return runner.concurrent(ctx, opt.Concurrency) +} + +type Options struct { + Concurrency int +} + +func getDefaultOptions() *Options { + opt := &Options{ + Concurrency: 1, + } + return opt +} diff --git a/cmd/devnet/scenarios/scenario.go b/cmd/devnet/scenarios/scenario.go new file mode 100644 index 00000000000..1446bb53c49 --- /dev/null +++ b/cmd/devnet/scenarios/scenario.go @@ -0,0 +1,150 @@ +package scenarios + +import ( + "context" + "errors" + "fmt" + "path" + "reflect" + "regexp" + "runtime" + "unicode" +) + +var ( + ErrUnmatchedStepArgumentNumber = errors.New("func received more arguments than expected") + ErrCannotConvert = errors.New("cannot convert argument") + ErrUnsupportedArgumentType = errors.New("unsupported argument type") +) + +var stepRunnerRegistry = map[reflect.Value]*stepRunner{} + +type stepHandler struct { + handler reflect.Value + matchExpressions []string +} + +func RegisterStepHandlers(handlers ...stepHandler) error { + for _, h := range handlers { + var exprs []*regexp.Regexp + + if kind := h.handler.Kind(); kind != reflect.Func { + return fmt.Errorf("Can't register non-function %s as step handler", kind) + } + + if len(h.matchExpressions) == 0 { + name := path.Ext(runtime.FuncForPC(h.handler.Pointer()).Name())[1:] + + if unicode.IsLower(rune(name[0])) { + return fmt.Errorf("Can't register unexported function %s as step handler", name) + } + + h.matchExpressions = []string{ + name, + } + } + + for _, e := range h.matchExpressions { + exp, err := regexp.Compile(e) + + if err != nil { + return err + } + + exprs = append(exprs, exp) + } + + stepRunnerRegistry[h.handler] = &stepRunner{ + Handler: h.handler, + Exprs: exprs, + } + } + + return nil +} + +func MustRegisterStepHandlers(handlers ...stepHandler) { + if err := RegisterStepHandlers(handlers...); err != nil { + panic(fmt.Errorf("Step handler registration failed: %w", err)) + } +} + +func StepHandler(handler interface{}, matchExpressions ...string) stepHandler { + return stepHandler{reflect.ValueOf(handler), matchExpressions} +} + +type Scenario struct { + Id string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Steps []*Step `json:"steps"` +} + +type Step struct { + Id string `json:"id"` + Args []interface{} `json:"args,omitempty"` + Text string `json:"text"` + Description string `json:"description,omitempty"` +} + +type stepRunner struct { + Exprs []*regexp.Regexp + Handler reflect.Value + // multistep related + Nested bool + Undefined []string +} + +var typeOfBytes = reflect.TypeOf([]byte(nil)) + +var typeOfContext = reflect.TypeOf((*context.Context)(nil)).Elem() + +func (c *stepRunner) Run(ctx context.Context, args []interface{}) (context.Context, interface{}) { + var values = make([]reflect.Value, 0, len(args)) + + typ := c.Handler.Type() + numIn := typ.NumIn() + hasCtxIn := numIn > 0 && typ.In(0).Implements(typeOfContext) + + if hasCtxIn { + values = append(values, reflect.ValueOf(ctx)) + numIn-- + } + + if len(args) < numIn { + return ctx, fmt.Errorf("Expected %d arguments, matched %d from step", typ.NumIn(), len(args)) + } + + for _, arg := range args { + values = append(values, reflect.ValueOf(arg)) + } + + res := c.Handler.Call(values) + + if len(res) == 0 { + return ctx, nil + } + + r := res[0].Interface() + + if rctx, ok := r.(context.Context); ok { + if len(res) == 1 { + return rctx, nil + } + + res = res[1:] + ctx = rctx + } + + if len(res) == 1 { + return ctx, res[0].Interface() + } + + var results = make([]interface{}, 0, len(res)) + + for _, value := range res { + results = append(results, value.Interface()) + } + + return ctx, results +} diff --git a/cmd/devnet/scenarios/stack.go b/cmd/devnet/scenarios/stack.go new file mode 100644 index 00000000000..3b50082fd8f --- /dev/null +++ b/cmd/devnet/scenarios/stack.go @@ -0,0 +1,141 @@ +package scenarios + +import ( + "fmt" + "go/build" + "io" + "path" + "path/filepath" + "runtime" + "strings" +) + +// Frame represents a program counter inside a stack frame. +type stackFrame uintptr + +// pc returns the program counter for this frame; +// multiple frames may have the same PC value. +func (f stackFrame) pc() uintptr { return uintptr(f) - 1 } + +// file returns the full path to the file that contains the +// function for this Frame's pc. +func (f stackFrame) file() string { + fn := runtime.FuncForPC(f.pc()) + if fn == nil { + return "unknown" + } + file, _ := fn.FileLine(f.pc()) + return file +} + +func trimGoPath(file string) string { + for _, p := range filepath.SplitList(build.Default.GOPATH) { + file = strings.Replace(file, filepath.Join(p, "src")+string(filepath.Separator), "", 1) + } + return file +} + +// line returns the line number of source code of the +// function for this Frame's pc. +func (f stackFrame) line() int { + fn := runtime.FuncForPC(f.pc()) + if fn == nil { + return 0 + } + _, line := fn.FileLine(f.pc()) + return line +} + +// Format formats the frame according to the fmt.Formatter interface. +// +// %s source file +// %d source line +// %n function name +// %v equivalent to %s:%d +// +// Format accepts flags that alter the printing of some verbs, as follows: +// +// %+s path of source file relative to the compile time GOPATH +// %+v equivalent to %+s:%d +func (f stackFrame) Format(s fmt.State, verb rune) { + funcname := func(name string) string { + i := strings.LastIndex(name, "/") + name = name[i+1:] + i = strings.Index(name, ".") + return name[i+1:] + } + + switch verb { + case 's': + switch { + case s.Flag('+'): + pc := f.pc() + fn := runtime.FuncForPC(pc) + if fn == nil { + io.WriteString(s, "unknown") + } else { + file, _ := fn.FileLine(pc) + fmt.Fprintf(s, "%s\n\t%s", fn.Name(), trimGoPath(file)) + } + default: + io.WriteString(s, path.Base(f.file())) + } + case 'd': + fmt.Fprintf(s, "%d", f.line()) + case 'n': + name := runtime.FuncForPC(f.pc()).Name() + io.WriteString(s, funcname(name)) + case 'v': + f.Format(s, 's') + io.WriteString(s, ":") + f.Format(s, 'd') + } +} + +// stack represents a stack of program counters. +type stack []uintptr + +func (s *stack) Format(st fmt.State, verb rune) { + switch verb { + case 'v': + switch { + case st.Flag('+'): + for _, pc := range *s { + f := stackFrame(pc) + fmt.Fprintf(st, "\n%+v", f) + } + } + } +} + +func callStack() *stack { + const depth = 32 + var pcs [depth]uintptr + n := runtime.Callers(3, pcs[:]) + var st stack = pcs[0:n] + return &st +} + +// fundamental is an error that has a message and a stack, but no caller. +type traceError struct { + msg string + *stack +} + +func (f *traceError) Error() string { return f.msg } + +func (f *traceError) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + io.WriteString(s, f.msg) + f.stack.Format(s, verb) + return + } + fallthrough + case 's': + io.WriteString(s, f.msg) + case 'q': + fmt.Fprintf(s, "%q", f.msg) + } +} diff --git a/cmd/devnet/scenarios/suite.go b/cmd/devnet/scenarios/suite.go new file mode 100644 index 00000000000..6062b19b2da --- /dev/null +++ b/cmd/devnet/scenarios/suite.go @@ -0,0 +1,378 @@ +package scenarios + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +type SimulationContext struct { + suite *suite +} + +type BeforeScenarioHook func(context.Context, *Scenario) (context.Context, error) +type AfterScenarioHook func(context.Context, *Scenario, error) (context.Context, error) +type BeforeStepHook func(context.Context, *Step) (context.Context, error) +type AfterStepHook func(context.Context, *Step, StepStatus, error) (context.Context, error) + +var TimeNowFunc = func() time.Time { + return time.Now() +} + +type suite struct { + stepRunners []*stepRunner + failed bool + randomize bool + strict bool + stopOnFailure bool + testingT *testing.T + defaultContext context.Context + + // suite event handlers + beforeScenarioHandlers []BeforeScenarioHook + afterScenarioHandlers []AfterScenarioHook + beforeStepHandlers []BeforeStepHook + afterStepHandlers []AfterStepHook +} + +func (s *suite) runSteps(ctx context.Context, scenario *Scenario, steps []*Step) (context.Context, []StepResult, error) { + var results = make([]StepResult, 0, len(steps)) + var err error + + for i, step := range steps { + isLast := i == len(steps)-1 + isFirst := i == 0 + + var stepResult StepResult + + ctx, stepResult = s.runStep(ctx, scenario, step, err, isFirst, isLast) + + switch { + case stepResult.Err == nil: + case errors.Is(stepResult.Err, ErrUndefined): + // do not overwrite failed error + if err == nil { + err = stepResult.Err + } + default: + err = stepResult.Err + } + + results = append(results, stepResult) + } + + return ctx, results, err +} + +func (s *suite) runStep(ctx context.Context, scenario *Scenario, step *Step, prevStepErr error, isFirst, isLast bool) (rctx context.Context, sr StepResult) { + var match *stepRunner + + sr = StepResult{Status: Undefined} + rctx = ctx + + // user multistep definitions may panic + defer func() { + if e := recover(); e != nil { + sr.Err = &traceError{ + msg: fmt.Sprintf("%v", e), + stack: callStack(), + } + } + + earlyReturn := prevStepErr != nil || sr.Err == ErrUndefined + + if !earlyReturn { + sr = NewStepResult(scenario.Id, step) + } + + // Run after step handlers. + rctx, sr.Err = s.runAfterStepHooks(ctx, step, sr.Status, sr.Err) + + // Trigger after scenario on failing or last step to attach possible hook error to step. + if isLast || (sr.Status != Skipped && sr.Status != Undefined && sr.Err != nil) { + rctx, sr.Err = s.runAfterScenarioHooks(rctx, scenario, sr.Err) + } + + if earlyReturn { + return + } + + switch sr.Err { + case nil: + sr.Status = Passed + default: + sr.Status = Failed + } + }() + + // run before scenario handlers + if isFirst { + ctx, sr.Err = s.runBeforeScenarioHooks(ctx, scenario) + } + + // run before step handlers + ctx, sr.Err = s.runBeforeStepHooks(ctx, step, sr.Err) + + if sr.Err != nil { + sr = NewStepResult(step.Id, step) + sr.Status = Failed + return ctx, sr + } + + ctx, undef, match, err := s.maybeUndefined(ctx, step.Text, step.Args) + + if err != nil { + return ctx, sr + } else if len(undef) > 0 { + sr = NewStepResult(scenario.Id, step) + sr.Status = Undefined + sr.Err = ErrUndefined + return ctx, sr + } + + if prevStepErr != nil { + sr = NewStepResult(scenario.Id, step) + sr.Status = Skipped + return ctx, sr + } + + ctx, sr.Err = s.maybeSubSteps(match.Run(ctx, step.Args)) + + return ctx, sr +} + +func (s *suite) maybeUndefined(ctx context.Context, text string, args []interface{}) (context.Context, []string, *stepRunner, error) { + step := s.matchStep(text) + + if nil == step { + return ctx, []string{text}, nil, nil + } + + var undefined []string + + if !step.Nested { + return ctx, undefined, step, nil + } + + ctx, steps := step.Run(ctx, args) + + for _, next := range steps.([]Step) { + ctx, undef, _, err := s.maybeUndefined(ctx, next.Text, nil) + if err != nil { + return ctx, undefined, nil, err + } + undefined = append(undefined, undef...) + } + + return ctx, undefined, nil, nil +} + +func (s *suite) matchStep(text string) *stepRunner { + for _, r := range s.stepRunners { + for _, expr := range r.Exprs { + if m := expr.FindStringSubmatch(text); len(m) > 0 { + return r + } + } + } + + for _, r := range stepRunnerRegistry { + for _, expr := range r.Exprs { + if m := expr.FindStringSubmatch(text); len(m) > 0 { + return r + } + } + } + + return nil +} + +func (s *suite) maybeSubSteps(ctx context.Context, result interface{}) (context.Context, error) { + if nil == result { + return ctx, nil + } + + if err, ok := result.(error); ok { + return ctx, err + } + + steps, ok := result.([]Step) + if !ok { + return ctx, fmt.Errorf("unexpected error, should have been []string: %T - %+v", result, result) + } + + var err error + + for _, step := range steps { + if def := s.matchStep(step.Text); def == nil { + return ctx, ErrUndefined + } else if ctx, err = s.maybeSubSteps(def.Run(ctx, step.Args)); err != nil { + return ctx, fmt.Errorf("%s: %+v", step.Text, err) + } + } + return ctx, nil +} + +func (s *suite) runScenario(scenario *Scenario) (sr *ScenarioResult, err error) { + ctx := s.defaultContext + if ctx == nil { + ctx = context.Background() + } + + ctx, cancel := context.WithCancel(ctx) + + defer cancel() + + if len(scenario.Steps) == 0 { + return &ScenarioResult{ScenarioId: scenario.Id, StartedAt: TimeNowFunc()}, ErrUndefined + } + + // Before scenario hooks are called in context of first evaluated step + // so that error from handler can be added to step. + + sr = &ScenarioResult{ScenarioId: scenario.Id, StartedAt: TimeNowFunc()} + + // scenario + if s.testingT != nil { + // Running scenario as a subtest. + s.testingT.Run(scenario.Name, func(t *testing.T) { + ctx, sr.StepResults, err = s.runSteps(ctx, scenario, scenario.Steps) + if s.shouldFail(err) { + t.Error(err) + } + }) + } else { + ctx, sr.StepResults, err = s.runSteps(ctx, scenario, scenario.Steps) + } + + // After scenario handlers are called in context of last evaluated step + // so that error from handler can be added to step. + + return sr, err +} + +func (s *suite) shouldFail(err error) bool { + if err == nil { + return false + } + + if errors.Is(err, ErrUndefined) { + return s.strict + } + + return true +} + +func (s *suite) runBeforeStepHooks(ctx context.Context, step *Step, err error) (context.Context, error) { + hooksFailed := false + + for _, f := range s.beforeStepHandlers { + hctx, herr := f(ctx, step) + if herr != nil { + hooksFailed = true + + if err == nil { + err = herr + } else { + err = fmt.Errorf("%v, %w", herr, err) + } + } + + if hctx != nil { + ctx = hctx + } + } + + if hooksFailed { + err = fmt.Errorf("before step hook failed: %w", err) + } + + return ctx, err +} + +func (s *suite) runAfterStepHooks(ctx context.Context, step *Step, status StepStatus, err error) (context.Context, error) { + for _, f := range s.afterStepHandlers { + hctx, herr := f(ctx, step, status, err) + + // Adding hook error to resulting error without breaking hooks loop. + if herr != nil { + if err == nil { + err = herr + } else { + err = fmt.Errorf("%v, %w", herr, err) + } + } + + if hctx != nil { + ctx = hctx + } + } + + return ctx, err +} + +func (s *suite) runBeforeScenarioHooks(ctx context.Context, scenario *Scenario) (context.Context, error) { + var err error + + // run before scenario handlers + for _, f := range s.beforeScenarioHandlers { + hctx, herr := f(ctx, scenario) + if herr != nil { + if err == nil { + err = herr + } else { + err = fmt.Errorf("%v, %w", herr, err) + } + } + + if hctx != nil { + ctx = hctx + } + } + + if err != nil { + err = fmt.Errorf("before scenario hook failed: %w", err) + } + + return ctx, err +} + +func (s *suite) runAfterScenarioHooks(ctx context.Context, scenario *Scenario, lastStepErr error) (context.Context, error) { + err := lastStepErr + + hooksFailed := false + isStepErr := true + + // run after scenario handlers + for _, f := range s.afterScenarioHandlers { + hctx, herr := f(ctx, scenario, err) + + // Adding hook error to resulting error without breaking hooks loop. + if herr != nil { + hooksFailed = true + + if err == nil { + isStepErr = false + err = herr + } else { + if isStepErr { + err = fmt.Errorf("step error: %w", err) + isStepErr = false + } + err = fmt.Errorf("%v, %w", herr, err) + } + } + + if hctx != nil { + ctx = hctx + } + } + + if hooksFailed { + err = fmt.Errorf("after scenario hook failed: %w", err) + } + + return ctx, err +} diff --git a/cmd/devnet/services/account.go b/cmd/devnet/services/account.go index cb1af862352..e8d7d04ed3b 100644 --- a/cmd/devnet/services/account.go +++ b/cmd/devnet/services/account.go @@ -5,11 +5,11 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon/cmd/devnet/node" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/requests" ) -func GetNonce(node *node.Node, address libcommon.Address) (uint64, error) { +func GetNonce(node devnet.Node, address libcommon.Address) (uint64, error) { res, err := node.GetTransactionCount(address, requests.BlockNumbers.Latest) if err != nil { return 0, fmt.Errorf("failed to get transaction count for address 0x%x: %v", address, err) diff --git a/cmd/devnet/services/block.go b/cmd/devnet/services/block.go index bad6572283a..416ed65a945 100644 --- a/cmd/devnet/services/block.go +++ b/cmd/devnet/services/block.go @@ -11,12 +11,15 @@ import ( "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/accounts/abi/bind" + "github.com/ledgerwatch/erigon/accounts/abi/bind/backends" "github.com/ledgerwatch/erigon/cmd/devnet/contracts" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" - "github.com/ledgerwatch/erigon/cmd/devnet/models" - "github.com/ledgerwatch/erigon/cmd/devnet/node" "github.com/ledgerwatch/erigon/cmd/devnet/requests" + "github.com/ledgerwatch/erigon/common/hexutil" + "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rpc" ) @@ -24,21 +27,39 @@ import ( const gasPrice = 912_345_678 const gasAmount = 875_000_000 +const ( + // hexPrivateKey is the hex value for the private key + hexPrivateKey = "26e86e45f6fc45ec6e2ecd128cec80fa1d1505e5507dcd2ae58c3130a7a97b48" + // DevAddress is the developer address for sending + DevAddress = "0x67b1d87101671b127f5f8714789C7192f7ad340e" + + // SolContractMethodSignature is the function signature for the event in the solidity contract definition + SolContractMethodSignature = "SubscriptionEvent()" +) + var ( + // DevSignedPrivateKey is the signed private key for signing transactions + DevSignedPrivateKey, _ = crypto.HexToECDSA(hexPrivateKey) + // gspec is the geth dev genesis block + gspec = core.DeveloperGenesisBlock(uint64(0), libcommon.HexToAddress(DevAddress)) + // ContractBackend is a simulated backend created using a simulated blockchain + ContractBackend = backends.NewSimulatedBackendWithConfig(gspec.Alloc, gspec.Config, 1_000_000) + signer = types.LatestSigner(params.AllCliqueProtocolChanges) ) -func CreateManyEIP1559TransactionsRefWithBaseFee(node *node.Node, addr string, startingNonce *uint64, logger log.Logger) ([]*types.Transaction, []*types.Transaction, error) { +func CreateManyEIP1559TransactionsRefWithBaseFee(ctx context.Context, addr string, startingNonce *uint64, logger log.Logger) ([]types.Transaction, []types.Transaction, error) { toAddress := libcommon.HexToAddress(addr) - baseFeePerGas, err := BaseFeeFromBlock(node, logger) + baseFeePerGas, err := BaseFeeFromBlock(ctx) if err != nil { return nil, nil, fmt.Errorf("failed BaseFeeFromBlock: %v", err) } - logger.Info("BaseFeePerGas", "val", baseFeePerGas) + devnet.Logger(ctx).Info("BaseFeePerGas", "val", baseFeePerGas) + + lowerBaseFeeTransactions, higherBaseFeeTransactions, err := signEIP1559TxsLowerAndHigherThanBaseFee2(ctx, 1, 1, baseFeePerGas, startingNonce, toAddress) - lowerBaseFeeTransactions, higherBaseFeeTransactions, err := signEIP1559TxsLowerAndHigherThanBaseFee2(1, 1, baseFeePerGas, startingNonce, toAddress, logger) if err != nil { return nil, nil, fmt.Errorf("failed signEIP1559TxsLowerAndHigherThanBaseFee2: %v", err) } @@ -46,17 +67,18 @@ func CreateManyEIP1559TransactionsRefWithBaseFee(node *node.Node, addr string, s return lowerBaseFeeTransactions, higherBaseFeeTransactions, nil } -func CreateManyEIP1559TransactionsRefWithBaseFee2(node *node.Node, addr string, startingNonce *uint64, logger log.Logger) ([]*types.Transaction, []*types.Transaction, error) { +func CreateManyEIP1559TransactionsRefWithBaseFee2(ctx context.Context, addr string, startingNonce *uint64) ([]types.Transaction, []types.Transaction, error) { toAddress := libcommon.HexToAddress(addr) - baseFeePerGas, err := BaseFeeFromBlock(node, logger) + baseFeePerGas, err := BaseFeeFromBlock(ctx) if err != nil { return nil, nil, fmt.Errorf("failed BaseFeeFromBlock: %v", err) } - logger.Info("BaseFeePerGas2", "val", baseFeePerGas) + devnet.Logger(ctx).Info("BaseFeePerGas2", "val", baseFeePerGas) + + lowerBaseFeeTransactions, higherBaseFeeTransactions, err := signEIP1559TxsLowerAndHigherThanBaseFee2(ctx, 100, 100, baseFeePerGas, startingNonce, toAddress) - lowerBaseFeeTransactions, higherBaseFeeTransactions, err := signEIP1559TxsLowerAndHigherThanBaseFee2(100, 100, baseFeePerGas, startingNonce, toAddress, logger) if err != nil { return nil, nil, fmt.Errorf("failed signEIP1559TxsLowerAndHigherThanBaseFee2: %v", err) } @@ -64,45 +86,31 @@ func CreateManyEIP1559TransactionsRefWithBaseFee2(node *node.Node, addr string, return lowerBaseFeeTransactions, higherBaseFeeTransactions, nil } -// CreateTransaction creates a transaction depending on the type of transaction being passed -func CreateTransaction(txType models.TransactionType, addr string, value, nonce uint64) (*types.Transaction, libcommon.Address, *contracts.Subscription, *bind.TransactOpts, error) { - switch txType { - case models.NonContractTx: - tx, address, err := createNonContractTx(addr, value, nonce) - if err != nil { - return nil, libcommon.Address{}, nil, nil, fmt.Errorf("failed to create non-contract transaction: %v", err) - } - return tx, address, nil, nil, nil - case models.ContractTx: - return createContractTx(nonce) - default: - return nil, libcommon.Address{}, nil, nil, models.ErrInvalidTransactionType - } -} - // createNonContractTx returns a signed transaction and the recipient address -func createNonContractTx(addr string, value, nonce uint64) (*types.Transaction, libcommon.Address, error) { +func CreateTransaction(addr string, value, nonce uint64) (types.Transaction, libcommon.Address, error) { toAddress := libcommon.HexToAddress(addr) // create a new transaction using the parameters to send transaction := types.NewTransaction(nonce, toAddress, uint256.NewInt(value), params.TxGas, uint256.NewInt(gasPrice), nil) // sign the transaction using the developer 0signed private key - signedTx, err := types.SignTx(transaction, *signer, models.DevSignedPrivateKey) + signedTx, err := types.SignTx(transaction, *signer, DevSignedPrivateKey) if err != nil { return nil, libcommon.Address{}, fmt.Errorf("failed to sign non-contract transaction: %v", err) } - return &signedTx, toAddress, nil + return signedTx, toAddress, nil } -func signEIP1559TxsLowerAndHigherThanBaseFee2(amountLower, amountHigher int, baseFeePerGas uint64, nonce *uint64, toAddress libcommon.Address, logger log.Logger) ([]*types.Transaction, []*types.Transaction, error) { - higherBaseFeeTransactions, err := signEIP1559TxsHigherThanBaseFee(amountHigher, baseFeePerGas, nonce, toAddress, logger) +func signEIP1559TxsLowerAndHigherThanBaseFee2(ctx context.Context, amountLower, amountHigher int, baseFeePerGas uint64, nonce *uint64, toAddress libcommon.Address) ([]types.Transaction, []types.Transaction, error) { + higherBaseFeeTransactions, err := signEIP1559TxsHigherThanBaseFee(ctx, amountHigher, baseFeePerGas, nonce, toAddress) + if err != nil { return nil, nil, fmt.Errorf("failed signEIP1559TxsHigherThanBaseFee: %v", err) } - lowerBaseFeeTransactions, err := signEIP1559TxsLowerThanBaseFee(amountLower, baseFeePerGas, nonce, toAddress, logger) + lowerBaseFeeTransactions, err := signEIP1559TxsLowerThanBaseFee(ctx, amountLower, baseFeePerGas, nonce, toAddress) + if err != nil { return nil, nil, fmt.Errorf("failed signEIP1559TxsLowerThanBaseFee: %v", err) } @@ -111,8 +119,8 @@ func signEIP1559TxsLowerAndHigherThanBaseFee2(amountLower, amountHigher int, bas } // signEIP1559TxsLowerThanBaseFee creates n number of transactions with gasFeeCap lower than baseFeePerGas -func signEIP1559TxsLowerThanBaseFee(n int, baseFeePerGas uint64, nonce *uint64, toAddress libcommon.Address, logger log.Logger) ([]*types.Transaction, error) { - var signedTransactions []*types.Transaction +func signEIP1559TxsLowerThanBaseFee(ctx context.Context, n int, baseFeePerGas uint64, nonce *uint64, toAddress libcommon.Address) ([]types.Transaction, error) { + var signedTransactions []types.Transaction var ( minFeeCap = baseFeePerGas - 300_000_000 @@ -132,14 +140,15 @@ func signEIP1559TxsLowerThanBaseFee(n int, baseFeePerGas uint64, nonce *uint64, transaction := types.NewEIP1559Transaction(*signer.ChainID(), *nonce, toAddress, uint256.NewInt(value), uint64(210_000), uint256.NewInt(gasPrice), new(uint256.Int), uint256.NewInt(gasFeeCap), nil) - logger.Info("LOWER", "transaction", i, "nonce", transaction.Nonce, "value", transaction.Value, "feecap", transaction.FeeCap) + devnet.Logger(ctx).Info("LOWER", "transaction", i, "nonce", transaction.Nonce, "value", transaction.Value, "feecap", transaction.FeeCap) + + signedTransaction, err := types.SignTx(transaction, *signer, DevSignedPrivateKey) - signedTransaction, err := types.SignTx(transaction, *signer, models.DevSignedPrivateKey) if err != nil { return nil, err } - signedTransactions = append(signedTransactions, &signedTransaction) + signedTransactions = append(signedTransactions, signedTransaction) *nonce++ } @@ -147,8 +156,8 @@ func signEIP1559TxsLowerThanBaseFee(n int, baseFeePerGas uint64, nonce *uint64, } // signEIP1559TxsHigherThanBaseFee creates amount number of transactions with gasFeeCap higher than baseFeePerGas -func signEIP1559TxsHigherThanBaseFee(n int, baseFeePerGas uint64, nonce *uint64, toAddress libcommon.Address, logger log.Logger) ([]*types.Transaction, error) { - var signedTransactions []*types.Transaction +func signEIP1559TxsHigherThanBaseFee(ctx context.Context, n int, baseFeePerGas uint64, nonce *uint64, toAddress libcommon.Address) ([]types.Transaction, error) { + var signedTransactions []types.Transaction var ( minFeeCap = baseFeePerGas @@ -168,22 +177,22 @@ func signEIP1559TxsHigherThanBaseFee(n int, baseFeePerGas uint64, nonce *uint64, transaction := types.NewEIP1559Transaction(*signer.ChainID(), *nonce, toAddress, uint256.NewInt(value), uint64(210_000), uint256.NewInt(gasPrice), new(uint256.Int), uint256.NewInt(gasFeeCap), nil) - logger.Info("HIGHER", "transaction", i, "nonce", transaction.Nonce, "value", transaction.Value, "feecap", transaction.FeeCap) + devnet.Logger(ctx).Info("HIGHER", "transaction", i, "nonce", transaction.Nonce, "value", transaction.Value, "feecap", transaction.FeeCap) - signedTransaction, err := types.SignTx(transaction, *signer, models.DevSignedPrivateKey) + signedTransaction, err := types.SignTx(transaction, *signer, DevSignedPrivateKey) if err != nil { return nil, err } - signedTransactions = append(signedTransactions, &signedTransaction) + signedTransactions = append(signedTransactions, signedTransaction) *nonce++ } return signedTransactions, nil } -// createContractTx creates and signs a transaction using the developer address, returns the contract and the signed transaction -func createContractTx(nonce uint64) (*types.Transaction, libcommon.Address, *contracts.Subscription, *bind.TransactOpts, error) { +// DeploySubsriptionContract creates and signs a transaction using the developer address, returns the contract and the signed transaction +func DeploySubsriptionContract(nonce uint64) (types.Transaction, libcommon.Address, *contracts.Subscription, *bind.TransactOpts, error) { // initialize transactOpts transactOpts, err := initializeTransactOps(nonce) if err != nil { @@ -191,25 +200,25 @@ func createContractTx(nonce uint64) (*types.Transaction, libcommon.Address, *con } // deploy the contract and get the contract handler - address, txToSign, subscriptionContract, err := contracts.DeploySubscription(transactOpts, models.ContractBackend) + address, txToSign, subscriptionContract, err := contracts.DeploySubscription(transactOpts, ContractBackend) if err != nil { return nil, libcommon.Address{}, nil, nil, fmt.Errorf("failed to deploy subscription: %v", err) } // sign the transaction with the private key - signedTx, err := types.SignTx(txToSign, *signer, models.DevSignedPrivateKey) + signedTx, err := types.SignTx(txToSign, *signer, DevSignedPrivateKey) if err != nil { return nil, libcommon.Address{}, nil, nil, fmt.Errorf("failed to sign tx: %v", err) } - return &signedTx, address, subscriptionContract, transactOpts, nil + return signedTx, address, subscriptionContract, transactOpts, nil } // initializeTransactOps initializes the transactOpts object for a contract transaction func initializeTransactOps(nonce uint64) (*bind.TransactOpts, error) { var chainID = big.NewInt(1337) - transactOpts, err := bind.NewKeyedTransactorWithChainID(models.DevSignedPrivateKey, chainID) + transactOpts, err := bind.NewKeyedTransactorWithChainID(DevSignedPrivateKey, chainID) if err != nil { return nil, fmt.Errorf("cannot create transactor with chainID %d, error: %v", chainID, err) } @@ -221,13 +230,20 @@ func initializeTransactOps(nonce uint64) (*bind.TransactOpts, error) { return transactOpts, nil } +// Block represents a simple block for queries +type Block struct { + Number *hexutil.Big + Transactions []libcommon.Hash + BlockHash libcommon.Hash +} + // txHashInBlock checks if the block with block number has the transaction hash in its list of transactions func txHashInBlock(client *rpc.Client, hashmap map[libcommon.Hash]bool, blockNumber string, txToBlockMap map[libcommon.Hash]string, logger log.Logger) (uint64, int, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // releases the resources held by the context var ( - currBlock models.Block + currBlock Block numFound int ) err := client.CallContext(ctx, &currBlock, string(requests.Methods.ETHGetBlockByNumber), blockNumber, false) @@ -244,7 +260,7 @@ func txHashInBlock(client *rpc.Client, hashmap map[libcommon.Hash]bool, blockNum txToBlockMap[txnHash] = blockNumber delete(hashmap, txnHash) if len(hashmap) == 0 { - return requests.HexToInt(blockNumber), numFound, nil + return devnetutils.HexToInt(blockNumber), numFound, nil } } } @@ -253,7 +269,7 @@ func txHashInBlock(client *rpc.Client, hashmap map[libcommon.Hash]bool, blockNum } // EmitFallbackEvent emits an event from the contract using the fallback method -func EmitFallbackEvent(node *node.Node, subContract *contracts.Subscription, opts *bind.TransactOpts, logger log.Logger) (*libcommon.Hash, error) { +func EmitFallbackEvent(node devnet.Node, subContract *contracts.Subscription, opts *bind.TransactOpts, logger log.Logger) (*libcommon.Hash, error) { logger.Info("EMITTING EVENT FROM FALLBACK...") // adding one to the nonce before initiating another transaction @@ -264,12 +280,12 @@ func EmitFallbackEvent(node *node.Node, subContract *contracts.Subscription, opt return nil, fmt.Errorf("failed to emit event from fallback: %v", err) } - signedTx, err := types.SignTx(tx, *signer, models.DevSignedPrivateKey) + signedTx, err := types.SignTx(tx, *signer, DevSignedPrivateKey) if err != nil { return nil, fmt.Errorf("failed to sign fallback transaction: %v", err) } - hash, err := node.SendTransaction(&signedTx) + hash, err := node.SendTransaction(signedTx) if err != nil { return nil, fmt.Errorf("failed to send fallback transaction: %v", err) } @@ -277,9 +293,9 @@ func EmitFallbackEvent(node *node.Node, subContract *contracts.Subscription, opt return hash, nil } -func BaseFeeFromBlock(node *node.Node, logger log.Logger) (uint64, error) { +func BaseFeeFromBlock(ctx context.Context) (uint64, error) { var val uint64 - res, err := node.GetBlockByNumberDetails("latest", false) + res, err := devnet.SelectNode(ctx).GetBlockByNumberDetails("latest", false) if err != nil { return 0, fmt.Errorf("failed to get base fee from block: %v\n", err) } @@ -287,21 +303,23 @@ func BaseFeeFromBlock(node *node.Node, logger log.Logger) (uint64, error) { if v, ok := res["baseFeePerGas"]; !ok { return val, fmt.Errorf("baseFeePerGas field missing from response") } else { - val = requests.HexToInt(v.(string)) + val = devnetutils.HexToInt(v.(string)) } return val, err } -func SendManyTransactions(node *node.Node, signedTransactions []*types.Transaction, logger log.Logger) ([]*libcommon.Hash, error) { +func SendManyTransactions(ctx context.Context, signedTransactions []types.Transaction) ([]*libcommon.Hash, error) { + logger := devnet.Logger(ctx) + logger.Info("Sending multiple transactions to the txpool...") hashes := make([]*libcommon.Hash, len(signedTransactions)) for idx, tx := range signedTransactions { - hash, err := node.SendTransaction(tx) + hash, err := devnet.SelectNode(ctx).SendTransaction(tx) if err != nil { logger.Error("failed SendTransaction", "error", err) - return nil, err + //return nil, err } hashes[idx] = hash } diff --git a/cmd/devnet/services/event.go b/cmd/devnet/services/event.go index 4b000ca5cfb..3625122be0b 100644 --- a/cmd/devnet/services/event.go +++ b/cmd/devnet/services/event.go @@ -6,34 +6,43 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" "github.com/ledgerwatch/erigon/cmd/devnet/devnetutils" - "github.com/ledgerwatch/erigon/cmd/devnet/models" "github.com/ledgerwatch/erigon/cmd/devnet/requests" "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/log/v3" ) -func InitSubscriptions(methods []requests.SubMethod, logger log.Logger) { +var ( + // MaxNumberOfBlockChecks is the max number of blocks to look for a transaction in + MaxNumberOfEmptyBlockChecks = 25 + Subscriptions *map[requests.SubMethod]*Subscription +) + +// Subscription houses the client subscription, name and channel for its delivery +type Subscription struct { + Client *rpc.Client + ClientSub *rpc.ClientSubscription + Name requests.SubMethod + SubChan chan interface{} +} + +// NewSubscription returns a new Subscription instance +func NewSubscription(name requests.SubMethod) *Subscription { + return &Subscription{ + Name: name, + SubChan: make(chan interface{}), + } +} + +func InitSubscriptions(ctx context.Context, methods []requests.SubMethod) { + logger := devnet.Logger(ctx) + logger.Info("CONNECTING TO WEBSOCKETS AND SUBSCRIBING TO METHODS...") if err := subscribeAll(methods, logger); err != nil { logger.Error("failed to subscribe to all methods", "error", err) return } - - // Initializing subscription methods - logger.Info("INITIATE LISTENS ON SUBSCRIPTION CHANNELS") - models.NewHeadsChan = make(chan interface{}) - - go func() { - methodSub := (*models.MethodSubscriptionMap)[requests.Methods.ETHNewHeads] - if methodSub == nil { - logger.Error("method subscription should not be nil") - return - } - - block := <-methodSub.SubChan - models.NewHeadsChan <- block - }() } func SearchReservesForTransactionHash(hashes map[libcommon.Hash]bool, logger log.Logger) (*map[libcommon.Hash]string, error) { @@ -47,8 +56,8 @@ func SearchReservesForTransactionHash(hashes map[libcommon.Hash]bool, logger log } // subscribe connects to a websocket client and returns the subscription handler and a channel buffer -func subscribe(client *rpc.Client, method requests.SubMethod, args ...interface{}) (*models.MethodSubscription, error) { - methodSub := models.NewMethodSubscription(method) +func subscribe(client *rpc.Client, method requests.SubMethod, args ...interface{}) (*Subscription, error) { + methodSub := NewSubscription(method) namespace, subMethod, err := devnetutils.NamespaceAndSubMethodFromMethod(string(method)) if err != nil { @@ -67,7 +76,7 @@ func subscribe(client *rpc.Client, method requests.SubMethod, args ...interface{ return methodSub, nil } -func subscribeToMethod(method requests.SubMethod, logger log.Logger) (*models.MethodSubscription, error) { +func subscribeToMethod(method requests.SubMethod, logger log.Logger) (*Subscription, error) { client, err := rpc.DialWebsocket(context.Background(), "ws://localhost:8545", "", logger) if err != nil { return nil, fmt.Errorf("failed to dial websocket: %v", err) @@ -90,28 +99,42 @@ func searchBlockForHashes(hashmap map[libcommon.Hash]bool, logger log.Logger) (* txToBlock := make(map[libcommon.Hash]string, len(hashmap)) - toFind := len(hashmap) - methodSub := (*models.MethodSubscriptionMap)[requests.Methods.ETHNewHeads] + methodSub := (*Subscriptions)[requests.Methods.ETHNewHeads] if methodSub == nil { return nil, fmt.Errorf("client subscription should not be nil") } + headsSub := (*Subscriptions)[requests.Methods.ETHNewHeads] + + // get a block from the new heads channel + if headsSub == nil { + return nil, fmt.Errorf("no block heads subscription") + } + var blockCount int for { - // get a block from the new heads channel - block := <-models.NewHeadsChan - blockCount++ // increment the number of blocks seen to check against the max number of blocks to iterate over + block := <-headsSub.SubChan blockNum := block.(map[string]interface{})["number"].(string) _, numFound, foundErr := txHashInBlock(methodSub.Client, hashmap, blockNum, txToBlock, logger) + if foundErr != nil { return nil, fmt.Errorf("failed to find hash in block with number %q: %v", foundErr, blockNum) } - toFind -= numFound // remove the amount of found txs from the amount we're looking for - if toFind == 0 { // this means we have found all the txs we're looking for + + if len(hashmap) == 0 { // this means we have found all the txs we're looking for logger.Info("All the transactions created have been included in blocks") return &txToBlock, nil } - if blockCount == models.MaxNumberOfBlockChecks { + + if numFound == 0 { + blockCount++ // increment the number of blocks seen to check against the max number of blocks to iterate over + } + + if blockCount == MaxNumberOfEmptyBlockChecks { + for h := range hashmap { + logger.Error("Missing Tx", "txHash", h) + } + return nil, fmt.Errorf("timeout when searching for tx") } } @@ -119,10 +142,10 @@ func searchBlockForHashes(hashmap map[libcommon.Hash]bool, logger log.Logger) (* // UnsubscribeAll closes all the client subscriptions and empties their global subscription channel func UnsubscribeAll() { - if models.MethodSubscriptionMap == nil { + if Subscriptions == nil { return } - for _, methodSub := range *models.MethodSubscriptionMap { + for _, methodSub := range *Subscriptions { if methodSub != nil { methodSub.ClientSub.Unsubscribe() for len(methodSub.SubChan) > 0 { @@ -135,14 +158,14 @@ func UnsubscribeAll() { // subscribeAll subscribes to the range of methods provided func subscribeAll(methods []requests.SubMethod, logger log.Logger) error { - m := make(map[requests.SubMethod]*models.MethodSubscription) - models.MethodSubscriptionMap = &m + m := make(map[requests.SubMethod]*Subscription) + Subscriptions = &m for _, method := range methods { sub, err := subscribeToMethod(method, logger) if err != nil { return err } - (*models.MethodSubscriptionMap)[method] = sub + (*Subscriptions)[method] = sub } return nil diff --git a/cmd/devnet/services/tx.go b/cmd/devnet/services/tx.go index b8037d42eca..9c574665526 100644 --- a/cmd/devnet/services/tx.go +++ b/cmd/devnet/services/tx.go @@ -1,30 +1,34 @@ package services import ( - "github.com/ledgerwatch/erigon/cmd/devnet/node" - "github.com/ledgerwatch/log/v3" + "context" + + "github.com/ledgerwatch/erigon/cmd/devnet/devnet" ) -func CheckTxPoolContent(node *node.Node, expectedPendingSize, expectedQueuedSize, expectedBaseFeeSize int, logger log.Logger) { - pendingSize, queuedSize, baseFeeSize, err := node.TxpoolContent() +func CheckTxPoolContent(ctx context.Context, expectedPendingSize, expectedQueuedSize, expectedBaseFeeSize int) { + pendingSize, queuedSize, baseFeeSize, err := devnet.SelectMiner(ctx).TxpoolContent() + + logger := devnet.Logger(ctx) + if err != nil { logger.Error("FAILURE getting txpool content", "error", err) return } - if pendingSize != expectedPendingSize { + if expectedPendingSize >= 0 && pendingSize != expectedPendingSize { logger.Error("FAILURE mismatched pending subpool size", "expected", expectedPendingSize, "got", pendingSize) return } - if queuedSize != expectedQueuedSize { + if expectedQueuedSize >= 0 && queuedSize != expectedQueuedSize { logger.Error("FAILURE mismatched queued subpool size", "expected", expectedQueuedSize, "got", queuedSize) return } - if baseFeeSize != expectedBaseFeeSize { + if expectedBaseFeeSize >= 0 && baseFeeSize != expectedBaseFeeSize { logger.Error("FAILURE mismatched basefee subpool size", "expected", expectedBaseFeeSize, "got", baseFeeSize) } - logger.Info("SUCCESS => subpool sizes", "pending", pendingSize, "queued", queuedSize, "basefee", baseFeeSize) + logger.Info("Subpool sizes", "pending", pendingSize, "queued", queuedSize, "basefee", baseFeeSize) } diff --git a/cmd/downloader/readme.md b/cmd/downloader/readme.md index 01ca8a25df6..6502a59b570 100644 --- a/cmd/downloader/readme.md +++ b/cmd/downloader/readme.md @@ -81,7 +81,7 @@ Erigon does: - wait for download of all snapshots - when .seg available - automatically create .idx files - secondary indices, for example to find block by hash - then switch to normal staged sync (which doesn't require connection to Downloader) -- ensure that snapshot dwnloading happening only once: even if new Erigon version does include new pre-verified snapshot +- ensure that snapshot downloading happens only once: even if new Erigon version does include new pre-verified snapshot hashes, Erigon will not download them (to avoid unpredictable downtime) - but Erigon may produce them by self. Downloader does: @@ -95,7 +95,7 @@ Technical details: - To prevent attack - .idx creation using random Seed - all nodes will have different .idx file (and same .seg files) - If you add/remove any .seg file manually, also need remove `/snapshots/db` folder -## How to verify that .seg files have same checksum withch current .torrent files +## How to verify that .seg files have the same checksum as current .torrent files ``` # Use it if you see weird behavior, bugs, bans, hardware issues, etc... diff --git a/cmd/erigon-el-mock/main.go b/cmd/erigon-el-mock/main.go index be31d630739..98ec8e66a8f 100644 --- a/cmd/erigon-el-mock/main.go +++ b/cmd/erigon-el-mock/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "flag" "net" @@ -9,16 +8,13 @@ import ( "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/gointerfaces/execution" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/erigon-lib/kv/memdb" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" - "github.com/ledgerwatch/erigon/eth/ethconfig" - "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/log/v3" "google.golang.org/grpc" - "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" ) func main() { @@ -43,23 +39,10 @@ func main() { return } } - blockReader, blockWriter := blocksIO(db) - execution.RegisterExecutionServer(s, NewEth1Execution(db, blockReader, blockWriter)) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) + execution.RegisterExecutionServer(s, NewEth1Execution(db, blockReader)) log.Info("Serving mock Execution layer.") if err := s.Serve(lis); err != nil { log.Error("failed to serve", "err", err) } } - -func blocksIO(db kv.RoDB) (services.FullBlockReader, *blockio.BlockWriter) { - var histV3 bool - if err := db.View(context.Background(), func(tx kv.Tx) error { - histV3, _ = kvcfg.HistoryV3.Enabled(tx) - return nil - }); err != nil { - panic(err) - } - br := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) - bw := blockio.NewBlockWriter(histV3) - return br, bw -} diff --git a/cmd/erigon-el-mock/server.go b/cmd/erigon-el-mock/server.go index 1b7921d3765..9f9ab68b48f 100644 --- a/cmd/erigon-el-mock/server.go +++ b/cmd/erigon-el-mock/server.go @@ -14,7 +14,6 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces/execution" types2 "github.com/ledgerwatch/erigon-lib/gointerfaces/types" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/core/rawdb" @@ -27,15 +26,13 @@ type Eth1Execution struct { db kv.RwDB blockReader services.FullBlockReader - blockWriter *blockio.BlockWriter mu sync.Mutex } -func NewEth1Execution(db kv.RwDB, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter) *Eth1Execution { +func NewEth1Execution(db kv.RwDB, blockReader services.FullBlockReader) *Eth1Execution { return &Eth1Execution{ db: db, blockReader: blockReader, - blockWriter: blockWriter, } } @@ -53,7 +50,7 @@ func (e *Eth1Execution) InsertHeaders(ctx context.Context, req *execution.Insert if err != nil { return nil, err } - if err := e.blockWriter.WriteHeader(tx, h); err != nil { + if err := rawdb.WriteHeader(tx, h); err != nil { return nil, err } } @@ -88,7 +85,7 @@ func (e *Eth1Execution) InsertBodies(ctx context.Context, req *execution.InsertB Amount: withdrawal.Amount, }) } - if _, err := e.blockWriter.WriteRawBodyIfNotExists(tx, gointerfaces.ConvertH256ToHash(body.BlockHash), + if _, err := rawdb.WriteRawBodyIfNotExists(tx, gointerfaces.ConvertH256ToHash(body.BlockHash), body.BlockNumber, &types.RawBody{ Transactions: body.Transactions, Uncles: uncles, diff --git a/cmd/erigon-el/backend/backend.go b/cmd/erigon-el/backend/backend.go index c7896e5073d..9aa8b2f08d2 100644 --- a/cmd/erigon-el/backend/backend.go +++ b/cmd/erigon-el/backend/backend.go @@ -16,6 +16,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/turbo/snapshotsync/snap" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" @@ -83,7 +84,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/engineapi" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/shards" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" stages2 "github.com/ledgerwatch/erigon/turbo/stages" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" ) @@ -155,7 +155,7 @@ type Ethereum struct { blockWriter *blockio.BlockWriter agg *libstate.AggregatorV3 - blockSnapshots *snapshotsync.RoSnapshots + blockSnapshots *freezeblocks.RoSnapshots logger log.Logger } @@ -229,7 +229,7 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( logger: logger, } var ( - allSnapshots *snapshotsync.RoSnapshots + allSnapshots *freezeblocks.RoSnapshots ) blockReader, blockWriter, allSnapshots, agg, err := setUpBlockReader(ctx, chainKv, config.Dirs, config.Snapshot, config.HistoryV3, logger) if err != nil { @@ -251,7 +251,7 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( genesisSpec = nil } var genesisErr error - chainConfig, genesis, genesisErr = core.WriteGenesisBlock(tx, genesisSpec, config.OverrideShanghaiTime, tmpdir, logger, blockWriter) + chainConfig, genesis, genesisErr = core.WriteGenesisBlock(tx, genesisSpec, config.OverrideShanghaiTime, tmpdir, logger) if _, ok := genesisErr.(*chain.ConfigCompatError); genesisErr != nil && !ok { return genesisErr } @@ -379,7 +379,7 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( inMemoryExecution := func(batch kv.RwTx, header *types.Header, body *types.RawBody, unwindPoint uint64, headersChain []*types.Header, bodiesChain []*types.RawBody, notifications *shards.Notifications) error { // Needs its own notifications to not update RPC daemon and txpool about pending blocks - stateSync, err := stages2.NewInMemoryExecution(backend.sentryCtx, backend.chainDB, config, backend.sentriesClient, dirs, notifications, allSnapshots, backend.agg, log.New() /* discard logging */) + stateSync, err := stages2.NewInMemoryExecution(backend.sentryCtx, backend.chainDB, config, backend.sentriesClient, dirs, notifications, blockReader, blockWriter, backend.agg, log.New() /* discard logging */) if err != nil { return err } @@ -501,7 +501,7 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( // proof-of-work mining mining := stagedsync.New( stagedsync.MiningStages(backend.sentryCtx, - stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miner, *backend.chainConfig, backend.engine, backend.txPool2, backend.txPool2DB, nil, tmpdir, backend.blockReader), + stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miner, *backend.chainConfig, backend.engine, backend.txPool2DB, nil, tmpdir, backend.blockReader), stagedsync.StageMiningExecCfg(backend.chainDB, miner, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, nil, 0, backend.txPool2, backend.txPool2DB, false, blockReader), stagedsync.StageHashStateCfg(backend.chainDB, dirs, config.HistoryV3), stagedsync.StageTrieCfg(backend.chainDB, false, true, true, tmpdir, backend.blockReader, nil, config.HistoryV3, backend.agg), @@ -519,7 +519,7 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( miningStatePos.MiningConfig.Etherbase = param.SuggestedFeeRecipient proposingSync := stagedsync.New( stagedsync.MiningStages(backend.sentryCtx, - stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miningStatePos, *backend.chainConfig, backend.engine, backend.txPool2, backend.txPool2DB, param, tmpdir, backend.blockReader), + stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miningStatePos, *backend.chainConfig, backend.engine, backend.txPool2DB, param, tmpdir, backend.blockReader), stagedsync.StageMiningExecCfg(backend.chainDB, miningStatePos, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, interrupt, param.PayloadId, backend.txPool2, backend.txPool2DB, param.NoTxPool, blockReader), stagedsync.StageHashStateCfg(backend.chainDB, dirs, config.HistoryV3), stagedsync.StageTrieCfg(backend.chainDB, false, true, true, tmpdir, backend.blockReader, nil, config.HistoryV3, backend.agg), @@ -638,8 +638,9 @@ func NewBackend(stack *node.Node, config *ethconfig.Config, logger log.Logger) ( return nil, err } + blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, backend.notifications.Events, logger) backend.stagedSync, err = stages3.NewStagedSync(backend.sentryCtx, backend.chainDB, stack.Config().P2P, config, - backend.sentriesClient, backend.notifications, backend.downloaderClient, backend.agg, backend.forkValidator, logger, backend.blockReader, backend.blockWriter) + backend.sentriesClient, backend.notifications, backend.downloaderClient, backend.agg, backend.forkValidator, logger, backend.blockReader, backend.blockWriter, blockRetire) if err != nil { return nil, err } @@ -916,13 +917,13 @@ func (s *Ethereum) setUpSnapDownloader(ctx context.Context, downloaderCfg *downl return err } -func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig ethconfig.Snapshot, histV3 bool, logger log.Logger) (services.FullBlockReader, *blockio.BlockWriter, *snapshotsync.RoSnapshots, *libstate.AggregatorV3, error) { - allSnapshots := snapshotsync.NewRoSnapshots(snConfig, dirs.Snap, logger) +func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig ethconfig.BlocksFreezing, histV3 bool, logger log.Logger) (services.FullBlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *libstate.AggregatorV3, error) { + allSnapshots := freezeblocks.NewRoSnapshots(snConfig, dirs.Snap, logger) var err error if !snConfig.NoDownloader { allSnapshots.OptimisticalyReopenWithDB(db) } - blockReader := snapshotsync.NewBlockReader(allSnapshots) + blockReader := freezeblocks.NewBlockReader(allSnapshots) blockWriter := blockio.NewBlockWriter(histV3) dir.MustExist(dirs.SnapHistory) diff --git a/cmd/erigon-el/eth1/execution.go b/cmd/erigon-el/eth1/execution.go index 6f764a91f2e..9fbe71a79ed 100644 --- a/cmd/erigon-el/eth1/execution.go +++ b/cmd/erigon-el/eth1/execution.go @@ -58,7 +58,7 @@ func (e *Eth1Execution) InsertHeaders(ctx context.Context, req *execution.Insert if err != nil { return nil, err } - if err := e.blockWriter.WriteHeader(tx, h); err != nil { + if err := rawdb.WriteHeader(tx, h); err != nil { return nil, err } } @@ -84,7 +84,7 @@ func (e *Eth1Execution) InsertBodies(ctx context.Context, req *execution.InsertB uncles = append(uncles, h) } // Withdrawals processing - if _, err := e.blockWriter.WriteRawBodyIfNotExists(tx, gointerfaces.ConvertH256ToHash(body.BlockHash), + if _, err := rawdb.WriteRawBodyIfNotExists(tx, gointerfaces.ConvertH256ToHash(body.BlockHash), body.BlockNumber, &types.RawBody{ Transactions: body.Transactions, Uncles: uncles, diff --git a/cmd/erigon-el/stages/stages.go b/cmd/erigon-el/stages/stages.go index 1f2f0341502..24274484ee0 100644 --- a/cmd/erigon-el/stages/stages.go +++ b/cmd/erigon-el/stages/stages.go @@ -18,7 +18,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/engineapi" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/shards" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" ) func nullStage(firstCycle bool, badBlockUnwind bool, s *stagedsync.StageState, u stagedsync.Unwinder, tx kv.RwTx, logger log.Logger) error { @@ -45,9 +44,9 @@ func NewStagedSync( logger log.Logger, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter, + blockRetire services.BlockRetire, ) (*stagedsync.Sync, error) { dirs := cfg.Dirs - blockRetire := snapshotsync.NewBlockRetire(1, dirs.Tmp, blockReader, blockWriter, db, snapDownloader, notifications.Events, logger) // During Import we don't want other services like header requests, body requests etc. to be running. // Hence we run it in the test mode. @@ -60,7 +59,7 @@ func NewStagedSync( stagedsync.StageCumulativeIndexCfg(db, blockReader), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(db, controlServer.Bd, controlServer.SendBodyRequest, controlServer.Penalize, controlServer.BroadcastNewBlock, cfg.Sync.BodyDownloadTimeoutSeconds, *controlServer.ChainConfig, blockReader, cfg.HistoryV3, blockWriter), - stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockRetire, blockWriter, blockReader, controlServer.Hd), + stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg( db, cfg.Prune, diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index dd326f5b194..0b390e0c39c 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -24,11 +24,13 @@ import ( "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/math" "github.com/ledgerwatch/erigon/consensus/ethash" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/rpchelper" ) type Prestate struct { @@ -73,9 +75,9 @@ type stEnvMarshaling struct { BaseFee *math.HexOrDecimal256 } -func MakePreState(chainRules *chain.Rules, tx kv.RwTx, accounts types.GenesisAlloc) (*state.PlainStateReader, *state.PlainStateWriter) { +func MakePreState(chainRules *chain.Rules, tx kv.RwTx, accounts types.GenesisAlloc) (state.StateReader, *state.PlainStateWriter) { var blockNr uint64 = 0 - stateReader, stateWriter := state.NewPlainStateReader(tx), state.NewPlainStateWriter(tx, tx, blockNr) + stateReader, stateWriter := rpchelper.NewLatestStateReader(tx), state.NewPlainStateWriter(tx, tx, blockNr) statedb := state.New(stateReader) //ibs for addr, a := range accounts { statedb.SetCode(addr, a.Code) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 5801c72d113..9fc2261e6bb 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -28,6 +28,8 @@ import ( "path/filepath" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/log/v3" "github.com/urfave/cli/v2" @@ -37,8 +39,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" - "github.com/ledgerwatch/erigon-lib/kv/memdb" - "github.com/ledgerwatch/erigon/cmd/rpcdaemon/commands" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/math" @@ -221,7 +221,7 @@ func Main(ctx *cli.Context) error { txsWithKeys = inputData.Txs } // We may have to sign the transactions. - signer := types.MakeSigner(chainConfig, prestate.Env.Number) + signer := types.MakeSigner(chainConfig, prestate.Env.Number, prestate.Env.Timestamp) if txs, err = signUnsignedTransactions(txsWithKeys, *signer); err != nil { return NewError(ErrorJson, fmt.Errorf("failed signing transactions: %v", err)) @@ -293,7 +293,8 @@ func Main(ctx *cli.Context) error { } return h } - db := memdb.New("" /* tmpDir */) + + _, db, _ := temporal.NewTestDB(nil, datadir.New(""), nil) defer db.Close() tx, err := db.BeginRw(context.Background()) @@ -446,13 +447,13 @@ func getTransaction(txJson commands.RPCTransaction) (types.Transaction, error) { dynamicFeeTx := types.DynamicFeeTransaction{ CommonTx: types.CommonTx{ - ChainID: chainId, - Nonce: uint64(txJson.Nonce), - To: txJson.To, - Value: value, - Gas: uint64(txJson.Gas), - Data: txJson.Input, + Nonce: uint64(txJson.Nonce), + To: txJson.To, + Value: value, + Gas: uint64(txJson.Gas), + Data: txJson.Input, }, + ChainID: chainId, Tip: tip, FeeCap: feeCap, AccessList: *txJson.Accesses, diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go index 63c789a0cf9..dcb757be04a 100644 --- a/cmd/hack/hack.go +++ b/cmd/hack/hack.go @@ -33,6 +33,7 @@ import ( librlp "github.com/ledgerwatch/erigon-lib/rlp" "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" @@ -55,7 +56,6 @@ import ( "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/logging" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" ) var ( @@ -138,7 +138,7 @@ func blocksIO(db kv.RoDB) (services.FullBlockReader, *blockio.BlockWriter) { }); err != nil { panic(err) } - br := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) + br := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) bw := blockio.NewBlockWriter(histV3) return br, bw } @@ -311,7 +311,7 @@ func dumpStorage() { db := mdbx.MustOpen(paths.DefaultDataDir() + "/geth/chaindata") defer db.Close() if err := db.View(context.Background(), func(tx kv.Tx) error { - return tx.ForEach(kv.StorageHistory, nil, func(k, v []byte) error { + return tx.ForEach(kv.E2StorageHistory, nil, func(k, v []byte) error { fmt.Printf("%x %x\n", k, v) return nil }) @@ -329,7 +329,7 @@ func printBucket(chaindata string) { fb := bufio.NewWriter(f) defer fb.Flush() if err := db.View(context.Background(), func(tx kv.Tx) error { - c, err := tx.Cursor(kv.StorageHistory) + c, err := tx.Cursor(kv.E2StorageHistory) if err != nil { return err } @@ -534,7 +534,7 @@ func extractHeaders(chaindata string, block uint64, blockTotalOrOffset int64) er } func extractBodies(datadir string) error { - snaps := snapshotsync.NewRoSnapshots(ethconfig.Snapshot{ + snaps := freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{ Enabled: true, KeepBlocks: true, Produce: false, diff --git a/cmd/integration/commands/refetence_db.go b/cmd/integration/commands/refetence_db.go index 3e8fcb7b595..5213b23b11f 100644 --- a/cmd/integration/commands/refetence_db.go +++ b/cmd/integration/commands/refetence_db.go @@ -34,8 +34,8 @@ var stateBuckets = []string{ kv.Code, kv.TrieOfAccounts, kv.TrieOfStorage, - kv.AccountsHistory, - kv.StorageHistory, + kv.E2AccountsHistory, + kv.E2StorageHistory, kv.TxLookup, kv.ContractTEVMCode, } diff --git a/cmd/integration/commands/reset_state.go b/cmd/integration/commands/reset_state.go index ff75a7bf6bc..eb60d6d5fb0 100644 --- a/cmd/integration/commands/reset_state.go +++ b/cmd/integration/commands/reset_state.go @@ -13,6 +13,7 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/state" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" @@ -21,7 +22,6 @@ import ( "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/ethdb/prune" "github.com/ledgerwatch/erigon/turbo/debug" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" ) var cmdResetState = &cobra.Command{ @@ -78,7 +78,7 @@ func init() { rootCmd.AddCommand(cmdResetState) } -func printStages(tx kv.Tx, snapshots *snapshotsync.RoSnapshots, agg *state.AggregatorV3) error { +func printStages(tx kv.Tx, snapshots *freezeblocks.RoSnapshots, agg *state.AggregatorV3) error { var err error var progress uint64 w := new(tabwriter.Writer) diff --git a/cmd/integration/commands/root.go b/cmd/integration/commands/root.go index 8535815dfc8..44687db4d5b 100644 --- a/cmd/integration/commands/root.go +++ b/cmd/integration/commands/root.go @@ -5,14 +5,15 @@ import ( "path/filepath" "github.com/c2h5oh/datasize" - "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/kvcfg" - kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" "github.com/torquem-ch/mdbx-go/mdbx" "golang.org/x/sync/semaphore" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/erigon/cmd/utils" "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/systemcontracts" diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 3deddac2d05..6c4527fbd63 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -12,6 +12,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/ledgerwatch/erigon/core/rawdb/blockio" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/secp256k1" "github.com/spf13/cobra" @@ -27,7 +28,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" libstate "github.com/ledgerwatch/erigon-lib/state" - "github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb" "github.com/ledgerwatch/erigon/cmd/sentry/sentry" "github.com/ledgerwatch/erigon/consensus" @@ -37,7 +37,6 @@ import ( "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/eth/ethconfig" - "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/erigon/eth/ethconsensusconfig" "github.com/ledgerwatch/erigon/eth/integrity" "github.com/ledgerwatch/erigon/eth/stagedsync" @@ -49,7 +48,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/shards" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/erigon/turbo/snapshotsync/snap" stages2 "github.com/ledgerwatch/erigon/turbo/stages" ) @@ -706,7 +704,7 @@ func stageHeaders(db kv.RwDB, ctx context.Context, logger log.Logger) error { if reset { dirs := datadir.New(datadirCli) - if err := reset2.ResetBlocks(tx, db, sn, agg, br, bw, dirs, *chainConfig, engine, logger); err != nil { + if err := reset2.ResetBlocks(tx, db, agg, br, bw, dirs, *chainConfig, engine, logger); err != nil { return err } return nil @@ -731,7 +729,7 @@ func stageHeaders(db kv.RwDB, ctx context.Context, logger log.Logger) error { return fmt.Errorf("re-read Headers progress: %w", err) } { // hard-unwind stage_body also - if err := bw.TruncateBlocks(ctx, tx, progress+1); err != nil { + if err := rawdb.TruncateBlocks(ctx, tx, progress+1); err != nil { return err } progressBodies, err := stages.GetStageProgress(tx, stages.Bodies) @@ -748,7 +746,7 @@ func stageHeaders(db kv.RwDB, ctx context.Context, logger log.Logger) error { if err = rawdb.TruncateCanonicalHash(tx, progress+1, false); err != nil { return err } - if err = bw.TruncateTd(tx, progress+1); err != nil { + if err = rawdb.TruncateTd(tx, progress+1); err != nil { return err } hash, err := br.CanonicalHash(ctx, tx, progress-1) @@ -811,10 +809,9 @@ func stageSenders(db kv.RwDB, ctx context.Context, logger log.Logger) error { must(sync.SetCurrentStage(stages.Senders)) - br, bw := blocksIO(db, logger) + br, _ := blocksIO(db, logger) if reset { - _, bw := blocksIO(db, logger) - return db.Update(ctx, func(tx kv.RwTx) error { return reset2.ResetSenders(ctx, db, tx, bw) }) + return db.Update(ctx, func(tx kv.RwTx) error { return reset2.ResetSenders(ctx, db, tx) }) } tx, err := db.BeginRw(ctx) @@ -846,7 +843,7 @@ func stageSenders(db kv.RwDB, ctx context.Context, logger log.Logger) error { if txs.Len() == 0 { continue } - signer := types.MakeSigner(chainConfig, i) + signer := types.MakeSigner(chainConfig, i, h.Time) for j := 0; j < txs.Len(); j++ { from, err := signer.Sender(txs[j]) if err != nil { @@ -866,17 +863,12 @@ func stageSenders(db kv.RwDB, ctx context.Context, logger log.Logger) error { s := stage(sync, tx, nil, stages.Senders) logger.Info("Stage", "name", s.ID, "progress", s.BlockNumber) - var blockRetire *snapshotsync.BlockRetire - if sn.Cfg().Enabled { - blockRetire = snapshotsync.NewBlockRetire(estimate.CompressSnapshot.Workers(), tmpdir, br, bw, db, nil, nil, logger) - } - pm, err := prune.Get(tx) if err != nil { return err } - cfg := stagedsync.StageSendersCfg(db, chainConfig, false, tmpdir, pm, blockRetire, bw, br, nil) + cfg := stagedsync.StageSendersCfg(db, chainConfig, false, tmpdir, pm, br, nil) if unwind > 0 { u := sync.NewUnwindState(stages.Senders, s.BlockNumber-unwind, s.BlockNumber) if err = stagedsync.UnwindSendersStage(u, tx, cfg, ctx); err != nil { @@ -1362,10 +1354,10 @@ func removeMigration(db kv.RwDB, ctx context.Context) error { } var openSnapshotOnce sync.Once -var _allSnapshotsSingleton *snapshotsync.RoSnapshots +var _allSnapshotsSingleton *freezeblocks.RoSnapshots var _aggSingleton *libstate.AggregatorV3 -func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*snapshotsync.RoSnapshots, *libstate.AggregatorV3) { +func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*freezeblocks.RoSnapshots, *libstate.AggregatorV3) { openSnapshotOnce.Do(func() { var useSnapshots bool _ = db.View(context.Background(), func(tx kv.Tx) error { @@ -1376,7 +1368,7 @@ func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*snapshot dir.MustExist(dirs.SnapHistory) snapCfg := ethconfig.NewSnapCfg(useSnapshots, true, true) - _allSnapshotsSingleton = snapshotsync.NewRoSnapshots(snapCfg, dirs.Snap, logger) + _allSnapshotsSingleton = freezeblocks.NewRoSnapshots(snapCfg, dirs.Snap, logger) var err error _aggSingleton, err = libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db, logger) @@ -1413,7 +1405,7 @@ func blocksIO(db kv.RoDB, logger log.Logger) (services.FullBlockReader, *blockio openBlockReaderOnce.Do(func() { sn, _ := allSnapshots(context.Background(), db, logger) histV3 := kvcfg.HistoryV3.FromDB(db) - _blockReaderSingleton = snapshotsync.NewBlockReader(sn) + _blockReaderSingleton = freezeblocks.NewBlockReader(sn) _blockWriterSingleton = blockio.NewBlockWriter(histV3) }) return _blockReaderSingleton, _blockWriterSingleton @@ -1422,7 +1414,7 @@ func blocksIO(db kv.RoDB, logger log.Logger) (services.FullBlockReader, *blockio var openDomainsOnce sync.Once var _aggDomainSingleton *libstate.Aggregator -func allDomains(ctx context.Context, db kv.RoDB, stepSize uint64, mode libstate.CommitmentMode, trie commitment.TrieVariant, logger log.Logger) (*snapshotsync.RoSnapshots, *libstate.Aggregator) { +func allDomains(ctx context.Context, db kv.RoDB, stepSize uint64, mode libstate.CommitmentMode, trie commitment.TrieVariant, logger log.Logger) (*freezeblocks.RoSnapshots, *libstate.Aggregator) { openDomainsOnce.Do(func() { var useSnapshots bool _ = db.View(context.Background(), func(tx kv.Tx) error { @@ -1433,7 +1425,7 @@ func allDomains(ctx context.Context, db kv.RoDB, stepSize uint64, mode libstate. dir.MustExist(dirs.SnapHistory) snapCfg := ethconfig.NewSnapCfg(useSnapshots, true, true) - _allSnapshotsSingleton = snapshotsync.NewRoSnapshots(snapCfg, dirs.Snap, logger) + _allSnapshotsSingleton = freezeblocks.NewRoSnapshots(snapCfg, dirs.Snap, logger) var err error _aggDomainSingleton, err = libstate.NewAggregator(filepath.Join(dirs.DataDir, "state"), dirs.Tmp, stepSize, mode, trie, logger) @@ -1461,7 +1453,7 @@ func allDomains(ctx context.Context, db kv.RoDB, stepSize uint64, mode libstate. return _allSnapshotsSingleton, _aggDomainSingleton } -func newDomains(ctx context.Context, db kv.RwDB, stepSize uint64, mode libstate.CommitmentMode, trie commitment.TrieVariant, logger log.Logger) (consensus.Engine, ethconfig.Config, *snapshotsync.RoSnapshots, *libstate.Aggregator) { +func newDomains(ctx context.Context, db kv.RwDB, stepSize uint64, mode libstate.CommitmentMode, trie commitment.TrieVariant, logger log.Logger) (consensus.Engine, ethconfig.Config, *freezeblocks.RoSnapshots, *libstate.Aggregator) { historyV3, pm := kvcfg.HistoryV3.FromDB(db), fromdb.PruneMode(db) //events := shards.NewEvents() genesis := core.GenesisBlockByChainName(chain) @@ -1527,7 +1519,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig, engine := initConsensusEngine(chainConfig, cfg.Dirs.DataDir, db, logger) - blockReader, _ := blocksIO(db, logger) + blockReader, blockWriter := blocksIO(db, logger) sentryControlServer, err := sentry.NewMultiClient( db, "", @@ -1547,7 +1539,10 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig, panic(err) } - stages := stages2.NewDefaultStages(context.Background(), db, p2p.Config{}, &cfg, sentryControlServer, &shards.Notifications{}, nil, blockReader, agg, nil, logger) + notifications := &shards.Notifications{} + blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, db, notifications.Events, logger) + + stages := stages2.NewDefaultStages(context.Background(), db, p2p.Config{}, &cfg, sentryControlServer, notifications, nil, blockReader, blockRetire, agg, nil, logger) sync := stagedsync.New(stages, stagedsync.DefaultUnwindOrder, stagedsync.DefaultPruneOrder, logger) miner := stagedsync.NewMiningState(&cfg.Miner) @@ -1558,7 +1553,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig, }() miningSync := stagedsync.New( stagedsync.MiningStages(ctx, - stagedsync.StageMiningCreateBlockCfg(db, miner, *chainConfig, engine, nil, nil, nil, dirs.Tmp, blockReader), + stagedsync.StageMiningCreateBlockCfg(db, miner, *chainConfig, engine, nil, nil, dirs.Tmp, blockReader), stagedsync.StageMiningExecCfg(db, miner, events, *chainConfig, engine, &vm.Config{}, dirs.Tmp, nil, 0, nil, nil, false, blockReader), stagedsync.StageHashStateCfg(db, dirs, historyV3), stagedsync.StageTrieCfg(db, false, true, false, dirs.Tmp, blockReader, nil, historyV3, agg), diff --git a/cmd/integration/commands/state_stages.go b/cmd/integration/commands/state_stages.go index fe762ac7d33..a65a7163428 100644 --- a/cmd/integration/commands/state_stages.go +++ b/cmd/integration/commands/state_stages.go @@ -360,7 +360,7 @@ func syncBySmallSteps(db kv.RwDB, miningConfig params.MiningConfig, ctx context. miner.MiningConfig.ExtraData = nextBlock.Extra() miningStages.MockExecFunc(stages.MiningCreateBlock, func(firstCycle bool, badBlockUnwind bool, s *stagedsync.StageState, u stagedsync.Unwinder, tx kv.RwTx, logger log.Logger) error { err = stagedsync.SpawnMiningCreateBlockStage(s, tx, - stagedsync.StageMiningCreateBlockCfg(db, miner, *chainConfig, engine, nil, nil, nil, dirs.Tmp, br), + stagedsync.StageMiningCreateBlockCfg(db, miner, *chainConfig, engine, nil, nil, dirs.Tmp, br), quit, logger) if err != nil { return err diff --git a/cmd/pics/state.go b/cmd/pics/state.go index ae33a3640ec..81c40e87fdc 100644 --- a/cmd/pics/state.go +++ b/cmd/pics/state.go @@ -72,8 +72,8 @@ import ( var bucketLabels = map[string]string{ kv.Receipts: "Receipts", kv.Log: "Event Logs", - kv.AccountsHistory: "History Of Accounts", - kv.StorageHistory: "History Of Storage", + kv.E2AccountsHistory: "History Of Accounts", + kv.E2StorageHistory: "History Of Storage", kv.Headers: "Headers", kv.HeaderCanonical: "Canonical headers", kv.HeaderTD: "Headers TD", @@ -283,7 +283,7 @@ func initialState1() error { GasLimit: 10000000, } // this code generates a log - signer = types.MakeSigner(params.AllProtocolChanges, 1) + signer = types.MakeSigner(params.AllProtocolChanges, 1, 0) ) m := stages.MockWithGenesis(nil, gspec, key, false) defer m.DB.Close() @@ -430,13 +430,13 @@ func initialState1() error { // BLOCKS for i := 0; i < chain.Length(); i++ { - if err = m2.InsertChain(chain.Slice(i, i+1)); err != nil { + if err = m2.InsertChain(chain.Slice(i, i+1), nil); err != nil { return err } if err = stateDatabaseComparison(m.DB, m2.DB, i+1); err != nil { return err } - if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { + if err = m.InsertChain(chain.Slice(i, i+1), nil); err != nil { return err } } diff --git a/cmd/prometheus/Readme.md b/cmd/prometheus/Readme.md index 3b89fcd4c8c..853eb91b2a8 100644 --- a/cmd/prometheus/Readme.md +++ b/cmd/prometheus/Readme.md @@ -2,7 +2,7 @@ Add flag `--metrics` to Erigon or any other process (add `--metrics.addr` if nee Add hosts to collecting metrics in: `./cmd/prometheus/prometheus.yml` -Run Grafana and Prometheus: `docker-compose up -d prometheus grafana` or `make prometheus` +Run Grafana and Prometheus: `docker compose up -d prometheus grafana` or `make prometheus` Go to: [localhost:3000](localhost:3000), admin/admin diff --git a/cmd/rpcdaemon/README.md b/cmd/rpcdaemon/README.md index 02a897a4b74..6b03a3fc88d 100644 --- a/cmd/rpcdaemon/README.md +++ b/cmd/rpcdaemon/README.md @@ -19,7 +19,7 @@ ## Introduction -Erigon's `rpcdaemon` runs in its own seperate process. +Erigon's `rpcdaemon` runs in its own separate process. This brings many benefits including easier development, the ability to run multiple daemons at once, and the ability to run the daemon remotely. It is possible to run the daemon locally as well (read-only) if both processes have access to @@ -92,7 +92,7 @@ Configuration of the health check is sent as POST body of the method. Not adding a check disables that. -**`min_peer_count`** -- checks for mimimum of healthy node peers. Requires +**`min_peer_count`** -- checks for minimum of healthy node peers. Requires `net` namespace to be listed in `http.api`. **`known_block`** -- sets up the block that node has to know about. Requires @@ -184,7 +184,7 @@ Next options available (by `--prune` flag): By default data pruned after 90K blocks, can change it by flags like `--prune.history.after=100_000` -Some methods, if not found historical data in DB, can fallback to old blocks re-execution - but it require `h`. +Some methods, if not found historical data in DB, can fallback to old blocks re-execution - but it requires `h`. ### RPC Implementation Status @@ -355,7 +355,7 @@ Erigon and RPC daemon nodes that are supposed to work together): counterparts. 3. For each Erigon instance and each RPC daemon instance, generate a key pair. If you are lazy, you can generate one pair for all Erigon nodes, and one pair for all RPC daemons, and copy these keys around. -4. Using the CA private key, create cerificate file for each public key generated on the previous step. This +4. Using the CA private key, create certificate file for each public key generated on the previous step. This effectively "inducts" these keys into the "cluster of trust". 5. On each instance, deploy 3 files - CA certificate, instance key, and certificate signed by CA for this instance key. @@ -429,8 +429,8 @@ daemon needs to be started with these extra options: ``` **WARNING** Normally, the "client side" (which in our case is RPC daemon), verifies that the host name of the server -matches the "Common Name" attribute of the "server" cerificate. At this stage, this verification is turned off, and it -will be turned on again once we have updated the instruction above on how to properly generate cerificates with "Common +matches the "Common Name" attribute of the "server" certificate. At this stage, this verification is turned off, and it +will be turned on again once we have updated the instruction above on how to properly generate certificates with "Common Name". When running Erigon instance in the Google Cloud, for example, you need to specify the **Internal IP** in diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 7d564e60485..aca03c64699 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -25,6 +25,7 @@ import ( "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/logging" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/turbo/snapshotsync/snap" "github.com/ledgerwatch/erigon-lib/direct" @@ -57,8 +58,6 @@ import ( "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" - // Force-load native and js packages, to trigger registration _ "github.com/ledgerwatch/erigon/eth/tracers/js" _ "github.com/ledgerwatch/erigon/eth/tracers/native" @@ -294,7 +293,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, } // Configure DB first - var allSnapshots *snapshotsync.RoSnapshots + var allSnapshots *freezeblocks.RoSnapshots onNewSnapshot := func() {} if cfg.WithDatadir { var rwKv kv.RwDB @@ -337,7 +336,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, } // Configure sapshots - allSnapshots = snapshotsync.NewRoSnapshots(cfg.Snap, cfg.Dirs.Snap, logger) + allSnapshots = freezeblocks.NewRoSnapshots(cfg.Snap, cfg.Dirs.Snap, logger) // To povide good UX - immediatly can read snapshots after RPCDaemon start, even if Erigon is down // Erigon does store list of snapshots in db: means RPCDaemon can read this list now, but read by `remoteKvClient.Snapshots` after establish grpc connection allSnapshots.OptimisticReopenWithDB(db) @@ -384,7 +383,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, }() } onNewSnapshot() - blockReader = snapshotsync.NewBlockReader(allSnapshots) + blockReader = freezeblocks.NewBlockReader(allSnapshots) var histV3Enabled bool _ = db.View(ctx, func(tx kv.Tx) error { @@ -448,7 +447,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, txPoolService := rpcservices.NewTxPoolService(txPool) if !cfg.WithDatadir { - blockReader = snapshotsync.NewRemoteBlockReader(remoteBackendClient) + blockReader = freezeblocks.NewRemoteBlockReader(remoteBackendClient) } remoteEth := rpcservices.NewRemoteBackend(remoteBackendClient, db, blockReader) @@ -493,7 +492,6 @@ func startRegularRpcServer(ctx context.Context, cfg httpcfg.HttpCfg, rpcAPI []rp // register apis and create handler stack httpEndpoint := fmt.Sprintf("%s:%d", cfg.HttpListenAddress, cfg.HttpPort) - logger.Trace("TraceRequests = %t\n", cfg.TraceRequests) srv := rpc.NewServer(cfg.RpcBatchConcurrency, cfg.TraceRequests, cfg.RpcStreamingDisable, logger) allowListForRPC, err := parseAllowListForRPC(cfg.RpcAllowListFilePath) @@ -613,7 +611,6 @@ type engineInfo struct { } func startAuthenticatedRpcServer(cfg httpcfg.HttpCfg, rpcAPI []rpc.API, logger log.Logger) (*engineInfo, error) { - logger.Trace("TraceRequests = %t\n", cfg.TraceRequests) srv := rpc.NewServer(cfg.RpcBatchConcurrency, cfg.TraceRequests, cfg.RpcStreamingDisable, logger) engineListener, engineSrv, engineHttpEndpoint, err := createEngineListener(cfg, rpcAPI, logger) diff --git a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go index 818e9f33fa4..8a72c81c4e8 100644 --- a/cmd/rpcdaemon/cli/httpcfg/http_cfg.go +++ b/cmd/rpcdaemon/cli/httpcfg/http_cfg.go @@ -39,7 +39,7 @@ type HttpCfg struct { TraceCompatibility bool // Bug for bug compatibility for trace_ routines with OpenEthereum TxPoolApiAddr string StateCache kvcache.CoherentConfig - Snap ethconfig.Snapshot + Snap ethconfig.BlocksFreezing Sync ethconfig.Sync // GRPC server diff --git a/cmd/rpcdaemon/commands/call_traces_test.go b/cmd/rpcdaemon/commands/call_traces_test.go index 520c0e5ee4f..7e45325bbc6 100644 --- a/cmd/rpcdaemon/commands/call_traces_test.go +++ b/cmd/rpcdaemon/commands/call_traces_test.go @@ -8,14 +8,11 @@ import ( "github.com/holiman/uint256" jsoniter "github.com/json-iterator/go" "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/valyala/fastjson" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/cli/httpcfg" - "github.com/ledgerwatch/erigon/rpc/rpccfg" - "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/types" @@ -52,14 +49,10 @@ func TestCallTraceOneByOne(t *testing.T) { t.Fatalf("generate chain: %v", err) } - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewTraceAPI( - NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), - m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) // Insert blocks 1 by 1, to tirgget possible "off by one" errors for i := 0; i < chain.Length(); i++ { - if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { + if err = m.InsertChain(chain.Slice(i, i+1), nil); err != nil { t.Fatalf("inserting chain: %v", err) } } @@ -101,11 +94,9 @@ func TestCallTraceUnwind(t *testing.T) { t.Fatalf("generate chainB: %v", err) } - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) - if err = m.InsertChain(chainA); err != nil { + if err = m.InsertChain(chainA, nil); err != nil { t.Fatalf("inserting chainA: %v", err) } stream := jsoniter.ConfigDefault.BorrowStream(nil) @@ -124,7 +115,7 @@ func TestCallTraceUnwind(t *testing.T) { } assert.Equal(t, []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, blockNumbersFromTraces(t, stream.Buffer())) - if err = m.InsertChain(chainB.Slice(0, 12)); err != nil { + if err = m.InsertChain(chainB.Slice(0, 12), nil); err != nil { t.Fatalf("inserting chainB: %v", err) } stream.Reset(nil) @@ -139,7 +130,7 @@ func TestCallTraceUnwind(t *testing.T) { } assert.Equal(t, []int{1, 2, 3, 4, 5, 11, 12}, blockNumbersFromTraces(t, stream.Buffer())) - if err = m.InsertChain(chainB.Slice(12, 20)); err != nil { + if err = m.InsertChain(chainB.Slice(12, 20), nil); err != nil { t.Fatalf("inserting chainB: %v", err) } stream.Reset(nil) @@ -164,12 +155,10 @@ func TestFilterNoAddresses(t *testing.T) { if err != nil { t.Fatalf("generate chain: %v", err) } - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) // Insert blocks 1 by 1, to tirgget possible "off by one" errors for i := 0; i < chain.Length(); i++ { - if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { + if err = m.InsertChain(chain.Slice(i, i+1), nil); err != nil { t.Fatalf("inserting chain: %v", err) } } @@ -190,9 +179,7 @@ func TestFilterNoAddresses(t *testing.T) { func TestFilterAddressIntersection(t *testing.T) { m := stages.Mock(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) toAddress1, toAddress2, other := common.Address{1}, common.Address{2}, common.Address{3} @@ -218,7 +205,7 @@ func TestFilterAddressIntersection(t *testing.T) { }, false /* intermediateHashes */) require.NoError(t, err, "generate chain") - err = m.InsertChain(chain) + err = m.InsertChain(chain, nil) require.NoError(t, err, "inserting chain") fromBlock, toBlock := uint64(1), uint64(15) diff --git a/cmd/rpcdaemon/commands/corner_cases_support_test.go b/cmd/rpcdaemon/commands/corner_cases_support_test.go index 443156d268b..0009ef480a9 100644 --- a/cmd/rpcdaemon/commands/corner_cases_support_test.go +++ b/cmd/rpcdaemon/commands/corner_cases_support_test.go @@ -5,13 +5,11 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/kv/kvcache" - "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/require" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/rpc" - "github.com/ledgerwatch/erigon/rpc/rpccfg" + "github.com/ledgerwatch/log/v3" ) // TestNotFoundMustReturnNil - next methods - when record not found in db - must return nil instead of error @@ -19,11 +17,7 @@ import ( func TestNotFoundMustReturnNil(t *testing.T) { require := require.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI( - NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) ctx := context.Background() diff --git a/cmd/rpcdaemon/commands/daemon.go b/cmd/rpcdaemon/commands/daemon.go index ffed8c410e6..b5cf5479798 100644 --- a/cmd/rpcdaemon/commands/daemon.go +++ b/cmd/rpcdaemon/commands/daemon.go @@ -31,7 +31,7 @@ func APIList(db kv.RoDB, borDb kv.RoDB, eth rpchelper.ApiBackend, txPool txpool. web3Impl := NewWeb3APIImpl(eth) dbImpl := NewDBAPIImpl() /* deprecated */ adminImpl := NewAdminAPI(eth) - parityImpl := NewParityAPIImpl(db) + parityImpl := NewParityAPIImpl(base, db) borImpl := NewBorAPI(base, db, borDb) // bor (consensus) specific otsImpl := NewOtterscanAPI(base, db) gqlImpl := NewGraphQLAPI(base, db) diff --git a/cmd/rpcdaemon/commands/debug_api.go b/cmd/rpcdaemon/commands/debug_api.go index 14f29485afa..0df89ed0207 100644 --- a/cmd/rpcdaemon/commands/debug_api.go +++ b/cmd/rpcdaemon/commands/debug_api.go @@ -15,7 +15,6 @@ import ( "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/eth/tracers" @@ -212,7 +211,7 @@ func (api *PrivateDebugAPIImpl) GetModifiedAccountsByNumber(ctx context.Context, // getModifiedAccountsV3 returns a list of addresses that were modified in the block range // [startNum:endNum) func getModifiedAccountsV3(tx kv.TemporalTx, startTxNum, endTxNum uint64) ([]common.Address, error) { - it, err := tx.HistoryRange(temporal.AccountsHistory, int(startTxNum), int(endTxNum), order.Asc, kv.Unlim) + it, err := tx.HistoryRange(kv.AccountsHistory, int(startTxNum), int(endTxNum), order.Asc, kv.Unlim) if err != nil { return nil, err } @@ -311,7 +310,7 @@ func (api *PrivateDebugAPIImpl) AccountAt(ctx context.Context, blockHash common. return nil, err } ttx := tx.(kv.TemporalTx) - v, ok, err := ttx.DomainGetAsOf(temporal.AccountsDomain, address[:], nil, minTxNum+txIndex+1) + v, ok, err := ttx.DomainGetAsOf(kv.AccountsDomain, address[:], nil, minTxNum+txIndex+1) if err != nil { return nil, err } @@ -328,7 +327,7 @@ func (api *PrivateDebugAPIImpl) AccountAt(ctx context.Context, blockHash common. result.Nonce = hexutil.Uint64(a.Nonce) result.CodeHash = a.CodeHash - code, _, err := ttx.DomainGetAsOf(temporal.CodeDomain, address[:], a.CodeHash[:], minTxNum+txIndex) + code, _, err := ttx.DomainGetAsOf(kv.CodeDomain, address[:], a.CodeHash[:], minTxNum+txIndex) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/commands/debug_api_test.go b/cmd/rpcdaemon/commands/debug_api_test.go index bcd58513d94..b4286888dd4 100644 --- a/cmd/rpcdaemon/commands/debug_api_test.go +++ b/cmd/rpcdaemon/commands/debug_api_test.go @@ -15,7 +15,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" common2 "github.com/ledgerwatch/erigon/common" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/tracers" "github.com/ledgerwatch/erigon/rpc" @@ -52,9 +51,8 @@ var debugTraceTransactionNoRefundTests = []struct { func TestTraceBlockByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) + baseApi := NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000, log.New()) api := NewPrivateDebugAPI(baseApi, m.DB, 0) for _, tt := range debugTraceTransactionTests { @@ -100,12 +98,8 @@ func TestTraceBlockByNumber(t *testing.T) { func TestTraceBlockByHash(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000, log.New()) - api := NewPrivateDebugAPI(baseApi, m.DB, 0) + ethApi := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer stream := jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096) @@ -136,11 +130,7 @@ func TestTraceBlockByHash(t *testing.T) { func TestTraceTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - api := NewPrivateDebugAPI(base, m.DB, 0) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer stream := jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096) @@ -153,7 +143,7 @@ func TestTraceTransaction(t *testing.T) { } var er ethapi.ExecutionResult if err = json.Unmarshal(buf.Bytes(), &er); err != nil { - t.Fatalf("parsing result: %v", err) + t.Fatalf("parsing result: %v, %s", err, buf.String()) } if er.Gas != tt.gas { t.Errorf("wrong gas for transaction %s, got %d, expected %d", tt.txHash, er.Gas, tt.gas) @@ -169,11 +159,7 @@ func TestTraceTransaction(t *testing.T) { func TestTraceTransactionNoRefund(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - agg := m.HistoryV3Components() - api := NewPrivateDebugAPI( - NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), - m.DB, 0) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) for _, tt := range debugTraceTransactionNoRefundTests { var buf bytes.Buffer stream := jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096) @@ -203,16 +189,12 @@ func TestTraceTransactionNoRefund(t *testing.T) { func TestStorageRangeAt(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - agg := m.HistoryV3Components() - api := NewPrivateDebugAPI( - NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), - m.DB, 0) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) t.Run("invalid addr", func(t *testing.T) { var block4 *types.Block var err error err = m.DB.View(m.Ctx, func(tx kv.Tx) error { - block4, err = br.BlockByNumber(m.Ctx, tx, 4) + block4, err = m.BlockReader.BlockByNumber(m.Ctx, tx, 4) return err }) require.NoError(t, err) @@ -225,7 +207,7 @@ func TestStorageRangeAt(t *testing.T) { t.Run("block 4, addr 1", func(t *testing.T) { var block4 *types.Block err := m.DB.View(m.Ctx, func(tx kv.Tx) error { - block4, _ = br.BlockByNumber(m.Ctx, tx, 4) + block4, _ = m.BlockReader.BlockByNumber(m.Ctx, tx, 4) return nil }) require.NoError(t, err) @@ -246,7 +228,7 @@ func TestStorageRangeAt(t *testing.T) { t.Run("block latest, addr 1", func(t *testing.T) { var latestBlock *types.Block err := m.DB.View(m.Ctx, func(tx kv.Tx) (err error) { - latestBlock, err = br.CurrentBlock(tx) + latestBlock, err = m.BlockReader.CurrentBlock(tx) return err }) require.NoError(t, err) @@ -301,11 +283,7 @@ func TestStorageRangeAt(t *testing.T) { func TestAccountRange(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - agg := m.HistoryV3Components() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - api := NewPrivateDebugAPI(base, m.DB, 0) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) t.Run("valid account", func(t *testing.T) { addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf55") @@ -364,11 +342,7 @@ func TestAccountRange(t *testing.T) { func TestGetModifiedAccountsByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - agg := m.HistoryV3Components() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - api := NewPrivateDebugAPI(base, m.DB, 0) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) t.Run("correct input", func(t *testing.T) { n, n2 := rpc.BlockNumber(1), rpc.BlockNumber(2) @@ -433,10 +407,10 @@ func TestMapTxNum2BlockNum(t *testing.T) { defer dbtx.Rollback() tx := dbtx.(kv.TemporalTx) - txNums, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], 1024, -1, order.Desc, kv.Unlim) + txNums, err := tx.IndexRange(kv.LogAddrIdx, addr[:], 1024, -1, order.Desc, kv.Unlim) require.NoError(t, err) txNumsIter := MapDescendTxNum2BlockNum(tx, txNums) - expectTxNums, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], 1024, -1, order.Desc, kv.Unlim) + expectTxNums, err := tx.IndexRange(kv.LogAddrIdx, addr[:], 1024, -1, order.Desc, kv.Unlim) require.NoError(t, err) checkIter(t, expectTxNums, txNumsIter) }) @@ -446,10 +420,10 @@ func TestMapTxNum2BlockNum(t *testing.T) { defer dbtx.Rollback() tx := dbtx.(kv.TemporalTx) - txNums, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], 0, 1024, order.Asc, kv.Unlim) + txNums, err := tx.IndexRange(kv.LogAddrIdx, addr[:], 0, 1024, order.Asc, kv.Unlim) require.NoError(t, err) txNumsIter := MapDescendTxNum2BlockNum(tx, txNums) - expectTxNums, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], 0, 1024, order.Asc, kv.Unlim) + expectTxNums, err := tx.IndexRange(kv.LogAddrIdx, addr[:], 0, 1024, order.Asc, kv.Unlim) require.NoError(t, err) checkIter(t, expectTxNums, txNumsIter) }) @@ -459,10 +433,10 @@ func TestMapTxNum2BlockNum(t *testing.T) { defer dbtx.Rollback() tx := dbtx.(kv.TemporalTx) - txNums, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], 0, 1024, order.Asc, 2) + txNums, err := tx.IndexRange(kv.LogAddrIdx, addr[:], 0, 1024, order.Asc, 2) require.NoError(t, err) txNumsIter := MapDescendTxNum2BlockNum(tx, txNums) - expectTxNums, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], 0, 1024, order.Asc, 2) + expectTxNums, err := tx.IndexRange(kv.LogAddrIdx, addr[:], 0, 1024, order.Asc, 2) require.NoError(t, err) checkIter(t, expectTxNums, txNumsIter) }) @@ -470,19 +444,15 @@ func TestMapTxNum2BlockNum(t *testing.T) { func TestAccountAt(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - api := NewPrivateDebugAPI(base, m.DB, 0) + api := NewPrivateDebugAPI(newBaseApiForTest(m), m.DB, 0) var blockHash0, blockHash1, blockHash3, blockHash10, blockHash12 common.Hash _ = m.DB.View(m.Ctx, func(tx kv.Tx) error { - blockHash0, _ = br.CanonicalHash(m.Ctx, tx, 0) - blockHash1, _ = br.CanonicalHash(m.Ctx, tx, 1) - blockHash3, _ = br.CanonicalHash(m.Ctx, tx, 3) - blockHash10, _ = br.CanonicalHash(m.Ctx, tx, 10) - blockHash12, _ = br.CanonicalHash(m.Ctx, tx, 12) + blockHash0, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 0) + blockHash1, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 1) + blockHash3, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 3) + blockHash10, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 10) + blockHash12, _ = m.BlockReader.CanonicalHash(m.Ctx, tx, 12) _, _, _, _, _ = blockHash0, blockHash1, blockHash3, blockHash10, blockHash12 return nil }) diff --git a/cmd/rpcdaemon/commands/erigon_block.go b/cmd/rpcdaemon/commands/erigon_block.go index df969cc0973..f1ada108f22 100644 --- a/cmd/rpcdaemon/commands/erigon_block.go +++ b/cmd/rpcdaemon/commands/erigon_block.go @@ -11,6 +11,8 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/order" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" "github.com/ledgerwatch/erigon/turbo/services" @@ -207,11 +209,55 @@ func (api *ErigonImpl) GetBalanceChangesInBlock(ctx context.Context, blockNrOrHa } defer tx.Rollback() + balancesMapping := make(map[common.Address]*hexutil.Big) + latestState, err := rpchelper.CreateStateReader(ctx, tx, blockNrOrHash, 0, api.filters, api.stateCache, api.historyV3(tx), "") + if err != nil { + return nil, err + } + blockNumber, _, _, err := rpchelper.GetBlockNumber(blockNrOrHash, tx, api.filters) if err != nil { return nil, err } + if api.historyV3(tx) { + minTxNum, _ := rawdbv3.TxNums.Min(tx, blockNumber) + it, err := tx.(kv.TemporalTx).HistoryRange(kv.AccountsHistory, int(minTxNum), -1, order.Asc, -1) + if err != nil { + return nil, err + } + for it.HasNext() { + addressBytes, v, err := it.Next() + if err != nil { + return nil, err + } + + var oldAcc accounts.Account + if len(v) > 0 { + if err = accounts.DeserialiseV3(&oldAcc, v); err != nil { + return nil, err + } + } + oldBalance := oldAcc.Balance + + address := common.BytesToAddress(addressBytes) + newAcc, err := latestState.ReadAccountData(address) + if err != nil { + return nil, err + } + + newBalance := uint256.NewInt(0) + if newAcc != nil { + newBalance = &newAcc.Balance + } + + if !oldBalance.Eq(newBalance) { + newBalanceDesc := (*hexutil.Big)(newBalance.ToBig()) + balancesMapping[address] = newBalanceDesc + } + } + } + c, err := tx.Cursor(kv.AccountChangeSet) if err != nil { return nil, err @@ -222,13 +268,6 @@ func (api *ErigonImpl) GetBalanceChangesInBlock(ctx context.Context, blockNrOrHa decodeFn := historyv2.Mapper[kv.AccountChangeSet].Decode - balancesMapping := make(map[common.Address]*hexutil.Big) - - newReader, err := rpchelper.CreateStateReader(ctx, tx, blockNrOrHash, 0, api.filters, api.stateCache, api.historyV3(tx), "") - if err != nil { - return nil, err - } - for dbKey, dbValue, err := c.Seek(startkey); bytes.Equal(dbKey, startkey) && dbKey != nil; dbKey, dbValue, err = c.Next() { if err != nil { return nil, err @@ -237,16 +276,14 @@ func (api *ErigonImpl) GetBalanceChangesInBlock(ctx context.Context, blockNrOrHa if err != nil { return nil, err } - var oldAcc accounts.Account if err = oldAcc.DecodeForStorage(v); err != nil { return nil, err } oldBalance := oldAcc.Balance - address := common.BytesToAddress(addressBytes) - newAcc, err := newReader.ReadAccountData(address) + newAcc, err := latestState.ReadAccountData(address) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/commands/erigon_receipts_test.go b/cmd/rpcdaemon/commands/erigon_receipts_test.go index 939c8fc8633..29c5df7e928 100644 --- a/cmd/rpcdaemon/commands/erigon_receipts_test.go +++ b/cmd/rpcdaemon/commands/erigon_receipts_test.go @@ -9,7 +9,6 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,7 +21,6 @@ import ( "github.com/ledgerwatch/erigon/eth/filters" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rpc" - "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/stages" "github.com/ledgerwatch/log/v3" ) @@ -30,11 +28,8 @@ import ( func TestGetLogs(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - agg := m.HistoryV3Components() - baseApi := NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) { - ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + ethApi := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) logs, err := ethApi.GetLogs(context.Background(), filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)}) assert.NoError(err) @@ -63,11 +58,8 @@ func TestGetLogs(t *testing.T) { func TestErigonGetLatestLogs(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) db := m.DB - agg := m.HistoryV3Components() - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), db, nil) + api := NewErigonAPI(newBaseApiForTest(m), db, nil) expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) expectedErigonLogs := make([]*types.ErigonLog, 0) @@ -98,11 +90,8 @@ func TestErigonGetLatestLogs(t *testing.T) { func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) db := m.DB - agg := m.HistoryV3Components() - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), db, nil) + api := NewErigonAPI(newBaseApiForTest(m), db, nil) expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) expectedErigonLogs := make([]*types.ErigonLog, 0) @@ -189,10 +178,7 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) { } // Assemble the test environment m := mockWithGenerator(t, 4, generator) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) expect := map[uint64]string{ 0: `[]`, @@ -228,7 +214,7 @@ func mockWithGenerator(t *testing.T, blocks int, generator func(int, *core.Block }, testKey, false) if blocks > 0 { chain, _ := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator, true) - err := m.InsertChain(chain) + err := m.InsertChain(chain, nil) require.NoError(t, err) } return m diff --git a/cmd/rpcdaemon/commands/eth_api.go b/cmd/rpcdaemon/commands/eth_api.go index 27b8fd3be26..0a3479bf0fb 100644 --- a/cmd/rpcdaemon/commands/eth_api.go +++ b/cmd/rpcdaemon/commands/eth_api.go @@ -472,16 +472,18 @@ func newRPCTransaction(tx types.Transaction, blockHash common.Hash, blockNumber result.V = (*hexutil.Big)(common.Big0) result.R = (*hexutil.Big)(common.Big0) result.S = (*hexutil.Big)(common.Big0) - case *types.SignedBlobTx: - result.Tip = (*hexutil.Big)(t.GetTip().ToBig()) - result.FeeCap = (*hexutil.Big)(t.GetFeeCap().ToBig()) - result.MaxFeePerDataGas = (*hexutil.Big)(t.GetMaxFeePerDataGas().ToBig()) - result.V = (*hexutil.Big)(t.Signature.GetV().ToBig()) - result.R = (*hexutil.Big)(t.Signature.GetR().ToBig()) - result.S = (*hexutil.Big)(t.Signature.GetS().ToBig()) - al := t.GetAccessList() - result.Accesses = &al + case *types.BlobTx: + chainId.Set(t.ChainID) + result.ChainID = (*hexutil.Big)(chainId.ToBig()) + result.Tip = (*hexutil.Big)(t.Tip.ToBig()) + result.FeeCap = (*hexutil.Big)(t.FeeCap.ToBig()) + result.V = (*hexutil.Big)(t.V.ToBig()) + result.R = (*hexutil.Big)(t.R.ToBig()) + result.S = (*hexutil.Big)(t.S.ToBig()) + result.Accesses = &t.AccessList result.GasPrice = computeGasPrice(tx, blockHash, baseFee) + result.MaxFeePerDataGas = (*hexutil.Big)(t.MaxFeePerDataGas.ToBig()) + result.BlobVersionedHashes = t.BlobVersionedHashes } signer := types.LatestSignerForChainID(chainId.ToBig()) result.From, _ = tx.Sender(*signer) diff --git a/cmd/rpcdaemon/commands/eth_api_test.go b/cmd/rpcdaemon/commands/eth_api_test.go index 1a1a2d5a9d6..883919ede54 100644 --- a/cmd/rpcdaemon/commands/eth_api_test.go +++ b/cmd/rpcdaemon/commands/eth_api_test.go @@ -7,6 +7,7 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/turbo/stages" "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/assert" @@ -20,15 +21,18 @@ import ( "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" ) +func newBaseApiForTest(m *stages.MockSentry) *BaseAPI { + agg := m.HistoryV3Components() + stateCache := kvcache.New(kvcache.DefaultCoherentConfig) + return NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) +} + func TestGetBalanceChangesInBlock(t *testing.T) { assert := assert.New(t) myBlockNum := rpc.BlockNumberOrHashWithNumber(0) m, _, _ := rpcdaemontest.CreateTestSentry(t) - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) db := m.DB - agg := m.HistoryV3Components() - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), db, nil) + api := NewErigonAPI(newBaseApiForTest(m), db, nil) balances, err := api.GetBalanceChangesInBlock(context.Background(), myBlockNum) if err != nil { t.Errorf("calling GetBalanceChangesInBlock resulted in an error: %v", err) @@ -49,9 +53,8 @@ func TestGetTransactionReceipt(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) db := m.DB agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), db, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), db, nil, nil, nil, 5000000, 100_000, log.New()) // Call GetTransactionReceipt for transaction which is not in the database if _, err := api.GetTransactionReceipt(context.Background(), common.Hash{}); err != nil { t.Errorf("calling GetTransactionReceipt with empty hash: %v", err) @@ -60,10 +63,7 @@ func TestGetTransactionReceipt(t *testing.T) { func TestGetTransactionReceiptUnprotected(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) // Call GetTransactionReceipt for un-protected transaction if _, err := api.GetTransactionReceipt(context.Background(), common.HexToHash("0x3f3cb8a0e13ed2481f97f53f7095b9cbc78b6ffb779f2d3e565146371a8830ea")); err != nil { t.Errorf("calling GetTransactionReceipt for unprotected tx: %v", err) @@ -75,10 +75,7 @@ func TestGetTransactionReceiptUnprotected(t *testing.T) { func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithNumber(0)) @@ -92,10 +89,7 @@ func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), false)) @@ -109,10 +103,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), true)) @@ -125,10 +116,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue(t *testing.T) { func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *core.BlockGen) { @@ -149,10 +137,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *core.BlockGen) { @@ -174,10 +159,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testing.T) { assert := assert.New(t) m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -196,10 +178,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock( func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -215,10 +194,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t * func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -241,10 +217,7 @@ func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testi func TestCall_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing.T) { m, _, orphanedChain := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") diff --git a/cmd/rpcdaemon/commands/eth_block.go b/cmd/rpcdaemon/commands/eth_block.go index bbc52b72e69..0b8a4f393d3 100644 --- a/cmd/rpcdaemon/commands/eth_block.go +++ b/cmd/rpcdaemon/commands/eth_block.go @@ -112,7 +112,7 @@ func (api *APIImpl) CallBundle(ctx context.Context, txHashes []common.Hash, stat Coinbase: coinbase, } - signer := types.MakeSigner(chainConfig, blockNumber) + signer := types.MakeSigner(chainConfig, blockNumber, timestamp) rules := chainConfig.Rules(blockNumber, timestamp) firstMsg, err := txs[0].AsMessage(*signer, nil, rules) if err != nil { diff --git a/cmd/rpcdaemon/commands/eth_block_test.go b/cmd/rpcdaemon/commands/eth_block_test.go index 45dfc61d583..8398eddeb36 100644 --- a/cmd/rpcdaemon/commands/eth_block_test.go +++ b/cmd/rpcdaemon/commands/eth_block_test.go @@ -25,10 +25,7 @@ import ( // Gets the latest block number with the latest tag func TestGetBlockByNumberWithLatestTag(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) b, err := api.GetBlockByNumber(context.Background(), rpc.LatestBlockNumber, false) expected := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") if err != nil { @@ -39,16 +36,13 @@ func TestGetBlockByNumberWithLatestTag(t *testing.T) { func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) tx, err := m.DB.BeginRw(ctx) if err != nil { t.Errorf("could not begin read write transaction: %s", err) } latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") - latestBlock, err := br.BlockByHash(ctx, tx, latestBlockHash) + latestBlock, err := m.BlockReader.BlockByHash(ctx, tx, latestBlockHash) if err != nil { tx.Rollback() t.Errorf("couldn't retrieve latest block") @@ -61,7 +55,7 @@ func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { } tx.Commit() - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) block, err := api.GetBlockByNumber(ctx, rpc.LatestBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -73,7 +67,6 @@ func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { func TestGetBlockByNumberWithPendingTag(t *testing.T) { m := stages.MockWithTxPool(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) @@ -93,7 +86,7 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { RplBlock: rlpBlock, }) - api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) b, err := api.GetBlockByNumber(context.Background(), rpc.PendingBlockNumber, false) if err != nil { t.Errorf("error getting block number with pending tag: %s", err) @@ -103,11 +96,8 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { func TestGetBlockByNumber_WithFinalizedTag_NoFinalizedBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) if _, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false); err != nil { assert.ErrorIs(t, rpchelper.UnknownBlockError, err) } @@ -115,16 +105,13 @@ func TestGetBlockByNumber_WithFinalizedTag_NoFinalizedBlockInDb(t *testing.T) { func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) tx, err := m.DB.BeginRw(ctx) if err != nil { t.Errorf("could not begin read write transaction: %s", err) } latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") - latestBlock, err := br.BlockByHash(ctx, tx, latestBlockHash) + latestBlock, err := m.BlockReader.BlockByHash(ctx, tx, latestBlockHash) if err != nil { tx.Rollback() t.Errorf("couldn't retrieve latest block") @@ -137,7 +124,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) } tx.Commit() - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) block, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -148,11 +135,8 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) func TestGetBlockByNumber_WithSafeTag_NoSafeBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) if _, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false); err != nil { assert.ErrorIs(t, rpchelper.UnknownBlockError, err) } @@ -160,16 +144,13 @@ func TestGetBlockByNumber_WithSafeTag_NoSafeBlockInDb(t *testing.T) { func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) tx, err := m.DB.BeginRw(ctx) if err != nil { t.Errorf("could not begin read write transaction: %s", err) } latestBlockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") - latestBlock, err := br.BlockByHash(ctx, tx, latestBlockHash) + latestBlock, err := m.BlockReader.BlockByHash(ctx, tx, latestBlockHash) if err != nil { tx.Rollback() t.Errorf("couldn't retrieve latest block") @@ -182,7 +163,7 @@ func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { } tx.Commit() - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) block, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -193,12 +174,9 @@ func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { func TestGetBlockTransactionCountByHash(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -210,7 +188,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { tx.Rollback() t.Errorf("failed reading block by hash: %s", err) } - bodyWithTx, err := br.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) + bodyWithTx, err := m.BlockReader.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) if err != nil { tx.Rollback() t.Errorf("failed getting body with transactions: %s", err) @@ -229,12 +207,8 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") tx, err := m.DB.BeginRw(ctx) @@ -246,7 +220,7 @@ func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { tx.Rollback() t.Errorf("failed reading block by hash: %s", err) } - bodyWithTx, err := br.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) + bodyWithTx, err := m.BlockReader.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) if err != nil { tx.Rollback() t.Errorf("failed getting body with transactions: %s", err) @@ -265,11 +239,8 @@ func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { func TestGetBlockTransactionCountByNumber(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -281,7 +252,7 @@ func TestGetBlockTransactionCountByNumber(t *testing.T) { tx.Rollback() t.Errorf("failed reading block by hash: %s", err) } - bodyWithTx, err := br.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) + bodyWithTx, err := m.BlockReader.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) if err != nil { tx.Rollback() t.Errorf("failed getting body with transactions: %s", err) @@ -300,11 +271,8 @@ func TestGetBlockTransactionCountByNumber(t *testing.T) { func TestGetBlockTransactionCountByNumber_ZeroTx(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() ctx := context.Background() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") @@ -317,7 +285,7 @@ func TestGetBlockTransactionCountByNumber_ZeroTx(t *testing.T) { tx.Rollback() t.Errorf("failed reading block by hash: %s", err) } - bodyWithTx, err := br.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) + bodyWithTx, err := m.BlockReader.BodyWithTransactions(ctx, tx, blockHash, header.Number.Uint64()) if err != nil { tx.Rollback() t.Errorf("failed getting body with transactions: %s", err) diff --git a/cmd/rpcdaemon/commands/eth_callMany.go b/cmd/rpcdaemon/commands/eth_callMany.go index 652921367ec..4f979d331d7 100644 --- a/cmd/rpcdaemon/commands/eth_callMany.go +++ b/cmd/rpcdaemon/commands/eth_callMany.go @@ -173,7 +173,7 @@ func (api *APIImpl) CallMany(ctx context.Context, bundles []Bundle, simulateCont // Get a new instance of the EVM evm = vm.NewEVM(blockCtx, txCtx, st, chainConfig, vm.Config{Debug: false}) - signer := types.MakeSigner(chainConfig, blockNum) + signer := types.MakeSigner(chainConfig, blockNum, blockCtx.Time) rules := chainConfig.Rules(blockNum, blockCtx.Time) timeoutMilliSeconds := int64(5000) diff --git a/cmd/rpcdaemon/commands/eth_call_test.go b/cmd/rpcdaemon/commands/eth_call_test.go index 7bcf081384f..cd3469a9cf2 100644 --- a/cmd/rpcdaemon/commands/eth_call_test.go +++ b/cmd/rpcdaemon/commands/eth_call_test.go @@ -38,12 +38,11 @@ import ( func TestEstimateGas(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, stages.Mock(t)) mining := txpool.NewMiningClient(conn) ff := rpchelper.New(ctx, nil, nil, mining, func() {}, m.Log) - api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) var from = libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = libcommon.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") if _, err := api.EstimateGas(context.Background(), ðapi.CallArgs{ @@ -57,9 +56,8 @@ func TestEstimateGas(t *testing.T) { func TestEthCallNonCanonical(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) var from = libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = libcommon.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") if _, err := api.Call(context.Background(), ethapi.CallArgs{ @@ -77,14 +75,8 @@ func TestEthCallToPrunedBlock(t *testing.T) { ethCallBlockNumber := rpc.BlockNumber(2) m, bankAddress, contractAddress := chainWithDeployedContract(t) - br, _ := m.NewBlocksIO() - doPrune(t, m.DB, pruneTo) - - agg := m.HistoryV3Components() - - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) callData := hexutil.MustDecode("0x2e64cec1") callDataBytes := hexutility.Bytes(callData) @@ -102,15 +94,10 @@ func TestGetProof(t *testing.T) { maxGetProofRewindBlockCount = 1 // Note, this is unsafe for parallel tests, but, this test is the only consumer for now m, bankAddr, contractAddr := chainWithDeployedContract(t) - br, _ := m.NewBlocksIO() - if m.HistoryV3 { t.Skip("not supported by Erigon3") } - agg := m.HistoryV3Components() - - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) key := func(b byte) libcommon.Hash { result := libcommon.Hash{} @@ -231,18 +218,14 @@ func TestGetProof(t *testing.T) { func TestGetBlockByTimestampLatestTime(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() tx, err := m.DB.BeginRo(ctx) if err != nil { t.Errorf("fail at beginning tx") } defer tx.Rollback() + api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil) - - latestBlock, err := br.CurrentBlock(tx) + latestBlock, err := m.BlockReader.CurrentBlock(tx) require.NoError(t, err) response, err := ethapi.RPCMarshalBlockDeprecated(latestBlock, true, false, nil) @@ -270,18 +253,14 @@ func TestGetBlockByTimestampLatestTime(t *testing.T) { func TestGetBlockByTimestampOldestTime(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() tx, err := m.DB.BeginRo(ctx) if err != nil { t.Errorf("failed at beginning tx") } defer tx.Rollback() + api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil) - - oldestBlock, err := br.BlockByNumber(m.Ctx, tx, 0) + oldestBlock, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 0) if err != nil { t.Error("couldn't retrieve oldest block") } @@ -312,18 +291,14 @@ func TestGetBlockByTimestampOldestTime(t *testing.T) { func TestGetBlockByTimeHigherThanLatestBlock(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() tx, err := m.DB.BeginRo(ctx) if err != nil { t.Errorf("fail at beginning tx") } defer tx.Rollback() + api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil) - - latestBlock, err := br.CurrentBlock(tx) + latestBlock, err := m.BlockReader.CurrentBlock(tx) require.NoError(t, err) response, err := ethapi.RPCMarshalBlockDeprecated(latestBlock, true, false, nil) @@ -352,16 +327,12 @@ func TestGetBlockByTimeHigherThanLatestBlock(t *testing.T) { func TestGetBlockByTimeMiddle(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() tx, err := m.DB.BeginRo(ctx) if err != nil { t.Errorf("fail at beginning tx") } defer tx.Rollback() - - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) currentHeader := rawdb.ReadCurrentHeader(tx) oldestHeader, err := api._blockReader.HeaderByNumber(ctx, tx, 0) @@ -373,7 +344,7 @@ func TestGetBlockByTimeMiddle(t *testing.T) { } middleNumber := (currentHeader.Number.Uint64() + oldestHeader.Number.Uint64()) / 2 - middleBlock, err := br.BlockByNumber(m.Ctx, tx, middleNumber) + middleBlock, err := m.BlockReader.BlockByNumber(m.Ctx, tx, middleNumber) if err != nil { t.Error("couldn't retrieve middle block") } @@ -403,19 +374,15 @@ func TestGetBlockByTimeMiddle(t *testing.T) { func TestGetBlockByTimestamp(t *testing.T) { ctx := context.Background() m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() tx, err := m.DB.BeginRo(ctx) if err != nil { t.Errorf("fail at beginning tx") } defer tx.Rollback() - - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil) + api := NewErigonAPI(newBaseApiForTest(m), m.DB, nil) highestBlockNumber := rawdb.ReadCurrentHeader(tx).Number - pickedBlock, err := br.BlockByNumber(m.Ctx, tx, highestBlockNumber.Uint64()/3) + pickedBlock, err := m.BlockReader.BlockByNumber(m.Ctx, tx, highestBlockNumber.Uint64()/3) if err != nil { t.Errorf("couldn't get block %v", pickedBlock.Number()) } @@ -557,7 +524,7 @@ func chainWithDeployedContract(t *testing.T) (*stages.MockSentry, libcommon.Addr t.Fatalf("generate blocks: %v", err) } - err = m.InsertChain(chain) + err = m.InsertChain(chain, nil) assert.NoError(t, err) tx, err := db.BeginRo(context.Background()) diff --git a/cmd/rpcdaemon/commands/eth_filters.go b/cmd/rpcdaemon/commands/eth_filters.go index 0e55bcca22a..938e3267c50 100644 --- a/cmd/rpcdaemon/commands/eth_filters.go +++ b/cmd/rpcdaemon/commands/eth_filters.go @@ -146,12 +146,11 @@ func (api *APIImpl) NewHeads(ctx context.Context) (*rpc.Subscription, error) { if h != nil { err := notifier.Notify(rpcSub.ID, h) if err != nil { - log.Warn("error while notifying subscription", "err", err) - return + log.Warn("[rpc] error while notifying subscription", "err", err) } } if !ok { - log.Warn("new heads channel was closed") + log.Warn("[rpc] new heads channel was closed") return } case <-rpcSub.Err(): @@ -187,13 +186,12 @@ func (api *APIImpl) NewPendingTransactions(ctx context.Context) (*rpc.Subscripti if t != nil { err := notifier.Notify(rpcSub.ID, t.Hash()) if err != nil { - log.Warn("error while notifying subscription", "err", err) - return + log.Warn("[rpc] error while notifying subscription", "err", err) } } } if !ok { - log.Warn("new pending transactions channel was closed") + log.Warn("[rpc] new pending transactions channel was closed") return } case <-rpcSub.Err(): @@ -229,13 +227,12 @@ func (api *APIImpl) NewPendingTransactionsWithBody(ctx context.Context) (*rpc.Su if t != nil { err := notifier.Notify(rpcSub.ID, t) if err != nil { - log.Warn("error while notifying subscription", "err", err) - return + log.Warn("[rpc] error while notifying subscription", "err", err) } } } if !ok { - log.Warn("new pending transactions channel was closed") + log.Warn("[rpc] new pending transactions channel was closed") return } case <-rpcSub.Err(): @@ -274,12 +271,11 @@ func (api *APIImpl) Logs(ctx context.Context, crit filters.FilterCriteria) (*rpc } err := notifier.Notify(rpcSub.ID, h) if err != nil { - log.Warn("error while notifying subscription", "err", err) - return + log.Warn("[rpc] error while notifying subscription", "err", err) } } if !ok { - log.Warn("log channel was closed") + log.Warn("[rpc] log channel was closed") return } case <-rpcSub.Err(): diff --git a/cmd/rpcdaemon/commands/eth_filters_test.go b/cmd/rpcdaemon/commands/eth_filters_test.go index fbe068a4cb2..aa565c18b75 100644 --- a/cmd/rpcdaemon/commands/eth_filters_test.go +++ b/cmd/rpcdaemon/commands/eth_filters_test.go @@ -26,12 +26,11 @@ func TestNewFilters(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, stages.Mock(t)) mining := txpool.NewMiningClient(conn) ff := rpchelper.New(ctx, nil, nil, mining, func() {}, m.Log) - api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) ptf, err := api.NewPendingTransactionFilter(ctx) assert.Nil(err) diff --git a/cmd/rpcdaemon/commands/eth_mining_test.go b/cmd/rpcdaemon/commands/eth_mining_test.go index d02951f451f..f9ee3ff03e0 100644 --- a/cmd/rpcdaemon/commands/eth_mining_test.go +++ b/cmd/rpcdaemon/commands/eth_mining_test.go @@ -26,8 +26,7 @@ func TestPendingBlock(t *testing.T) { ff := rpchelper.New(ctx, nil, nil, mining, func() {}, m.Log) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) engine := ethash.NewFaker() - br, _ := m.NewBlocksIO() - api := NewEthAPI(NewBaseApi(ff, stateCache, br, nil, false, rpccfg.DefaultEvmCallTimeout, engine, + api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, nil, false, rpccfg.DefaultEvmCallTimeout, engine, m.Dirs, nil, nil), nil, nil, nil, mining, 5000000, 100_000, log.New()) expect := uint64(12345) b, err := rlp.EncodeToBytes(types.NewBlockWithHeader(&types.Header{Number: big.NewInt(int64(expect))})) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index 2431546ed4d..f0a5ef85502 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -23,7 +23,6 @@ import ( "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/core/vm/evmtypes" @@ -507,7 +506,7 @@ func (e *intraBlockExec) changeBlock(header *types.Header) { e.blockHash = header.Hash() e.header = header e.rules = e.chainConfig.Rules(e.blockNum, header.Time) - e.signer = types.MakeSigner(e.chainConfig, e.blockNum) + e.signer = types.MakeSigner(e.chainConfig, e.blockNum, header.Time) e.vmConfig.SkipAnalysis = core.SkipAnalysis(e.chainConfig, e.blockNum) } @@ -550,7 +549,7 @@ func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]common.Hash, from, to uint64 var topicsUnion iter.U64 for _, topic := range sub { - it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), int(from), int(to), order.Asc, kv.Unlim) + it, err := tx.IndexRange(kv.LogTopicIdx, topic.Bytes(), int(from), int(to), order.Asc, kv.Unlim) if err != nil { return nil, err } @@ -568,7 +567,7 @@ func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]common.Hash, from, to uint64 func getAddrsBitmapV3(tx kv.TemporalTx, addrs []common.Address, from, to uint64) (res iter.U64, err error) { for _, addr := range addrs { - it, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], int(from), int(to), true, kv.Unlim) + it, err := tx.IndexRange(kv.LogAddrIdx, addr[:], int(from), int(to), true, kv.Unlim) if err != nil { return nil, err } @@ -722,12 +721,6 @@ func marshalReceipt(receipt *types.Receipt, txn types.Transaction, chainConfig * if t.Protected() { chainId = types.DeriveChainId(&t.V).ToBig() } - case *types.AccessListTx: - chainId = t.ChainID.ToBig() - case *types.DynamicFeeTransaction: - chainId = t.ChainID.ToBig() - // case *types.SignedBlobTx: // TODO: needs eip-4844 signer - // chainId = t.GetChainID().ToBig() case *types.DepositTx: // Deposit TX does not have chain ID default: diff --git a/cmd/rpcdaemon/commands/eth_subscribe_test.go b/cmd/rpcdaemon/commands/eth_subscribe_test.go index 8697c676a2e..959200d364c 100644 --- a/cmd/rpcdaemon/commands/eth_subscribe_test.go +++ b/cmd/rpcdaemon/commands/eth_subscribe_test.go @@ -22,7 +22,6 @@ import ( func TestEthSubscribe(t *testing.T) { m, require := stages.Mock(t), require.New(t) - br, _ := m.NewBlocksIO() chain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 7, func(i int, b *core.BlockGen) { b.SetCoinbase(libcommon.Address{1}) }, false /* intermediateHashes */) @@ -42,19 +41,19 @@ func TestEthSubscribe(t *testing.T) { ctx := context.Background() logger := log.New() - backendServer := privateapi.NewEthBackendServer(ctx, nil, m.DB, m.Notifications.Events, br, nil, nil, nil, false, logger) + backendServer := privateapi.NewEthBackendServer(ctx, nil, m.DB, m.Notifications.Events, m.BlockReader, nil, nil, nil, false, logger) backendClient := direct.NewEthBackendClientDirect(backendServer) - backend := rpcservices.NewRemoteBackend(backendClient, m.DB, br) + backend := rpcservices.NewRemoteBackend(backendClient, m.DB, m.BlockReader) ff := rpchelper.New(ctx, backend, nil, nil, func() {}, m.Log) newHeads, id := ff.SubscribeNewHeads(16) defer ff.UnsubscribeHeads(id) - initialCycle := true + initialCycle := stages.MockInsertAsInitialCycle highestSeenHeader := chain.TopBlock.NumberU64() - hook := stages.NewHook(m.Ctx, m.Notifications, m.Sync, br, m.ChainConfig, m.Log, m.UpdateHead) - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, logger, nil, hook); err != nil { + hook := stages.NewHook(m.Ctx, m.Notifications, m.Sync, m.BlockReader, m.ChainConfig, m.Log, m.UpdateHead) + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, logger, m.BlockReader, hook); err != nil { t.Fatal(err) } diff --git a/cmd/rpcdaemon/commands/eth_system_test.go b/cmd/rpcdaemon/commands/eth_system_test.go index 8c18267f82f..cd90fc840aa 100644 --- a/cmd/rpcdaemon/commands/eth_system_test.go +++ b/cmd/rpcdaemon/commands/eth_system_test.go @@ -8,8 +8,6 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/kv/kvcache" - "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/core" @@ -42,10 +40,7 @@ func TestGasPrice(t *testing.T) { t.Run(testCase.description, func(t *testing.T) { m := createGasPriceTestKV(t, testCase.chainSize) defer m.DB.Close() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - br, _ := m.NewBlocksIO() - base := NewBaseApi(nil, stateCache, br, nil, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - eth := NewEthAPI(base, m.DB, nil, nil, nil, 5000000, 100_000, log.New()) + eth := NewEthAPI(newBaseApiForTest(m), m.DB, nil, nil, nil, 5000000, 100_000, log.New()) ctx := context.Background() result, err := eth.GasPrice(ctx) @@ -86,7 +81,7 @@ func createGasPriceTestKV(t *testing.T, chainSize int) *stages.MockSentry { t.Error(err) } // Construct testing chain - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Error(err) } diff --git a/cmd/rpcdaemon/commands/gen_traces_test.go b/cmd/rpcdaemon/commands/gen_traces_test.go index 8842e96314b..e311a50c231 100644 --- a/cmd/rpcdaemon/commands/gen_traces_test.go +++ b/cmd/rpcdaemon/commands/gen_traces_test.go @@ -28,9 +28,8 @@ Testing tracing RPC API by generating patters of contracts invoking one another func TestGeneratedDebugApi(t *testing.T) { m := rpcdaemontest.CreateTestSentryForTraces(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) + baseApi := NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) api := NewPrivateDebugAPI(baseApi, m.DB, 0) var buf bytes.Buffer stream := jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096) @@ -116,9 +115,8 @@ func TestGeneratedDebugApi(t *testing.T) { func TestGeneratedTraceApi(t *testing.T) { m := rpcdaemontest.CreateTestSentryForTraces(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) + baseApi := NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) api := NewTraceAPI(baseApi, m.DB, &httpcfg.HttpCfg{}) traces, err := api.Block(context.Background(), rpc.BlockNumber(1), new(bool)) if err != nil { @@ -274,11 +272,7 @@ func TestGeneratedTraceApi(t *testing.T) { func TestGeneratedTraceApiCollision(t *testing.T) { m := rpcdaemontest.CreateTestSentryForTracesCollision(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) - api := NewTraceAPI(baseApi, m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) traces, err := api.Transaction(context.Background(), common.HexToHash("0xb2b9fa4c999c1c8370ce1fbd1c4315a9ce7f8421fe2ebed8a9051ff2e4e7e3da"), new(bool)) if err != nil { t.Errorf("trace_block %d: %v", 0, err) diff --git a/cmd/rpcdaemon/commands/get_chain_config_test.go b/cmd/rpcdaemon/commands/get_chain_config_test.go index 83541a6c780..8912e25a91e 100644 --- a/cmd/rpcdaemon/commands/get_chain_config_test.go +++ b/cmd/rpcdaemon/commands/get_chain_config_test.go @@ -4,13 +4,14 @@ import ( "context" "testing" - "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon/core" + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/log/v3" ) func TestGetChainConfig(t *testing.T) { - db := memdb.NewTestDB(t) + _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) config, _, err := core.CommitGenesisBlock(db, core.MainnetGenesisBlock(), "", log.New()) if err != nil { t.Fatalf("setting up genensis block: %v", err) diff --git a/cmd/rpcdaemon/commands/otterscan_api.go b/cmd/rpcdaemon/commands/otterscan_api.go index 0afb6b12f79..981f87698b6 100644 --- a/cmd/rpcdaemon/commands/otterscan_api.go +++ b/cmd/rpcdaemon/commands/otterscan_api.go @@ -17,7 +17,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/iter" "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/vm/evmtypes" "github.com/ledgerwatch/erigon/eth/tracers" @@ -469,11 +468,11 @@ func (api *OtterscanAPIImpl) searchTransactionsBeforeV3(tx kv.TemporalTx, ctx co if err != nil { return nil, err } - itTo, err := tx.IndexRange(temporal.TracesToIdx, addr[:], int(fromTxNum), -1, order.Desc, kv.Unlim) + itTo, err := tx.IndexRange(kv.TracesToIdx, addr[:], int(fromTxNum), -1, order.Desc, kv.Unlim) if err != nil { return nil, err } - itFrom, err := tx.IndexRange(temporal.TracesFromIdx, addr[:], int(fromTxNum), -1, order.Desc, kv.Unlim) + itFrom, err := tx.IndexRange(kv.TracesFromIdx, addr[:], int(fromTxNum), -1, order.Desc, kv.Unlim) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/commands/otterscan_contract_creator.go b/cmd/rpcdaemon/commands/otterscan_contract_creator.go index 40c337a0a4d..62da99226d3 100644 --- a/cmd/rpcdaemon/commands/otterscan_contract_creator.go +++ b/cmd/rpcdaemon/commands/otterscan_contract_creator.go @@ -13,12 +13,10 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" - "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/log/v3" - "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types/accounts" + "github.com/ledgerwatch/erigon/turbo/rpchelper" ) type ContractCreatorData struct { @@ -33,8 +31,8 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common } defer tx.Rollback() - reader := state.NewPlainStateReader(tx) - plainStateAcc, err := reader.ReadAccountData(addr) + latestState := rpchelper.NewLatestStateReader(tx) + plainStateAcc, err := latestState.ReadAccountData(addr) if err != nil { return nil, err } @@ -57,14 +55,6 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common var acc accounts.Account if api.historyV3(tx) { ttx := tx.(kv.TemporalTx) - headNumber, err := stages.GetStageProgress(tx, stages.Execution) - if err != nil { - return nil, err - } - lastTxNum, err := rawdbv3.TxNums.Max(tx, headNumber) - if err != nil { - return nil, err - } // Contract; search for creation tx; navigate forward on AccountsHistory/ChangeSets // @@ -75,7 +65,7 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common // so it is optimal to search from the beginning even if the contract has multiple // incarnations. var prevTxnID, nextTxnID uint64 - it, err := ttx.IndexRange(temporal.AccountsHistoryIdx, addr[:], 0, int(lastTxNum+1), order.Asc, kv.Unlim) + it, err := ttx.IndexRange(kv.AccountsHistoryIdx, addr[:], 0, -1, order.Asc, kv.Unlim) if err != nil { return nil, err } @@ -90,12 +80,11 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common continue } - v, ok, err := ttx.HistoryGet(temporal.AccountsHistory, addr[:], txnID) + v, ok, err := ttx.HistoryGet(kv.AccountsHistory, addr[:], txnID) if err != nil { - log.Error("Unexpected error, couldn't find changeset", "txNum", i, "addr", addr) + log.Error("Unexpected error, couldn't find changeset", "txNum", txnID, "addr", addr) return nil, err } - fmt.Printf("i: %d, %t, %x\n", i, ok, v) if !ok { err = fmt.Errorf("couldn't find history txnID=%v addr=%v", txnID, addr) @@ -107,7 +96,7 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common continue } - if err := acc.DecodeForStorage(v); err != nil { + if err := accounts.DeserialiseV3(&acc, v); err != nil { return nil, err } // Found the shard where the incarnation change happens; ignore all next index values @@ -132,7 +121,7 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common // can be replaced by full-scan over ttx.HistoryRange([prevTxnID, nextTxnID])? idx := sort.Search(int(nextTxnID-prevTxnID), func(i int) bool { txnID := uint64(i) + prevTxnID - v, ok, err := ttx.HistoryGet(temporal.AccountsHistory, addr[:], txnID) + v, ok, err := ttx.HistoryGet(kv.AccountsHistory, addr[:], txnID) if err != nil { log.Error("[rpc] Unexpected error, couldn't find changeset", "txNum", i, "addr", addr) panic(err) @@ -145,7 +134,7 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common return false } - if err := acc.DecodeForStorage(v); err != nil { + if err := accounts.DeserialiseV3(&acc, v); err != nil { searchErr = err return false } @@ -195,7 +184,7 @@ func (api *OtterscanAPIImpl) GetContractCreator(ctx context.Context, addr common // dozens of states changes due to ETH deposits/withdraw after contract creation, // so it is optimal to search from the beginning even if the contract has multiple // incarnations. - accHistory, err := tx.Cursor(kv.AccountsHistory) + accHistory, err := tx.Cursor(kv.E2AccountsHistory) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go b/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go index 523ccffa981..91c46c3617f 100644 --- a/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go +++ b/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go @@ -5,15 +5,12 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" - "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/stretchr/testify/require" ) func TestGetContractCreator(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB) + api := NewOtterscanAPI(newBaseApiForTest(m), m.DB) addr := libcommon.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") expectCreator := libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") diff --git a/cmd/rpcdaemon/commands/otterscan_generic_tracer.go b/cmd/rpcdaemon/commands/otterscan_generic_tracer.go index f75bf7694e7..8ea642b4d27 100644 --- a/cmd/rpcdaemon/commands/otterscan_generic_tracer.go +++ b/cmd/rpcdaemon/commands/otterscan_generic_tracer.go @@ -63,7 +63,6 @@ func (api *OtterscanAPIImpl) genericTracer(dbtx kv.Tx, ctx context.Context, bloc cachedWriter := state.NewCachedWriter(noop, stateCache) ibs := state.New(cachedReader) - signer := types.MakeSigner(chainConfig, blockNum) getHeader := func(hash common.Hash, number uint64) *types.Header { h, e := api._blockReader.Header(ctx, dbtx, hash, number) @@ -83,6 +82,7 @@ func (api *OtterscanAPIImpl) genericTracer(dbtx kv.Tx, ctx context.Context, bloc header := block.Header() rules := chainConfig.Rules(block.NumberU64(), header.Time) + signer := types.MakeSigner(chainConfig, blockNum, header.Time) for idx, tx := range block.Transactions() { ibs.SetTxContext(tx.Hash(), block.Hash(), idx) diff --git a/cmd/rpcdaemon/commands/otterscan_search_backward_test.go b/cmd/rpcdaemon/commands/otterscan_search_backward_test.go index 369a267a375..882176e8d5d 100644 --- a/cmd/rpcdaemon/commands/otterscan_search_backward_test.go +++ b/cmd/rpcdaemon/commands/otterscan_search_backward_test.go @@ -8,7 +8,6 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/common/hexutil" - "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/stretchr/testify/require" ) @@ -149,9 +148,7 @@ func TestBackwardBlockProviderWithMultipleChunksBlockNotFound(t *testing.T) { func TestSearchTransactionsBefore(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB) + api := NewOtterscanAPI(newBaseApiForTest(m), m.DB) addr := libcommon.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") t.Run("small page size", func(t *testing.T) { diff --git a/cmd/rpcdaemon/commands/otterscan_search_trace.go b/cmd/rpcdaemon/commands/otterscan_search_trace.go index 71a000df825..7ecf1b11f90 100644 --- a/cmd/rpcdaemon/commands/otterscan_search_trace.go +++ b/cmd/rpcdaemon/commands/otterscan_search_trace.go @@ -79,7 +79,7 @@ func (api *OtterscanAPIImpl) traceBlock(dbtx kv.Tx, ctx context.Context, blockNu cachedWriter := state.NewCachedWriter(noop, stateCache) ibs := state.New(cachedReader) - signer := types.MakeSigner(chainConfig, blockNum) + signer := types.MakeSigner(chainConfig, blockNum, block.Time()) getHeader := func(hash common.Hash, number uint64) *types.Header { h, e := api._blockReader.Header(ctx, dbtx, hash, number) diff --git a/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce.go b/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce.go index 06381ef9ed8..30db1a45d10 100644 --- a/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce.go +++ b/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce.go @@ -13,7 +13,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/core/types/accounts" @@ -29,7 +28,7 @@ func (api *OtterscanAPIImpl) GetTransactionBySenderAndNonce(ctx context.Context, var acc accounts.Account if api.historyV3(tx) { ttx := tx.(kv.TemporalTx) - it, err := ttx.IndexRange(temporal.AccountsHistoryIdx, addr[:], -1, -1, order.Asc, kv.Unlim) + it, err := ttx.IndexRange(kv.AccountsHistoryIdx, addr[:], -1, -1, order.Asc, kv.Unlim) if err != nil { return nil, err } @@ -46,7 +45,7 @@ func (api *OtterscanAPIImpl) GetTransactionBySenderAndNonce(ctx context.Context, continue } - v, ok, err := ttx.HistoryGet(temporal.AccountsHistory, addr[:], txnID) + v, ok, err := ttx.HistoryGet(kv.AccountsHistory, addr[:], txnID) if err != nil { log.Error("Unexpected error, couldn't find changeset", "txNum", i, "addr", addr) return nil, err @@ -62,7 +61,7 @@ func (api *OtterscanAPIImpl) GetTransactionBySenderAndNonce(ctx context.Context, continue } - if err := acc.DecodeForStorage(v); err != nil { + if err := accounts.DeserialiseV3(&acc, v); err != nil { return nil, err } // Desired nonce was found in this chunk @@ -86,7 +85,7 @@ func (api *OtterscanAPIImpl) GetTransactionBySenderAndNonce(ctx context.Context, // can be replaced by full-scan over ttx.HistoryRange([prevTxnID, nextTxnID])? idx := sort.Search(int(nextTxnID-prevTxnID), func(i int) bool { txnID := uint64(i) + prevTxnID - v, ok, err := ttx.HistoryGet(temporal.AccountsHistory, addr[:], txnID) + v, ok, err := ttx.HistoryGet(kv.AccountsHistory, addr[:], txnID) if err != nil { log.Error("[rpc] Unexpected error, couldn't find changeset", "txNum", i, "addr", addr) panic(err) @@ -99,10 +98,11 @@ func (api *OtterscanAPIImpl) GetTransactionBySenderAndNonce(ctx context.Context, return false } - if err := acc.DecodeForStorage(v); err != nil { + if err := accounts.DeserialiseV3(&acc, v); err != nil { searchErr = err return false } + // Since the state contains the nonce BEFORE the block changes, we look for // the block when the nonce changed to be > the desired once, which means the // previous history block contains the actual change; it may contain multiple @@ -150,7 +150,7 @@ func (api *OtterscanAPIImpl) GetTransactionBySenderAndNonce(ctx context.Context, return &txHash, nil } - accHistoryC, err := tx.Cursor(kv.AccountsHistory) + accHistoryC, err := tx.Cursor(kv.E2AccountsHistory) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go b/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go index e644d5e939f..6d39bdab465 100644 --- a/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go +++ b/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go @@ -12,8 +12,7 @@ import ( func TestGetTransactionBySenderAndNonce(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB) + api := NewOtterscanAPI(NewBaseApi(nil, nil, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB) addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") expectCreator := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") diff --git a/cmd/rpcdaemon/commands/parity_api.go b/cmd/rpcdaemon/commands/parity_api.go index 604b232f03b..9e128a8c976 100644 --- a/cmd/rpcdaemon/commands/parity_api.go +++ b/cmd/rpcdaemon/commands/parity_api.go @@ -9,8 +9,12 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/order" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" + "github.com/ledgerwatch/erigon/common" + "github.com/ledgerwatch/erigon/core/rawdb" + "github.com/ledgerwatch/erigon/turbo/rpchelper" - "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/rpc" ) @@ -25,13 +29,15 @@ type ParityAPI interface { // ParityAPIImpl data structure to store things needed for parity_ commands type ParityAPIImpl struct { + *BaseAPI db kv.RoDB } // NewParityAPIImpl returns ParityAPIImpl instance -func NewParityAPIImpl(db kv.RoDB) *ParityAPIImpl { +func NewParityAPIImpl(base *BaseAPI, db kv.RoDB) *ParityAPIImpl { return &ParityAPIImpl{ - db: db, + BaseAPI: base, + db: db, } } @@ -40,19 +46,45 @@ func (api *ParityAPIImpl) ListStorageKeys(ctx context.Context, account libcommon if err := api.checkBlockNumber(blockNumberOrTag); err != nil { return nil, err } + keys := make([]hexutility.Bytes, 0) tx, err := api.db.BeginRo(ctx) if err != nil { return nil, fmt.Errorf("listStorageKeys cannot open tx: %w", err) } defer tx.Rollback() - a, err := state.NewPlainStateReader(tx).ReadAccountData(account) + a, err := rpchelper.NewLatestStateReader(tx).ReadAccountData(account) if err != nil { return nil, err } else if a == nil { return nil, fmt.Errorf("acc not found") } + if api.historyV3(tx) { + bn := rawdb.ReadCurrentBlockNumber(tx) + minTxNum, err := rawdbv3.TxNums.Min(tx, *bn) + if err != nil { + return nil, err + } + + from := account[:] + if offset != nil { + from = append(from, *offset...) + } + to, _ := kv.NextSubtree(account[:]) + r, err := tx.(kv.TemporalTx).DomainRange(kv.StorageDomain, from, to, minTxNum, order.Asc, quantity) + if err != nil { + return nil, err + } + for r.HasNext() { + k, _, err := r.Next() + if err != nil { + return nil, err + } + keys = append(keys, common.CopyBytes(k[20:])) + } + return keys, nil + } b := make([]byte, 8) binary.BigEndian.PutUint64(b, a.GetIncarnation()) seekBytes := append(account.Bytes(), b...) @@ -62,7 +94,6 @@ func (api *ParityAPIImpl) ListStorageKeys(ctx context.Context, account libcommon return nil, err } defer c.Close() - keys := make([]hexutility.Bytes, 0) var v []byte var seekVal []byte if offset != nil { diff --git a/cmd/rpcdaemon/commands/parity_api_test.go b/cmd/rpcdaemon/commands/parity_api_test.go index 1b23060b846..6a03b0f0d77 100644 --- a/cmd/rpcdaemon/commands/parity_api_test.go +++ b/cmd/rpcdaemon/commands/parity_api_test.go @@ -5,6 +5,7 @@ import ( "fmt" "testing" + "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/stretchr/testify/assert" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -20,7 +21,9 @@ var latestBlock = rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) func TestParityAPIImpl_ListStorageKeys_NoOffset(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewParityAPIImpl(m.DB) + agg := m.HistoryV3Components() + baseApi := NewBaseApi(nil, nil, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) + api := NewParityAPIImpl(baseApi, m.DB) answers := []string{ "0000000000000000000000000000000000000000000000000000000000000000", "0000000000000000000000000000000000000000000000000000000000000002", @@ -42,7 +45,7 @@ func TestParityAPIImpl_ListStorageKeys_NoOffset(t *testing.T) { func TestParityAPIImpl_ListStorageKeys_WithOffset_ExistingPrefix(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewParityAPIImpl(m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) answers := []string{ "29d05770ca9ee7088a64e18c8e5160fc62c3c2179dc8ef9b4dbc970c9e51b4d8", "29edc84535d98b29835079d685b97b41ee8e831e343cc80793057e462353a26d", @@ -66,7 +69,7 @@ func TestParityAPIImpl_ListStorageKeys_WithOffset_ExistingPrefix(t *testing.T) { func TestParityAPIImpl_ListStorageKeys_WithOffset_NonExistingPrefix(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewParityAPIImpl(m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) answers := []string{ "4644be453c81744b6842ddf615d7fca0e14a23b09734be63d44c23452de95631", "4974416255391052161ba8184fe652f3bf8c915592c65f7de127af8e637dce5d", @@ -87,7 +90,7 @@ func TestParityAPIImpl_ListStorageKeys_WithOffset_NonExistingPrefix(t *testing.T func TestParityAPIImpl_ListStorageKeys_WithOffset_EmptyResponse(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewParityAPIImpl(m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) addr := libcommon.HexToAddress("0x920fd5070602feaea2e251e9e7238b6c376bcae5") offset := common.Hex2Bytes("ff") b := hexutility.Bytes(offset) @@ -101,7 +104,7 @@ func TestParityAPIImpl_ListStorageKeys_WithOffset_EmptyResponse(t *testing.T) { func TestParityAPIImpl_ListStorageKeys_AccNotFound(t *testing.T) { assert := assert.New(t) m, _, _ := rpcdaemontest.CreateTestSentry(t) - api := NewParityAPIImpl(m.DB) + api := NewParityAPIImpl(newBaseApiForTest(m), m.DB) addr := libcommon.HexToAddress("0x920fd5070602feaea2e251e9e7238b6c376bcaef") _, err := api.ListStorageKeys(context.Background(), addr, 2, nil, latestBlock) assert.Error(err, fmt.Errorf("acc not found")) diff --git a/cmd/rpcdaemon/commands/send_transaction.go b/cmd/rpcdaemon/commands/send_transaction.go index 27c90f47f25..fbb6fd00c8a 100644 --- a/cmd/rpcdaemon/commands/send_transaction.go +++ b/cmd/rpcdaemon/commands/send_transaction.go @@ -11,9 +11,7 @@ import ( txPoolProto "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon/common/hexutil" - "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/types" - "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" ) @@ -40,27 +38,16 @@ func (api *APIImpl) SendRawTransaction(ctx context.Context, encodedTx hexutility if !txn.Protected() { return common.Hash{}, errors.New("only replay-protected (EIP-155) transactions allowed over RPC") } - hash := txn.Hash() - res, err := api.txPool.Add(ctx, &txPoolProto.AddRequest{RlpTxs: [][]byte{encodedTx}}) - if err != nil { - return common.Hash{}, err - } - - if res.Imported[0] != txPoolProto.ImportResult_SUCCESS { - return hash, fmt.Errorf("%s: %s", txPoolProto.ImportResult_name[int32(res.Imported[0])], res.Errors[0]) - } + // this has been moved to prior to adding of transactions to capture the + // pre state of the db - which is used for logging in the messages below tx, err := api.db.BeginRo(ctx) if err != nil { return common.Hash{}, err } + defer tx.Rollback() - // Print a log with full txn details for manual investigations and interventions - blockNum := rawdb.ReadCurrentBlockNumber(tx) - if blockNum == nil { - return common.Hash{}, err - } cc, err := api.chainConfig(tx) if err != nil { return common.Hash{}, err @@ -73,17 +60,14 @@ func (api *APIImpl) SendRawTransaction(ctx context.Context, encodedTx hexutility return common.Hash{}, fmt.Errorf("invalid chain id, expected: %d got: %d", chainId, *txnChainId) } - signer := types.MakeSigner(cc, *blockNum) - from, err := txn.Sender(*signer) + hash := txn.Hash() + res, err := api.txPool.Add(ctx, &txPoolProto.AddRequest{RlpTxs: [][]byte{encodedTx}}) if err != nil { return common.Hash{}, err } - if txn.GetTo() == nil { - addr := crypto.CreateAddress(from, txn.GetNonce()) - api.logger.Info("Submitted contract creation", "hash", txn.Hash().Hex(), "from", from, "nonce", txn.GetNonce(), "contract", addr.Hex(), "value", txn.GetValue()) - } else { - api.logger.Info("Submitted transaction", "hash", txn.Hash().Hex(), "from", from, "nonce", txn.GetNonce(), "recipient", txn.GetTo(), "value", txn.GetValue()) + if res.Imported[0] != txPoolProto.ImportResult_SUCCESS { + return hash, fmt.Errorf("%s: %s", txPoolProto.ImportResult_name[int32(res.Imported[0])], res.Errors[0]) } return txn.Hash(), nil diff --git a/cmd/rpcdaemon/commands/send_transaction_test.go b/cmd/rpcdaemon/commands/send_transaction_test.go index eb6668ebf2c..5ec4a2484a6 100644 --- a/cmd/rpcdaemon/commands/send_transaction_test.go +++ b/cmd/rpcdaemon/commands/send_transaction_test.go @@ -11,6 +11,7 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv/kvcache" + "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/require" @@ -22,11 +23,16 @@ import ( "github.com/ledgerwatch/erigon/eth/protocols/eth" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" - "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/stages" ) +func newBaseApiForTest(m *stages.MockSentry) *commands.BaseAPI { + agg := m.HistoryV3Components() + stateCache := kvcache.New(kvcache.DefaultCoherentConfig) + return commands.NewBaseApi(nil, stateCache, m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil) +} + func TestSendRawTransaction(t *testing.T) { t.Skip("Flaky test") m, require := stages.Mock(t), require.New(t) @@ -60,8 +66,8 @@ func TestSendRawTransaction(t *testing.T) { } m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed - initialCycle := true - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, logger, nil, nil); err != nil { + initialCycle := stages.MockInsertAsInitialCycle + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, logger, m.BlockReader, nil); err != nil { t.Fatal(err) } } @@ -73,9 +79,7 @@ func TestSendRawTransaction(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) txPool := txpool.NewTxpoolClient(conn) ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}, m.Log) - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - br, _ := m.NewBlocksIO() - api := commands.NewEthAPI(commands.NewBaseApi(ff, stateCache, br, nil, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, nil, txPool, nil, 5000000, 100_000, logger) + api := commands.NewEthAPI(newBaseApiForTest(m), m.DB, nil, txPool, nil, 5000000, 100_000, logger) buf := bytes.NewBuffer(nil) err = txn.MarshalBinary(buf) diff --git a/cmd/rpcdaemon/commands/storage_range.go b/cmd/rpcdaemon/commands/storage_range.go index 975ec976ae8..5534659a3c3 100644 --- a/cmd/rpcdaemon/commands/storage_range.go +++ b/cmd/rpcdaemon/commands/storage_range.go @@ -8,7 +8,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/erigon/common" - "github.com/ledgerwatch/erigon/core/state/temporal" ) // StorageRangeResult is the result of a debug_storageRangeAt API call. @@ -54,7 +53,7 @@ func storageRangeAtV3(ttx kv.TemporalTx, contractAddress libcommon.Address, star fromKey := append(libcommon.Copy(contractAddress.Bytes()), start...) toKey, _ := kv.NextSubtree(contractAddress.Bytes()) - r, err := ttx.DomainRange(temporal.StorageDomain, fromKey, toKey, txNum, order.Asc, maxResult+1) + r, err := ttx.DomainRange(kv.StorageDomain, fromKey, toKey, txNum, order.Asc, maxResult+1) if err != nil { return StorageRangeResult{}, err } diff --git a/cmd/rpcdaemon/commands/trace_adhoc.go b/cmd/rpcdaemon/commands/trace_adhoc.go index d1a54256813..1a01928b570 100644 --- a/cmd/rpcdaemon/commands/trace_adhoc.go +++ b/cmd/rpcdaemon/commands/trace_adhoc.go @@ -754,8 +754,9 @@ func (api *TraceAPIImpl) ReplayTransaction(ctx context.Context, txHash libcommon } } + signer := types.MakeSigner(chainConfig, blockNum, block.Time()) // Returns an array of trace arrays, one trace array for each transaction - traces, _, err := api.callManyTransactions(ctx, tx, block, traceTypes, int(txnIndex), *gasBailOut, types.MakeSigner(chainConfig, blockNum), chainConfig) + traces, _, err := api.callManyTransactions(ctx, tx, block, traceTypes, int(txnIndex), *gasBailOut, signer, chainConfig) if err != nil { return nil, err } @@ -836,8 +837,9 @@ func (api *TraceAPIImpl) ReplayBlockTransactions(ctx context.Context, blockNrOrH } } + signer := types.MakeSigner(chainConfig, blockNumber, block.Time()) // Returns an array of trace arrays, one trace array for each transaction - traces, _, err := api.callManyTransactions(ctx, tx, block, traceTypes, -1 /* all tx indices */, *gasBailOut, types.MakeSigner(chainConfig, blockNumber), chainConfig) + traces, _, err := api.callManyTransactions(ctx, tx, block, traceTypes, -1 /* all tx indices */, *gasBailOut, signer, chainConfig) if err != nil { return nil, err } diff --git a/cmd/rpcdaemon/commands/trace_adhoc_test.go b/cmd/rpcdaemon/commands/trace_adhoc_test.go index 411f28615bc..8b8073b35fb 100644 --- a/cmd/rpcdaemon/commands/trace_adhoc_test.go +++ b/cmd/rpcdaemon/commands/trace_adhoc_test.go @@ -7,23 +7,17 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/require" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/cli/httpcfg" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/rpc" - "github.com/ledgerwatch/erigon/rpc/rpccfg" ) func TestEmptyQuery(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - br, _ := m.NewBlocksIO() - - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage("[]"), &rpc.BlockNumberOrHash{BlockNumber: &latest}) @@ -39,11 +33,7 @@ func TestEmptyQuery(t *testing.T) { } func TestCoinbaseBalance(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - br, _ := m.NewBlocksIO() - - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage(` @@ -69,14 +59,10 @@ func TestCoinbaseBalance(t *testing.T) { func TestReplayTransaction(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - br, _ := m.NewBlocksIO() - - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) var txnHash libcommon.Hash if err := m.DB.View(context.Background(), func(tx kv.Tx) error { - b, err := br.BlockByNumber(m.Ctx, tx, 6) + b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 6) if err != nil { return err } @@ -100,11 +86,7 @@ func TestReplayTransaction(t *testing.T) { func TestReplayBlockTransactions(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) - agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - - stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(newBaseApiForTest(m), m.DB, &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database n := rpc.BlockNumber(6) diff --git a/cmd/rpcdaemon/commands/trace_filtering.go b/cmd/rpcdaemon/commands/trace_filtering.go index 64b7311144c..74b7b2b6c74 100644 --- a/cmd/rpcdaemon/commands/trace_filtering.go +++ b/cmd/rpcdaemon/commands/trace_filtering.go @@ -22,7 +22,6 @@ import ( "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/eth/stagedsync" @@ -85,8 +84,9 @@ func (api *TraceAPIImpl) Transaction(ctx context.Context, txHash common.Hash, ga hash := block.Hash() + signer := types.MakeSigner(chainConfig, blockNumber, block.Time()) // Returns an array of trace arrays, one trace array for each transaction - traces, _, err := api.callManyTransactions(ctx, tx, block, []string{TraceTypeTrace}, txIndex, *gasBailOut, types.MakeSigner(chainConfig, blockNumber), chainConfig) + traces, _, err := api.callManyTransactions(ctx, tx, block, []string{TraceTypeTrace}, txIndex, *gasBailOut, signer, chainConfig) if err != nil { return nil, err } @@ -179,7 +179,8 @@ func (api *TraceAPIImpl) Block(ctx context.Context, blockNr rpc.BlockNumber, gas if err != nil { return nil, err } - traces, syscall, err := api.callManyTransactions(ctx, tx, block, []string{TraceTypeTrace}, -1 /* all tx indices */, *gasBailOut /* gasBailOut */, types.MakeSigner(cfg, blockNum), cfg) + signer := types.MakeSigner(cfg, blockNum, block.Time()) + traces, syscall, err := api.callManyTransactions(ctx, tx, block, []string{TraceTypeTrace}, -1 /* all tx indices */, *gasBailOut /* gasBailOut */, signer, cfg) if err != nil { return nil, err } @@ -281,7 +282,7 @@ func traceFilterBitmapsV3(tx kv.TemporalTx, req TraceFilterRequest, from, to uin for _, addr := range req.FromAddress { if addr != nil { - it, err := tx.IndexRange(temporal.TracesFromIdx, addr.Bytes(), int(from), int(to), order.Asc, kv.Unlim) + it, err := tx.IndexRange(kv.TracesFromIdx, addr.Bytes(), int(from), int(to), order.Asc, kv.Unlim) if errors.Is(err, ethdb.ErrKeyNotFound) { continue } @@ -292,7 +293,7 @@ func traceFilterBitmapsV3(tx kv.TemporalTx, req TraceFilterRequest, from, to uin for _, addr := range req.ToAddress { if addr != nil { - it, err := tx.IndexRange(temporal.TracesToIdx, addr.Bytes(), int(from), int(to), order.Asc, kv.Unlim) + it, err := tx.IndexRange(kv.TracesToIdx, addr.Bytes(), int(from), int(to), order.Asc, kv.Unlim) if errors.Is(err, ethdb.ErrKeyNotFound) { continue } @@ -413,7 +414,8 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, gas blockHash := block.Hash() blockNumber := block.NumberU64() txs := block.Transactions() - t, syscall, tErr := api.callManyTransactions(ctx, dbtx, block, []string{TraceTypeTrace}, -1 /* all tx indices */, *gasBailOut, types.MakeSigner(chainConfig, b), chainConfig) + signer := types.MakeSigner(chainConfig, b, block.Time()) + t, syscall, tErr := api.callManyTransactions(ctx, dbtx, block, []string{TraceTypeTrace}, -1 /* all tx indices */, *gasBailOut, signer, chainConfig) if tErr != nil { if first { first = false @@ -608,7 +610,7 @@ func (api *TraceAPIImpl) filterV3(ctx context.Context, dbtx kv.TemporalTx, fromB } lastBlockHash = lastHeader.Hash() - lastSigner = types.MakeSigner(chainConfig, blockNum) + lastSigner = types.MakeSigner(chainConfig, blockNum, lastHeader.Time) lastRules = chainConfig.Rules(blockNum, lastHeader.Time) } if isFnalTxn { diff --git a/cmd/rpcdaemon/commands/tracing.go b/cmd/rpcdaemon/commands/tracing.go index 1e4bbab7e65..9dd9e11ef72 100644 --- a/cmd/rpcdaemon/commands/tracing.go +++ b/cmd/rpcdaemon/commands/tracing.go @@ -121,7 +121,7 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp return err } - signer := types.MakeSigner(chainConfig, block.NumberU64()) + signer := types.MakeSigner(chainConfig, block.NumberU64(), block.Time()) rules := chainConfig.Rules(block.NumberU64(), block.Time()) stream.WriteArrayStart() @@ -477,7 +477,7 @@ func (api *PrivateDebugAPIImpl) TraceCallMany(ctx context.Context, bundles []Bun // Get a new instance of the EVM evm = vm.NewEVM(blockCtx, txCtx, st, chainConfig, vm.Config{Debug: false}) - signer := types.MakeSigner(chainConfig, blockNum) + signer := types.MakeSigner(chainConfig, blockNum, block.Time()) rules := chainConfig.Rules(blockNum, blockCtx.Time) // Setup the gas pool (also for unmetered requests) diff --git a/cmd/rpcdaemon/commands/txpool_api_test.go b/cmd/rpcdaemon/commands/txpool_api_test.go index 9fb2fdeef42..9371a3c9901 100644 --- a/cmd/rpcdaemon/commands/txpool_api_test.go +++ b/cmd/rpcdaemon/commands/txpool_api_test.go @@ -28,15 +28,14 @@ func TestTxPoolContent(t *testing.T) { b.SetCoinbase(libcommon.Address{1}) }, false /* intermediateHashes */) require.NoError(err) - err = m.InsertChain(chain) + err = m.InsertChain(chain, nil) require.NoError(err) ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, m) txPool := txpool.NewTxpoolClient(conn) ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}, m.Log) agg := m.HistoryV3Components() - br, _ := m.NewBlocksIO() - api := NewTxPoolAPI(NewBaseApi(ff, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, txPool) + api := NewTxPoolAPI(NewBaseApi(ff, kvcache.New(kvcache.DefaultCoherentConfig), m.BlockReader, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, m.Dirs, nil, nil), m.DB, txPool) expectValue := uint64(1234) txn, err := types.SignTx(types.NewTransaction(0, libcommon.Address{1}, uint256.NewInt(expectValue), params.TxGas, uint256.NewInt(10*params.GWei), nil), *types.LatestSignerForChainID(m.ChainConfig.ChainID), m.Key) diff --git a/cmd/rpcdaemon/graphql/graph/generated.go b/cmd/rpcdaemon/graphql/graph/generated.go index e9c2d74631b..2fe792cabcb 100644 --- a/cmd/rpcdaemon/graphql/graph/generated.go +++ b/cmd/rpcdaemon/graphql/graph/generated.go @@ -192,7 +192,7 @@ func (e *executableSchema) Schema() *ast.Schema { } func (e *executableSchema) Complexity(typeName, field string, childComplexity int, rawArgs map[string]interface{}) (int, bool) { - ec := executionContext{nil, e} + ec := executionContext{nil, e, 0, 0, nil} _ = ec switch typeName + "." + field { @@ -932,7 +932,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { rc := graphql.GetOperationContext(ctx) - ec := executionContext{rc, e} + ec := executionContext{rc, e, 0, 0, make(chan graphql.DeferredResult)} inputUnmarshalMap := graphql.BuildUnmarshalerMap( ec.unmarshalInputBlockFilterCriteria, ec.unmarshalInputCallData, @@ -943,18 +943,33 @@ func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { switch rc.Operation.Operation { case ast.Query: return func(ctx context.Context) *graphql.Response { - if !first { - return nil + var response graphql.Response + var data graphql.Marshaler + if first { + first = false + ctx = graphql.WithUnmarshalerMap(ctx, inputUnmarshalMap) + data = ec._Query(ctx, rc.Operation.SelectionSet) + } else { + if atomic.LoadInt32(&ec.pendingDeferred) > 0 { + result := <-ec.deferredResults + atomic.AddInt32(&ec.pendingDeferred, -1) + data = result.Result + response.Path = result.Path + response.Label = result.Label + response.Errors = result.Errors + } else { + return nil + } } - first = false - ctx = graphql.WithUnmarshalerMap(ctx, inputUnmarshalMap) - data := ec._Query(ctx, rc.Operation.SelectionSet) var buf bytes.Buffer data.MarshalGQL(&buf) - - return &graphql.Response{ - Data: buf.Bytes(), + response.Data = buf.Bytes() + if atomic.LoadInt32(&ec.deferred) > 0 { + hasNext := atomic.LoadInt32(&ec.pendingDeferred) > 0 + response.HasNext = &hasNext } + + return &response } case ast.Mutation: return func(ctx context.Context) *graphql.Response { @@ -980,6 +995,28 @@ func (e *executableSchema) Exec(ctx context.Context) graphql.ResponseHandler { type executionContext struct { *graphql.OperationContext *executableSchema + deferred int32 + pendingDeferred int32 + deferredResults chan graphql.DeferredResult +} + +func (ec *executionContext) processDeferredGroup(dg graphql.DeferredGroup) { + atomic.AddInt32(&ec.pendingDeferred, 1) + go func() { + ctx := graphql.WithFreshResponseContext(dg.Context) + dg.FieldSet.Dispatch(ctx) + ds := graphql.DeferredResult{ + Path: dg.Path, + Label: dg.Label, + Result: dg.FieldSet, + Errors: graphql.GetErrors(ctx), + } + // null fields should bubble up + if dg.FieldSet.Invalids > 0 { + ds.Result = graphql.Null + } + ec.deferredResults <- ds + }() } func (ec *executionContext) introspectSchema() (*introspection.Schema, error) { @@ -8216,18 +8253,20 @@ func (ec *executionContext) unmarshalInputBlockFilterCriteria(ctx context.Contex var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("addresses")) - it.Addresses, err = ec.unmarshalOAddress2ᚕstringᚄ(ctx, v) + data, err := ec.unmarshalOAddress2ᚕstringᚄ(ctx, v) if err != nil { return it, err } + it.Addresses = data case "topics": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("topics")) - it.Topics, err = ec.unmarshalOBytes322ᚕᚕstringᚄ(ctx, v) + data, err := ec.unmarshalOBytes322ᚕᚕstringᚄ(ctx, v) if err != nil { return it, err } + it.Topics = data } } @@ -8252,66 +8291,74 @@ func (ec *executionContext) unmarshalInputCallData(ctx context.Context, obj inte var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("from")) - it.From, err = ec.unmarshalOAddress2ᚖstring(ctx, v) + data, err := ec.unmarshalOAddress2ᚖstring(ctx, v) if err != nil { return it, err } + it.From = data case "to": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("to")) - it.To, err = ec.unmarshalOAddress2ᚖstring(ctx, v) + data, err := ec.unmarshalOAddress2ᚖstring(ctx, v) if err != nil { return it, err } + it.To = data case "gas": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("gas")) - it.Gas, err = ec.unmarshalOLong2ᚖuint64(ctx, v) + data, err := ec.unmarshalOLong2ᚖuint64(ctx, v) if err != nil { return it, err } + it.Gas = data case "gasPrice": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("gasPrice")) - it.GasPrice, err = ec.unmarshalOBigInt2ᚖstring(ctx, v) + data, err := ec.unmarshalOBigInt2ᚖstring(ctx, v) if err != nil { return it, err } + it.GasPrice = data case "maxFeePerGas": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("maxFeePerGas")) - it.MaxFeePerGas, err = ec.unmarshalOBigInt2ᚖstring(ctx, v) + data, err := ec.unmarshalOBigInt2ᚖstring(ctx, v) if err != nil { return it, err } + it.MaxFeePerGas = data case "maxPriorityFeePerGas": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("maxPriorityFeePerGas")) - it.MaxPriorityFeePerGas, err = ec.unmarshalOBigInt2ᚖstring(ctx, v) + data, err := ec.unmarshalOBigInt2ᚖstring(ctx, v) if err != nil { return it, err } + it.MaxPriorityFeePerGas = data case "value": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("value")) - it.Value, err = ec.unmarshalOBigInt2ᚖstring(ctx, v) + data, err := ec.unmarshalOBigInt2ᚖstring(ctx, v) if err != nil { return it, err } + it.Value = data case "data": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("data")) - it.Data, err = ec.unmarshalOBytes2ᚖstring(ctx, v) + data, err := ec.unmarshalOBytes2ᚖstring(ctx, v) if err != nil { return it, err } + it.Data = data } } @@ -8336,34 +8383,38 @@ func (ec *executionContext) unmarshalInputFilterCriteria(ctx context.Context, ob var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("fromBlock")) - it.FromBlock, err = ec.unmarshalOLong2ᚖuint64(ctx, v) + data, err := ec.unmarshalOLong2ᚖuint64(ctx, v) if err != nil { return it, err } + it.FromBlock = data case "toBlock": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("toBlock")) - it.ToBlock, err = ec.unmarshalOLong2ᚖuint64(ctx, v) + data, err := ec.unmarshalOLong2ᚖuint64(ctx, v) if err != nil { return it, err } + it.ToBlock = data case "addresses": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("addresses")) - it.Addresses, err = ec.unmarshalOAddress2ᚕstringᚄ(ctx, v) + data, err := ec.unmarshalOAddress2ᚕstringᚄ(ctx, v) if err != nil { return it, err } + it.Addresses = data case "topics": var err error ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("topics")) - it.Topics, err = ec.unmarshalOBytes322ᚕᚕstringᚄ(ctx, v) + data, err := ec.unmarshalOBytes322ᚕᚕstringᚄ(ctx, v) if err != nil { return it, err } + it.Topics = data } } @@ -8382,34 +8433,43 @@ var accessTupleImplementors = []string{"AccessTuple"} func (ec *executionContext) _AccessTuple(ctx context.Context, sel ast.SelectionSet, obj *model.AccessTuple) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, accessTupleImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("AccessTuple") case "address": - out.Values[i] = ec._AccessTuple_address(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "storageKeys": - out.Values[i] = ec._AccessTuple_storageKeys(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8417,55 +8477,58 @@ var accountImplementors = []string{"Account"} func (ec *executionContext) _Account(ctx context.Context, sel ast.SelectionSet, obj *model.Account) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, accountImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Account") case "address": - out.Values[i] = ec._Account_address(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "balance": - out.Values[i] = ec._Account_balance(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "transactionCount": - out.Values[i] = ec._Account_transactionCount(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "code": - out.Values[i] = ec._Account_code(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "storage": - out.Values[i] = ec._Account_storage(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8473,207 +8536,158 @@ var blockImplementors = []string{"Block"} func (ec *executionContext) _Block(ctx context.Context, sel ast.SelectionSet, obj *model.Block) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, blockImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Block") case "number": - out.Values[i] = ec._Block_number(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "hash": - out.Values[i] = ec._Block_hash(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "parent": - out.Values[i] = ec._Block_parent(ctx, field, obj) - case "nonce": - out.Values[i] = ec._Block_nonce(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "transactionsRoot": - out.Values[i] = ec._Block_transactionsRoot(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "transactionCount": - out.Values[i] = ec._Block_transactionCount(ctx, field, obj) - case "stateRoot": - out.Values[i] = ec._Block_stateRoot(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "receiptsRoot": - out.Values[i] = ec._Block_receiptsRoot(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "miner": - out.Values[i] = ec._Block_miner(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "extraData": - out.Values[i] = ec._Block_extraData(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "gasLimit": - out.Values[i] = ec._Block_gasLimit(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "gasUsed": - out.Values[i] = ec._Block_gasUsed(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "baseFeePerGas": - out.Values[i] = ec._Block_baseFeePerGas(ctx, field, obj) - case "nextBaseFeePerGas": - out.Values[i] = ec._Block_nextBaseFeePerGas(ctx, field, obj) - case "timestamp": - out.Values[i] = ec._Block_timestamp(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "logsBloom": - out.Values[i] = ec._Block_logsBloom(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "mixHash": - out.Values[i] = ec._Block_mixHash(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "difficulty": - out.Values[i] = ec._Block_difficulty(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "totalDifficulty": - out.Values[i] = ec._Block_totalDifficulty(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "ommerCount": - out.Values[i] = ec._Block_ommerCount(ctx, field, obj) - case "ommers": - out.Values[i] = ec._Block_ommers(ctx, field, obj) - case "ommerAt": - out.Values[i] = ec._Block_ommerAt(ctx, field, obj) - case "ommerHash": - out.Values[i] = ec._Block_ommerHash(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "transactions": - out.Values[i] = ec._Block_transactions(ctx, field, obj) - case "transactionAt": - out.Values[i] = ec._Block_transactionAt(ctx, field, obj) - case "logs": - out.Values[i] = ec._Block_logs(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "account": - out.Values[i] = ec._Block_account(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "call": - out.Values[i] = ec._Block_call(ctx, field, obj) - case "estimateGas": - out.Values[i] = ec._Block_estimateGas(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "rawHeader": - out.Values[i] = ec._Block_rawHeader(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "raw": - out.Values[i] = ec._Block_raw(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8681,41 +8695,48 @@ var callResultImplementors = []string{"CallResult"} func (ec *executionContext) _CallResult(ctx context.Context, sel ast.SelectionSet, obj *model.CallResult) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, callResultImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("CallResult") case "data": - out.Values[i] = ec._CallResult_data(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "gasUsed": - out.Values[i] = ec._CallResult_gasUsed(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "status": - out.Values[i] = ec._CallResult_status(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8723,55 +8744,58 @@ var logImplementors = []string{"Log"} func (ec *executionContext) _Log(ctx context.Context, sel ast.SelectionSet, obj *model.Log) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, logImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Log") case "index": - out.Values[i] = ec._Log_index(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "account": - out.Values[i] = ec._Log_account(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "topics": - out.Values[i] = ec._Log_topics(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "data": - out.Values[i] = ec._Log_data(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "transaction": - out.Values[i] = ec._Log_transaction(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8784,7 +8808,7 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) }) out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { innerCtx := graphql.WithRootFieldContext(ctx, &graphql.RootFieldContext{ Object: field.Name, @@ -8795,22 +8819,32 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) case "__typename": out.Values[i] = graphql.MarshalString("Mutation") case "sendRawTransaction": - out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { return ec._Mutation_sendRawTransaction(ctx, field) }) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8818,49 +8852,52 @@ var pendingImplementors = []string{"Pending"} func (ec *executionContext) _Pending(ctx context.Context, sel ast.SelectionSet, obj *model.Pending) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, pendingImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Pending") case "transactionCount": - out.Values[i] = ec._Pending_transactionCount(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "transactions": - out.Values[i] = ec._Pending_transactions(ctx, field, obj) - case "account": - out.Values[i] = ec._Pending_account(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "call": - out.Values[i] = ec._Pending_call(ctx, field, obj) - case "estimateGas": - out.Values[i] = ec._Pending_estimateGas(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -8873,7 +8910,7 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }) out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { innerCtx := graphql.WithRootFieldContext(ctx, &graphql.RootFieldContext{ Object: field.Name, @@ -8886,7 +8923,7 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr case "block": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -8897,16 +8934,15 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "blocks": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -8914,22 +8950,21 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }() res = ec._Query_blocks(ctx, field) if res == graphql.Null { - atomic.AddUint32(&invalids, 1) + atomic.AddUint32(&fs.Invalids, 1) } return res } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "pending": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -8937,22 +8972,21 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }() res = ec._Query_pending(ctx, field) if res == graphql.Null { - atomic.AddUint32(&invalids, 1) + atomic.AddUint32(&fs.Invalids, 1) } return res } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "transaction": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -8963,16 +8997,15 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "logs": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -8980,22 +9013,21 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }() res = ec._Query_logs(ctx, field) if res == graphql.Null { - atomic.AddUint32(&invalids, 1) + atomic.AddUint32(&fs.Invalids, 1) } return res } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "gasPrice": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -9003,22 +9035,21 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }() res = ec._Query_gasPrice(ctx, field) if res == graphql.Null { - atomic.AddUint32(&invalids, 1) + atomic.AddUint32(&fs.Invalids, 1) } return res } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "maxPriorityFeePerGas": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -9026,22 +9057,21 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }() res = ec._Query_maxPriorityFeePerGas(ctx, field) if res == graphql.Null { - atomic.AddUint32(&invalids, 1) + atomic.AddUint32(&fs.Invalids, 1) } return res } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "syncing": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -9052,16 +9082,15 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "chainID": field := field - innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { defer func() { if r := recover(); r != nil { ec.Error(ctx, ec.Recover(ctx, r)) @@ -9069,38 +9098,45 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr }() res = ec._Query_chainID(ctx, field) if res == graphql.Null { - atomic.AddUint32(&invalids, 1) + atomic.AddUint32(&fs.Invalids, 1) } return res } rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + return ec.OperationContext.RootResolverMiddleware(ctx, + func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func() graphql.Marshaler { - return rrm(innerCtx) - }) + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "__type": - out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { return ec._Query___type(ctx, field) }) - case "__schema": - out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) { return ec._Query___schema(ctx, field) }) - default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9108,41 +9144,48 @@ var syncStateImplementors = []string{"SyncState"} func (ec *executionContext) _SyncState(ctx context.Context, sel ast.SelectionSet, obj *model.SyncState) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, syncStateImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("SyncState") case "startingBlock": - out.Values[i] = ec._SyncState_startingBlock(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "currentBlock": - out.Values[i] = ec._SyncState_currentBlock(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "highestBlock": - out.Values[i] = ec._SyncState_highestBlock(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9150,160 +9193,121 @@ var transactionImplementors = []string{"Transaction"} func (ec *executionContext) _Transaction(ctx context.Context, sel ast.SelectionSet, obj *model.Transaction) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, transactionImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Transaction") case "hash": - out.Values[i] = ec._Transaction_hash(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "nonce": - out.Values[i] = ec._Transaction_nonce(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "index": - out.Values[i] = ec._Transaction_index(ctx, field, obj) - case "from": - out.Values[i] = ec._Transaction_from(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "to": - out.Values[i] = ec._Transaction_to(ctx, field, obj) - case "value": - out.Values[i] = ec._Transaction_value(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "gasPrice": - out.Values[i] = ec._Transaction_gasPrice(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "maxFeePerGas": - out.Values[i] = ec._Transaction_maxFeePerGas(ctx, field, obj) - case "maxPriorityFeePerGas": - out.Values[i] = ec._Transaction_maxPriorityFeePerGas(ctx, field, obj) - case "effectiveTip": - out.Values[i] = ec._Transaction_effectiveTip(ctx, field, obj) - case "gas": - out.Values[i] = ec._Transaction_gas(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "inputData": - out.Values[i] = ec._Transaction_inputData(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "block": - out.Values[i] = ec._Transaction_block(ctx, field, obj) - case "status": - out.Values[i] = ec._Transaction_status(ctx, field, obj) - case "gasUsed": - out.Values[i] = ec._Transaction_gasUsed(ctx, field, obj) - case "cumulativeGasUsed": - out.Values[i] = ec._Transaction_cumulativeGasUsed(ctx, field, obj) - case "effectiveGasPrice": - out.Values[i] = ec._Transaction_effectiveGasPrice(ctx, field, obj) - case "createdContract": - out.Values[i] = ec._Transaction_createdContract(ctx, field, obj) - case "logs": - out.Values[i] = ec._Transaction_logs(ctx, field, obj) - case "r": - out.Values[i] = ec._Transaction_r(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "s": - out.Values[i] = ec._Transaction_s(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "v": - out.Values[i] = ec._Transaction_v(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "type": - out.Values[i] = ec._Transaction_type(ctx, field, obj) - case "accessList": - out.Values[i] = ec._Transaction_accessList(ctx, field, obj) - case "raw": - out.Values[i] = ec._Transaction_raw(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "rawReceipt": - out.Values[i] = ec._Transaction_rawReceipt(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9311,52 +9315,55 @@ var __DirectiveImplementors = []string{"__Directive"} func (ec *executionContext) ___Directive(ctx context.Context, sel ast.SelectionSet, obj *introspection.Directive) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, __DirectiveImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("__Directive") case "name": - out.Values[i] = ec.___Directive_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "description": - out.Values[i] = ec.___Directive_description(ctx, field, obj) - case "locations": - out.Values[i] = ec.___Directive_locations(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "args": - out.Values[i] = ec.___Directive_args(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "isRepeatable": - out.Values[i] = ec.___Directive_isRepeatable(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9364,42 +9371,47 @@ var __EnumValueImplementors = []string{"__EnumValue"} func (ec *executionContext) ___EnumValue(ctx context.Context, sel ast.SelectionSet, obj *introspection.EnumValue) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, __EnumValueImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("__EnumValue") case "name": - out.Values[i] = ec.___EnumValue_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "description": - out.Values[i] = ec.___EnumValue_description(ctx, field, obj) - case "isDeprecated": - out.Values[i] = ec.___EnumValue_isDeprecated(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "deprecationReason": - out.Values[i] = ec.___EnumValue_deprecationReason(ctx, field, obj) - default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9407,56 +9419,57 @@ var __FieldImplementors = []string{"__Field"} func (ec *executionContext) ___Field(ctx context.Context, sel ast.SelectionSet, obj *introspection.Field) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, __FieldImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("__Field") case "name": - out.Values[i] = ec.___Field_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "description": - out.Values[i] = ec.___Field_description(ctx, field, obj) - case "args": - out.Values[i] = ec.___Field_args(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "type": - out.Values[i] = ec.___Field_type(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "isDeprecated": - out.Values[i] = ec.___Field_isDeprecated(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "deprecationReason": - out.Values[i] = ec.___Field_deprecationReason(ctx, field, obj) - default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9464,42 +9477,47 @@ var __InputValueImplementors = []string{"__InputValue"} func (ec *executionContext) ___InputValue(ctx context.Context, sel ast.SelectionSet, obj *introspection.InputValue) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, __InputValueImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("__InputValue") case "name": - out.Values[i] = ec.___InputValue_name(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "description": - out.Values[i] = ec.___InputValue_description(ctx, field, obj) - case "type": - out.Values[i] = ec.___InputValue_type(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "defaultValue": - out.Values[i] = ec.___InputValue_defaultValue(ctx, field, obj) - default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9507,53 +9525,54 @@ var __SchemaImplementors = []string{"__Schema"} func (ec *executionContext) ___Schema(ctx context.Context, sel ast.SelectionSet, obj *introspection.Schema) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, __SchemaImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("__Schema") case "description": - out.Values[i] = ec.___Schema_description(ctx, field, obj) - case "types": - out.Values[i] = ec.___Schema_types(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "queryType": - out.Values[i] = ec.___Schema_queryType(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "mutationType": - out.Values[i] = ec.___Schema_mutationType(ctx, field, obj) - case "subscriptionType": - out.Values[i] = ec.___Schema_subscriptionType(ctx, field, obj) - case "directives": - out.Values[i] = ec.___Schema_directives(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } @@ -9561,63 +9580,56 @@ var __TypeImplementors = []string{"__Type"} func (ec *executionContext) ___Type(ctx context.Context, sel ast.SelectionSet, obj *introspection.Type) graphql.Marshaler { fields := graphql.CollectFields(ec.OperationContext, sel, __TypeImplementors) + out := graphql.NewFieldSet(fields) - var invalids uint32 + deferred := make(map[string]*graphql.FieldSet) for i, field := range fields { switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("__Type") case "kind": - out.Values[i] = ec.___Type_kind(ctx, field, obj) - if out.Values[i] == graphql.Null { - invalids++ + out.Invalids++ } case "name": - out.Values[i] = ec.___Type_name(ctx, field, obj) - case "description": - out.Values[i] = ec.___Type_description(ctx, field, obj) - case "fields": - out.Values[i] = ec.___Type_fields(ctx, field, obj) - case "interfaces": - out.Values[i] = ec.___Type_interfaces(ctx, field, obj) - case "possibleTypes": - out.Values[i] = ec.___Type_possibleTypes(ctx, field, obj) - case "enumValues": - out.Values[i] = ec.___Type_enumValues(ctx, field, obj) - case "inputFields": - out.Values[i] = ec.___Type_inputFields(ctx, field, obj) - case "ofType": - out.Values[i] = ec.___Type_ofType(ctx, field, obj) - case "specifiedByURL": - out.Values[i] = ec.___Type_specifiedByURL(ctx, field, obj) - default: panic("unknown field " + strconv.Quote(field.Name)) } } - out.Dispatch() - if invalids > 0 { + out.Dispatch(ctx) + if out.Invalids > 0 { return graphql.Null } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + return out } diff --git a/cmd/rpcdaemon/graphql/graph/model/models_gen.go b/cmd/rpcdaemon/graphql/graph/model/models_gen.go index ae67ddff784..67bfe18e0de 100644 --- a/cmd/rpcdaemon/graphql/graph/model/models_gen.go +++ b/cmd/rpcdaemon/graphql/graph/model/models_gen.go @@ -18,51 +18,51 @@ type Account struct { type Block struct { Number uint64 `json:"number"` Hash string `json:"hash"` - Parent *Block `json:"parent"` + Parent *Block `json:"parent,omitempty"` Nonce string `json:"nonce"` TransactionsRoot string `json:"transactionsRoot"` - TransactionCount *int `json:"transactionCount"` + TransactionCount *int `json:"transactionCount,omitempty"` StateRoot string `json:"stateRoot"` ReceiptsRoot string `json:"receiptsRoot"` Miner *Account `json:"miner"` ExtraData string `json:"extraData"` GasLimit uint64 `json:"gasLimit"` GasUsed uint64 `json:"gasUsed"` - BaseFeePerGas *string `json:"baseFeePerGas"` - NextBaseFeePerGas *string `json:"nextBaseFeePerGas"` + BaseFeePerGas *string `json:"baseFeePerGas,omitempty"` + NextBaseFeePerGas *string `json:"nextBaseFeePerGas,omitempty"` Timestamp string `json:"timestamp"` LogsBloom string `json:"logsBloom"` MixHash string `json:"mixHash"` Difficulty string `json:"difficulty"` TotalDifficulty string `json:"totalDifficulty"` - OmmerCount *int `json:"ommerCount"` - Ommers []*Block `json:"ommers"` - OmmerAt *Block `json:"ommerAt"` + OmmerCount *int `json:"ommerCount,omitempty"` + Ommers []*Block `json:"ommers,omitempty"` + OmmerAt *Block `json:"ommerAt,omitempty"` OmmerHash string `json:"ommerHash"` - Transactions []*Transaction `json:"transactions"` - TransactionAt *Transaction `json:"transactionAt"` + Transactions []*Transaction `json:"transactions,omitempty"` + TransactionAt *Transaction `json:"transactionAt,omitempty"` Logs []*Log `json:"logs"` Account *Account `json:"account"` - Call *CallResult `json:"call"` + Call *CallResult `json:"call,omitempty"` EstimateGas uint64 `json:"estimateGas"` RawHeader string `json:"rawHeader"` Raw string `json:"raw"` } type BlockFilterCriteria struct { - Addresses []string `json:"addresses"` - Topics [][]string `json:"topics"` + Addresses []string `json:"addresses,omitempty"` + Topics [][]string `json:"topics,omitempty"` } type CallData struct { - From *string `json:"from"` - To *string `json:"to"` - Gas *uint64 `json:"gas"` - GasPrice *string `json:"gasPrice"` - MaxFeePerGas *string `json:"maxFeePerGas"` - MaxPriorityFeePerGas *string `json:"maxPriorityFeePerGas"` - Value *string `json:"value"` - Data *string `json:"data"` + From *string `json:"from,omitempty"` + To *string `json:"to,omitempty"` + Gas *uint64 `json:"gas,omitempty"` + GasPrice *string `json:"gasPrice,omitempty"` + MaxFeePerGas *string `json:"maxFeePerGas,omitempty"` + MaxPriorityFeePerGas *string `json:"maxPriorityFeePerGas,omitempty"` + Value *string `json:"value,omitempty"` + Data *string `json:"data,omitempty"` } type CallResult struct { @@ -72,10 +72,10 @@ type CallResult struct { } type FilterCriteria struct { - FromBlock *uint64 `json:"fromBlock"` - ToBlock *uint64 `json:"toBlock"` - Addresses []string `json:"addresses"` - Topics [][]string `json:"topics"` + FromBlock *uint64 `json:"fromBlock,omitempty"` + ToBlock *uint64 `json:"toBlock,omitempty"` + Addresses []string `json:"addresses,omitempty"` + Topics [][]string `json:"topics,omitempty"` } type Log struct { @@ -88,9 +88,9 @@ type Log struct { type Pending struct { TransactionCount int `json:"transactionCount"` - Transactions []*Transaction `json:"transactions"` + Transactions []*Transaction `json:"transactions,omitempty"` Account *Account `json:"account"` - Call *CallResult `json:"call"` + Call *CallResult `json:"call,omitempty"` EstimateGas uint64 `json:"estimateGas"` } @@ -103,28 +103,28 @@ type SyncState struct { type Transaction struct { Hash string `json:"hash"` Nonce string `json:"nonce"` - Index *int `json:"index"` + Index *int `json:"index,omitempty"` From *Account `json:"from"` - To *Account `json:"to"` + To *Account `json:"to,omitempty"` Value string `json:"value"` GasPrice string `json:"gasPrice"` - MaxFeePerGas *string `json:"maxFeePerGas"` - MaxPriorityFeePerGas *string `json:"maxPriorityFeePerGas"` - EffectiveTip *string `json:"effectiveTip"` + MaxFeePerGas *string `json:"maxFeePerGas,omitempty"` + MaxPriorityFeePerGas *string `json:"maxPriorityFeePerGas,omitempty"` + EffectiveTip *string `json:"effectiveTip,omitempty"` Gas uint64 `json:"gas"` InputData string `json:"inputData"` - Block *Block `json:"block"` - Status *uint64 `json:"status"` - GasUsed *uint64 `json:"gasUsed"` - CumulativeGasUsed *uint64 `json:"cumulativeGasUsed"` - EffectiveGasPrice *string `json:"effectiveGasPrice"` - CreatedContract *Account `json:"createdContract"` - Logs []*Log `json:"logs"` + Block *Block `json:"block,omitempty"` + Status *uint64 `json:"status,omitempty"` + GasUsed *uint64 `json:"gasUsed,omitempty"` + CumulativeGasUsed *uint64 `json:"cumulativeGasUsed,omitempty"` + EffectiveGasPrice *string `json:"effectiveGasPrice,omitempty"` + CreatedContract *Account `json:"createdContract,omitempty"` + Logs []*Log `json:"logs,omitempty"` R string `json:"r"` S string `json:"s"` V string `json:"v"` - Type *int `json:"type"` - AccessList []*AccessTuple `json:"accessList"` + Type *int `json:"type,omitempty"` + AccessList []*AccessTuple `json:"accessList,omitempty"` Raw string `json:"raw"` RawReceipt string `json:"rawReceipt"` } diff --git a/cmd/rpcdaemon/graphql/graph/schema.resolvers.go b/cmd/rpcdaemon/graphql/graph/schema.resolvers.go index 82d7d3e2ad9..14947154688 100644 --- a/cmd/rpcdaemon/graphql/graph/schema.resolvers.go +++ b/cmd/rpcdaemon/graphql/graph/schema.resolvers.go @@ -2,7 +2,7 @@ package graph // This file will be automatically regenerated based on the schema, any resolver implementations // will be copied through when generating and any unknown code will be moved to the end. -// Code generated by github.com/99designs/gqlgen version v0.17.26 +// Code generated by github.com/99designs/gqlgen version v0.17.33 import ( "context" diff --git a/cmd/rpcdaemon/graphql/graphql.go b/cmd/rpcdaemon/graphql/graphql.go index 9dc18b93151..748cc51eaca 100644 --- a/cmd/rpcdaemon/graphql/graphql.go +++ b/cmd/rpcdaemon/graphql/graphql.go @@ -6,6 +6,7 @@ import ( "github.com/99designs/gqlgen/graphql/handler" "github.com/99designs/gqlgen/graphql/playground" + "github.com/ledgerwatch/erigon/cmd/rpcdaemon/commands" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/graphql/graph" "github.com/ledgerwatch/erigon/rpc" diff --git a/cmd/rpcdaemon/rpcdaemontest/test_util.go b/cmd/rpcdaemon/rpcdaemontest/test_util.go index 0ccd61d185d..f2d7c1e119b 100644 --- a/cmd/rpcdaemon/rpcdaemontest/test_util.go +++ b/cmd/rpcdaemon/rpcdaemontest/test_util.go @@ -101,10 +101,10 @@ func CreateTestSentry(t *testing.T) (*stages.MockSentry, *core.ChainPack, []*cor t.Fatal(err) } - if err = m.InsertChain(orphanedChain); err != nil { + if err = m.InsertChain(orphanedChain, nil); err != nil { t.Fatal(err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatal(err) } @@ -294,9 +294,8 @@ func CreateTestGrpcConn(t *testing.T, m *stages.MockSentry) (context.Context, *g ethashApi := apis[1].Service.(*ethash.API) server := grpc.NewServer() - br, _ := m.NewBlocksIO() remote.RegisterETHBACKENDServer(server, privateapi.NewEthBackendServer(ctx, nil, m.DB, m.Notifications.Events, - br, nil, nil, nil, false, log.New())) + m.BlockReader, nil, nil, nil, false, log.New())) txpool.RegisterTxpoolServer(server, m.TxPoolGrpcServer) txpool.RegisterMiningServer(server, privateapi.NewMiningServer(ctx, &IsMiningMock{}, ethashApi, m.Log)) listener := bufconn.Listen(1024 * 1024) @@ -428,7 +427,7 @@ func CreateTestSentryForTraces(t *testing.T) *stages.MockSentry { t.Fatalf("generate blocks: %v", err) } - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } return m @@ -534,7 +533,7 @@ func CreateTestSentryForTracesCollision(t *testing.T) *stages.MockSentry { t.Fatalf("generate blocks: %v", err) } // Import the canonical chain - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } diff --git a/cmd/rpcdaemon/rpcservices/eth_backend.go b/cmd/rpcdaemon/rpcservices/eth_backend.go index e3cbd935cb7..70151649a11 100644 --- a/cmd/rpcdaemon/rpcservices/eth_backend.go +++ b/cmd/rpcdaemon/rpcservices/eth_backend.go @@ -14,6 +14,7 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces/remote" types2 "github.com/ledgerwatch/erigon-lib/gointerfaces/types" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/log/v3" "google.golang.org/grpc" "google.golang.org/grpc/status" @@ -45,6 +46,12 @@ func NewRemoteBackend(client remote.ETHBACKENDClient, db kv.RoDB, blockReader se } } +func (back *RemoteBackend) CanPruneTo(currentBlockInDB uint64) (canPruneBlocksTo uint64) { + return back.blockReader.CanPruneTo(currentBlockInDB) +} +func (back *RemoteBackend) HeadersRange(ctx context.Context, walker func(header *types.Header) error) error { + panic("not implemented") +} func (back *RemoteBackend) CurrentBlock(db kv.Tx) (*types.Block, error) { panic("not implemented") } @@ -73,11 +80,12 @@ func (back *RemoteBackend) BlockByHash(ctx context.Context, db kv.Tx, hash commo block, _, err := back.BlockWithSenders(ctx, db, hash, *number) return block, err } -func (back *RemoteBackend) TxsV3Enabled() bool { - panic("not implemented") -} -func (back *RemoteBackend) Snapshots() services.BlockSnapshots { - panic("not implemented") +func (back *RemoteBackend) TxsV3Enabled() bool { panic("not implemented") } +func (back *RemoteBackend) Snapshots() services.BlockSnapshots { panic("not implemented") } +func (back *RemoteBackend) FrozenBlocks() uint64 { return back.blockReader.FrozenBlocks() } +func (back *RemoteBackend) FrozenFiles() (list []string) { return back.blockReader.FrozenFiles() } +func (back *RemoteBackend) FreezingCfg() ethconfig.BlocksFreezing { + return back.blockReader.FreezingCfg() } func (back *RemoteBackend) EnsureVersionCompatibility() bool { versionReply, err := back.remoteEthBackend.Version(context.Background(), &emptypb.Empty{}, grpc.WaitForReady(true)) diff --git a/cmd/rpctest/rpctest/account_range_verify.go b/cmd/rpctest/rpctest/account_range_verify.go index b167b538430..dcb89c619ae 100644 --- a/cmd/rpctest/rpctest/account_range_verify.go +++ b/cmd/rpctest/rpctest/account_range_verify.go @@ -86,7 +86,7 @@ func CompareAccountRange(logger log.Logger, erigonURL, gethURL, tmpDataDir, geth if innerErr != nil { return innerErr } - err = db.Put(kv.AccountsHistory, addr.Bytes(), b) + err = db.Put(kv.E2AccountsHistory, addr.Bytes(), b) if err != nil { return err } @@ -127,13 +127,13 @@ func CompareAccountRange(logger log.Logger, erigonURL, gethURL, tmpDataDir, geth log.Error(err.Error()) return } - tgCursor, err := tgTx.Cursor(kv.AccountsHistory) + tgCursor, err := tgTx.Cursor(kv.E2AccountsHistory) if err != nil { log.Error(err.Error()) return } defer tgCursor.Close() - gethCursor, err := gethTx.Cursor(kv.AccountsHistory) + gethCursor, err := gethTx.Cursor(kv.E2AccountsHistory) if err != nil { log.Error(err.Error()) return diff --git a/cmd/sentinel/cli/cliSettings.go b/cmd/sentinel/cli/cliSettings.go index 5763008f30b..4aeff652ed8 100644 --- a/cmd/sentinel/cli/cliSettings.go +++ b/cmd/sentinel/cli/cliSettings.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "time" "github.com/ledgerwatch/erigon/cl/phase1/core/rawdb" "github.com/ledgerwatch/erigon/cl/phase1/core/state" @@ -17,27 +18,30 @@ import ( ) type ConsensusClientCliCfg struct { - GenesisCfg *clparams.GenesisConfig `json:"genesisCfg"` - BeaconCfg *clparams.BeaconChainConfig `json:"beaconCfg"` - NetworkCfg *clparams.NetworkConfig `json:"networkCfg"` - BeaconDataCfg *rawdb.BeaconDataConfig `json:"beaconDataConfig"` - Port uint `json:"port"` - Addr string `json:"address"` - ServerAddr string `json:"serverAddr"` - ServerProtocol string `json:"serverProtocol"` - ServerTcpPort uint `json:"serverTcpPort"` - LogLvl uint `json:"logLevel"` - NoDiscovery bool `json:"noDiscovery"` - CheckpointUri string `json:"checkpointUri"` - Chaindata string `json:"chaindata"` - ErigonPrivateApi string `json:"erigonPrivateApi"` - TransitionChain bool `json:"transitionChain"` - NetworkType clparams.NetworkType `json:"networkType"` - InitialSync bool `json:"initialSync"` - BeaconAddr string `json:"beaconAddr"` - BeaconProtocol string `json:"beaconProtocol"` - RecordMode bool `json:"recordMode"` - RecordDir string `json:"recordDir"` + GenesisCfg *clparams.GenesisConfig `json:"genesisCfg"` + BeaconCfg *clparams.BeaconChainConfig `json:"beaconCfg"` + NetworkCfg *clparams.NetworkConfig `json:"networkCfg"` + BeaconDataCfg *rawdb.BeaconDataConfig `json:"beaconDataConfig"` + Port uint `json:"port"` + Addr string `json:"address"` + ServerAddr string `json:"serverAddr"` + ServerProtocol string `json:"serverProtocol"` + ServerTcpPort uint `json:"serverTcpPort"` + LogLvl uint `json:"logLevel"` + NoDiscovery bool `json:"noDiscovery"` + CheckpointUri string `json:"checkpointUri"` + Chaindata string `json:"chaindata"` + ErigonPrivateApi string `json:"erigonPrivateApi"` + TransitionChain bool `json:"transitionChain"` + NetworkType clparams.NetworkType `json:"networkType"` + InitialSync bool `json:"initialSync"` + NoBeaconApi bool `json:"noBeaconApi"` + BeaconApiReadTimeout time.Duration `json:"beaconApiReadTimeout"` + BeaconApiWriteTimeout time.Duration `json:"beaconApiWriteTimeout"` + BeaconAddr string `json:"beaconAddr"` + BeaconProtocol string `json:"beaconProtocol"` + RecordMode bool `json:"recordMode"` + RecordDir string `json:"recordDir"` InitalState *state.BeaconState } @@ -75,6 +79,9 @@ func SetupConsensusClientCfg(ctx *cli.Context) (*ConsensusClientCliCfg, error) { cfg.ServerAddr = fmt.Sprintf("%s:%d", ctx.String(flags.SentinelServerAddr.Name), ctx.Int(flags.SentinelServerPort.Name)) cfg.ServerProtocol = "tcp" + cfg.NoBeaconApi = ctx.Bool(flags.NoBeaconApi.Name) + cfg.BeaconApiReadTimeout = time.Duration(ctx.Uint64(flags.BeaconApiReadTimeout.Name)) * time.Second + cfg.BeaconApiWriteTimeout = time.Duration(ctx.Uint(flags.BeaconApiWriteTimeout.Name)) * time.Second cfg.BeaconAddr = fmt.Sprintf("%s:%d", ctx.String(flags.BeaconApiAddr.Name), ctx.Int(flags.BeaconApiPort.Name)) cfg.BeaconProtocol = "tcp" cfg.RecordMode = ctx.Bool(flags.RecordModeFlag.Name) diff --git a/cmd/sentinel/cli/flags/defaultFlags.go b/cmd/sentinel/cli/flags/defaultFlags.go index 47bd69075e3..d9e8831b30b 100644 --- a/cmd/sentinel/cli/flags/defaultFlags.go +++ b/cmd/sentinel/cli/flags/defaultFlags.go @@ -7,6 +7,9 @@ var CLDefaultFlags = []cli.Flag{ &SentinelDiscoveryAddr, &SentinelServerPort, &SentinelServerAddr, + &NoBeaconApi, + &BeaconApiReadTimeout, + &BeaconApiWriteTimeout, &BeaconApiPort, &BeaconApiAddr, &Chain, diff --git a/cmd/sentinel/cli/flags/flags.go b/cmd/sentinel/cli/flags/flags.go index d8ac288f142..a71663d61f2 100644 --- a/cmd/sentinel/cli/flags/flags.go +++ b/cmd/sentinel/cli/flags/flags.go @@ -1,6 +1,8 @@ package flags -import "github.com/urfave/cli/v2" +import ( + "github.com/urfave/cli/v2" +) var ( SentinelDiscoveryPort = cli.IntFlag{ @@ -28,6 +30,21 @@ var ( Usage: "sets the lightclient server host addr", Value: "localhost", } + NoBeaconApi = cli.BoolFlag{ + Name: "no-beacon-api", + Usage: "turn off the beacon api", + Value: false, + } + BeaconApiReadTimeout = cli.Uint64Flag{ + Name: "beacon.api.read.timeout", + Usage: "Sets the seconds for a read time out in the beacon api", + Value: 5, + } + BeaconApiWriteTimeout = cli.Uint64Flag{ + Name: "beacon.api.write.timeout", + Usage: "Sets the seconds for a write time out in the beacon api", + Value: 5, + } BeaconApiAddr = cli.StringFlag{ Name: "beacon.api.addr", Usage: "sets the host to listen for beacon api requests", diff --git a/cmd/sentinel/sentinel/service/service.go b/cmd/sentinel/sentinel/service/service.go index 0e997c4f28e..1550730cc8f 100644 --- a/cmd/sentinel/sentinel/service/service.go +++ b/cmd/sentinel/sentinel/service/service.go @@ -253,35 +253,6 @@ func (s *SentinelServer) ListenToGossip() { } } -var sentinelLoopInterval = 4 * time.Hour - -func (s *SentinelServer) startServerBackgroundLoop() { - var err error - ticker := time.NewTicker(sentinelLoopInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - s.mu.Lock() - peers := s.sentinel.PeersList() - s.sentinel.Stop() - status := s.sentinel.Status() - s.sentinel, err = createSentinel(s.sentinel.Config(), s.sentinel.DB(), s.logger) - if err != nil { - log.Warn("Could not coordinate sentinel", "err", err) - continue - } - s.sentinel.SetStatus(status) - for _, peer := range peers { - s.sentinel.ConnectWithPeer(s.ctx, peer, true) - } - s.mu.Unlock() - case <-s.ctx.Done(): - return - } - } -} - func (s *SentinelServer) handleGossipPacket(pkt *pubsub.Message) error { var err error s.logger.Trace("[Sentinel Gossip] Received Packet", "topic", pkt.Topic) diff --git a/cmd/sentinel/sentinel/service/start.go b/cmd/sentinel/sentinel/service/start.go index 0a73bafd3eb..dcfcfeb153e 100644 --- a/cmd/sentinel/sentinel/service/start.go +++ b/cmd/sentinel/sentinel/service/start.go @@ -111,7 +111,6 @@ func StartServe(server *SentinelServer, srvCfg *ServerConfig, creds credentials. // Create a gRPC server gRPCserver := grpc.NewServer(grpc.Creds(creds)) go server.ListenToGossip() - go server.startServerBackgroundLoop() // Regiser our server as a gRPC server sentinelrpc.RegisterSentinelServer(gRPCserver, server) if err := gRPCserver.Serve(lis); err != nil { diff --git a/cmd/sentry/sentry/sentry_grpc_server_test.go b/cmd/sentry/sentry/sentry_grpc_server_test.go index f9e1639da08..9940a9361a2 100644 --- a/cmd/sentry/sentry/sentry_grpc_server_test.go +++ b/cmd/sentry/sentry/sentry_grpc_server_test.go @@ -7,19 +7,19 @@ import ( "time" "github.com/holiman/uint256" - "github.com/stretchr/testify/require" - "github.com/ledgerwatch/erigon-lib/chain" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/direct" "github.com/ledgerwatch/erigon-lib/gointerfaces" proto_sentry "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/stretchr/testify/require" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/forkid" "github.com/ledgerwatch/erigon/core/rawdb" + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/p2p" ) @@ -69,8 +69,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { SpuriousDragonBlock: big.NewInt(2), ByzantiumBlock: big.NewInt(3), } - dbNoFork = memdb.NewTestDB(t) - dbProFork = memdb.NewTestDB(t) + _, dbNoFork, _ = temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, dbProFork, _ = temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) gspecNoFork = &types.Genesis{Config: configNoFork} gspecProFork = &types.Genesis{Config: configProFork} @@ -162,7 +162,7 @@ func TestSentryServerImpl_SetStatusInitPanic(t *testing.T) { }() configNoFork := &chain.Config{HomesteadBlock: big.NewInt(1), ChainID: big.NewInt(1)} - dbNoFork := memdb.NewTestDB(t) + _, dbNoFork, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) gspecNoFork := &types.Genesis{Config: configNoFork} genesisNoFork := core.MustCommitGenesis(gspecNoFork, dbNoFork, "") ss := &GrpcServer{p2p: &p2p.Config{}} diff --git a/cmd/state/commands/check_change_sets.go b/cmd/state/commands/check_change_sets.go index d8016386bf5..8410a27f760 100644 --- a/cmd/state/commands/check_change_sets.go +++ b/cmd/state/commands/check_change_sets.go @@ -16,6 +16,7 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" @@ -26,7 +27,6 @@ import ( "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/turbo/debug" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" ) var ( @@ -78,12 +78,12 @@ func CheckChangeSets(genesis *types.Genesis, blockNum uint64, chaindata string, if err != nil { return err } - allSnapshots := snapshotsync.NewRoSnapshots(ethconfig.NewSnapCfg(true, false, true), path.Join(datadirCli, "snapshots"), logger) + allSnapshots := freezeblocks.NewRoSnapshots(ethconfig.NewSnapCfg(true, false, true), path.Join(datadirCli, "snapshots"), logger) defer allSnapshots.Close() if err := allSnapshots.ReopenFolder(); err != nil { return fmt.Errorf("reopen snapshot segments: %w", err) } - blockReader := snapshotsync.NewBlockReader(allSnapshots) + blockReader := freezeblocks.NewBlockReader(allSnapshots) chainDb := db defer chainDb.Close() diff --git a/cmd/state/commands/erigon4.go b/cmd/state/commands/erigon4.go index 90e3175bebe..30481b9a3aa 100644 --- a/cmd/state/commands/erigon4.go +++ b/cmd/state/commands/erigon4.go @@ -15,6 +15,7 @@ import ( "github.com/VictoriaMetrics/metrics" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" @@ -42,7 +43,6 @@ import ( "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" ) var ( @@ -236,12 +236,12 @@ func Erigon4(genesis *types.Genesis, chainConfig *chain2.Config, logger log.Logg }() var blockReader services.FullBlockReader - var allSnapshots = snapshotsync.NewRoSnapshots(ethconfig.NewSnapCfg(true, false, true), path.Join(datadirCli, "snapshots"), logger) + var allSnapshots = freezeblocks.NewRoSnapshots(ethconfig.NewSnapCfg(true, false, true), path.Join(datadirCli, "snapshots"), logger) defer allSnapshots.Close() if err := allSnapshots.ReopenFolder(); err != nil { return fmt.Errorf("reopen snapshot segments: %w", err) } - blockReader = snapshotsync.NewBlockReader(allSnapshots) + blockReader = freezeblocks.NewBlockReader(allSnapshots) engine := initConsensusEngine(chainConfig, allSnapshots, logger) getHeader := func(hash libcommon.Hash, number uint64) *types.Header { @@ -603,7 +603,7 @@ func (ww *StateWriterV4) CreateContract(address libcommon.Address) error { return nil } -func initConsensusEngine(cc *chain2.Config, snapshots *snapshotsync.RoSnapshots, logger log.Logger) (engine consensus.Engine) { +func initConsensusEngine(cc *chain2.Config, snapshots *freezeblocks.RoSnapshots, logger log.Logger) (engine consensus.Engine) { config := ethconfig.Defaults var consensusConfig interface{} diff --git a/cmd/state/commands/global_flags_vars.go b/cmd/state/commands/global_flags_vars.go index 2df1b0d8803..dd81e19aee6 100644 --- a/cmd/state/commands/global_flags_vars.go +++ b/cmd/state/commands/global_flags_vars.go @@ -49,7 +49,7 @@ func withCSBucket(cmd *cobra.Command) { } func withIndexBucket(cmd *cobra.Command) { - cmd.Flags().StringVar(&indexBucket, "index-bucket", kv.AccountsHistory, kv.AccountsHistory+" for account and "+kv.StorageHistory+" for storage") + cmd.Flags().StringVar(&indexBucket, "index-bucket", kv.E2AccountsHistory, kv.E2AccountsHistory+" for account and "+kv.E2StorageHistory+" for storage") } func withSnapshotBlocks(cmd *cobra.Command) { diff --git a/cmd/state/commands/opcode_tracer.go b/cmd/state/commands/opcode_tracer.go index fc806d78dc5..7644eda0009 100644 --- a/cmd/state/commands/opcode_tracer.go +++ b/cmd/state/commands/opcode_tracer.go @@ -21,7 +21,7 @@ import ( "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/rpchelper" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" @@ -430,7 +430,7 @@ func OpcodeTracer(genesis *types.Genesis, blockNum uint64, chaindata string, num } return nil }) - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) chainConfig := genesis.Config vmConfig := vm.Config{Tracer: ot, Debug: true} diff --git a/cmd/state/commands/state_root.go b/cmd/state/commands/state_root.go index f4f51428fcf..1f6773ab1ac 100644 --- a/cmd/state/commands/state_root.go +++ b/cmd/state/commands/state_root.go @@ -19,7 +19,7 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" @@ -60,7 +60,7 @@ func blocksIO(db kv.RoDB) (services.FullBlockReader, *blockio.BlockWriter) { }); err != nil { panic(err) } - br := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) + br := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) bw := blockio.NewBlockWriter(histV3) return br, bw } diff --git a/cmd/state/exec22/txtask.go b/cmd/state/exec22/txtask.go index fa99c574801..9225d260fd9 100644 --- a/cmd/state/exec22/txtask.go +++ b/cmd/state/exec22/txtask.go @@ -7,9 +7,10 @@ import ( "time" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/state" + "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/core/vm/evmtypes" diff --git a/cmd/state/stats/index_stats.go b/cmd/state/stats/index_stats.go index 72e2b6194a4..0b275436c3c 100644 --- a/cmd/state/stats/index_stats.go +++ b/cmd/state/stats/index_stats.go @@ -24,7 +24,7 @@ func IndexStats(chaindata string, indexBucket string, statsFile string) error { db := mdbx.MustOpen(chaindata) startTime := time.Now() lenOfKey := length.Addr - if strings.HasPrefix(indexBucket, kv.StorageHistory) { + if strings.HasPrefix(indexBucket, kv.E2StorageHistory) { lenOfKey = length.Addr + length.Hash + length.Incarnation } diff --git a/cmd/state/verify/verify_txlookup.go b/cmd/state/verify/verify_txlookup.go index 9dd5a76079b..d4a9e2415e3 100644 --- a/cmd/state/verify/verify_txlookup.go +++ b/cmd/state/verify/verify_txlookup.go @@ -16,7 +16,7 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" ) @@ -28,7 +28,7 @@ func blocksIO(db kv.RoDB) (services.FullBlockReader, *blockio.BlockWriter) { }); err != nil { panic(err) } - br := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) + br := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) bw := blockio.NewBlockWriter(histV3) return br, bw } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 12b6c6ae99d..9d5d5a93068 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -339,12 +339,12 @@ var ( } HTTPVirtualHostsFlag = cli.StringFlag{ Name: "http.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts 'any' or '*' as wildcard.", Value: strings.Join(nodecfg.DefaultConfig.HTTPVirtualHosts, ","), } AuthRpcVirtualHostsFlag = cli.StringFlag{ Name: "authrpc.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept Engine API requests (server enforced). Accepts '*' wildcard.", + Usage: "Comma separated list of virtual hostnames from which to accept Engine API requests (server enforced). Accepts 'any' or '*' as wildcard.", Value: strings.Join(nodecfg.DefaultConfig.HTTPVirtualHosts, ","), } HTTPApiFlag = cli.StringFlag{ diff --git a/consensus/aura/aura_test.go b/consensus/aura/aura_test.go index d3c9da5c6c9..1862ac9360b 100644 --- a/consensus/aura/aura_test.go +++ b/consensus/aura/aura_test.go @@ -53,6 +53,6 @@ func TestEmptyBlock(t *testing.T) { blocks[0] = block chain := &core.ChainPack{Headers: headers, Blocks: blocks, Receipts: receipts, TopBlock: block} - err = m.InsertChain(chain) + err = m.InsertChain(chain, nil) require.NoError(err) } diff --git a/consensus/bor/api.go b/consensus/bor/api.go index ca78de76494..bb8c933c12a 100644 --- a/consensus/bor/api.go +++ b/consensus/bor/api.go @@ -8,11 +8,12 @@ import ( "strconv" "sync" - "github.com/hashicorp/golang-lru/v2" - "github.com/ledgerwatch/erigon-lib/common" "github.com/xsleonard/go-merkle" "golang.org/x/crypto/sha3" + lru "github.com/hashicorp/golang-lru/arc/v2" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/consensus" "github.com/ledgerwatch/erigon/consensus/bor/valset" "github.com/ledgerwatch/erigon/core/types" diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index b8c4e01b319..27e64d7494e 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -16,7 +16,7 @@ import ( "time" "github.com/google/btree" - lru "github.com/hashicorp/golang-lru/v2" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon-lib/chain" @@ -41,9 +41,11 @@ import ( ) const ( - checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database - inmemorySnapshots = 128 // Number of recent vote snapshots to keep in memory - inmemorySignatures = 4096 // Number of recent block signatures to keep in memory + spanLength = 6400 // Number of blocks in a span + zerothSpanEnd = 255 // End block of 0th span + snapshotPersistInterval = 1024 // Number of blocks after which to persist the vote snapshot to the database + inmemorySnapshots = 128 // Number of recent vote snapshots to keep in memory + inmemorySignatures = 4096 // Number of recent block signatures to keep in memory ) // Bor protocol constants. @@ -481,7 +483,14 @@ func (c *Bor) verifyCascadingFields(chain consensus.ChainHeaderReader, header *t } // Verify the validator list match the local contract - if isSprintStart(number+1, c.config.CalculateSprint(number)) { + // + // Note: Here we fetch the data from span instead of contract + // as done in bor client. The contract (validator set) returns + // a fixed span for 0th span i.e. 0 - 255 blocks. Hence, the + // contract data and span data won't match for that. Skip validating + // for 0th span. TODO: Remove `number > zerothSpanEnd` check + // once we start fetching validator data from contract. + if number > zerothSpanEnd && isSprintStart(number+1, c.config.CalculateSprint(number)) { producerSet, err := c.spanner.GetCurrentProducers(number+1, c.authorizedSigner.Load().signer, c.getSpanForBlock) if err != nil { @@ -544,8 +553,8 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash li break } - // If an on-disk checkpoint snapshot can be found, use that - if number%checkpointInterval == 0 { + // If an on-disk snapshot can be found, use that + if number%snapshotPersistInterval == 0 { if s, err := loadSnapshot(c.config, c.signatures, c.DB, hash); err == nil { c.logger.Trace("Loaded snapshot from disk", "number", number, "hash", hash) @@ -625,8 +634,8 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash li c.recents.Add(snap.Hash, snap) - // If we've generated a new checkpoint snapshot, save to disk - if snap.Number%checkpointInterval == 0 && len(headers) > 0 { + // If we've generated a new persistent snapshot, save to disk + if snap.Number%snapshotPersistInterval == 0 && len(headers) > 0 { if err = snap.store(c.DB); err != nil { return nil, err } @@ -738,8 +747,12 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header, s header.Extra = header.Extra[:extraVanity] // get validator set if number + // Note: headers.Extra has producer set and not validator set. The bor + // client calls `GetCurrentValidators` because it makes a contract call + // where it fetches producers internally. As we fetch data from span + // in Erigon, use directly the `GetCurrentProducers` function. if isSprintStart(number+1, c.config.CalculateSprint(number)) { - newValidators, err := c.spanner.GetCurrentValidators(number+1, c.authorizedSigner.Load().signer, c.getSpanForBlock) + newValidators, err := c.spanner.GetCurrentProducers(number+1, c.authorizedSigner.Load().signer, c.getSpanForBlock) if err != nil { return errUnknownValidators } @@ -1103,42 +1116,33 @@ func (c *Bor) needToCommitSpan(currentSpan *span.Span, headerNumber uint64) bool } func (c *Bor) getSpanForBlock(blockNum uint64) (*span.HeimdallSpan, error) { - c.logger.Info("Getting span", "for block", blockNum) + c.logger.Debug("Getting span", "for block", blockNum) var borSpan *span.HeimdallSpan c.spanCache.AscendGreaterOrEqual(&span.HeimdallSpan{Span: span.Span{EndBlock: blockNum}}, func(item btree.Item) bool { borSpan = item.(*span.HeimdallSpan) return false }) - if borSpan == nil { - // Span with high enough block number is not loaded - var spanID uint64 - if c.spanCache.Len() > 0 { - spanID = c.spanCache.Max().(*span.HeimdallSpan).ID + 1 - } - for borSpan == nil || borSpan.EndBlock < blockNum { - c.logger.Info("Span with high enough block number is not loaded", "fetching span", spanID) - response, err := c.HeimdallClient.Span(c.execCtx, spanID) - if err != nil { - return nil, err - } - borSpan = response - c.spanCache.ReplaceOrInsert(borSpan) - spanID++ - } - } else { - for borSpan.StartBlock > blockNum { - // Span wit low enough block number is not loaded - var spanID = borSpan.ID - 1 - c.logger.Info("Span with low enough block number is not loaded", "fetching span", spanID) - response, err := c.HeimdallClient.Span(c.execCtx, spanID) - if err != nil { - return nil, err - } - borSpan = response - c.spanCache.ReplaceOrInsert(borSpan) - } + if borSpan != nil && borSpan.StartBlock <= blockNum && borSpan.EndBlock >= blockNum { + return borSpan, nil + } + + // Span with given block block number is not loaded + // As span has fixed set of blocks (except 0th span), we can + // formulate it and get the exact ID we'd need to fetch. + var spanID uint64 + if blockNum > zerothSpanEnd { + spanID = 1 + (blockNum-zerothSpanEnd-1)/spanLength + } + + c.logger.Info("Span with given block number is not loaded", "fetching span", spanID) + + response, err := c.HeimdallClient.Span(c.execCtx, spanID) + if err != nil { + return nil, err } + borSpan = response + c.spanCache.ReplaceOrInsert(borSpan) for c.spanCache.Len() > 128 { c.spanCache.DeleteMin() diff --git a/consensus/bor/snapshot.go b/consensus/bor/snapshot.go index 736ef1f29a2..0e8825dd5bb 100644 --- a/consensus/bor/snapshot.go +++ b/consensus/bor/snapshot.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" - lru "github.com/hashicorp/golang-lru/v2" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/ledgerwatch/erigon-lib/chain" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" @@ -139,10 +139,11 @@ func (s *Snapshot) apply(headers []*types.Header, logger log.Logger) (*Snapshot, for _, header := range headers { // Remove any votes on checkpoint blocks number := header.Number.Uint64() + sprintLen := s.config.CalculateSprint(number) // Delete the oldest signer from the recent list to allow it signing again - if number >= s.config.CalculateSprint(number) { - delete(snap.Recents, number-s.config.CalculateSprint(number)) + if number >= sprintLen { + delete(snap.Recents, number-sprintLen) } // Resolve the authorization key and check against signers @@ -164,7 +165,7 @@ func (s *Snapshot) apply(headers []*types.Header, logger log.Logger) (*Snapshot, snap.Recents[number] = signer // change validator set and change proposer - if number > 0 && (number+1)%s.config.CalculateSprint(number) == 0 { + if number > 0 && (number+1)%sprintLen == 0 { if err := validateHeaderExtraField(header.Extra); err != nil { return nil, err } diff --git a/consensus/bor/snapshot_test.go b/consensus/bor/snapshot_test.go index 69aead915d0..a9b886fad38 100644 --- a/consensus/bor/snapshot_test.go +++ b/consensus/bor/snapshot_test.go @@ -8,7 +8,7 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/consensus/bor/valset" "github.com/ledgerwatch/log/v3" - crand "github.com/maticnetwork/crand" + "github.com/maticnetwork/crand" "github.com/stretchr/testify/require" ) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 432d57cd6a6..88515977752 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -29,7 +29,7 @@ import ( "time" "github.com/goccy/go-json" - lru "github.com/hashicorp/golang-lru/v2" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/log/v3" diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 6b9a4d08f21..43bb77bd5a5 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -59,12 +59,11 @@ func TestReimportMirroredState(t *testing.T) { } copy(genspec.ExtraData[clique.ExtraVanity:], addr[:]) m := stages.MockWithGenesisEngine(t, genspec, engine, false) - br, _ := m.NewBlocksIO() // Generate a batch of blocks, each properly signed getHeader := func(hash libcommon.Hash, number uint64) (h *types.Header) { if err := m.DB.View(m.Ctx, func(tx kv.Tx) (err error) { - h, err = br.Header(m.Ctx, tx, hash, number) + h, err = m.BlockReader.Header(m.Ctx, tx, hash, number) return err }); err != nil { panic(err) @@ -106,11 +105,11 @@ func TestReimportMirroredState(t *testing.T) { } // Insert the first two blocks and make sure the chain is valid - if err := m.InsertChain(chain.Slice(0, 2)); err != nil { + if err := m.InsertChain(chain.Slice(0, 2), nil); err != nil { t.Fatalf("failed to insert initial blocks: %v", err) } if err := m.DB.View(m.Ctx, func(tx kv.Tx) error { - if head, err1 := br.BlockByHash(m.Ctx, tx, rawdb.ReadHeadHeaderHash(tx)); err1 != nil { + if head, err1 := m.BlockReader.BlockByHash(m.Ctx, tx, rawdb.ReadHeadHeaderHash(tx)); err1 != nil { t.Errorf("could not read chain head: %v", err1) } else if head.NumberU64() != 2 { t.Errorf("chain head mismatch: have %d, want %d", head.NumberU64(), 2) @@ -123,11 +122,11 @@ func TestReimportMirroredState(t *testing.T) { // Simulate a crash by creating a new chain on top of the database, without // flushing the dirty states out. Insert the last block, triggering a sidechain // reimport. - if err := m.InsertChain(chain.Slice(2, chain.Length())); err != nil { + if err := m.InsertChain(chain.Slice(2, chain.Length()), nil); err != nil { t.Fatalf("failed to insert final block: %v", err) } if err := m.DB.View(m.Ctx, func(tx kv.Tx) error { - if head, err1 := br.CurrentBlock(tx); err1 != nil { + if head, err1 := m.BlockReader.CurrentBlock(tx); err1 != nil { t.Errorf("could not read chain head: %v", err1) } else if head.NumberU64() != 3 { t.Errorf("chain head mismatch: have %d, want %d", head.NumberU64(), 3) diff --git a/consensus/clique/snapshot.go b/consensus/clique/snapshot.go index 0d9dfcd0626..c478d6ec3b1 100644 --- a/consensus/clique/snapshot.go +++ b/consensus/clique/snapshot.go @@ -25,7 +25,7 @@ import ( "time" "github.com/goccy/go-json" - "github.com/hashicorp/golang-lru/v2" + lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 7af161303c6..a5c99a0e685 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -428,7 +428,6 @@ func TestClique(t *testing.T) { engine.FakeDiff = true // Create a pristine blockchain with the genesis injected m := stages.MockWithGenesisEngine(t, genesis, engine, false) - blockReader, _ := m.NewBlocksIO() chain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, len(tt.votes), func(j int, gen *core.BlockGen) { // Cast the vote contained in this block @@ -477,7 +476,7 @@ func TestClique(t *testing.T) { chainX.Headers[k] = b.Header() } chainX.TopBlock = batches[j][len(batches[j])-1] - if err = m.InsertChain(chainX); err != nil { + if err = m.InsertChain(chainX, nil); err != nil { t.Errorf("test %d: failed to import batch %d, %v", i, j, err) failed = true break @@ -493,7 +492,7 @@ func TestClique(t *testing.T) { chainX.Headers[k] = b.Header() } chainX.TopBlock = batches[len(batches)-1][len(batches[len(batches)-1])-1] - err = m.InsertChain(chainX) + err = m.InsertChain(chainX, nil) if tt.failure != nil && err == nil { t.Errorf("test %d: expected failure", i) } @@ -509,7 +508,7 @@ func TestClique(t *testing.T) { var snap *clique.Snapshot if err := m.DB.View(context.Background(), func(tx kv.Tx) error { - snap, err = engine.Snapshot(stagedsync.ChainReader{Cfg: config, Db: tx, BlockReader: blockReader}, head.NumberU64(), head.Hash(), nil) + snap, err = engine.Snapshot(stagedsync.ChainReader{Cfg: config, Db: tx, BlockReader: m.BlockReader}, head.NumberU64(), head.Hash(), nil) if err != nil { return err } diff --git a/consensus/merge/merge.go b/consensus/merge/merge.go index 3709fba37f9..ac23d0007ce 100644 --- a/consensus/merge/merge.go +++ b/consensus/merge/merge.go @@ -174,7 +174,7 @@ func (s *Merge) FinalizeAndAssemble(config *chain.Config, header *types.Header, return nil, nil, nil, err } if config.IsCancun(header.Time) { - dataGasUsed := uint64(misc.CountBlobs(txs) * params.DataGasPerBlob) + dataGasUsed := uint64(misc.CountBlobs(txs)) * params.DataGasPerBlob header.DataGasUsed = &dataGasUsed } return types.NewBlock(header, outTxs, uncles, outReceipts, withdrawals), outTxs, outReceipts, nil diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 322a864d394..c82625b3dac 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -42,17 +42,16 @@ func TestHeaderVerification(t *testing.T) { if err != nil { t.Fatalf("genetate chain: %v", err) } - blockReader, _ := m.NewBlocksIO() // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces for i := 0; i < chain.Length(); i++ { if err := m.DB.View(context.Background(), func(tx kv.Tx) error { for j, valid := range []bool{true, false} { if valid { engine := ethash.NewFaker() - err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainConfig, Db: tx, BlockReader: blockReader}, chain.Headers[i], true) + err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainConfig, Db: tx, BlockReader: m.BlockReader}, chain.Headers[i], true) } else { engine := ethash.NewFakeFailer(chain.Headers[i].Number.Uint64()) - err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainConfig, Db: tx, BlockReader: blockReader}, chain.Headers[i], true) + err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainConfig, Db: tx, BlockReader: m.BlockReader}, chain.Headers[i], true) } if (err == nil) != valid { t.Errorf("test %d.%d: validity mismatch: have %v, want %v", i, j, err, valid) @@ -62,7 +61,7 @@ func TestHeaderVerification(t *testing.T) { }); err != nil { panic(err) } - if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { + if err = m.InsertChain(chain.Slice(i, i+1), nil); err != nil { t.Fatalf("test %d: error inserting the block: %v", i, err) } @@ -84,17 +83,16 @@ func TestHeaderWithSealVerification(t *testing.T) { t.Fatalf("genetate chain: %v", err) } - blockReader, _ := m.NewBlocksIO() // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces for i := 0; i < chain.Length(); i++ { if err := m.DB.View(context.Background(), func(tx kv.Tx) error { for j, valid := range []bool{true, false} { if valid { engine := ethash.NewFaker() - err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainAuraConfig, Db: tx, BlockReader: blockReader}, chain.Headers[i], true) + err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainAuraConfig, Db: tx, BlockReader: m.BlockReader}, chain.Headers[i], true) } else { engine := ethash.NewFakeFailer(chain.Headers[i].Number.Uint64()) - err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainAuraConfig, Db: tx, BlockReader: blockReader}, chain.Headers[i], true) + err = engine.VerifyHeader(stagedsync.ChainReader{Cfg: *params.TestChainAuraConfig, Db: tx, BlockReader: m.BlockReader}, chain.Headers[i], true) } if (err == nil) != valid { t.Errorf("test %d.%d: validity mismatch: have %v, want %v", i, j, err, valid) @@ -104,7 +102,7 @@ func TestHeaderWithSealVerification(t *testing.T) { }); err != nil { panic(err) } - if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { + if err = m.InsertChain(chain.Slice(i, i+1), nil); err != nil { t.Fatalf("test %d: error inserting the block: %v", i, err) } diff --git a/core/blockchain.go b/core/blockchain.go index 364d128a68a..0b16d699c94 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -117,7 +117,6 @@ func ExecuteBlockEphemerally( vmConfig.Tracer = tracer writeTrace = true } - receipt, _, err := ApplyTransaction(chainConfig, blockHashFunc, engine, nil, gp, ibs, noop, header, tx, usedGas, *vmConfig) if writeTrace { if ftracer, ok := vmConfig.Tracer.(vm.FlushableTracer); ok { diff --git a/core/chain_makers.go b/core/chain_makers.go index 42933b54fff..12332876baa 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -25,8 +25,6 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon/core/systemcontracts" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/common" @@ -34,9 +32,12 @@ import ( "github.com/ledgerwatch/erigon/consensus/merge" "github.com/ledgerwatch/erigon/consensus/misc" "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/systemcontracts" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/trie" ) @@ -355,13 +356,11 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E var txNum uint64 for i := 0; i < n; i++ { - var stateReader state.StateReader + stateReader := rpchelper.NewLatestStateReader(tx) var stateWriter state.StateWriter - if ethconfig.EnableHistoryV4InTest { panic("implement me on v4") } else { - stateReader = state.NewPlainStateReader(tx) stateWriter = state.NewPlainStateWriter(tx, nil, parent.NumberU64()+uint64(i)+1) } ibs := state.New(stateReader) diff --git a/core/genesis_test.go b/core/genesis_test.go index ebd6d0af429..05c7d4d778e 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -9,9 +9,7 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon/core" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/stretchr/testify/assert" @@ -27,7 +25,7 @@ import ( func TestGenesisBlockHashes(t *testing.T) { logger := log.New() - _, db, _ := temporal.NewTestDB(t, context.Background(), datadir.New(t.TempDir()), nil, logger) + _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) check := func(network string) { genesis := core.GenesisBlockByChainName(network) tx, err := db.BeginRw(context.Background()) @@ -35,12 +33,7 @@ func TestGenesisBlockHashes(t *testing.T) { t.Fatal(err) } defer tx.Rollback() - histV3, err := kvcfg.HistoryV3.Enabled(tx) - if err != nil { - panic(err) - } - blockWriter := blockio.NewBlockWriter(histV3) - _, block, err := core.WriteGenesisBlock(tx, genesis, nil, "", logger, blockWriter) + _, block, err := core.WriteGenesisBlock(tx, genesis, nil, "", logger) require.NoError(t, err) expect := params.GenesisHashByChainName(network) require.NotNil(t, expect, network) @@ -81,24 +74,19 @@ func TestGenesisBlockRoots(t *testing.T) { func TestCommitGenesisIdempotency(t *testing.T) { logger := log.New() - _, db, _ := temporal.NewTestDB(t, context.Background(), datadir.New(t.TempDir()), nil, logger) + _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) tx, err := db.BeginRw(context.Background()) require.NoError(t, err) defer tx.Rollback() - histV3, err := kvcfg.HistoryV3.Enabled(tx) - if err != nil { - panic(err) - } - blockWriter := blockio.NewBlockWriter(histV3) genesis := core.GenesisBlockByChainName(networkname.MainnetChainName) - _, _, err = core.WriteGenesisBlock(tx, genesis, nil, "", logger, blockWriter) + _, _, err = core.WriteGenesisBlock(tx, genesis, nil, "", logger) require.NoError(t, err) seq, err := tx.ReadSequence(kv.EthTx) require.NoError(t, err) require.Equal(t, uint64(2), seq) - _, _, err = core.WriteGenesisBlock(tx, genesis, nil, "", logger, blockWriter) + _, _, err = core.WriteGenesisBlock(tx, genesis, nil, "", logger) require.NoError(t, err) seq, err = tx.ReadSequence(kv.EthTx) require.NoError(t, err) @@ -123,7 +111,7 @@ func TestAllocConstructor(t *testing.T) { }, } - historyV3, db, _ := temporal.NewTestDB(t, context.Background(), datadir.New(t.TempDir()), nil, logger) + historyV3, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) _, _, err := core.CommitGenesisBlock(db, genSpec, "", logger) require.NoError(err) diff --git a/core/genesis_write.go b/core/genesis_write.go index 0dd8366850e..34ef842718a 100644 --- a/core/genesis_write.go +++ b/core/genesis_write.go @@ -27,15 +27,13 @@ import ( "github.com/c2h5oh/datasize" "github.com/holiman/uint256" - "github.com/ledgerwatch/erigon-lib/kv/kvcfg" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" @@ -76,12 +74,7 @@ func CommitGenesisBlockWithOverride(db kv.RwDB, genesis *types.Genesis, override return nil, nil, err } defer tx.Rollback() - histV3, err := kvcfg.HistoryV3.Enabled(tx) - if err != nil { - return nil, nil, err - } - blockWriter := blockio.NewBlockWriter(histV3) - c, b, err := WriteGenesisBlock(tx, genesis, overrideShanghaiTime, tmpDir, logger, blockWriter) + c, b, err := WriteGenesisBlock(tx, genesis, overrideShanghaiTime, tmpDir, logger) if err != nil { return c, b, err } @@ -92,7 +85,7 @@ func CommitGenesisBlockWithOverride(db kv.RwDB, genesis *types.Genesis, override return c, b, nil } -func WriteGenesisBlock(tx kv.RwTx, genesis *types.Genesis, overrideShanghaiTime *big.Int, tmpDir string, logger log.Logger, blockWriter *blockio.BlockWriter) (*chain.Config, *types.Block, error) { +func WriteGenesisBlock(tx kv.RwTx, genesis *types.Genesis, overrideShanghaiTime *big.Int, tmpDir string, logger log.Logger) (*chain.Config, *types.Block, error) { var storedBlock *types.Block if genesis != nil && genesis.Config == nil { return params.AllProtocolChanges, nil, types.ErrGenesisNoConfig @@ -124,7 +117,7 @@ func WriteGenesisBlock(tx kv.RwTx, genesis *types.Genesis, overrideShanghaiTime custom = false } applyOverrides(genesis.Config) - block, _, err1 := write(tx, genesis, tmpDir, blockWriter) + block, _, err1 := write(tx, genesis, tmpDir) if err1 != nil { return genesis.Config, nil, err1 } @@ -145,11 +138,10 @@ func WriteGenesisBlock(tx kv.RwTx, genesis *types.Genesis, overrideShanghaiTime return genesis.Config, block, &types.GenesisMismatchError{Stored: storedHash, New: hash} } } - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) number := rawdb.ReadHeaderNumber(tx, storedHash) if number != nil { var err error - storedBlock, _, err = blockReader.BlockWithSenders(context.Background(), tx, storedHash, *number) + storedBlock, _, err = rawdb.ReadBlockWithSenders(tx, storedHash, *number) if err != nil { return genesis.Config, nil, err } @@ -215,6 +207,11 @@ func WriteGenesisState(g *types.Genesis, tx kv.RwTx, tmpDir string) (*types.Bloc if err != nil { return nil, nil, err } + histV3, err := kvcfg.HistoryV3.Enabled(tx) + if err != nil { + panic(err) + } + var stateWriter state.StateWriter if ethconfig.EnableHistoryV4InTest { panic("implement me") @@ -242,12 +239,14 @@ func WriteGenesisState(g *types.Genesis, tx kv.RwTx, tmpDir string) (*types.Bloc if err := statedb.CommitBlock(&chain.Rules{}, stateWriter); err != nil { return nil, statedb, fmt.Errorf("cannot write state: %w", err) } - if csw, ok := stateWriter.(state.WriterWithChangeSets); ok { - if err := csw.WriteChangeSets(); err != nil { - return nil, statedb, fmt.Errorf("cannot write change sets: %w", err) - } - if err := csw.WriteHistory(); err != nil { - return nil, statedb, fmt.Errorf("cannot write history: %w", err) + if !histV3 { + if csw, ok := stateWriter.(state.WriterWithChangeSets); ok { + if err := csw.WriteChangeSets(); err != nil { + return nil, statedb, fmt.Errorf("cannot write change sets: %w", err) + } + if err := csw.WriteHistory(); err != nil { + return nil, statedb, fmt.Errorf("cannot write history: %w", err) + } } } return block, statedb, nil @@ -258,12 +257,7 @@ func MustCommitGenesis(g *types.Genesis, db kv.RwDB, tmpDir string) *types.Block panic(err) } defer tx.Rollback() - histV3, err := kvcfg.HistoryV3.Enabled(tx) - if err != nil { - panic(err) - } - blockWriter := blockio.NewBlockWriter(histV3) - block, _, err := write(tx, g, tmpDir, blockWriter) + block, _, err := write(tx, g, tmpDir) if err != nil { panic(err) } @@ -276,7 +270,7 @@ func MustCommitGenesis(g *types.Genesis, db kv.RwDB, tmpDir string) *types.Block // Write writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. -func write(tx kv.RwTx, g *types.Genesis, tmpDir string, blockWriter *blockio.BlockWriter) (*types.Block, *state.IntraBlockState, error) { +func write(tx kv.RwTx, g *types.Genesis, tmpDir string) (*types.Block, *state.IntraBlockState, error) { block, statedb, err2 := WriteGenesisState(g, tx, tmpDir) if err2 != nil { return block, statedb, err2 @@ -289,10 +283,10 @@ func write(tx kv.RwTx, g *types.Genesis, tmpDir string, blockWriter *blockio.Blo return nil, nil, err } - if err := blockWriter.WriteBlock(tx, block); err != nil { + if err := rawdb.WriteBlock(tx, block); err != nil { return nil, nil, err } - if err := blockWriter.WriteTd(tx, block.Hash(), block.NumberU64(), g.Difficulty); err != nil { + if err := rawdb.WriteTd(tx, block.Hash(), block.NumberU64(), g.Difficulty); err != nil { return nil, nil, err } if err := rawdbv3.TxNums.WriteForGenesis(tx, 1); err != nil { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index a18a4a5418b..f0ac56ed6be 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -385,10 +385,16 @@ func ReadStorageBody(db kv.Getter, hash libcommon.Hash, number uint64) (types.Bo return *bodyForStorage, nil } -func CanonicalTxnByID(db kv.Getter, id uint64) (types.Transaction, error) { - txIdKey := make([]byte, 8) - binary.BigEndian.PutUint64(txIdKey, id) - v, err := db.GetOne(kv.EthTx, txIdKey) +func TxnByIdxInBlock(db kv.Getter, blockHash libcommon.Hash, blockNum uint64, txIdxInBlock int) (types.Transaction, error) { + b, err := ReadBodyForStorageByKey(db, dbutils.BlockBodyKey(blockNum, blockHash)) + if err != nil { + return nil, err + } + if b == nil { + return nil, nil + } + + v, err := db.GetOne(kv.EthTx, hexutility.EncodeTs(b.BaseTxId+1+uint64(txIdxInBlock))) if err != nil { return nil, err } diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index cc652bf0563..8dce1fb0312 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -46,7 +46,7 @@ func TestHeaderStorage(t *testing.T) { require.NoError(t, err) defer tx.Rollback() ctx := m.Ctx - br, bw := m.NewBlocksIO() + br := m.BlockReader // Create a test header to move around the database and make sure it's really new header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} @@ -56,7 +56,7 @@ func TestHeaderStorage(t *testing.T) { t.Fatalf("Non existent header returned: %v", entry) } // Write and verify the header in the database - bw.WriteHeader(tx, header) + rawdb.WriteHeader(tx, header) if entry, _ := br.Header(ctx, tx, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != header.Hash() { @@ -86,7 +86,7 @@ func TestBodyStorage(t *testing.T) { require.NoError(t, err) defer tx.Rollback() ctx := m.Ctx - br, bw := m.NewBlocksIO() + br := m.BlockReader require := require.New(t) var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -99,7 +99,7 @@ func TestBodyStorage(t *testing.T) { } // prepare db so it works with our test - signer1 := types.MakeSigner(params.MainnetChainConfig, 1) + signer1 := types.MakeSigner(params.MainnetChainConfig, 1, 0) body := &types.Body{ Transactions: []types.Transaction{ mustSign(types.NewTransaction(1, testAddr, u256.Num1, 1, u256.Num1, nil), *signer1), @@ -118,8 +118,8 @@ func TestBodyStorage(t *testing.T) { t.Fatalf("Non existent body returned: %v", entry) } require.NoError(rawdb.WriteCanonicalHash(tx, header.Hash(), 1)) - require.NoError(bw.WriteHeader(tx, header)) - require.NoError(bw.WriteBody(tx, header.Hash(), 1, body)) + require.NoError(rawdb.WriteHeader(tx, header)) + require.NoError(rawdb.WriteBody(tx, header.Hash(), 1, body)) if entry, _ := br.BodyWithTransactions(ctx, tx, header.Hash(), 1); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { @@ -155,7 +155,7 @@ func TestBlockStorage(t *testing.T) { require.NoError(err) defer tx.Rollback() ctx := m.Ctx - br, bw := m.NewBlocksIO() + br, bw := m.BlocksIO() // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(&types.Header{ @@ -176,7 +176,7 @@ func TestBlockStorage(t *testing.T) { } // Write and verify the block in the database - err = bw.WriteBlock(tx, block) + err = rawdb.WriteBlock(tx, block) if err != nil { t.Fatalf("Could not write block: %v", err) } @@ -190,7 +190,7 @@ func TestBlockStorage(t *testing.T) { } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if err := bw.TruncateBlocks(context.Background(), tx, 2); err != nil { + if err := rawdb.TruncateBlocks(context.Background(), tx, 2); err != nil { t.Fatal(err) } if entry, _ := br.BodyWithTransactions(ctx, tx, block.Hash(), block.NumberU64()); entry == nil { @@ -199,7 +199,7 @@ func TestBlockStorage(t *testing.T) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body()) } // Delete the block and verify the execution - if err := bw.TruncateBlocks(context.Background(), tx, block.NumberU64()); err != nil { + if err := rawdb.TruncateBlocks(context.Background(), tx, block.NumberU64()); err != nil { t.Fatal(err) } //if err := DeleteBlock(tx, block.Hash(), block.NumberU64()); err != nil { @@ -216,7 +216,7 @@ func TestBlockStorage(t *testing.T) { } // write again and delete it as old one - require.NoError(bw.WriteBlock(tx, block)) + require.NoError(rawdb.WriteBlock(tx, block)) { // mark chain as bad @@ -264,7 +264,7 @@ func TestPartialBlockStorage(t *testing.T) { require.NoError(t, err) defer tx.Rollback() ctx := m.Ctx - br, bw := m.NewBlocksIO() + br := m.BlockReader block := types.NewBlockWithHeader(&types.Header{ Extra: []byte("test block"), @@ -275,14 +275,14 @@ func TestPartialBlockStorage(t *testing.T) { header := block.Header() // Not identical to struct literal above, due to other fields // Store a header and check that it's not recognized as a block - bw.WriteHeader(tx, header) + rawdb.WriteHeader(tx, header) if entry, _, _ := br.BlockWithSenders(ctx, tx, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } rawdb.DeleteHeader(tx, block.Hash(), block.NumberU64()) // Store a body and check that it's not recognized as a block - if err := bw.WriteBody(tx, block.Hash(), block.NumberU64(), block.Body()); err != nil { + if err := rawdb.WriteBody(tx, block.Hash(), block.NumberU64(), block.Body()); err != nil { t.Fatal(err) } if entry, _, _ := br.BlockWithSenders(ctx, tx, block.Hash(), block.NumberU64()); entry != nil { @@ -291,8 +291,8 @@ func TestPartialBlockStorage(t *testing.T) { rawdb.DeleteBody(tx, block.Hash(), block.NumberU64()) // Store a header and a body separately and check reassembly - bw.WriteHeader(tx, header) - if err := bw.WriteBody(tx, block.Hash(), block.NumberU64(), block.Body()); err != nil { + rawdb.WriteHeader(tx, header) + if err := rawdb.WriteBody(tx, block.Hash(), block.NumberU64(), block.Body()); err != nil { t.Fatal(err) } @@ -309,7 +309,6 @@ func TestTdStorage(t *testing.T) { tx, err := m.DB.BeginRw(m.Ctx) require.NoError(t, err) defer tx.Rollback() - _, bw := m.NewBlocksIO() // Create a test TD to move around the database and make sure it's really new hash, td := libcommon.Hash{}, big.NewInt(314) @@ -321,7 +320,7 @@ func TestTdStorage(t *testing.T) { t.Fatalf("Non existent TD returned: %v", entry) } // Write and verify the TD in the database - err = bw.WriteTd(tx, hash, 0, td) + err = rawdb.WriteTd(tx, hash, 0, td) if err != nil { t.Fatalf("WriteTd failed: %v", err) } @@ -335,7 +334,7 @@ func TestTdStorage(t *testing.T) { t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td) } // Delete the TD and verify the execution - err = bw.TruncateTd(tx, 0) + err = rawdb.TruncateTd(tx, 0) if err != nil { t.Fatalf("DeleteTd failed: %v", err) } @@ -354,7 +353,7 @@ func TestCanonicalMappingStorage(t *testing.T) { tx, err := m.DB.BeginRw(m.Ctx) require.NoError(t, err) defer tx.Rollback() - br, bw := m.NewBlocksIO() + br := m.BlockReader // Create a test canonical number and assinged hash to move around hash, number := libcommon.Hash{0: 0xff}, uint64(314) @@ -366,7 +365,7 @@ func TestCanonicalMappingStorage(t *testing.T) { t.Fatalf("Non existent canonical mapping returned: %v", entry) } // Write and verify the TD in the database - err = bw.WriteCanonicalHash(tx, hash, number) + err = rawdb.WriteCanonicalHash(tx, hash, number) if err != nil { t.Fatalf("WriteCanoncalHash failed: %v", err) } @@ -449,7 +448,7 @@ func TestBlockReceiptStorage(t *testing.T) { tx, err := m.DB.BeginRw(m.Ctx) require.NoError(t, err) defer tx.Rollback() - br, bw := m.NewBlocksIO() + br := m.BlockReader require := require.New(t) ctx := m.Ctx @@ -492,9 +491,9 @@ func TestBlockReceiptStorage(t *testing.T) { hash := header.Hash() //libcommon.BytesToHash([]byte{0x03, 0x14}) rawdb.WriteCanonicalHash(tx, header.Hash(), header.Number.Uint64()) - bw.WriteHeader(tx, header) + rawdb.WriteHeader(tx, header) // Insert the body that corresponds to the receipts - require.NoError(bw.WriteBody(tx, hash, 1, body)) + require.NoError(rawdb.WriteBody(tx, hash, 1, body)) require.NoError(rawdb.WriteSenders(tx, hash, 1, body.SendersFromTxs())) // Insert the receipt slice into the database and check presence @@ -523,9 +522,9 @@ func TestBlockReceiptStorage(t *testing.T) { if err := checkReceiptsRLP(rawdb.ReadRawReceipts(tx, 1), receipts); err != nil { t.Fatal(err) } - bw.WriteHeader(tx, header) + rawdb.WriteHeader(tx, header) // Sanity check that body alone without the receipt is a full purge - require.NoError(bw.WriteBody(tx, hash, 1, body)) + require.NoError(rawdb.WriteBody(tx, hash, 1, body)) require.NoError(rawdb.TruncateReceipts(tx, 1)) b, senders, err = br.BlockWithSenders(ctx, tx, hash, 1) require.NoError(err) @@ -542,7 +541,7 @@ func TestBlockWithdrawalsStorage(t *testing.T) { tx, err := m.DB.BeginRw(m.Ctx) require.NoError(err) defer tx.Rollback() - br, bw := m.NewBlocksIO() + br, bw := m.BlocksIO() ctx := context.Background() // create fake withdrawals @@ -585,15 +584,15 @@ func TestBlockWithdrawalsStorage(t *testing.T) { // Write withdrawals to block wBlock := types.NewBlockFromStorage(block.Hash(), block.Header(), block.Transactions(), block.Uncles(), withdrawals) - if err := bw.WriteHeader(tx, wBlock.HeaderNoCopy()); err != nil { + if err := rawdb.WriteHeader(tx, wBlock.HeaderNoCopy()); err != nil { t.Fatalf("Could not write body: %v", err) } - if err := bw.WriteBody(tx, wBlock.Hash(), wBlock.NumberU64(), wBlock.Body()); err != nil { + if err := rawdb.WriteBody(tx, wBlock.Hash(), wBlock.NumberU64(), wBlock.Body()); err != nil { t.Fatalf("Could not write body: %v", err) } // Write and verify the block in the database - err = bw.WriteBlock(tx, wBlock) + err = rawdb.WriteBlock(tx, wBlock) if err != nil { t.Fatalf("Could not write block: %v", err) } @@ -607,7 +606,7 @@ func TestBlockWithdrawalsStorage(t *testing.T) { } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if err := bw.TruncateBlocks(context.Background(), tx, 2); err != nil { + if err := rawdb.TruncateBlocks(context.Background(), tx, 2); err != nil { t.Fatal(err) } entry, _ := br.BodyWithTransactions(ctx, tx, block.Hash(), block.NumberU64()) @@ -639,7 +638,7 @@ func TestBlockWithdrawalsStorage(t *testing.T) { require.Equal(uint64(1001), rw2.Amount) // Delete the block and verify the execution - if err := bw.TruncateBlocks(context.Background(), tx, block.NumberU64()); err != nil { + if err := rawdb.TruncateBlocks(context.Background(), tx, block.NumberU64()); err != nil { t.Fatal(err) } //if err := DeleteBlock(tx, block.Hash(), block.NumberU64()); err != nil { @@ -656,7 +655,7 @@ func TestBlockWithdrawalsStorage(t *testing.T) { } // write again and delete it as old one - if err := bw.WriteBlock(tx, block); err != nil { + if err := rawdb.WriteBlock(tx, block); err != nil { t.Fatalf("Could not write block: %v", err) } // prune: [1: N) diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index e9b916d64d1..49ec779c273 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -50,7 +50,7 @@ func TestLookupStorage(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { m := stages.Mock(t) - br, bw := m.NewBlocksIO() + br := m.BlockReader tx, err := m.DB.BeginRw(m.Ctx) require.NoError(t, err) defer tx.Rollback() @@ -72,7 +72,7 @@ func TestLookupStorage(t *testing.T) { if err := rawdb.WriteCanonicalHash(tx, block.Hash(), block.NumberU64()); err != nil { t.Fatal(err) } - if err := bw.WriteBlock(tx, block); err != nil { + if err := rawdb.WriteBlock(tx, block); err != nil { t.Fatal(err) } if err := rawdb.WriteSenders(tx, block.Hash(), block.NumberU64(), block.Body().SendersFromTxs()); err != nil { diff --git a/core/rawdb/blockio/block_writer.go b/core/rawdb/blockio/block_writer.go index 90118794a78..a4a9e7702f0 100644 --- a/core/rawdb/blockio/block_writer.go +++ b/core/rawdb/blockio/block_writer.go @@ -3,7 +3,6 @@ package blockio import ( "context" "encoding/binary" - "math/big" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" @@ -12,7 +11,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/common/dbutils" "github.com/ledgerwatch/erigon/core/rawdb" - "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/turbo/backup" "github.com/ledgerwatch/log/v3" ) @@ -34,22 +32,6 @@ func NewBlockWriter(historyV3 bool) *BlockWriter { return &BlockWriter{historyV3: historyV3, txsV3: true} } -func (w *BlockWriter) WriteBlock(tx kv.RwTx, block *types.Block) error { - return rawdb.WriteBlock(tx, block) -} -func (w *BlockWriter) WriteHeader(tx kv.RwTx, header *types.Header) error { - return rawdb.WriteHeader(tx, header) -} -func (w *BlockWriter) WriteHeaderRaw(tx kv.StatelessRwTx, number uint64, hash common.Hash, headerRlp []byte, skipIndexing bool) error { - return rawdb.WriteHeaderRaw(tx, number, hash, headerRlp, skipIndexing) -} -func (w *BlockWriter) WriteCanonicalHash(tx kv.RwTx, hash common.Hash, number uint64) error { - return rawdb.WriteCanonicalHash(tx, hash, number) -} -func (w *BlockWriter) WriteTd(db kv.Putter, hash common.Hash, number uint64, td *big.Int) error { - return rawdb.WriteTd(db, hash, number, td) -} - func (w *BlockWriter) FillHeaderNumberIndex(logPrefix string, tx kv.RwTx, tmpDir string, from, to uint64, ctx context.Context, logger log.Logger) error { startKey := make([]byte, 8) binary.BigEndian.PutUint64(startKey, from) @@ -97,12 +79,6 @@ func extractHeaders(k []byte, _ []byte, next etl.ExtractNextFunc) error { return next(k, k[8:], k[:8]) } -func (w *BlockWriter) WriteRawBodyIfNotExists(tx kv.RwTx, hash common.Hash, number uint64, body *types.RawBody) (ok bool, err error) { - return rawdb.WriteRawBodyIfNotExists(tx, hash, number, body) -} -func (w *BlockWriter) WriteBody(tx kv.RwTx, hash common.Hash, number uint64, body *types.Body) error { - return rawdb.WriteBody(tx, hash, number, body) -} func (w *BlockWriter) TruncateBodies(db kv.RoDB, tx kv.RwTx, from uint64) error { fromB := hexutility.EncodeTs(from) if err := tx.ForEach(kv.BlockBody, fromB, func(k, _ []byte) error { return tx.Delete(kv.BlockBody, k) }); err != nil { @@ -126,16 +102,6 @@ func (w *BlockWriter) TruncateBodies(db kv.RoDB, tx kv.RwTx, from uint64) error return nil } -func (w *BlockWriter) TruncateBlocks(ctx context.Context, tx kv.RwTx, blockFrom uint64) error { - return rawdb.TruncateBlocks(ctx, tx, blockFrom) -} -func (w *BlockWriter) TruncateTd(tx kv.RwTx, blockFrom uint64) error { - return rawdb.TruncateTd(tx, blockFrom) -} -func (w *BlockWriter) ResetSenders(ctx context.Context, db kv.RoDB, tx kv.RwTx) error { - return backup.ClearTables(ctx, db, tx, kv.Senders) -} - // PruneBlocks - [1, to) old blocks after moving it to snapshots. // keeps genesis in db // doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs diff --git a/core/rawdb/rawdbhelpers/rawdbhelpers.go b/core/rawdb/rawdbhelpers/rawdbhelpers.go index 63f72faba2c..195c3950810 100644 --- a/core/rawdb/rawdbhelpers/rawdbhelpers.go +++ b/core/rawdb/rawdbhelpers/rawdbhelpers.go @@ -8,8 +8,8 @@ import ( ) func IdxStepsCountV3(tx kv.Tx) float64 { - fst, _ := kv.FirstKey(tx, kv.TracesToKeys) - lst, _ := kv.LastKey(tx, kv.TracesToKeys) + fst, _ := kv.FirstKey(tx, kv.TblTracesToKeys) + lst, _ := kv.LastKey(tx, kv.TblTracesToKeys) if len(fst) > 0 && len(lst) > 0 { fstTxNum := binary.BigEndian.Uint64(fst) lstTxNum := binary.BigEndian.Uint64(lst) diff --git a/core/rawdb/rawdbreset/reset_stages.go b/core/rawdb/rawdbreset/reset_stages.go index 4a0271db918..93938e78e90 100644 --- a/core/rawdb/rawdbreset/reset_stages.go +++ b/core/rawdb/rawdbreset/reset_stages.go @@ -17,7 +17,6 @@ import ( "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/turbo/backup" "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/log/v3" ) @@ -51,10 +50,10 @@ func ResetState(db kv.RwDB, ctx context.Context, chain string, tmpDir string) er return nil } -func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, agg *state.AggregatorV3, +func ResetBlocks(tx kv.RwTx, db kv.RoDB, agg *state.AggregatorV3, br services.FullBlockReader, bw *blockio.BlockWriter, dirs datadir.Dirs, cc chain.Config, engine consensus.Engine, logger log.Logger) error { // keep Genesis - if err := bw.TruncateBlocks(context.Background(), tx, 1); err != nil { + if err := rawdb.TruncateBlocks(context.Background(), tx, 1); err != nil { return err } if err := stages.SaveStageProgress(tx, stages.Bodies, 1); err != nil { @@ -71,7 +70,7 @@ func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, ag if err := rawdb.TruncateCanonicalHash(tx, 1, false); err != nil { return err } - if err := bw.TruncateTd(tx, 1); err != nil { + if err := rawdb.TruncateTd(tx, 1); err != nil { return err } hash, err := rawdb.ReadCanonicalHash(tx, 0) @@ -87,20 +86,20 @@ func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, ag return err } - if snapshots != nil && snapshots.Cfg().Enabled && snapshots.BlocksAvailable() > 0 { - if err := stagedsync.FillDBFromSnapshots("fillind_db_from_snapshots", context.Background(), tx, dirs, snapshots, br, agg, logger); err != nil { + if br.FreezingCfg().Enabled && br.FrozenBlocks() > 0 { + if err := stagedsync.FillDBFromSnapshots("fillind_db_from_snapshots", context.Background(), tx, dirs, br, agg, logger); err != nil { return err } - _ = stages.SaveStageProgress(tx, stages.Snapshots, snapshots.BlocksAvailable()) - _ = stages.SaveStageProgress(tx, stages.Headers, snapshots.BlocksAvailable()) - _ = stages.SaveStageProgress(tx, stages.Bodies, snapshots.BlocksAvailable()) - _ = stages.SaveStageProgress(tx, stages.Senders, snapshots.BlocksAvailable()) + _ = stages.SaveStageProgress(tx, stages.Snapshots, br.FrozenBlocks()) + _ = stages.SaveStageProgress(tx, stages.Headers, br.FrozenBlocks()) + _ = stages.SaveStageProgress(tx, stages.Bodies, br.FrozenBlocks()) + _ = stages.SaveStageProgress(tx, stages.Senders, br.FrozenBlocks()) } return nil } -func ResetSenders(ctx context.Context, db kv.RwDB, tx kv.RwTx, bw *blockio.BlockWriter) error { - if err := bw.ResetSenders(ctx, db, tx); err != nil { +func ResetSenders(ctx context.Context, db kv.RwDB, tx kv.RwTx) error { + if err := backup.ClearTables(ctx, db, tx, kv.Senders); err != nil { return nil } return clearStageProgress(tx, stages.Senders) @@ -172,8 +171,8 @@ var Tables = map[stages.SyncStage][]string{ stages.IntermediateHashes: {kv.TrieOfAccounts, kv.TrieOfStorage}, stages.CallTraces: {kv.CallFromIndex, kv.CallToIndex}, stages.LogIndex: {kv.LogAddressIndex, kv.LogTopicIndex}, - stages.AccountHistoryIndex: {kv.AccountsHistory}, - stages.StorageHistoryIndex: {kv.StorageHistory}, + stages.AccountHistoryIndex: {kv.E2AccountsHistory}, + stages.StorageHistoryIndex: {kv.E2StorageHistory}, stages.Finish: {}, } var stateBuckets = []string{ @@ -189,20 +188,20 @@ var stateHistoryBuckets = []string{ kv.CallTraceSet, } var stateHistoryV3Buckets = []string{ - kv.AccountHistoryKeys, kv.AccountIdx, kv.AccountHistoryVals, - kv.StorageKeys, kv.StorageVals, kv.StorageHistoryKeys, kv.StorageHistoryVals, kv.StorageIdx, - kv.CodeKeys, kv.CodeVals, kv.CodeHistoryKeys, kv.CodeHistoryVals, kv.CodeIdx, - kv.AccountHistoryKeys, kv.AccountIdx, kv.AccountHistoryVals, - kv.StorageHistoryKeys, kv.StorageIdx, kv.StorageHistoryVals, - kv.CodeHistoryKeys, kv.CodeIdx, kv.CodeHistoryVals, - kv.LogAddressKeys, kv.LogAddressIdx, - kv.LogTopicsKeys, kv.LogTopicsIdx, - kv.TracesFromKeys, kv.TracesFromIdx, - kv.TracesToKeys, kv.TracesToIdx, + kv.TblAccountHistoryKeys, kv.TblAccountIdx, kv.TblAccountHistoryVals, + kv.TblStorageKeys, kv.TblStorageVals, kv.TblStorageHistoryKeys, kv.TblStorageHistoryVals, kv.TblStorageIdx, + kv.TblCodeKeys, kv.TblCodeVals, kv.TblCodeHistoryKeys, kv.TblCodeHistoryVals, kv.TblCodeIdx, + kv.TblAccountHistoryKeys, kv.TblAccountIdx, kv.TblAccountHistoryVals, + kv.TblStorageHistoryKeys, kv.TblStorageIdx, kv.TblStorageHistoryVals, + kv.TblCodeHistoryKeys, kv.TblCodeIdx, kv.TblCodeHistoryVals, + kv.TblLogAddressKeys, kv.TblLogAddressIdx, + kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, + kv.TblTracesFromKeys, kv.TblTracesFromIdx, + kv.TblTracesToKeys, kv.TblTracesToIdx, } var stateHistoryV4Buckets = []string{ - kv.AccountKeys, kv.StorageKeys, kv.CodeKeys, - kv.CommitmentKeys, kv.CommitmentVals, kv.CommitmentHistoryKeys, kv.CommitmentHistoryVals, kv.CommitmentIdx, + kv.TblAccountKeys, kv.TblStorageKeys, kv.TblCodeKeys, + kv.TblCommitmentKeys, kv.TblCommitmentVals, kv.TblCommitmentHistoryKeys, kv.TblCommitmentHistoryVals, kv.TblCommitmentIdx, } func clearStageProgress(tx kv.RwTx, stagesList ...stages.SyncStage) error { diff --git a/core/rlp_test.go b/core/rlp_test.go index ce5dcadc76b..f73ec1d9e43 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -18,7 +18,6 @@ package core import ( - "context" "fmt" "math/big" "testing" @@ -26,7 +25,6 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/log/v3" "golang.org/x/crypto/sha3" "github.com/ledgerwatch/erigon/common/u256" @@ -38,8 +36,7 @@ import ( ) func getBlock(tb testing.TB, transactions int, uncles int, dataSize int, tmpDir string) *types.Block { - logger := log.New() - _, db, _ := temporal.NewTestDB(tb, context.Background(), datadir.New(tmpDir), nil, logger) + _, db, _ := temporal.NewTestDB(tb, datadir.New(tmpDir), nil) var ( aa = libcommon.HexToAddress("0x000000000000000000000000000000000000aaaa") // Generate a canonical chain to act as the main dataset diff --git a/core/state/access_list.go b/core/state/access_list.go index f5166136765..72f9e9a4c75 100644 --- a/core/state/access_list.go +++ b/core/state/access_list.go @@ -17,23 +17,23 @@ package state import ( - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" ) type accessList struct { - addresses map[libcommon.Address]int - slots []map[libcommon.Hash]struct{} + addresses map[common.Address]int + slots []map[common.Hash]struct{} } // ContainsAddress returns true if the address is in the access list. -func (al *accessList) ContainsAddress(address libcommon.Address) bool { +func (al *accessList) ContainsAddress(address common.Address) bool { _, ok := al.addresses[address] return ok } // Contains checks if a slot within an account is present in the access list, returning // separate flags for the presence of the account and the slot respectively. -func (al *accessList) Contains(address libcommon.Address, slot libcommon.Hash) (addressPresent bool, slotPresent bool) { +func (al *accessList) Contains(address common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) { idx, ok := al.addresses[address] if !ok { // no such address (and hence zero slots) @@ -50,7 +50,7 @@ func (al *accessList) Contains(address libcommon.Address, slot libcommon.Hash) ( // newAccessList creates a new accessList. func newAccessList() *accessList { return &accessList{ - addresses: make(map[libcommon.Address]int), + addresses: make(map[common.Address]int), } } @@ -60,9 +60,9 @@ func (al *accessList) Copy() *accessList { for k, v := range al.addresses { cp.addresses[k] = v } - cp.slots = make([]map[libcommon.Hash]struct{}, len(al.slots)) + cp.slots = make([]map[common.Hash]struct{}, len(al.slots)) for i, slotMap := range al.slots { - newSlotmap := make(map[libcommon.Hash]struct{}, len(slotMap)) + newSlotmap := make(map[common.Hash]struct{}, len(slotMap)) for k := range slotMap { newSlotmap[k] = struct{}{} } @@ -73,7 +73,7 @@ func (al *accessList) Copy() *accessList { // AddAddress adds an address to the access list, and returns 'true' if the operation // caused a change (addr was not previously in the list). -func (al *accessList) AddAddress(address libcommon.Address) bool { +func (al *accessList) AddAddress(address common.Address) bool { if _, present := al.addresses[address]; present { return false } @@ -86,12 +86,12 @@ func (al *accessList) AddAddress(address libcommon.Address) bool { // - address added // - slot added // For any 'true' value returned, a corresponding journal entry must be made. -func (al *accessList) AddSlot(address libcommon.Address, slot libcommon.Hash) (addrChange bool, slotChange bool) { +func (al *accessList) AddSlot(address common.Address, slot common.Hash) (addrChange bool, slotChange bool) { idx, addrPresent := al.addresses[address] if !addrPresent || idx == -1 { // Address not present, or addr present but no slots there al.addresses[address] = len(al.slots) - slotmap := map[libcommon.Hash]struct{}{slot: {}} + slotmap := map[common.Hash]struct{}{slot: {}} al.slots = append(al.slots, slotmap) return !addrPresent, true } @@ -110,7 +110,7 @@ func (al *accessList) AddSlot(address libcommon.Address, slot libcommon.Hash) (a // This operation needs to be performed in the same order as the addition happened. // This method is meant to be used by the journal, which maintains ordering of // operations. -func (al *accessList) DeleteSlot(address libcommon.Address, slot libcommon.Hash) { +func (al *accessList) DeleteSlot(address common.Address, slot common.Hash) { idx, addrOk := al.addresses[address] // There are two ways this can fail if !addrOk { @@ -131,6 +131,6 @@ func (al *accessList) DeleteSlot(address libcommon.Address, slot libcommon.Hash) // needs to be performed in the same order as the addition happened. // This method is meant to be used by the journal, which maintains ordering of // operations. -func (al *accessList) DeleteAddress(address libcommon.Address) { +func (al *accessList) DeleteAddress(address common.Address) { delete(al.addresses, address) } diff --git a/core/state/access_list_test.go b/core/state/access_list_test.go index 99b98ddafa8..03ad2511203 100644 --- a/core/state/access_list_test.go +++ b/core/state/access_list_test.go @@ -3,17 +3,17 @@ package state import ( "testing" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv/memdb" ) func verifyAddrs(t *testing.T, s *IntraBlockState, astrings ...string) { t.Helper() - // convert to libcommon.Address form - addresses := make([]libcommon.Address, 0, len(astrings)) - var addressMap = make(map[libcommon.Address]struct{}) + // convert to common.Address form + addresses := make([]common.Address, 0, len(astrings)) + var addressMap = make(map[common.Address]struct{}) for _, astring := range astrings { - address := libcommon.HexToAddress(astring) + address := common.HexToAddress(astring) addresses = append(addresses, address) addressMap[address] = struct{}{} } @@ -32,15 +32,15 @@ func verifyAddrs(t *testing.T, s *IntraBlockState, astrings ...string) { } func verifySlots(t *testing.T, s *IntraBlockState, addrString string, slotStrings ...string) { - if !s.AddressInAccessList(libcommon.HexToAddress(addrString)) { + if !s.AddressInAccessList(common.HexToAddress(addrString)) { t.Fatalf("scope missing address/slots %v", addrString) } - var address = libcommon.HexToAddress(addrString) - // convert to libcommon.Hash form - slots := make([]libcommon.Hash, 0, len(slotStrings)) - var slotMap = make(map[libcommon.Hash]struct{}) + var address = common.HexToAddress(addrString) + // convert to common.Hash form + slots := make([]common.Hash, 0, len(slotStrings)) + var slotMap = make(map[common.Hash]struct{}) for _, slotString := range slotStrings { - s := libcommon.HexToHash(slotString) + s := common.HexToHash(slotString) slots = append(slots, s) slotMap[s] = struct{}{} } @@ -64,8 +64,8 @@ func verifySlots(t *testing.T, s *IntraBlockState, addrString string, slotString func TestAccessList(t *testing.T) { // Some helpers - addr := libcommon.HexToAddress - slot := libcommon.HexToHash + addr := common.HexToAddress + slot := common.HexToHash _, tx := memdb.NewTestTx(t) state := New(NewPlainState(tx, 1, nil)) diff --git a/core/state/cached_reader.go b/core/state/cached_reader.go index 93727758d9f..196d3823344 100644 --- a/core/state/cached_reader.go +++ b/core/state/cached_reader.go @@ -3,7 +3,7 @@ package state import ( "bytes" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/turbo/shards" @@ -22,7 +22,7 @@ func NewCachedReader(r StateReader, cache *shards.StateCache) *CachedReader { } // ReadAccountData is called when an account needs to be fetched from the state -func (cr *CachedReader) ReadAccountData(address libcommon.Address) (*accounts.Account, error) { +func (cr *CachedReader) ReadAccountData(address common.Address) (*accounts.Account, error) { addrBytes := address.Bytes() if a, ok := cr.cache.GetAccount(addrBytes); ok { return a, nil @@ -40,7 +40,7 @@ func (cr *CachedReader) ReadAccountData(address libcommon.Address) (*accounts.Ac } // ReadAccountStorage is called when a storage item needs to be fetched from the state -func (cr *CachedReader) ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) { +func (cr *CachedReader) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { addrBytes := address.Bytes() if s, ok := cr.cache.GetStorage(addrBytes, incarnation, key.Bytes()); ok { return s, nil @@ -59,7 +59,7 @@ func (cr *CachedReader) ReadAccountStorage(address libcommon.Address, incarnatio // ReadAccountCode is called when code of an account needs to be fetched from the state // Usually, one of (address;incarnation) or codeHash is enough to uniquely identify the code -func (cr *CachedReader) ReadAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) ([]byte, error) { +func (cr *CachedReader) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { if bytes.Equal(codeHash[:], emptyCodeHash) { return nil, nil } @@ -76,13 +76,13 @@ func (cr *CachedReader) ReadAccountCode(address libcommon.Address, incarnation u return c, nil } -func (cr *CachedReader) ReadAccountCodeSize(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) (int, error) { +func (cr *CachedReader) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { c, err := cr.ReadAccountCode(address, incarnation, codeHash) return len(c), err } // ReadAccountIncarnation is called when incarnation of the account is required (to create and recreate contract) -func (cr *CachedReader) ReadAccountIncarnation(address libcommon.Address) (uint64, error) { +func (cr *CachedReader) ReadAccountIncarnation(address common.Address) (uint64, error) { deleted := cr.cache.GetDeletedAccount(address.Bytes()) if deleted != nil { return deleted.Incarnation, nil diff --git a/core/state/cached_reader2.go b/core/state/cached_reader2.go index 867e4580be5..58e63b4620f 100644 --- a/core/state/cached_reader2.go +++ b/core/state/cached_reader2.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/binary" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" @@ -25,7 +25,7 @@ func NewCachedReader2(cache kvcache.CacheView, tx kv.Tx) *CachedReader2 { } // ReadAccountData is called when an account needs to be fetched from the state -func (r *CachedReader2) ReadAccountData(address libcommon.Address) (*accounts.Account, error) { +func (r *CachedReader2) ReadAccountData(address common.Address) (*accounts.Account, error) { enc, err := r.cache.Get(address[:]) if err != nil { return nil, err @@ -40,7 +40,7 @@ func (r *CachedReader2) ReadAccountData(address libcommon.Address) (*accounts.Ac return &a, nil } -func (r *CachedReader2) ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) { +func (r *CachedReader2) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { compositeKey := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes()) enc, err := r.cache.Get(compositeKey) if err != nil { @@ -52,7 +52,7 @@ func (r *CachedReader2) ReadAccountStorage(address libcommon.Address, incarnatio return enc, nil } -func (r *CachedReader2) ReadAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) ([]byte, error) { +func (r *CachedReader2) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { if bytes.Equal(codeHash.Bytes(), emptyCodeHash) { return nil, nil } @@ -63,12 +63,12 @@ func (r *CachedReader2) ReadAccountCode(address libcommon.Address, incarnation u return code, err } -func (r *CachedReader2) ReadAccountCodeSize(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) (int, error) { +func (r *CachedReader2) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { code, err := r.ReadAccountCode(address, incarnation, codeHash) return len(code), err } -func (r *CachedReader2) ReadAccountIncarnation(address libcommon.Address) (uint64, error) { +func (r *CachedReader2) ReadAccountIncarnation(address common.Address) (uint64, error) { b, err := r.db.GetOne(kv.IncarnationMap, address.Bytes()) if err != nil { return 0, err diff --git a/core/state/cached_writer.go b/core/state/cached_writer.go index 9aec0d635ec..1cd916ae382 100644 --- a/core/state/cached_writer.go +++ b/core/state/cached_writer.go @@ -2,7 +2,7 @@ package state import ( "github.com/holiman/uint256" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/turbo/shards" @@ -19,7 +19,7 @@ func NewCachedWriter(w WriterWithChangeSets, cache *shards.StateCache) *CachedWr return &CachedWriter{w: w, cache: cache} } -func (cw *CachedWriter) UpdateAccountData(address libcommon.Address, original, account *accounts.Account) error { +func (cw *CachedWriter) UpdateAccountData(address common.Address, original, account *accounts.Account) error { if err := cw.w.UpdateAccountData(address, original, account); err != nil { return err } @@ -27,7 +27,7 @@ func (cw *CachedWriter) UpdateAccountData(address libcommon.Address, original, a return nil } -func (cw *CachedWriter) UpdateAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash, code []byte) error { +func (cw *CachedWriter) UpdateAccountCode(address common.Address, incarnation uint64, codeHash common.Hash, code []byte) error { if err := cw.w.UpdateAccountCode(address, incarnation, codeHash, code); err != nil { return err } @@ -35,7 +35,7 @@ func (cw *CachedWriter) UpdateAccountCode(address libcommon.Address, incarnation return nil } -func (cw *CachedWriter) DeleteAccount(address libcommon.Address, original *accounts.Account) error { +func (cw *CachedWriter) DeleteAccount(address common.Address, original *accounts.Account) error { if err := cw.w.DeleteAccount(address, original); err != nil { return err } @@ -43,7 +43,7 @@ func (cw *CachedWriter) DeleteAccount(address libcommon.Address, original *accou return nil } -func (cw *CachedWriter) WriteAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash, original, value *uint256.Int) error { +func (cw *CachedWriter) WriteAccountStorage(address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error { if err := cw.w.WriteAccountStorage(address, incarnation, key, original, value); err != nil { return err } @@ -58,7 +58,7 @@ func (cw *CachedWriter) WriteAccountStorage(address libcommon.Address, incarnati return nil } -func (cw *CachedWriter) CreateContract(address libcommon.Address) error { +func (cw *CachedWriter) CreateContract(address common.Address) error { return cw.w.CreateContract(address) } diff --git a/core/state/change_set_writer.go b/core/state/change_set_writer.go index b8e4123dad9..b3da878448f 100644 --- a/core/state/change_set_writer.go +++ b/core/state/change_set_writer.go @@ -161,7 +161,7 @@ func (w *ChangeSetWriter) WriteHistory() error { if err != nil { return err } - err = writeIndex(w.blockNumber, accountChanges, kv.AccountsHistory, w.db) + err = writeIndex(w.blockNumber, accountChanges, kv.E2AccountsHistory, w.db) if err != nil { return err } @@ -170,7 +170,7 @@ func (w *ChangeSetWriter) WriteHistory() error { if err != nil { return err } - err = writeIndex(w.blockNumber, storageChanges, kv.StorageHistory, w.db) + err = writeIndex(w.blockNumber, storageChanges, kv.E2StorageHistory, w.db) if err != nil { return err } diff --git a/core/state/database.go b/core/state/database.go index 1ddb74131a3..651581ebfd3 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -19,7 +19,7 @@ package state import ( "github.com/holiman/uint256" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/core/types/accounts" ) @@ -32,19 +32,19 @@ const ( ) type StateReader interface { - ReadAccountData(address libcommon.Address) (*accounts.Account, error) - ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) - ReadAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) ([]byte, error) - ReadAccountCodeSize(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) (int, error) - ReadAccountIncarnation(address libcommon.Address) (uint64, error) + ReadAccountData(address common.Address) (*accounts.Account, error) + ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) + ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) + ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) + ReadAccountIncarnation(address common.Address) (uint64, error) } type StateWriter interface { - UpdateAccountData(address libcommon.Address, original, account *accounts.Account) error - UpdateAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash, code []byte) error - DeleteAccount(address libcommon.Address, original *accounts.Account) error - WriteAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash, original, value *uint256.Int) error - CreateContract(address libcommon.Address) error + UpdateAccountData(address common.Address, original, account *accounts.Account) error + UpdateAccountCode(address common.Address, incarnation uint64, codeHash common.Hash, code []byte) error + DeleteAccount(address common.Address, original *accounts.Account) error + WriteAccountStorage(address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error + CreateContract(address common.Address) error } type WriterWithChangeSets interface { @@ -62,23 +62,23 @@ func NewNoopWriter() *NoopWriter { return noopWriter } -func (nw *NoopWriter) UpdateAccountData(address libcommon.Address, original, account *accounts.Account) error { +func (nw *NoopWriter) UpdateAccountData(address common.Address, original, account *accounts.Account) error { return nil } -func (nw *NoopWriter) DeleteAccount(address libcommon.Address, original *accounts.Account) error { +func (nw *NoopWriter) DeleteAccount(address common.Address, original *accounts.Account) error { return nil } -func (nw *NoopWriter) UpdateAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash, code []byte) error { +func (nw *NoopWriter) UpdateAccountCode(address common.Address, incarnation uint64, codeHash common.Hash, code []byte) error { return nil } -func (nw *NoopWriter) WriteAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash, original, value *uint256.Int) error { +func (nw *NoopWriter) WriteAccountStorage(address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error { return nil } -func (nw *NoopWriter) CreateContract(address libcommon.Address) error { +func (nw *NoopWriter) CreateContract(address common.Address) error { return nil } diff --git a/core/state/database_test.go b/core/state/database_test.go index e4097330204..c90f4075bd2 100644 --- a/core/state/database_test.go +++ b/core/state/database_test.go @@ -143,7 +143,7 @@ func TestCreate2Revive(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -157,7 +157,7 @@ func TestCreate2Revive(t *testing.T) { require.NoError(t, err) // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } @@ -179,7 +179,7 @@ func TestCreate2Revive(t *testing.T) { require.NoError(t, err) // BLOCK 3 - if err = m.InsertChain(chain.Slice(2, 3)); err != nil { + if err = m.InsertChain(chain.Slice(2, 3), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -192,7 +192,7 @@ func TestCreate2Revive(t *testing.T) { require.NoError(t, err) // BLOCK 4 - if err = m.InsertChain(chain.Slice(3, 4)); err != nil { + if err = m.InsertChain(chain.Slice(3, 4), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -349,7 +349,7 @@ func TestCreate2Polymorth(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -364,7 +364,7 @@ func TestCreate2Polymorth(t *testing.T) { require.NoError(t, err) // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } @@ -384,7 +384,7 @@ func TestCreate2Polymorth(t *testing.T) { require.NoError(t, err) // BLOCK 3 - if err = m.InsertChain(chain.Slice(2, 3)); err != nil { + if err = m.InsertChain(chain.Slice(2, 3), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -397,7 +397,7 @@ func TestCreate2Polymorth(t *testing.T) { require.NoError(t, err) // BLOCK 4 - if err = m.InsertChain(chain.Slice(3, 4)); err != nil { + if err = m.InsertChain(chain.Slice(3, 4), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -417,7 +417,7 @@ func TestCreate2Polymorth(t *testing.T) { require.NoError(t, err) // BLOCK 5 - if err = m.InsertChain(chain.Slice(4, 5)); err != nil { + if err = m.InsertChain(chain.Slice(4, 5), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -534,7 +534,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { }) require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -553,7 +553,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { require.NoError(t, err) // BLOCKS 2 + 3 - if err = m.InsertChain(chain.Slice(1, chain.Length())); err != nil { + if err = m.InsertChain(chain.Slice(1, chain.Length()), nil); err != nil { t.Fatal(err) } @@ -567,7 +567,7 @@ func TestReorgOverSelfDestruct(t *testing.T) { require.NoError(t, err) // REORG of block 2 and 3, and insert new (empty) BLOCK 2, 3, and 4 - if err = m.InsertChain(longerChain.Slice(1, 4)); err != nil { + if err = m.InsertChain(longerChain.Slice(1, 4), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -675,7 +675,7 @@ func TestReorgOverStateChange(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -694,12 +694,12 @@ func TestReorgOverStateChange(t *testing.T) { require.NoError(t, err) // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, chain.Length())); err != nil { + if err = m.InsertChain(chain.Slice(1, chain.Length()), nil); err != nil { t.Fatal(err) } // REORG of block 2 and 3, and insert new (empty) BLOCK 2, 3, and 4 - if err = m.InsertChain(longerChain.Slice(1, 3)); err != nil { + if err = m.InsertChain(longerChain.Slice(1, 3), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -802,7 +802,7 @@ func TestCreateOnExistingStorage(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -938,7 +938,7 @@ func TestEip2200Gas(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -1024,7 +1024,7 @@ func TestWrongIncarnation(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -1051,7 +1051,7 @@ func TestWrongIncarnation(t *testing.T) { require.NoError(t, err) // BLOCKS 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -1169,12 +1169,12 @@ func TestWrongIncarnation2(t *testing.T) { require.NoError(t, err) // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } // BLOCKS 2 - if err = m.InsertChain(chain.Slice(1, chain.Length())); err != nil { + if err = m.InsertChain(chain.Slice(1, chain.Length()), nil); err != nil { t.Fatal(err) } @@ -1199,7 +1199,7 @@ func TestWrongIncarnation2(t *testing.T) { }) require.NoError(t, err) // REORG of block 2 and 3, and insert new (empty) BLOCK 2, 3, and 4 - if err = m.InsertChain(longerChain.Slice(1, longerChain.Length())); err != nil { + if err = m.InsertChain(longerChain.Slice(1, longerChain.Length()), nil); err != nil { t.Fatal(err) } @@ -1472,7 +1472,7 @@ func TestRecreateAndRewind(t *testing.T) { } // BLOCKS 1 and 2 - if err = m.InsertChain(chain.Slice(0, 2)); err != nil { + if err = m.InsertChain(chain.Slice(0, 2), nil); err != nil { t.Fatal(err) } @@ -1493,7 +1493,7 @@ func TestRecreateAndRewind(t *testing.T) { require.NoError(t, err) // Block 3 and 4 - if err = m.InsertChain(chain.Slice(2, chain.Length())); err != nil { + if err = m.InsertChain(chain.Slice(2, chain.Length()), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -1512,7 +1512,7 @@ func TestRecreateAndRewind(t *testing.T) { require.NoError(t, err) // Reorg - if err = m.InsertChain(longerChain); err != nil { + if err = m.InsertChain(longerChain, nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -1572,10 +1572,10 @@ func TestTxLookupUnwind(t *testing.T) { if err != nil { t.Fatal(err) } - if err = m.InsertChain(chain1); err != nil { + if err = m.InsertChain(chain1, nil); err != nil { t.Fatal(err) } - if err = m.InsertChain(chain2); err != nil { + if err = m.InsertChain(chain2, nil); err != nil { t.Fatal(err) } var count uint64 diff --git a/core/state/db_state_writer.go b/core/state/db_state_writer.go index 4e2897982c9..0853e902062 100644 --- a/core/state/db_state_writer.go +++ b/core/state/db_state_writer.go @@ -161,7 +161,7 @@ func (dsw *DbStateWriter) WriteHistory() error { if err != nil { return err } - err = writeIndex(dsw.blockNr, accountChanges, kv.AccountsHistory, dsw.db.(ethdb.HasTx).Tx().(kv.RwTx)) + err = writeIndex(dsw.blockNr, accountChanges, kv.E2AccountsHistory, dsw.db.(ethdb.HasTx).Tx().(kv.RwTx)) if err != nil { return err } @@ -170,7 +170,7 @@ func (dsw *DbStateWriter) WriteHistory() error { if err != nil { return err } - err = writeIndex(dsw.blockNr, storageChanges, kv.StorageHistory, dsw.db.(ethdb.HasTx).Tx().(kv.RwTx)) + err = writeIndex(dsw.blockNr, storageChanges, kv.E2StorageHistory, dsw.db.(ethdb.HasTx).Tx().(kv.RwTx)) if err != nil { return err } diff --git a/core/state/dump.go b/core/state/dump.go index 62efa87d10b..d217b1442a7 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -29,7 +29,6 @@ import ( "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/dbutils" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/turbo/trie" @@ -163,7 +162,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo return nil, err } - it, err := ttx.DomainRange(temporal.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults+1) + it, err := ttx.DomainRange(kv.AccountsDomain, startAddress[:], nil, txNum, order.Asc, maxResults+1) if err != nil { return nil, err } @@ -172,10 +171,6 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo if err != nil { return nil, err } - //TODO: what to do in this case? maybe iterator must skip this values?? - if len(v) == 0 { - continue - } if maxResults > 0 && numberOfResults >= maxResults { if nextKey == nil { nextKey = make([]byte, len(k)) @@ -183,6 +178,9 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo copy(nextKey, k) break } + if len(v) == 0 { + continue + } if e := acc.DecodeForStorage(v); e != nil { return nil, fmt.Errorf("decoding %x for %x: %w", v, k, e) @@ -261,7 +259,7 @@ func (d *Dumper) DumpToCollector(c DumpCollector, excludeCode, excludeStorage bo if !excludeStorage { t := trie.New(libcommon.Hash{}) if d.historyV3 { - r, err := d.db.(kv.TemporalTx).DomainRange(temporal.StorageDomain, addr[:], nil, txNumForStorage, order.Asc, kv.Unlim) + r, err := d.db.(kv.TemporalTx).DomainRange(kv.StorageDomain, addr[:], nil, txNumForStorage, order.Asc, kv.Unlim) if err != nil { return nil, fmt.Errorf("walking over storage for %x: %w", addr, err) } diff --git a/core/state/history_reader_inc.go b/core/state/history_reader_inc.go index 035b219e33f..f3021a79cf5 100644 --- a/core/state/history_reader_inc.go +++ b/core/state/history_reader_inc.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "fmt" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" libstate "github.com/ledgerwatch/erigon-lib/state" @@ -51,7 +51,7 @@ func (hr *HistoryReaderInc) SetTrace(trace bool) { hr.trace = trace } -func (hr *HistoryReaderInc) ReadAccountData(address libcommon.Address) (*accounts.Account, error) { +func (hr *HistoryReaderInc) ReadAccountData(address common.Address) (*accounts.Account, error) { addr := address.Bytes() enc, noState, stateTxNum := hr.as.ReadAccountDataNoState(addr, hr.txNum) if hr.trace { @@ -116,7 +116,7 @@ func (hr *HistoryReaderInc) ReadAccountData(address libcommon.Address) (*account return &a, nil } -func (hr *HistoryReaderInc) ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) { +func (hr *HistoryReaderInc) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { addr, k := address.Bytes(), key.Bytes() var err error enc, noState, stateTxNum := hr.as.ReadAccountStorageNoState(addr, k, hr.txNum) @@ -174,7 +174,7 @@ func (hr *HistoryReaderInc) ReadAccountStorage(address libcommon.Address, incarn return enc, nil } -func (hr *HistoryReaderInc) ReadAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) ([]byte, error) { +func (hr *HistoryReaderInc) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { addr := address.Bytes() enc, noState, stateTxNum := hr.as.ReadAccountCodeNoState(addr, hr.txNum) var err error @@ -213,7 +213,7 @@ func (hr *HistoryReaderInc) ReadAccountCode(address libcommon.Address, incarnati return enc, nil } -func (hr *HistoryReaderInc) ReadAccountCodeSize(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) (int, error) { +func (hr *HistoryReaderInc) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { addr := address.Bytes() size, noState, stateTxNum := hr.as.ReadAccountCodeSizeNoState(addr, hr.txNum) if hr.trace { @@ -260,7 +260,7 @@ func (hr *HistoryReaderInc) ReadAccountCodeSize(address libcommon.Address, incar return size, nil } -func (hr *HistoryReaderInc) ReadAccountIncarnation(address libcommon.Address) (uint64, error) { +func (hr *HistoryReaderInc) ReadAccountIncarnation(address common.Address) (uint64, error) { return 0, nil } diff --git a/core/state/history_reader_v3.go b/core/state/history_reader_v3.go index 0de590599ad..855d46d6d84 100644 --- a/core/state/history_reader_v3.go +++ b/core/state/history_reader_v3.go @@ -4,9 +4,8 @@ import ( "encoding/binary" "fmt" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/eth/ethconfig" ) @@ -32,8 +31,8 @@ func (hr *HistoryReaderV3) SetTx(tx kv.Tx) { func (hr *HistoryReaderV3) SetTxNum(txNum uint64) { hr.txNum = txNum } func (hr *HistoryReaderV3) SetTrace(trace bool) { hr.trace = trace } -func (hr *HistoryReaderV3) ReadAccountData(address libcommon.Address) (*accounts.Account, error) { - enc, ok, err := hr.ttx.DomainGetAsOf(temporal.AccountsDomain, address.Bytes(), nil, hr.txNum) +func (hr *HistoryReaderV3) ReadAccountData(address common.Address) (*accounts.Account, error) { + enc, ok, err := hr.ttx.DomainGetAsOf(kv.AccountsDomain, address.Bytes(), nil, hr.txNum) if err != nil || !ok || len(enc) == 0 { if hr.trace { fmt.Printf("ReadAccountData [%x] => []\n", address) @@ -50,7 +49,7 @@ func (hr *HistoryReaderV3) ReadAccountData(address libcommon.Address) (*accounts return &a, nil } -func (hr *HistoryReaderV3) ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) { +func (hr *HistoryReaderV3) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { var acc []byte if ethconfig.EnableHistoryV4InTest { acc = address.Bytes() @@ -59,31 +58,31 @@ func (hr *HistoryReaderV3) ReadAccountStorage(address libcommon.Address, incarna copy(acc, address.Bytes()) binary.BigEndian.PutUint64(acc[20:], incarnation) } - enc, _, err := hr.ttx.DomainGetAsOf(temporal.StorageDomain, acc, key.Bytes(), hr.txNum) + enc, _, err := hr.ttx.DomainGetAsOf(kv.StorageDomain, acc, key.Bytes(), hr.txNum) if hr.trace { fmt.Printf("ReadAccountStorage [%x] [%x] => [%x]\n", address, *key, enc) } return enc, err } -func (hr *HistoryReaderV3) ReadAccountCode(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) ([]byte, error) { +func (hr *HistoryReaderV3) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { if codeHash == emptyCodeHashH { return nil, nil } - code, _, err := hr.ttx.DomainGetAsOf(temporal.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) + code, _, err := hr.ttx.DomainGetAsOf(kv.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) if hr.trace { fmt.Printf("ReadAccountCode [%x %x] => [%x]\n", address, codeHash, code) } return code, err } -func (hr *HistoryReaderV3) ReadAccountCodeSize(address libcommon.Address, incarnation uint64, codeHash libcommon.Hash) (int, error) { - enc, _, err := hr.ttx.DomainGetAsOf(temporal.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) +func (hr *HistoryReaderV3) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { + enc, _, err := hr.ttx.DomainGetAsOf(kv.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) return len(enc), err } -func (hr *HistoryReaderV3) ReadAccountIncarnation(address libcommon.Address) (uint64, error) { - enc, ok, err := hr.ttx.DomainGetAsOf(temporal.AccountsDomain, address.Bytes(), nil, hr.txNum) +func (hr *HistoryReaderV3) ReadAccountIncarnation(address common.Address) (uint64, error) { + enc, ok, err := hr.ttx.DomainGetAsOf(kv.AccountsDomain, address.Bytes(), nil, hr.txNum) if err != nil || !ok || len(enc) == 0 { if hr.trace { fmt.Printf("ReadAccountIncarnation [%x] => [0]\n", address) @@ -107,7 +106,7 @@ func (hr *HistoryReaderV3) ReadAccountIncarnation(address libcommon.Address) (ui } /* -func (s *HistoryReaderV3) ForEachStorage(addr libcommon.Address, startLocation libcommon.Hash, cb func(key, seckey libcommon.Hash, value uint256.Int) bool, maxResults int) error { +func (s *HistoryReaderV3) ForEachStorage(addr common.Address, startLocation common.Hash, cb func(key, seckey common.Hash, value uint256.Int) bool, maxResults int) error { acc, err := s.ReadAccountData(addr) if err != nil { return err @@ -144,7 +143,7 @@ func (s *HistoryReaderV3) ForEachStorage(addr libcommon.Address, startLocation l } numDeletes := st.Len() - overrideCounter - var lastKey libcommon.Hash + var lastKey common.Hash iterator := s.ac.IterateStorageHistory(startLocation.Bytes(), nil, s.txNum) for iterator.HasNext() { k, vs, p := iterator.Next() @@ -192,7 +191,7 @@ func (s *HistoryReaderV3) ForEachStorage(addr libcommon.Address, startLocation l */ /* -func (s *PlainState) ForEachStorage(addr libcommon.Address, startLocation libcommon.Hash, cb func(key, seckey libcommon.Hash, value uint256.Int) bool, maxResults int) error { +func (s *PlainState) ForEachStorage(addr common.Address, startLocation common.Hash, cb func(key, seckey common.Hash, value uint256.Int) bool, maxResults int) error { st := btree.New(16) var k [length.Addr + length.Incarnation + length.Hash]byte copy(k[:], addr[:]) @@ -207,7 +206,7 @@ func (s *PlainState) ForEachStorage(addr libcommon.Address, startLocation libcom } binary.BigEndian.PutUint64(k[length.Addr:], acc.Incarnation) copy(k[length.Addr+length.Incarnation:], startLocation[:]) - var lastKey libcommon.Hash + var lastKey common.Hash overrideCounter := 0 min := &storageItem{key: startLocation} if t, ok := s.storage[addr]; ok { diff --git a/core/state/history_test.go b/core/state/history_test.go index a1a545fb1af..ab243534bba 100644 --- a/core/state/history_test.go +++ b/core/state/history_test.go @@ -60,7 +60,7 @@ func TestMutationDeleteTimestamp(t *testing.T) { t.FailNow() } - index, err := bitmapdb.Get64(tx, kv.AccountsHistory, addr[0].Bytes(), 0, math.MaxUint32) + index, err := bitmapdb.Get64(tx, kv.E2AccountsHistory, addr[0].Bytes(), 0, math.MaxUint32) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestMutationDeleteTimestamp(t *testing.T) { t.Fatal("changeset must be deleted") } - found, err := tx.GetOne(kv.AccountsHistory, addr[0].Bytes()) + found, err := tx.GetOne(kv.E2AccountsHistory, addr[0].Bytes()) require.NoError(t, err) require.Nil(t, found, "account must be deleted") } @@ -110,7 +110,7 @@ func TestMutationCommit(t *testing.T) { t.Fatal("Accounts not equals") } - index, err := bitmapdb.Get64(tx, kv.AccountsHistory, addr.Bytes(), 0, math.MaxUint32) + index, err := bitmapdb.Get64(tx, kv.E2AccountsHistory, addr.Bytes(), 0, math.MaxUint32) if err != nil { t.Fatal(err) } @@ -136,9 +136,9 @@ func TestMutationCommit(t *testing.T) { } for k, v := range accHistoryStateStorage[i] { - c1, _ := tx.Cursor(kv.StorageHistory) + c1, _ := tx.Cursor(kv.E2StorageHistory) c2, _ := tx.CursorDupSort(kv.StorageChangeSet) - res, err := historyv2read.GetAsOf(tx, c1, c2, true /* storage */, dbutils.PlainGenerateCompositeStorageKey(addr.Bytes(), acc.Incarnation, k.Bytes()), 1) + res, _, err := historyv2read.GetAsOf(tx, c1, c2, true /* storage */, dbutils.PlainGenerateCompositeStorageKey(addr.Bytes(), acc.Incarnation, k.Bytes()), 1) if err != nil { t.Fatal(err) } diff --git a/core/state/history_walk.go b/core/state/history_walk.go index 782ac1744f6..f14446f122a 100644 --- a/core/state/history_walk.go +++ b/core/state/history_walk.go @@ -43,7 +43,7 @@ func WalkAsOfStorage(tx kv.Tx, address libcommon.Address, incarnation uint64, st ) //for historic data - shCursor, err := tx.Cursor(kv.StorageHistory) + shCursor, err := tx.Cursor(kv.E2StorageHistory) if err != nil { return err } @@ -150,7 +150,7 @@ func WalkAsOfAccounts(tx kv.Tx, startAddress libcommon.Address, timestamp uint64 return err } defer mainCursor.Close() - ahCursor, err := tx.Cursor(kv.AccountsHistory) + ahCursor, err := tx.Cursor(kv.E2AccountsHistory) if err != nil { return err } diff --git a/core/state/historyv2read/history.go b/core/state/historyv2read/history.go index c07ac0261db..b33680397ae 100644 --- a/core/state/historyv2read/history.go +++ b/core/state/historyv2read/history.go @@ -19,10 +19,13 @@ func RestoreCodeHash(tx kv.Getter, key, v []byte, force *libcommon.Hash) ([]byte } if force != nil { acc.CodeHash = *force - } else if acc.Incarnation > 0 && acc.IsEmptyCodeHash() { + v = make([]byte, acc.EncodingLengthForStorage()) + acc.EncodeForStorage(v) + return v, nil + } + if acc.Incarnation > 0 && acc.IsEmptyCodeHash() { var codeHash []byte var err error - prefix := make([]byte, length.Addr+length.BlockNum) copy(prefix, key) binary.BigEndian.PutUint64(prefix[length.Addr:], acc.Incarnation) @@ -33,29 +36,21 @@ func RestoreCodeHash(tx kv.Getter, key, v []byte, force *libcommon.Hash) ([]byte } if len(codeHash) > 0 { acc.CodeHash.SetBytes(codeHash) + v = make([]byte, acc.EncodingLengthForStorage()) + acc.EncodeForStorage(v) } - v = make([]byte, acc.EncodingLengthForStorage()) - acc.EncodeForStorage(v) } return v, nil } -func GetAsOf(tx kv.Tx, indexC kv.Cursor, changesC kv.CursorDupSort, storage bool, key []byte, timestamp uint64) ([]byte, error) { +func GetAsOf(tx kv.Tx, indexC kv.Cursor, changesC kv.CursorDupSort, storage bool, key []byte, timestamp uint64) (v []byte, fromHistory bool, err error) { v, ok, err := historyv2.FindByHistory(indexC, changesC, storage, key, timestamp) if err != nil { - return nil, err + return nil, true, err } if ok { - //restore codehash - if !storage { - //restore codehash - v, err = RestoreCodeHash(tx, key, v, nil) - if err != nil { - return nil, err - } - } - - return v, nil + return v, true, nil } - return tx.GetOne(kv.PlainState, key) + v, err = tx.GetOne(kv.PlainState, key) + return v, false, err } diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index bd57ece5c95..e331b657a71 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" types2 "github.com/ledgerwatch/erigon-lib/types" diff --git a/core/state/plain_readonly.go b/core/state/plain_readonly.go index dea77d8345d..433ffeccd25 100644 --- a/core/state/plain_readonly.go +++ b/core/state/plain_readonly.go @@ -69,8 +69,8 @@ func NewPlainState(tx kv.Tx, blockNr uint64, systemContractLookup map[libcommon. systemContractLookup: systemContractLookup, } - c1, _ := tx.Cursor(kv.AccountsHistory) - c2, _ := tx.Cursor(kv.StorageHistory) + c1, _ := tx.Cursor(kv.E2AccountsHistory) + c2, _ := tx.Cursor(kv.E2StorageHistory) c3, _ := tx.CursorDupSort(kv.AccountChangeSet) c4, _ := tx.CursorDupSort(kv.StorageChangeSet) @@ -97,7 +97,7 @@ func (s *PlainState) ForEachStorage(addr libcommon.Address, startLocation libcom st := btree.New(16) var k [length.Addr + length.Incarnation + length.Hash]byte copy(k[:], addr[:]) - accData, err := historyv2read.GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false /* storage */, addr[:], s.blockNr) + accData, _, err := historyv2read.GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false /* storage */, addr[:], s.blockNr) if err != nil { return err } @@ -170,7 +170,7 @@ func (s *PlainState) ForEachStorage(addr libcommon.Address, startLocation libcom } func (s *PlainState) ReadAccountData(address libcommon.Address) (*accounts.Account, error) { - enc, err := historyv2read.GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false /* storage */, address[:], s.blockNr) + enc, fromHistory, err := historyv2read.GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false /* storage */, address[:], s.blockNr) if err != nil { return nil, err } @@ -184,19 +184,21 @@ func (s *PlainState) ReadAccountData(address libcommon.Address) (*accounts.Accou if err = a.DecodeForStorage(enc); err != nil { return nil, err } - //restore codehash - if records, ok := s.systemContractLookup[address]; ok { - p := sort.Search(len(records), func(i int) bool { - return records[i].BlockNumber > s.blockNr - }) - a.CodeHash = records[p-1].CodeHash - } else if a.Incarnation > 0 && a.IsEmptyCodeHash() { - if codeHash, err1 := s.tx.GetOne(kv.PlainContractCode, dbutils.PlainGenerateStoragePrefix(address[:], a.Incarnation)); err1 == nil { - if len(codeHash) > 0 { - a.CodeHash = libcommon.BytesToHash(codeHash) + if fromHistory { + //restore codehash + if records, ok := s.systemContractLookup[address]; ok { + p := sort.Search(len(records), func(i int) bool { + return records[i].BlockNumber > s.blockNr + }) + a.CodeHash = records[p-1].CodeHash + } else if a.Incarnation > 0 && a.IsEmptyCodeHash() { + if codeHash, err1 := s.tx.GetOne(kv.PlainContractCode, dbutils.PlainGenerateStoragePrefix(address[:], a.Incarnation)); err1 == nil { + if len(codeHash) > 0 { + a.CodeHash = libcommon.BytesToHash(codeHash) + } + } else { + return nil, err1 } - } else { - return nil, err1 } } if s.trace { @@ -207,7 +209,7 @@ func (s *PlainState) ReadAccountData(address libcommon.Address) (*accounts.Accou func (s *PlainState) ReadAccountStorage(address libcommon.Address, incarnation uint64, key *libcommon.Hash) ([]byte, error) { compositeKey := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes()) - enc, err := historyv2read.GetAsOf(s.tx, s.storageHistoryC, s.storageChangesC, true /* storage */, compositeKey, s.blockNr) + enc, _, err := historyv2read.GetAsOf(s.tx, s.storageHistoryC, s.storageChangesC, true /* storage */, compositeKey, s.blockNr) if err != nil { return nil, err } @@ -243,7 +245,7 @@ func (s *PlainState) ReadAccountCodeSize(address libcommon.Address, incarnation } func (s *PlainState) ReadAccountIncarnation(address libcommon.Address) (uint64, error) { - enc, err := historyv2read.GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false /* storage */, address[:], s.blockNr+1) + enc, _, err := historyv2read.GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false /* storage */, address[:], s.blockNr+1) if err != nil { return 0, err } @@ -253,20 +255,20 @@ func (s *PlainState) ReadAccountIncarnation(address libcommon.Address) (uint64, } return 0, nil } - var acc accounts.Account - if err = acc.DecodeForStorage(enc); err != nil { + inc, err := accounts.DecodeIncarnationFromStorage(enc) + if err != nil { return 0, err } - if acc.Incarnation == 0 { + if inc == 0 { if s.trace { fmt.Printf("ReadAccountIncarnation [%x] => [%d]\n", address, 0) } return 0, nil } if s.trace { - fmt.Printf("ReadAccountIncarnation [%x] => [%d]\n", address, acc.Incarnation-1) + fmt.Printf("ReadAccountIncarnation [%x] => [%d]\n", address, inc-1) } - return acc.Incarnation - 1, nil + return inc - 1, nil } func (s *PlainState) UpdateAccountData(address libcommon.Address, original, account *accounts.Account) error { diff --git a/core/state/plain_state_writer.go b/core/state/plain_state_writer.go index 96eae0b06bf..5bb88315b06 100644 --- a/core/state/plain_state_writer.go +++ b/core/state/plain_state_writer.go @@ -130,6 +130,7 @@ func (w *PlainStateWriter) CreateContract(address libcommon.Address) error { func (w *PlainStateWriter) WriteChangeSets() error { if w.csw != nil { + return w.csw.WriteChangeSets() } diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index ed70c78f542..925f41137ae 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -483,20 +483,20 @@ func (rs *StateV3) ApplyHistory(txTask *exec22.TxTask, agg *libstate.AggregatorV } if txTask.TraceFroms != nil { for addr := range txTask.TraceFroms { - if err := agg.PutIdx(kv.TracesFromIdx, addr[:]); err != nil { + if err := agg.PutIdx(kv.TblTracesFromIdx, addr[:]); err != nil { return err } } } if txTask.TraceTos != nil { for addr := range txTask.TraceTos { - if err := agg.PutIdx(kv.TracesToIdx, addr[:]); err != nil { + if err := agg.PutIdx(kv.TblTracesToIdx, addr[:]); err != nil { return err } } } for _, log := range txTask.Logs { - if err := agg.PutIdx(kv.LogAddressIdx, log.Address[:]); err != nil { + if err := agg.PutIdx(kv.TblLogAddressIdx, log.Address[:]); err != nil { return err } for _, topic := range log.Topics { diff --git a/core/state/state_test.go b/core/state/state_test.go index ba9fa3244ad..7a81456b31f 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -23,7 +23,7 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/chain" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/memdb" @@ -33,7 +33,7 @@ import ( "github.com/ledgerwatch/erigon/crypto" ) -var toAddr = libcommon.BytesToAddress +var toAddr = common.BytesToAddress type StateSuite struct { kv kv.RwDB @@ -125,12 +125,12 @@ func (s *StateSuite) TearDownTest(c *checker.C) { } func (s *StateSuite) TestNull(c *checker.C) { - address := libcommon.HexToAddress("0x823140710bf13990e4500136726d8b55") + address := common.HexToAddress("0x823140710bf13990e4500136726d8b55") s.state.CreateAccount(address, true) //value := common.FromHex("0x823140710bf13990e4500136726d8b55") var value uint256.Int - s.state.SetState(address, &libcommon.Hash{}, value) + s.state.SetState(address, &common.Hash{}, value) err := s.state.FinalizeTx(&chain.Rules{}, s.w) c.Check(err, checker.IsNil) @@ -138,14 +138,14 @@ func (s *StateSuite) TestNull(c *checker.C) { err = s.state.CommitBlock(&chain.Rules{}, s.w) c.Check(err, checker.IsNil) - s.state.GetCommittedState(address, &libcommon.Hash{}, &value) + s.state.GetCommittedState(address, &common.Hash{}, &value) if !value.IsZero() { c.Errorf("expected empty hash. got %x", value) } } func (s *StateSuite) TestTouchDelete(c *checker.C) { - s.state.GetOrNewStateObject(libcommon.Address{}) + s.state.GetOrNewStateObject(common.Address{}) err := s.state.FinalizeTx(&chain.Rules{}, s.w) if err != nil { @@ -160,7 +160,7 @@ func (s *StateSuite) TestTouchDelete(c *checker.C) { s.state.Reset() snapshot := s.state.Snapshot() - s.state.AddBalance(libcommon.Address{}, new(uint256.Int)) + s.state.AddBalance(common.Address{}, new(uint256.Int)) if len(s.state.journal.dirties) != 1 { c.Fatal("expected one dirty state object") @@ -173,7 +173,7 @@ func (s *StateSuite) TestTouchDelete(c *checker.C) { func (s *StateSuite) TestSnapshot(c *checker.C) { stateobjaddr := toAddr([]byte("aa")) - var storageaddr libcommon.Hash + var storageaddr common.Hash data1 := uint256.NewInt(42) data2 := uint256.NewInt(43) @@ -192,14 +192,14 @@ func (s *StateSuite) TestSnapshot(c *checker.C) { s.state.GetState(stateobjaddr, &storageaddr, &value) c.Assert(value, checker.DeepEquals, data1) s.state.GetCommittedState(stateobjaddr, &storageaddr, &value) - c.Assert(value, checker.DeepEquals, libcommon.Hash{}) + c.Assert(value, checker.DeepEquals, common.Hash{}) // revert up to the genesis state and ensure correct content s.state.RevertToSnapshot(genesis) s.state.GetState(stateobjaddr, &storageaddr, &value) - c.Assert(value, checker.DeepEquals, libcommon.Hash{}) + c.Assert(value, checker.DeepEquals, common.Hash{}) s.state.GetCommittedState(stateobjaddr, &storageaddr, &value) - c.Assert(value, checker.DeepEquals, libcommon.Hash{}) + c.Assert(value, checker.DeepEquals, common.Hash{}) } func (s *StateSuite) TestSnapshotEmpty(c *checker.C) { @@ -215,7 +215,7 @@ func TestSnapshot2(t *testing.T) { stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) - var storageaddr libcommon.Hash + var storageaddr common.Hash data0 := uint256.NewInt(17) data1 := uint256.NewInt(18) diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index 56398a3308d..935a4b21025 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -4,7 +4,6 @@ import ( "context" "encoding/binary" "fmt" - "sort" "testing" "github.com/ledgerwatch/erigon-lib/common" @@ -213,35 +212,12 @@ func (tx *Tx) Commit() error { return tx.MdbxTx.Commit() } -const ( - AccountsDomain kv.Domain = "AccountsDomain" - StorageDomain kv.Domain = "StorageDomain" - CodeDomain kv.Domain = "CodeDomain" -) - -const ( - AccountsHistory kv.History = "AccountsHistory" - StorageHistory kv.History = "StorageHistory" - CodeHistory kv.History = "CodeHistory" -) - -const ( - AccountsHistoryIdx kv.InvertedIdx = "AccountsHistoryIdx" - StorageHistoryIdx kv.InvertedIdx = "StorageHistoryIdx" - CodeHistoryIdx kv.InvertedIdx = "CodeHistoryIdx" - - LogTopicIdx kv.InvertedIdx = "LogTopicIdx" - LogAddrIdx kv.InvertedIdx = "LogAddrIdx" - TracesFromIdx kv.InvertedIdx = "TracesFromIdx" - TracesToIdx kv.InvertedIdx = "TracesToIdx" -) - func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, asc order.By, limit int) (it iter.KV, err error) { if asc == order.Desc { panic("not supported yet") } switch name { - case AccountsDomain: + case kv.AccountsDomain: histStateIt := tx.aggCtx.AccountHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx) // TODO: somehow avoid common.Copy(k) - WalkAsOfIter is not zero-copy // Is histStateIt possible to increase keys lifetime to: 2 .Next() calls?? @@ -253,21 +229,23 @@ func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, if err != nil { return nil, nil, err } - var force *common.Hash - if tx.db.systemContractLookup != nil { - if records, ok := tx.db.systemContractLookup[common.BytesToAddress(k)]; ok { - p := sort.Search(len(records), func(i int) bool { - return records[i].TxNumber > asOfTs - }) - hash := records[p-1].CodeHash - force = &hash + /* + var force *common.Hash + if tx.db.systemContractLookup != nil { + if records, ok := tx.db.systemContractLookup[common.BytesToAddress(k)]; ok { + p := sort.Search(len(records), func(i int) bool { + return records[i].TxNumber > asOfTs + }) + hash := records[p-1].CodeHash + force = &hash + } } - } - v, err = tx.db.restoreCodeHash(tx.MdbxTx, k, v, force) - if err != nil { - return nil, nil, err - } - return k[:20], v, nil + v, err = tx.db.restoreCodeHash(tx.MdbxTx, k, v, force) + if err != nil { + return nil, nil, err + } + */ + return k[:20], common.Copy(v), nil }) lastestStateIt, err := tx.RangeAscend(kv.PlainState, fromKey, toKey, -1) // don't apply limit, because need filter if err != nil { @@ -278,7 +256,7 @@ func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, return len(k) == 20 }) it = iter.UnionKV(histStateIt2, latestStateIt2, limit) - case StorageDomain: + case kv.StorageDomain: storageIt := tx.aggCtx.StorageHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx) storageIt1 := iter.TransformKV(storageIt, func(k, v []byte) ([]byte, []byte, error) { return k, v, nil @@ -309,7 +287,7 @@ func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, return append(append([]byte{}, k[:20]...), k[28:]...), v, nil }) it = iter.UnionKV(storageIt1, it3, limit) - case CodeDomain: + case kv.CodeDomain: panic("not implemented yet") default: panic(fmt.Sprintf("unexpected: %s", name)) @@ -326,13 +304,13 @@ func (tx *Tx) DomainGet(name kv.Domain, key, key2 []byte) (v []byte, ok bool, er panic("implement me") } switch name { - case AccountsDomain: + case kv.AccountsDomain: v, err = tx.GetOne(kv.PlainState, key) return v, v != nil, err - case StorageDomain: + case kv.StorageDomain: v, err = tx.GetOne(kv.PlainState, append(common.Copy(key), key2...)) return v, v != nil, err - case CodeDomain: + case kv.CodeDomain: v, err = tx.GetOne(kv.Code, key2) return v, v != nil, err default: @@ -344,8 +322,8 @@ func (tx *Tx) DomainGetAsOf(name kv.Domain, key, key2 []byte, ts uint64) (v []by panic("implement me") } switch name { - case AccountsDomain: - v, ok, err = tx.HistoryGet(AccountsHistory, key, ts) + case kv.AccountsDomain: + v, ok, err = tx.HistoryGet(kv.AccountsHistory, key, ts) if err != nil { return nil, false, err } @@ -360,8 +338,8 @@ func (tx *Tx) DomainGetAsOf(name kv.Domain, key, key2 []byte, ts uint64) (v []by } } return v, v != nil, err - case StorageDomain: - v, ok, err = tx.HistoryGet(StorageHistory, append(key[:20], key2...), ts) + case kv.StorageDomain: + v, ok, err = tx.HistoryGet(kv.StorageHistory, append(key[:20], key2...), ts) if err != nil { return nil, false, err } @@ -370,8 +348,8 @@ func (tx *Tx) DomainGetAsOf(name kv.Domain, key, key2 []byte, ts uint64) (v []by } v, err = tx.GetOne(kv.PlainState, append(key, key2...)) return v, v != nil, err - case CodeDomain: - v, ok, err = tx.HistoryGet(CodeHistory, key, ts) + case kv.CodeDomain: + v, ok, err = tx.HistoryGet(kv.CodeHistory, key, ts) if err != nil { return nil, false, err } @@ -387,7 +365,7 @@ func (tx *Tx) DomainGetAsOf(name kv.Domain, key, key2 []byte, ts uint64) (v []by func (tx *Tx) HistoryGet(name kv.History, key []byte, ts uint64) (v []byte, ok bool, err error) { switch name { - case AccountsHistory: + case kv.AccountsHistory: v, ok, err = tx.aggCtx.ReadAccountDataNoStateWithRecent(key, ts, tx.MdbxTx) if err != nil { return nil, false, err @@ -395,34 +373,36 @@ func (tx *Tx) HistoryGet(name kv.History, key []byte, ts uint64) (v []byte, ok b if !ok || len(v) == 0 { return v, ok, nil } - v, err = tx.db.convertV3toV2(v) - if err != nil { - return nil, false, err - } - var force *common.Hash - if tx.db.systemContractLookup != nil { - if records, ok := tx.db.systemContractLookup[common.BytesToAddress(key)]; ok { - p := sort.Search(len(records), func(i int) bool { - return records[i].TxNumber > ts - }) - hash := records[p-1].CodeHash - force = &hash + /* + v, err = tx.db.convertV3toV2(v) + if err != nil { + return nil, false, err } - } - v, err = tx.db.restoreCodeHash(tx.MdbxTx, key, v, force) - if err != nil { - return nil, false, err - } - if len(v) > 0 { - v, err = tx.db.convertV2toV3(v) + var force *common.Hash + if tx.db.systemContractLookup != nil { + if records, ok := tx.db.systemContractLookup[common.BytesToAddress(key)]; ok { + p := sort.Search(len(records), func(i int) bool { + return records[i].TxNumber > ts + }) + hash := records[p-1].CodeHash + force = &hash + } + } + v, err = tx.db.restoreCodeHash(tx.MdbxTx, key, v, force) if err != nil { return nil, false, err } - } + if len(v) > 0 { + v, err = tx.db.convertV2toV3(v) + if err != nil { + return nil, false, err + } + } + */ return v, true, nil - case StorageHistory: + case kv.StorageHistory: return tx.aggCtx.ReadAccountStorageNoStateWithRecent2(key, ts, tx.MdbxTx) - case CodeHistory: + case kv.CodeHistory: return tx.aggCtx.ReadAccountCodeNoStateWithRecent(key, ts, tx.MdbxTx) default: panic(fmt.Sprintf("unexpected: %s", name)) @@ -431,19 +411,19 @@ func (tx *Tx) HistoryGet(name kv.History, key []byte, ts uint64) (v []byte, ok b func (tx *Tx) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int) (timestamps iter.U64, err error) { switch name { - case AccountsHistoryIdx: + case kv.AccountsHistoryIdx: timestamps, err = tx.aggCtx.AccountHistoyIdxRange(k, fromTs, toTs, asc, limit, tx) - case StorageHistoryIdx: + case kv.StorageHistoryIdx: timestamps, err = tx.aggCtx.StorageHistoyIdxRange(k, fromTs, toTs, asc, limit, tx) - case CodeHistoryIdx: + case kv.CodeHistoryIdx: timestamps, err = tx.aggCtx.CodeHistoyIdxRange(k, fromTs, toTs, asc, limit, tx) - case LogTopicIdx: + case kv.LogTopicIdx: timestamps, err = tx.aggCtx.LogTopicRange(k, fromTs, toTs, asc, limit, tx) - case LogAddrIdx: + case kv.LogAddrIdx: timestamps, err = tx.aggCtx.LogAddrRange(k, fromTs, toTs, asc, limit, tx) - case TracesFromIdx: + case kv.TracesFromIdx: timestamps, err = tx.aggCtx.TraceFromRange(k, fromTs, toTs, asc, limit, tx) - case TracesToIdx: + case kv.TracesToIdx: timestamps, err = tx.aggCtx.TraceToRange(k, fromTs, toTs, asc, limit, tx) default: return nil, fmt.Errorf("unexpected history name: %s", name) @@ -465,11 +445,11 @@ func (tx *Tx) HistoryRange(name kv.History, fromTs, toTs int, asc order.By, limi panic("not implemented yet") } switch name { - case AccountsHistory: + case kv.AccountsHistory: it, err = tx.aggCtx.AccountHistoryRange(fromTs, toTs, asc, limit, tx) - case StorageHistory: + case kv.StorageHistory: it, err = tx.aggCtx.StorageHistoryRange(fromTs, toTs, asc, limit, tx) - case CodeHistory: + case kv.CodeHistory: it, err = tx.aggCtx.CodeHistoryRange(fromTs, toTs, asc, limit, tx) default: return nil, fmt.Errorf("unexpected history name: %s", name) @@ -484,8 +464,10 @@ func (tx *Tx) HistoryRange(name kv.History, fromTs, toTs int, asc order.By, limi } // TODO: need remove `gspec` param (move SystemContractCodeLookup feature somewhere) -func NewTestDB(tb testing.TB, ctx context.Context, dirs datadir.Dirs, gspec *types.Genesis, logger log.Logger) (histV3 bool, db kv.RwDB, agg *state.AggregatorV3) { +func NewTestDB(tb testing.TB, dirs datadir.Dirs, gspec *types.Genesis) (histV3 bool, db kv.RwDB, agg *state.AggregatorV3) { historyV3 := ethconfig.EnableHistoryV3InTest + logger := log.New() + ctx := context.Background() if tb != nil { db = memdb.NewTestDB(tb) diff --git a/core/state_processor.go b/core/state_processor.go index 41c28e817eb..3476e100676 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -34,7 +34,7 @@ import ( // indicating the block was invalid. func applyTransaction(config *chain.Config, engine consensus.EngineReader, gp *GasPool, ibs *state.IntraBlockState, stateWriter state.StateWriter, header *types.Header, tx types.Transaction, usedGas *uint64, evm vm.VMInterface, cfg vm.Config) (*types.Receipt, []byte, error) { rules := evm.ChainRules() - msg, err := tx.AsMessage(*types.MakeSigner(config, header.Number.Uint64()), header.BaseFee, rules) + msg, err := tx.AsMessage(*types.MakeSigner(config, header.Number.Uint64(), header.Time), header.BaseFee, rules) if err != nil { return nil, nil, err } diff --git a/core/state_transition.go b/core/state_transition.go index 3a478530989..9963d93083a 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -20,9 +20,8 @@ import ( "fmt" "github.com/holiman/uint256" - libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/txpool" + "github.com/ledgerwatch/erigon-lib/txpool/txpoolcfg" types2 "github.com/ledgerwatch/erigon-lib/types" "github.com/ledgerwatch/erigon/common" cmath "github.com/ledgerwatch/erigon/common/math" @@ -152,8 +151,8 @@ func IntrinsicGas(data []byte, accessList types2.AccessList, isContractCreation } } - gas, status := txpool.CalcIntrinsicGas(dataLen, dataNonZeroLen, accessList, isContractCreation, isHomestead, isEIP2028, isEIP3860) - if status != txpool.Success { + gas, status := txpoolcfg.CalcIntrinsicGas(dataLen, dataNonZeroLen, accessList, isContractCreation, isHomestead, isEIP2028, isEIP3860) + if status != txpoolcfg.Success { return 0, ErrGasUintOverflow } return gas, nil diff --git a/core/types/access_list_tx.go b/core/types/access_list_tx.go index 5d43ce262ad..4188b58000f 100644 --- a/core/types/access_list_tx.go +++ b/core/types/access_list_tx.go @@ -25,6 +25,7 @@ import ( "math/bits" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" types2 "github.com/ledgerwatch/erigon-lib/types" @@ -95,7 +96,7 @@ func (tx AccessListTx) EncodingSize() int { envelopeSize := payloadSize // Add envelope size and type size if payloadSize >= 56 { - envelopeSize += (bits.Len(uint(payloadSize)) + 7) / 8 + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) } envelopeSize += 2 return envelopeSize @@ -135,7 +136,7 @@ func (tx AccessListTx) payloadSize() (payloadSize int, nonceLen, gasLen, accessL } default: if len(tx.Data) >= 56 { - payloadSize += (bits.Len(uint(len(tx.Data))) + 7) / 8 + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(len(tx.Data)))) } payloadSize += len(tx.Data) } @@ -143,7 +144,7 @@ func (tx AccessListTx) payloadSize() (payloadSize int, nonceLen, gasLen, accessL payloadSize++ accessListLen = accessListSize(tx.AccessList) if accessListLen >= 56 { - payloadSize += (bits.Len(uint(accessListLen)) + 7) / 8 + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(accessListLen))) } payloadSize += accessListLen // size of V @@ -167,12 +168,12 @@ func accessListSize(al types2.AccessList) int { // Each storage key takes 33 bytes storageLen := 33 * len(tuple.StorageKeys) if storageLen >= 56 { - tupleLen += (bits.Len(uint(storageLen)) + 7) / 8 // BE encoding of the length of the storage keys + tupleLen += libcommon.BitLenToByteLen(bits.Len(uint(storageLen))) // BE encoding of the length of the storage keys } tupleLen += storageLen accessListLen++ if tupleLen >= 56 { - accessListLen += (bits.Len(uint(tupleLen)) + 7) / 8 // BE encoding of the length of the storage keys + accessListLen += libcommon.BitLenToByteLen(bits.Len(uint(tupleLen))) // BE encoding of the length of the storage keys } accessListLen += tupleLen } @@ -186,7 +187,7 @@ func encodeAccessList(al types2.AccessList, w io.Writer, b []byte) error { // Each storage key takes 33 bytes storageLen := 33 * len(tuple.StorageKeys) if storageLen >= 56 { - tupleLen += (bits.Len(uint(storageLen)) + 7) / 8 // BE encoding of the length of the storage keys + tupleLen += libcommon.BitLenToByteLen(bits.Len(uint(storageLen))) // BE encoding of the length of the storage keys } tupleLen += storageLen if err := EncodeStructSizePrefix(tupleLen, w, b); err != nil { @@ -217,7 +218,7 @@ func encodeAccessList(al types2.AccessList, w io.Writer, b []byte) error { func EncodeStructSizePrefix(size int, w io.Writer, b []byte) error { if size >= 56 { - beSize := (bits.Len(uint(size)) + 7) / 8 + beSize := libcommon.BitLenToByteLen(bits.Len(uint(size))) binary.BigEndian.PutUint64(b[1:], uint64(size)) b[8-beSize] = byte(beSize) + 247 if _, err := w.Write(b[8-beSize : 9]); err != nil { @@ -321,7 +322,7 @@ func (tx AccessListTx) EncodeRLP(w io.Writer) error { payloadSize, nonceLen, gasLen, accessListLen := tx.payloadSize() envelopeSize := payloadSize if payloadSize >= 56 { - envelopeSize += (bits.Len(uint(payloadSize)) + 7) / 8 + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) } // size of struct prefix and TxType envelopeSize += 2 diff --git a/core/types/accounts/account.go b/core/types/accounts/account.go index 8801c7277f5..1953249c652 100644 --- a/core/types/accounts/account.go +++ b/core/types/accounts/account.go @@ -62,7 +62,7 @@ func (a *Account) EncodingLengthForStorage() uint { } if a.Nonce > 0 { - structLength += uint((bits.Len64(a.Nonce)+7)/8) + 1 + structLength += uint(libcommon.BitLenToByteLen(bits.Len64(a.Nonce))) + 1 } if !a.IsEmptyCodeHash() { @@ -70,7 +70,7 @@ func (a *Account) EncodingLengthForStorage() uint { } if a.Incarnation > 0 { - structLength += uint((bits.Len64(a.Incarnation)+7)/8) + 1 + structLength += uint(libcommon.BitLenToByteLen(bits.Len64(a.Incarnation))) + 1 } return structLength @@ -94,7 +94,7 @@ func (a *Account) EncodingLengthForHashing() uint { return 1 + structLength } - lengthBytes := (bits.Len(structLength) + 7) / 8 + lengthBytes := libcommon.BitLenToByteLen(bits.Len(structLength)) return uint(1+lengthBytes) + structLength } @@ -104,7 +104,7 @@ func (a *Account) EncodeForStorage(buffer []byte) { var pos = 1 if a.Nonce > 0 { fieldSet = 1 - nonceBytes := (bits.Len64(a.Nonce) + 7) / 8 + nonceBytes := libcommon.BitLenToByteLen(bits.Len64(a.Nonce)) buffer[pos] = byte(nonceBytes) var nonce = a.Nonce for i := nonceBytes; i > 0; i-- { @@ -126,7 +126,7 @@ func (a *Account) EncodeForStorage(buffer []byte) { if a.Incarnation > 0 { fieldSet |= 4 - incarnationBytes := (bits.Len64(a.Incarnation) + 7) / 8 + incarnationBytes := libcommon.BitLenToByteLen(bits.Len64(a.Incarnation)) buffer[pos] = byte(incarnationBytes) var incarnation = a.Incarnation for i := incarnationBytes; i > 0; i-- { @@ -215,7 +215,7 @@ func (a *Account) EncodeForHashing(buffer []byte) { buffer[0] = byte(192 + structLength) pos = 1 } else { - lengthBytes := (bits.Len(structLength) + 7) / 8 + lengthBytes := libcommon.BitLenToByteLen(bits.Len(structLength)) buffer[0] = byte(247 + lengthBytes) for i := lengthBytes; i > 0; i-- { @@ -663,7 +663,7 @@ func SerialiseV3(a *Account) []byte { var l int l++ if a.Nonce > 0 { - l += (bits.Len64(a.Nonce) + 7) / 8 + l += libcommon.BitLenToByteLen(bits.Len64(a.Nonce)) } l++ if !a.Balance.IsZero() { @@ -675,7 +675,7 @@ func SerialiseV3(a *Account) []byte { } l++ if a.Incarnation > 0 { - l += (bits.Len64(a.Incarnation) + 7) / 8 + l += libcommon.BitLenToByteLen(bits.Len64(a.Incarnation)) } value := make([]byte, l) pos := 0 @@ -683,7 +683,7 @@ func SerialiseV3(a *Account) []byte { value[pos] = 0 pos++ } else { - nonceBytes := (bits.Len64(a.Nonce) + 7) / 8 + nonceBytes := libcommon.BitLenToByteLen(bits.Len64(a.Nonce)) value[pos] = byte(nonceBytes) var nonce = a.Nonce for i := nonceBytes; i > 0; i-- { @@ -714,7 +714,7 @@ func SerialiseV3(a *Account) []byte { if a.Incarnation == 0 { value[pos] = 0 } else { - incBytes := (bits.Len64(a.Incarnation) + 7) / 8 + incBytes := libcommon.BitLenToByteLen(bits.Len64(a.Incarnation)) value[pos] = byte(incBytes) var inc = a.Incarnation for i := incBytes; i > 0; i-- { @@ -728,7 +728,7 @@ func SerialiseV3(a *Account) []byte { func SerialiseV3Len(a *Account) (l int) { l++ if a.Nonce > 0 { - l += (bits.Len64(a.Nonce) + 7) / 8 + l += libcommon.BitLenToByteLen(bits.Len64(a.Nonce)) } l++ if !a.Balance.IsZero() { @@ -740,17 +740,18 @@ func SerialiseV3Len(a *Account) (l int) { } l++ if a.Incarnation > 0 { - l += (bits.Len64(a.Incarnation) + 7) / 8 + l += libcommon.BitLenToByteLen(bits.Len64(a.Incarnation)) } return l } + func SerialiseV3To(a *Account, value []byte) { pos := 0 if a.Nonce == 0 { value[pos] = 0 pos++ } else { - nonceBytes := (bits.Len64(a.Nonce) + 7) / 8 + nonceBytes := libcommon.BitLenToByteLen(bits.Len64(a.Nonce)) value[pos] = byte(nonceBytes) var nonce = a.Nonce for i := nonceBytes; i > 0; i-- { @@ -781,7 +782,7 @@ func SerialiseV3To(a *Account, value []byte) { if a.Incarnation == 0 { value[pos] = 0 } else { - incBytes := (bits.Len64(a.Incarnation) + 7) / 8 + incBytes := libcommon.BitLenToByteLen(bits.Len64(a.Incarnation)) value[pos] = byte(incBytes) var inc = a.Incarnation for i := incBytes; i > 0; i-- { diff --git a/core/types/blob_tx.go b/core/types/blob_tx.go new file mode 100644 index 00000000000..cd5d2e5bdea --- /dev/null +++ b/core/types/blob_tx.go @@ -0,0 +1,354 @@ +package types + +import ( + "fmt" + "io" + "math/big" + "math/bits" + + "github.com/holiman/uint256" + + "github.com/ledgerwatch/erigon-lib/chain" + libcommon "github.com/ledgerwatch/erigon-lib/common" + types2 "github.com/ledgerwatch/erigon-lib/types" + + "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/rlp" +) + +type BlobTx struct { + DynamicFeeTransaction + MaxFeePerDataGas *uint256.Int + BlobVersionedHashes []libcommon.Hash +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (stx BlobTx) copy() *BlobTx { + cpy := &BlobTx{ + DynamicFeeTransaction: *stx.DynamicFeeTransaction.copy(), + MaxFeePerDataGas: new(uint256.Int), + BlobVersionedHashes: make([]libcommon.Hash, len(stx.BlobVersionedHashes)), + } + copy(cpy.BlobVersionedHashes, stx.BlobVersionedHashes) + if stx.MaxFeePerDataGas != nil { + cpy.MaxFeePerDataGas.Set(stx.MaxFeePerDataGas) + } + return cpy +} + +func (stx BlobTx) Type() byte { return BlobTxType } + +func (stx BlobTx) GetDataHashes() []libcommon.Hash { + return stx.BlobVersionedHashes +} + +func (stx BlobTx) GetDataGas() uint64 { + return params.DataGasPerBlob * uint64(len(stx.BlobVersionedHashes)) +} + +func (stx BlobTx) AsMessage(s Signer, baseFee *big.Int, rules *chain.Rules) (Message, error) { + msg, err := stx.DynamicFeeTransaction.AsMessage(s, baseFee, rules) + if err != nil { + return Message{}, err + } + msg.dataHashes = stx.BlobVersionedHashes + return msg, err +} + +func (stx BlobTx) Hash() libcommon.Hash { + if hash := stx.hash.Load(); hash != nil { + return *hash.(*libcommon.Hash) + } + hash := prefixedRlpHash(BlobTxType, []interface{}{ + stx.ChainID, + stx.Nonce, + stx.Tip, + stx.FeeCap, + stx.Gas, + stx.To, + stx.Value, + stx.Data, + stx.AccessList, + stx.MaxFeePerDataGas, + stx.BlobVersionedHashes, + stx.V, stx.R, stx.S, + }) + stx.hash.Store(&hash) + return hash +} + +func (stx BlobTx) SigningHash(chainID *big.Int) libcommon.Hash { + return prefixedRlpHash( + BlobTxType, + []interface{}{ + chainID, + stx.Nonce, + stx.Tip, + stx.FeeCap, + stx.Gas, + stx.To, + stx.Value, + stx.Data, + stx.AccessList, + stx.MaxFeePerDataGas, + stx.BlobVersionedHashes, + }) +} + +func (stx BlobTx) payloadSize() (payloadSize, nonceLen, gasLen, accessListLen, blobHashesLen int) { + payloadSize, nonceLen, gasLen, accessListLen = stx.DynamicFeeTransaction.payloadSize() + // size of MaxFeePerDataGas + payloadSize++ + payloadSize += rlp.Uint256LenExcludingHead(stx.MaxFeePerDataGas) + // size of BlobVersionedHashes + payloadSize++ + blobHashesLen = blobVersionedHashesSize(stx.BlobVersionedHashes) + if blobHashesLen >= 56 { + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(blobHashesLen))) + } + payloadSize += blobHashesLen + return +} + +func blobVersionedHashesSize(hashes []libcommon.Hash) int { + return 33 * len(hashes) +} + +func encodeBlobVersionedHashes(hashes []libcommon.Hash, w io.Writer, b []byte) error { + for _, h := range hashes { + if err := rlp.EncodeString(h[:], w, b); err != nil { + return err + } + } + return nil +} + +func (stx BlobTx) encodePayload(w io.Writer, b []byte, payloadSize, nonceLen, gasLen, accessListLen, blobHashesLen int) error { + // prefix + if err := EncodeStructSizePrefix(payloadSize, w, b); err != nil { + return err + } + // encode ChainID + if err := stx.ChainID.EncodeRLP(w); err != nil { + return err + } + // encode Nonce + if err := rlp.EncodeInt(stx.Nonce, w, b); err != nil { + return err + } + // encode MaxPriorityFeePerGas + if err := stx.Tip.EncodeRLP(w); err != nil { + return err + } + // encode MaxFeePerGas + if err := stx.FeeCap.EncodeRLP(w); err != nil { + return err + } + // encode Gas + if err := rlp.EncodeInt(stx.Gas, w, b); err != nil { + return err + } + // encode To + if stx.To == nil { + b[0] = 128 + } else { + b[0] = 128 + 20 + } + if _, err := w.Write(b[:1]); err != nil { + return err + } + if stx.To != nil { + if _, err := w.Write(stx.To.Bytes()); err != nil { + return err + } + } + // encode Value + if err := stx.Value.EncodeRLP(w); err != nil { + return err + } + // encode Data + if err := rlp.EncodeString(stx.Data, w, b); err != nil { + return err + } + // prefix + if err := EncodeStructSizePrefix(accessListLen, w, b); err != nil { + return err + } + // encode AccessList + if err := encodeAccessList(stx.AccessList, w, b); err != nil { + return err + } + // encode MaxFeePerDataGas + if err := stx.MaxFeePerDataGas.EncodeRLP(w); err != nil { + return err + } + // prefix + if err := EncodeStructSizePrefix(blobHashesLen, w, b); err != nil { + return err + } + // encode BlobVersionedHashes + if err := encodeBlobVersionedHashes(stx.BlobVersionedHashes, w, b); err != nil { + return err + } + // encode y_parity + if err := stx.V.EncodeRLP(w); err != nil { + return err + } + // encode R + if err := stx.R.EncodeRLP(w); err != nil { + return err + } + // encode S + if err := stx.S.EncodeRLP(w); err != nil { + return err + } + return nil +} + +func (stx BlobTx) EncodeRLP(w io.Writer) error { + payloadSize, nonceLen, gasLen, accessListLen, blobHashesLen := stx.payloadSize() + envelopeSize := payloadSize + if payloadSize >= 56 { + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) + } + // size of struct prefix and TxType + envelopeSize += 2 + var b [33]byte + // envelope + if err := rlp.EncodeStringSizePrefix(envelopeSize, w, b[:]); err != nil { + return err + } + // encode TxType + b[0] = BlobTxType + if _, err := w.Write(b[:1]); err != nil { + return err + } + if err := stx.encodePayload(w, b[:], payloadSize, nonceLen, gasLen, accessListLen, blobHashesLen); err != nil { + return err + } + return nil +} + +func (stx BlobTx) MarshalBinary(w io.Writer) error { + payloadSize, nonceLen, gasLen, accessListLen, blobHashesLen := stx.payloadSize() + var b [33]byte + // encode TxType + b[0] = BlobTxType + if _, err := w.Write(b[:1]); err != nil { + return err + } + if err := stx.encodePayload(w, b[:], payloadSize, nonceLen, gasLen, accessListLen, blobHashesLen); err != nil { + return err + } + return nil +} + +func (stx *BlobTx) DecodeRLP(s *rlp.Stream) error { + _, err := s.List() + if err != nil { + return err + } + var b []byte + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.ChainID = new(uint256.Int).SetBytes(b) + + if stx.Nonce, err = s.Uint(); err != nil { + return err + } + + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.Tip = new(uint256.Int).SetBytes(b) + + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.FeeCap = new(uint256.Int).SetBytes(b) + + if stx.Gas, err = s.Uint(); err != nil { + return err + } + + if b, err = s.Bytes(); err != nil { + return err + } + if len(b) > 0 && len(b) != 20 { + return fmt.Errorf("wrong size for To: %d", len(b)) + } + if len(b) > 0 { + stx.To = &libcommon.Address{} + copy((*stx.To)[:], b) + } + + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.Value = new(uint256.Int).SetBytes(b) + + if stx.Data, err = s.Bytes(); err != nil { + return err + } + // decode AccessList + stx.AccessList = types2.AccessList{} + if err = decodeAccessList(&stx.AccessList, s); err != nil { + return err + } + + // decode MaxFeePerDataGas + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.MaxFeePerDataGas = new(uint256.Int).SetBytes(b) + + // decode BlobVersionedHashes + stx.BlobVersionedHashes = []libcommon.Hash{} + if err = decodeBlobVersionedHashes(stx.BlobVersionedHashes, s); err != nil { + return err + } + + // decode y_parity + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.V.SetBytes(b) + + // decode R + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.R.SetBytes(b) + + // decode S + if b, err = s.Uint256Bytes(); err != nil { + return err + } + stx.S.SetBytes(b) + return s.ListEnd() +} + +func decodeBlobVersionedHashes(hashes []libcommon.Hash, s *rlp.Stream) error { + _, err := s.List() + if err != nil { + return fmt.Errorf("open BlobVersionedHashes: %w", err) + } + var b []byte + _hash := libcommon.Hash{} + + for b, err = s.Bytes(); err == nil; b, err = s.Bytes() { + if len(b) == 32 { + copy((_hash)[:], b) + hashes = append(hashes, _hash) + } else { + return fmt.Errorf("wrong size for blobVersionedHashes: %d, %v", len(b), b[0]) + } + } + + if err = s.ListEnd(); err != nil { + return fmt.Errorf("close BlobVersionedHashes: %w", err) + } + + return nil +} diff --git a/core/types/blob_tx_wrapper.go b/core/types/blob_tx_wrapper.go index 2d1879dfd2e..cbfd40ab025 100644 --- a/core/types/blob_tx_wrapper.go +++ b/core/types/blob_tx_wrapper.go @@ -1,269 +1,152 @@ package types import ( - "bytes" - "encoding/hex" - "errors" "fmt" "io" "math/big" + "math/bits" "time" gokzg4844 "github.com/crate-crypto/go-kzg-4844" "github.com/holiman/uint256" - "github.com/protolambda/ztyp/codec" - "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/hexutility" libkzg "github.com/ledgerwatch/erigon-lib/crypto/kzg" types2 "github.com/ledgerwatch/erigon-lib/types" - "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" ) -// Compressed BLS12-381 G1 element -type KZGCommitment [48]byte +const ( + LEN_BLOB = params.FieldElementsPerBlob * 32 // 131072 + LEN_48 = 48 // KZGCommitment & KZGProof sizes +) -func (p *KZGCommitment) Deserialize(dr *codec.DecodingReader) error { - if p == nil { - return errors.New("nil pubkey") - } - _, err := dr.Read(p[:]) - return err -} +type KZGCommitment [LEN_48]byte // Compressed BLS12-381 G1 element +type KZGProof [LEN_48]byte +type Blob [LEN_BLOB]byte -func (p *KZGCommitment) Serialize(w *codec.EncodingWriter) error { - return w.Write(p[:]) -} - -func (KZGCommitment) ByteLength() uint64 { - return 48 -} +type BlobKzgs []KZGCommitment +type KZGProofs []KZGProof +type Blobs []Blob -func (KZGCommitment) FixedLength() uint64 { - return 48 +type BlobTxWrapper struct { + Tx BlobTx + Commitments BlobKzgs + Blobs Blobs + Proofs KZGProofs } -func (p KZGCommitment) MarshalText() ([]byte, error) { - return []byte("0x" + hex.EncodeToString(p[:])), nil -} +/* Blob methods */ -func (p KZGCommitment) String() string { - return "0x" + hex.EncodeToString(p[:]) +func (b *Blob) payloadSize() int { + size := 1 // 0xb7 + size += libcommon.BitLenToByteLen(bits.Len(LEN_BLOB)) // params.FieldElementsPerBlob * 32 = 131072 (length encoding size) + size += LEN_BLOB // byte_array it self + return size } -func (p *KZGCommitment) UnmarshalText(text []byte) error { - return hexutility.UnmarshalFixedText("KZGCommitment", text, p[:]) -} +/* BlobKzgs methods */ -func (c KZGCommitment) ComputeVersionedHash() libcommon.Hash { - return libcommon.Hash(libkzg.KZGToVersionedHash(gokzg4844.KZGCommitment(c))) +func (li BlobKzgs) copy() BlobKzgs { + cpy := make(BlobKzgs, len(li)) + copy(cpy, li) + return cpy } -// Compressed BLS12-381 G1 element -type KZGProof [48]byte - -func (p *KZGProof) Deserialize(dr *codec.DecodingReader) error { - if p == nil { - return errors.New("nil pubkey") +func (li BlobKzgs) payloadSize() int { + size := 49 * len(li) + if size >= 56 { + size += libcommon.BitLenToByteLen(bits.Len(uint(size))) // BE encoding of the length of hashes } - _, err := dr.Read(p[:]) - return err -} - -func (p *KZGProof) Serialize(w *codec.EncodingWriter) error { - return w.Write(p[:]) -} - -func (KZGProof) ByteLength() uint64 { - return 48 -} - -func (KZGProof) FixedLength() uint64 { - return 48 -} - -func (p KZGProof) MarshalText() ([]byte, error) { - return []byte("0x" + hex.EncodeToString(p[:])), nil -} - -func (p KZGProof) String() string { - return "0x" + hex.EncodeToString(p[:]) -} - -func (p *KZGProof) UnmarshalText(text []byte) error { - return hexutility.UnmarshalFixedText("KZGProof", text, p[:]) -} - -// BLSFieldElement is the raw bytes representation of a field element -type BLSFieldElement [32]byte - -func (p BLSFieldElement) MarshalText() ([]byte, error) { - return []byte("0x" + hex.EncodeToString(p[:])), nil + return size } -func (p BLSFieldElement) String() string { - return "0x" + hex.EncodeToString(p[:]) -} - -func (p *BLSFieldElement) UnmarshalText(text []byte) error { - return hexutility.UnmarshalFixedText("BLSFieldElement", text, p[:]) -} - -// Blob data -type Blob [params.FieldElementsPerBlob * 32]byte - -func (blob *Blob) Deserialize(dr *codec.DecodingReader) error { - if blob == nil { - return errors.New("cannot decode ssz into nil Blob") - } - - // We treat the blob as an opaque sequence of bytes - // and therefore we do not do any validation related to field - // elements - if _, err := dr.Read(blob[:]); err != nil { +func (li BlobKzgs) encodePayload(w io.Writer, b []byte, payloadSize int) error { + // prefix + if err := EncodeStructSizePrefix(payloadSize, w, b); err != nil { return err } - return nil -} - -func (blob *Blob) Serialize(w *codec.EncodingWriter) error { - return w.Write(blob[:]) -} - -func (blob *Blob) ByteLength() (out uint64) { - return params.FieldElementsPerBlob * 32 -} - -func (blob *Blob) FixedLength() uint64 { - return params.FieldElementsPerBlob * 32 -} - -func (blob *Blob) MarshalText() ([]byte, error) { - out := make([]byte, 2+params.FieldElementsPerBlob*32*2) - copy(out[:2], "0x") - hex.Encode(out[2:], blob[:]) - - return out, nil -} - -func (blob *Blob) String() string { - v, err := blob.MarshalText() - if err != nil { - return "" - } - return string(v) -} - -func (blob *Blob) UnmarshalText(text []byte) error { - if blob == nil { - return errors.New("cannot decode text into nil Blob") - } - l := 2 + params.FieldElementsPerBlob*32*2 - if len(text) != l { - return fmt.Errorf("expected %d characters but got %d", l, len(text)) - } - if !(text[0] == '0' && text[1] == 'x') { - return fmt.Errorf("expected '0x' prefix in Blob string") - } - if _, err := hex.Decode(blob[:], text[2:]); err != nil { - return fmt.Errorf("blob is not formatted correctly: %v", err) + b[0] = 128 + LEN_48 + for _, arr := range li { + if _, err := w.Write(b[:1]); err != nil { + return err + } + if _, err := w.Write(arr[:]); err != nil { + return err + } } - return nil } -type BlobKzgs []KZGCommitment - -func (li *BlobKzgs) Deserialize(dr *codec.DecodingReader) error { - return dr.List(func() codec.Deserializable { - i := len(*li) - *li = append(*li, KZGCommitment{}) - return &(*li)[i] - }, 48, params.MaxBlobsPerBlock) -} - -func (li BlobKzgs) Serialize(w *codec.EncodingWriter) error { - return w.List(func(i uint64) codec.Serializable { - return &li[i] - }, 48, uint64(len(li))) -} - -func (li BlobKzgs) ByteLength() uint64 { - return uint64(len(li)) * 48 -} - -func (li BlobKzgs) FixedLength() uint64 { - return 0 -} +/* KZGProofs methods */ -func (li BlobKzgs) copy() BlobKzgs { - cpy := make(BlobKzgs, len(li)) +func (li KZGProofs) copy() KZGProofs { + cpy := make(KZGProofs, len(li)) copy(cpy, li) return cpy } -type KZGProofs []KZGProof - -func (li *KZGProofs) Deserialize(dr *codec.DecodingReader) error { - return dr.List(func() codec.Deserializable { - i := len(*li) - *li = append(*li, KZGProof{}) - return &(*li)[i] - }, 48, params.MaxBlobsPerBlock) +func (li KZGProofs) payloadSize() int { + size := 49 * len(li) + if size >= 56 { + size += libcommon.BitLenToByteLen(bits.Len(uint(size))) // BE encoding of the length of hashes + } + return size } -func (li KZGProofs) Serialize(w *codec.EncodingWriter) error { - return w.List(func(i uint64) codec.Serializable { - return &li[i] - }, 48, uint64(len(li))) -} +func (li KZGProofs) encodePayload(w io.Writer, b []byte, payloadSize int) error { + // prefix + if err := EncodeStructSizePrefix(payloadSize, w, b); err != nil { + return err + } -func (li KZGProofs) ByteLength() uint64 { - return uint64(len(li)) * 48 + b[0] = 128 + LEN_48 + for _, arr := range li { + if _, err := w.Write(b[:1]); err != nil { + return err + } + if _, err := w.Write(arr[:]); err != nil { + return err + } + } + return nil } -func (li KZGProofs) FixedLength() uint64 { - return 0 -} +/* Blobs methods */ -func (li KZGProofs) copy() KZGProofs { - cpy := make(KZGProofs, len(li)) - copy(cpy, li) +func (blobs Blobs) copy() Blobs { + cpy := make(Blobs, len(blobs)) + copy(cpy, blobs) // each blob element is an array and gets deep-copied return cpy } -type Blobs []Blob - -func (a *Blobs) Deserialize(dr *codec.DecodingReader) error { - return dr.List(func() codec.Deserializable { - i := len(*a) - *a = append(*a, Blob{}) - return &(*a)[i] - }, params.FieldElementsPerBlob*32, params.FieldElementsPerBlob) -} - -func (a Blobs) Serialize(w *codec.EncodingWriter) error { - return w.List(func(i uint64) codec.Serializable { - return &a[i] - }, params.FieldElementsPerBlob*32, uint64(len(a))) +func (blobs Blobs) payloadSize() int { + total := 0 + if len(blobs) > 0 { + total = len(blobs) * blobs[0].payloadSize() + total += libcommon.BitLenToByteLen(bits.Len(uint(total))) + } + return total } -func (a Blobs) ByteLength() (out uint64) { - return uint64(len(a)) * params.FieldElementsPerBlob * 32 -} +func (blobs Blobs) encodePayload(w io.Writer, b []byte, payloadSize int) error { + // prefix + if err := EncodeStructSizePrefix(payloadSize, w, b); err != nil { + return err + } -func (a *Blobs) FixedLength() uint64 { - return 0 // it's a list, no fixed length -} + for _, arr := range blobs { + if err := rlp.EncodeStringSizePrefix(LEN_BLOB, w, b); err != nil { + return err + } + if _, err := w.Write(arr[:]); err != nil { + return err + } + } -func (blobs Blobs) copy() Blobs { - cpy := make(Blobs, len(blobs)) - copy(cpy, blobs) // each blob element is an array and gets deep-copied - return cpy + return nil } // Return KZG commitments, versioned hashes and the proofs that correspond to these blobs @@ -313,34 +196,15 @@ func toProofs(_proofs KZGProofs) []gokzg4844.KZGProof { return proofs } -// BlobTxWrapper is the "network representation" of a Blob transaction, that is it includes not -// only the SignedBlobTx but also all the associated blob data. -type BlobTxWrapper struct { - Tx SignedBlobTx - Commitments BlobKzgs - Blobs Blobs - Proofs KZGProofs -} - -func (txw *BlobTxWrapper) Deserialize(dr *codec.DecodingReader) error { - return dr.Container(&txw.Tx, &txw.Commitments, &txw.Blobs, &txw.Proofs) -} - -func (txw *BlobTxWrapper) Serialize(w *codec.EncodingWriter) error { - return w.Container(&txw.Tx, &txw.Commitments, &txw.Blobs, &txw.Proofs) -} - -func (txw *BlobTxWrapper) ByteLength() uint64 { - return codec.ContainerLength(&txw.Tx, &txw.Commitments, &txw.Blobs, &txw.Proofs) +func (c KZGCommitment) ComputeVersionedHash() libcommon.Hash { + return libcommon.Hash(libkzg.KZGToVersionedHash(gokzg4844.KZGCommitment(c))) } -func (txw *BlobTxWrapper) FixedLength() uint64 { - return 0 -} +/* BlobTxWrapper methods */ // validateBlobTransactionWrapper implements validate_blob_transaction_wrapper from EIP-4844 func (txw *BlobTxWrapper) ValidateBlobTransactionWrapper() error { - blobTx := txw.Tx.Message + blobTx := txw.Tx l1 := len(blobTx.BlobVersionedHashes) if l1 == 0 { return fmt.Errorf("a blob tx must contain at least one blob") @@ -354,7 +218,7 @@ func (txw *BlobTxWrapper) ValidateBlobTransactionWrapper() error { // the following check isn't strictly necessary as it would be caught by data gas processing // (and hence it is not explicitly in the spec for this function), but it doesn't hurt to fail // early in case we are getting spammed with too many blobs or there is a bug somewhere: - if l1 > params.MaxBlobsPerBlock { + if uint64(l1) > params.MaxBlobsPerBlock { return fmt.Errorf("number of blobs exceeds max: %v", l1) } kzgCtx := libkzg.Ctx() @@ -379,63 +243,129 @@ func (txw *BlobTxWrapper) GetTip() *uint256.Int { return txw.Tx.GetTip() } func (txw *BlobTxWrapper) GetEffectiveGasTip(baseFee *uint256.Int) *uint256.Int { return txw.Tx.GetEffectiveGasTip(baseFee) } -func (txw *BlobTxWrapper) GetFeeCap() *uint256.Int { return txw.Tx.GetFeeCap() } -func (txw *BlobTxWrapper) Cost() *uint256.Int { return txw.Tx.GetFeeCap() } +func (txw *BlobTxWrapper) GetFeeCap() *uint256.Int { return txw.Tx.GetFeeCap() } + +func (txw *BlobTxWrapper) Cost() *uint256.Int { return txw.Tx.GetFeeCap() } + func (txw *BlobTxWrapper) GetDataHashes() []libcommon.Hash { return txw.Tx.GetDataHashes() } -func (txw *BlobTxWrapper) GetGas() uint64 { return txw.Tx.GetGas() } -func (txw *BlobTxWrapper) GetDataGas() uint64 { return txw.Tx.GetDataGas() } -func (txw *BlobTxWrapper) GetValue() *uint256.Int { return txw.Tx.GetValue() } -func (txw *BlobTxWrapper) Time() time.Time { return txw.Tx.Time() } -func (txw *BlobTxWrapper) GetTo() *libcommon.Address { return txw.Tx.GetTo() } + +func (txw *BlobTxWrapper) GetGas() uint64 { return txw.Tx.GetGas() } +func (txw *BlobTxWrapper) GetDataGas() uint64 { return txw.Tx.GetDataGas() } +func (txw *BlobTxWrapper) GetValue() *uint256.Int { return txw.Tx.GetValue() } +func (txw *BlobTxWrapper) Time() time.Time { return txw.Tx.Time() } +func (txw *BlobTxWrapper) GetTo() *libcommon.Address { return txw.Tx.GetTo() } + func (txw *BlobTxWrapper) AsMessage(s Signer, baseFee *big.Int, rules *chain.Rules) (Message, error) { return txw.Tx.AsMessage(s, baseFee, rules) } func (txw *BlobTxWrapper) WithSignature(signer Signer, sig []byte) (Transaction, error) { return txw.Tx.WithSignature(signer, sig) } + func (txw *BlobTxWrapper) FakeSign(address libcommon.Address) (Transaction, error) { return txw.Tx.FakeSign(address) } + func (txw *BlobTxWrapper) Hash() libcommon.Hash { return txw.Tx.Hash() } + func (txw *BlobTxWrapper) SigningHash(chainID *big.Int) libcommon.Hash { return txw.Tx.SigningHash(chainID) } -func (txw *BlobTxWrapper) GetData() []byte { return txw.Tx.GetData() } + +func (txw *BlobTxWrapper) GetData() []byte { return txw.Tx.GetData() } + func (txw *BlobTxWrapper) GetAccessList() types2.AccessList { return txw.Tx.GetAccessList() } -func (txw *BlobTxWrapper) Protected() bool { return txw.Tx.Protected() } + +func (txw *BlobTxWrapper) Protected() bool { return txw.Tx.Protected() } + func (txw *BlobTxWrapper) RawSignatureValues() (*uint256.Int, *uint256.Int, *uint256.Int) { return txw.Tx.RawSignatureValues() } + func (txw *BlobTxWrapper) Sender(s Signer) (libcommon.Address, error) { return txw.Tx.Sender(s) } -func (txw *BlobTxWrapper) GetSender() (libcommon.Address, bool) { return txw.Tx.GetSender() } -func (txw *BlobTxWrapper) SetSender(address libcommon.Address) { txw.Tx.SetSender(address) } -func (txw *BlobTxWrapper) IsContractDeploy() bool { return txw.Tx.IsContractDeploy() } -func (txw *BlobTxWrapper) Unwrap() Transaction { return &txw.Tx } + +func (txw *BlobTxWrapper) GetSender() (libcommon.Address, bool) { return txw.Tx.GetSender() } + +func (txw *BlobTxWrapper) SetSender(address libcommon.Address) { txw.Tx.SetSender(address) } + +func (txw *BlobTxWrapper) IsContractDeploy() bool { return txw.Tx.IsContractDeploy() } + +func (txw *BlobTxWrapper) Unwrap() Transaction { return &txw.Tx } func (txw BlobTxWrapper) EncodingSize() int { - envelopeSize := int(codec.ContainerLength(&txw.Tx, &txw.Commitments, &txw.Blobs, &txw.Proofs)) - // Add type byte - envelopeSize++ + txSize, commitmentsSize, proofsSize, blobsSize := txw.payloadSize() + payloadSize := txSize + commitmentsSize + proofsSize + blobsSize + envelopeSize := payloadSize + // Add envelope size and type size + if payloadSize >= 56 { + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) + } + envelopeSize += 2 return envelopeSize } +func (txw BlobTxWrapper) payloadSize() (int, int, int, int) { + txSize, _, _, _, _ := txw.Tx.payloadSize() + commitmentsSize := txw.Commitments.payloadSize() + proofsSize := txw.Proofs.payloadSize() + blobsSize := txw.Blobs.payloadSize() + return txSize, commitmentsSize, proofsSize, blobsSize +} + +func (txw BlobTxWrapper) encodePayload(w io.Writer, b []byte, payloadSize, commitmentsSize, proofsSize, blobsSize int) error { + // prefix, encode txw payload size + if err := EncodeStructSizePrefix(payloadSize, w, b); err != nil { + return err + } + + txPayloadSize, nonceLen, gasLen, accessListLen, blobHashesLen := txw.Tx.payloadSize() + if err := txw.Tx.encodePayload(w, b, txPayloadSize, nonceLen, gasLen, accessListLen, blobHashesLen); err != nil { + return err + } + + // TODO: encode in order (see EIP-4844 updates) + if err := txw.Commitments.encodePayload(w, b, commitmentsSize); err != nil { + return err + } + if err := txw.Blobs.encodePayload(w, b, blobsSize); err != nil { + return err + } + txw.Proofs.encodePayload(w, b, proofsSize) + return nil +} + func (txw *BlobTxWrapper) MarshalBinary(w io.Writer) error { + return nil +} + +func (txw BlobTxWrapper) EncodeRLP(w io.Writer) error { + txSize, commitmentsSize, proofsSize, blobsSize := txw.payloadSize() + payloadSize := txSize + commitmentsSize + proofsSize + blobsSize + envelopeSize := payloadSize + if payloadSize >= 56 { + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) + } + // size of struct prefix and TxType + envelopeSize += 2 var b [33]byte + // envelope + if err := rlp.EncodeStringSizePrefix(envelopeSize, w, b[:]); err != nil { + return err + } // encode TxType b[0] = BlobTxType if _, err := w.Write(b[:1]); err != nil { return err } - wcodec := codec.NewEncodingWriter(w) - return txw.Serialize(wcodec) -} - -func (txw BlobTxWrapper) EncodeRLP(w io.Writer) error { - var buf bytes.Buffer - if err := txw.MarshalBinary(&buf); err != nil { + if err := txw.encodePayload(w, b[:], payloadSize, commitmentsSize, proofsSize, blobsSize); err != nil { return err } - return rlp.Encode(w, buf.Bytes()) + return nil +} + +func (txw BlobTxWrapper) DecodeRLP(s *rlp.Stream) error { + // TODO + return nil } func (txw *BlobTxWrapper) RollupDataGas() RollupGasData { diff --git a/core/types/block.go b/core/types/block.go index 278f3945d9b..ba52ca22ec8 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -107,10 +107,6 @@ type Header struct { VerkleKeyVals []verkle.KeyValuePair } -func bitsToBytes(bitLen int) (byteLen int) { - return (bitLen + 7) / 8 -} - func (h *Header) EncodingSize() int { encodingSize := 33 /* ParentHash */ + 33 /* UncleHash */ + 21 /* Coinbase */ + 33 /* Root */ + 33 /* TxHash */ + 33 /* ReceiptHash */ + 259 /* Bloom */ @@ -139,7 +135,7 @@ func (h *Header) EncodingSize() int { } default: if len(h.Extra) >= 56 { - encodingSize += bitsToBytes(bits.Len(uint(len(h.Extra)))) + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(len(h.Extra)))) } encodingSize += len(h.Extra) } @@ -147,7 +143,7 @@ func (h *Header) EncodingSize() int { if len(h.AuRaSeal) != 0 { encodingSize += 1 + rlp.IntLenExcludingHead(h.AuRaStep) + 1 + len(h.AuRaSeal) if len(h.AuRaSeal) >= 56 { - encodingSize += bitsToBytes(bits.Len(uint(len(h.AuRaSeal)))) + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(len(h.AuRaSeal)))) } } else { encodingSize += 33 /* MixDigest */ + 9 /* BlockNonce */ @@ -182,7 +178,7 @@ func (h *Header) EncodingSize() int { } default: if len(h.VerkleProof) >= 56 { - encodingSize += bitsToBytes(bits.Len(uint(len(h.VerkleProof)))) + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(len(h.VerkleProof)))) } encodingSize += len(h.VerkleProof) } @@ -537,9 +533,10 @@ var headerSize = common.StorageSize(reflect.TypeOf(Header{}).Size()) // Size returns the approximate memory used by all internal contents. It is used // to approximate and limit the memory consumption of various caches. func (h *Header) Size() common.StorageSize { - s := headerSize + common.StorageSize(len(h.Extra)+bitsToBytes(h.Difficulty.BitLen())+bitsToBytes(h.Number.BitLen())) + s := headerSize + s += common.StorageSize(len(h.Extra) + libcommon.BitLenToByteLen(h.Difficulty.BitLen()) + libcommon.BitLenToByteLen(h.Number.BitLen())) if h.BaseFee != nil { - s += common.StorageSize(bitsToBytes(h.BaseFee.BitLen())) + s += common.StorageSize(libcommon.BitLenToByteLen(h.BaseFee.BitLen())) } if h.WithdrawalsHash != nil { s += common.StorageSize(32) @@ -653,7 +650,7 @@ func (rb RawBody) payloadSize() (payloadSize, txsLen, unclesLen, withdrawalsLen txsLen += len(tx) } if txsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(txsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(txsLen))) } payloadSize += txsLen @@ -663,12 +660,12 @@ func (rb RawBody) payloadSize() (payloadSize, txsLen, unclesLen, withdrawalsLen unclesLen++ uncleLen := uncle.EncodingSize() if uncleLen >= 56 { - unclesLen += bitsToBytes(bits.Len(uint(uncleLen))) + unclesLen += libcommon.BitLenToByteLen(bits.Len(uint(uncleLen))) } unclesLen += uncleLen } if unclesLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(unclesLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(unclesLen))) } payloadSize += unclesLen @@ -679,12 +676,12 @@ func (rb RawBody) payloadSize() (payloadSize, txsLen, unclesLen, withdrawalsLen withdrawalsLen++ withdrawalLen := withdrawal.EncodingSize() if withdrawalLen >= 56 { - withdrawalLen += bitsToBytes(bits.Len(uint(withdrawalLen))) + withdrawalLen += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalLen))) } withdrawalsLen += withdrawalLen } if withdrawalsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(withdrawalsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalsLen))) } payloadSize += withdrawalsLen } @@ -817,12 +814,12 @@ func (bfs BodyForStorage) payloadSize() (payloadSize, unclesLen, withdrawalsLen unclesLen++ uncleLen := uncle.EncodingSize() if uncleLen >= 56 { - unclesLen += bitsToBytes(bits.Len(uint(uncleLen))) + unclesLen += libcommon.BitLenToByteLen(bits.Len(uint(uncleLen))) } unclesLen += uncleLen } if unclesLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(unclesLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(unclesLen))) } payloadSize += unclesLen @@ -833,12 +830,12 @@ func (bfs BodyForStorage) payloadSize() (payloadSize, unclesLen, withdrawalsLen withdrawalsLen++ withdrawalLen := withdrawal.EncodingSize() if withdrawalLen >= 56 { - withdrawalLen += bitsToBytes(bits.Len(uint(withdrawalLen))) + withdrawalLen += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalLen))) } withdrawalsLen += withdrawalLen } if withdrawalsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(withdrawalsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalsLen))) } payloadSize += withdrawalsLen } @@ -965,24 +962,14 @@ func (bb Body) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLen payloadSize++ for _, tx := range bb.Transactions { txsLen++ - var txLen int - switch t := tx.(type) { - case *LegacyTx: - txLen = t.EncodingSize() - case *AccessListTx: - txLen = t.EncodingSize() - case *DynamicFeeTransaction: - txLen = t.EncodingSize() - case *DepositTx: - txLen = t.EncodingSize() - } + txLen := tx.EncodingSize() if txLen >= 56 { - txsLen += bitsToBytes(bits.Len(uint(txLen))) + txsLen += libcommon.BitLenToByteLen(bits.Len(uint(txLen))) } txsLen += txLen } if txsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(txsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(txsLen))) } payloadSize += txsLen @@ -992,12 +979,12 @@ func (bb Body) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLen unclesLen++ uncleLen := uncle.EncodingSize() if uncleLen >= 56 { - unclesLen += bitsToBytes(bits.Len(uint(uncleLen))) + unclesLen += libcommon.BitLenToByteLen(bits.Len(uint(uncleLen))) } unclesLen += uncleLen } if unclesLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(unclesLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(unclesLen))) } payloadSize += unclesLen @@ -1008,12 +995,12 @@ func (bb Body) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLen withdrawalsLen++ withdrawalLen := withdrawal.EncodingSize() if withdrawalLen >= 56 { - withdrawalLen += bitsToBytes(bits.Len(uint(withdrawalLen))) + withdrawalLen += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalLen))) } withdrawalsLen += withdrawalLen } if withdrawalsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(withdrawalsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalsLen))) } payloadSize += withdrawalsLen } @@ -1033,23 +1020,8 @@ func (bb Body) EncodeRLP(w io.Writer) error { return err } for _, tx := range bb.Transactions { - switch t := tx.(type) { - case *LegacyTx: - if err := t.EncodeRLP(w); err != nil { - return err - } - case *AccessListTx: - if err := t.EncodeRLP(w); err != nil { - return err - } - case *DynamicFeeTransaction: - if err := t.EncodeRLP(w); err != nil { - return err - } - case *DepositTx: - if err := t.EncodeRLP(w); err != nil { - return err - } + if err := tx.EncodeRLP(w); err != nil { + return err } } // encode Uncles @@ -1331,7 +1303,7 @@ func (bb Block) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLe payloadSize++ headerLen := bb.header.EncodingSize() if headerLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(headerLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(headerLen))) } payloadSize += headerLen @@ -1341,12 +1313,12 @@ func (bb Block) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLe txsLen++ txLen := tx.EncodingSize() if txLen >= 56 { - txsLen += bitsToBytes(bits.Len(uint(txLen))) + txsLen += libcommon.BitLenToByteLen(bits.Len(uint(txLen))) } txsLen += txLen } if txsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(txsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(txsLen))) } payloadSize += txsLen @@ -1356,12 +1328,12 @@ func (bb Block) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLe unclesLen++ uncleLen := uncle.EncodingSize() if uncleLen >= 56 { - unclesLen += bitsToBytes(bits.Len(uint(uncleLen))) + unclesLen += libcommon.BitLenToByteLen(bits.Len(uint(uncleLen))) } unclesLen += uncleLen } if unclesLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(unclesLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(unclesLen))) } payloadSize += unclesLen @@ -1372,12 +1344,12 @@ func (bb Block) payloadSize() (payloadSize int, txsLen, unclesLen, withdrawalsLe withdrawalsLen++ withdrawalLen := withdrawal.EncodingSize() if withdrawalLen >= 56 { - withdrawalLen += bitsToBytes(bits.Len(uint(withdrawalLen))) + withdrawalLen += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalLen))) } withdrawalsLen += withdrawalLen } if withdrawalsLen >= 56 { - payloadSize += bitsToBytes(bits.Len(uint(withdrawalsLen))) + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(withdrawalsLen))) } payloadSize += withdrawalsLen } diff --git a/core/types/block_test.go b/core/types/block_test.go index cff99cf6048..5646d268861 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -115,12 +115,12 @@ func TestEIP1559BlockEncoding(t *testing.T) { feeCap, _ := uint256.FromBig(block.BaseFee()) var tx2 Transaction = &DynamicFeeTransaction{ CommonTx: CommonTx{ - ChainID: u256.Num1, - Nonce: 0, - To: &to, - Gas: 123457, - Data: []byte{}, + Nonce: 0, + To: &to, + Gas: 123457, + Data: []byte{}, }, + ChainID: u256.Num1, FeeCap: feeCap, Tip: u256.Num0, AccessList: accesses, @@ -318,12 +318,12 @@ func TestCanEncodeAndDecodeBodyTransactions(t *testing.T) { feeCap, _ := uint256.FromBig(big.NewInt(2)) var tx3 Transaction = &DynamicFeeTransaction{ CommonTx: CommonTx{ - ChainID: u256.Num1, - Nonce: 0, - To: &to, - Gas: 123457, - Data: []byte{40, 50, 60}, + Nonce: 0, + To: &to, + Gas: 123457, + Data: []byte{40, 50, 60}, }, + ChainID: u256.Num1, FeeCap: feeCap, Tip: u256.Num0, AccessList: accesses, diff --git a/core/types/dynamic_fee_tx.go b/core/types/dynamic_fee_tx.go index ba805031db4..3060368345d 100644 --- a/core/types/dynamic_fee_tx.go +++ b/core/types/dynamic_fee_tx.go @@ -24,6 +24,7 @@ import ( "math/bits" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" types2 "github.com/ledgerwatch/erigon-lib/types" @@ -35,6 +36,7 @@ import ( type DynamicFeeTransaction struct { CommonTx + ChainID *uint256.Int Tip *uint256.Int FeeCap *uint256.Int AccessList types2.AccessList @@ -78,14 +80,14 @@ func (tx DynamicFeeTransaction) copy() *DynamicFeeTransaction { TransactionMisc: TransactionMisc{ time: tx.time, }, - ChainID: new(uint256.Int), - Nonce: tx.Nonce, - To: tx.To, // TODO: copy pointed-to address - Data: common.CopyBytes(tx.Data), - Gas: tx.Gas, + Nonce: tx.Nonce, + To: tx.To, // TODO: copy pointed-to address + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, // These are copied below. Value: new(uint256.Int), }, + ChainID: new(uint256.Int), AccessList: make(types2.AccessList, len(tx.AccessList)), Tip: new(uint256.Int), FeeCap: new(uint256.Int), @@ -118,7 +120,7 @@ func (tx DynamicFeeTransaction) EncodingSize() int { envelopeSize := payloadSize // Add envelope size and type size if payloadSize >= 56 { - envelopeSize += (bits.Len(uint(payloadSize)) + 7) / 8 + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) } envelopeSize += 2 return envelopeSize @@ -160,7 +162,7 @@ func (tx DynamicFeeTransaction) payloadSize() (payloadSize int, nonceLen, gasLen } default: if len(tx.Data) >= 56 { - payloadSize += (bits.Len(uint(len(tx.Data))) + 7) / 8 + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(len(tx.Data)))) } payloadSize += len(tx.Data) } @@ -168,7 +170,7 @@ func (tx DynamicFeeTransaction) payloadSize() (payloadSize int, nonceLen, gasLen payloadSize++ accessListLen = accessListSize(tx.AccessList) if accessListLen >= 56 { - payloadSize += (bits.Len(uint(accessListLen)) + 7) / 8 + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(accessListLen))) } payloadSize += accessListLen // size of V @@ -296,7 +298,7 @@ func (tx DynamicFeeTransaction) EncodeRLP(w io.Writer) error { payloadSize, nonceLen, gasLen, accessListLen := tx.payloadSize() envelopeSize := payloadSize if payloadSize >= 56 { - envelopeSize += (bits.Len(uint(payloadSize)) + 7) / 8 + envelopeSize += libcommon.BitLenToByteLen(bits.Len(uint(payloadSize))) } // size of struct prefix and TxType envelopeSize += 2 @@ -459,6 +461,10 @@ func (tx DynamicFeeTransaction) RawSignatureValues() (*uint256.Int, *uint256.Int return &tx.V, &tx.R, &tx.S } +func (tx DynamicFeeTransaction) GetChainID() *uint256.Int { + return tx.ChainID +} + func (tx *DynamicFeeTransaction) Sender(signer Signer) (libcommon.Address, error) { if sc := tx.from.Load(); sc != nil { return sc.(libcommon.Address), nil @@ -475,15 +481,15 @@ func (tx *DynamicFeeTransaction) Sender(signer Signer) (libcommon.Address, error func NewEIP1559Transaction(chainID uint256.Int, nonce uint64, to libcommon.Address, amount *uint256.Int, gasLimit uint64, gasPrice *uint256.Int, gasTip *uint256.Int, gasFeeCap *uint256.Int, data []byte) *DynamicFeeTransaction { return &DynamicFeeTransaction{ CommonTx: CommonTx{ - ChainID: &chainID, - Nonce: nonce, - To: &to, - Value: amount, - Gas: gasLimit, - Data: data, + Nonce: nonce, + To: &to, + Value: amount, + Gas: gasLimit, + Data: data, }, - Tip: gasTip, - FeeCap: gasFeeCap, + ChainID: &chainID, + Tip: gasTip, + FeeCap: gasFeeCap, } } diff --git a/core/types/legacy_tx.go b/core/types/legacy_tx.go index f4116a49ceb..fe820788c5b 100644 --- a/core/types/legacy_tx.go +++ b/core/types/legacy_tx.go @@ -36,7 +36,6 @@ import ( type CommonTx struct { TransactionMisc - ChainID *uint256.Int Nonce uint64 // nonce of sender account Gas uint64 // gas limit To *libcommon.Address `rlp:"nil"` // nil means contract creation @@ -45,10 +44,6 @@ type CommonTx struct { V, R, S uint256.Int // signature values } -func (ct CommonTx) GetChainID() *uint256.Int { - return ct.ChainID -} - func (ct CommonTx) GetNonce() uint64 { return ct.Nonce } @@ -229,7 +224,7 @@ func (tx LegacyTx) payloadSize() (payloadSize int, nonceLen, gasLen int) { } default: if len(tx.Data) >= 56 { - payloadSize += (bits.Len(uint(len(tx.Data))) + 7) / 8 + payloadSize += libcommon.BitLenToByteLen(bits.Len(uint(len(tx.Data)))) } payloadSize += len(tx.Data) } diff --git a/core/types/receipt.go b/core/types/receipt.go index 6a5aa835674..699528356b0 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -305,7 +305,7 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error { } r.Type = b[0] switch r.Type { - case AccessListTxType, DynamicFeeTxType, DepositTxType: + case AccessListTxType, DynamicFeeTxType, DepositTxType, BlobTxType: if err := r.decodePayload(s); err != nil { return err } @@ -512,7 +512,9 @@ func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) { } case BlobTxType: w.WriteByte(BlobTxType) - rlp.Encode(w, data) + if err := rlp.Encode(w, data); err != nil { + panic(err) + } default: // For unsupported types, write nothing. Since this is for // DeriveSha, the error will be caught matching the derived hash diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 007ea72f654..9caa53a164a 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -322,7 +322,7 @@ func TestDeriveFields(t *testing.T) { t.Fatalf("DeriveFields(...) = %v, want ", err) } // Iterate over all the computed fields and check that they're correct - signer := MakeSigner(params.TestChainConfig, number.Uint64()) + signer := MakeSigner(params.TestChainConfig, number.Uint64(), 0) logIndex := uint(0) for i := range receipts { diff --git a/core/types/transaction.go b/core/types/transaction.go index df13d3aaa8d..234a160cf6b 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -205,9 +205,6 @@ func DecodeWrappedTransaction(data []byte) (Transaction, error) { } if data[0] < 0x80 { // the encoding is canonical, not RLP - // EIP-4844 tx differs from previous types of transactions in network - // encoding. It's SSZ encoded and includes blobs and kzgs. - // Previous types have no different encoding. return UnmarshalWrappedTransactionFromBinary(data) } s := rlp.NewStream(bytes.NewReader(data), uint64(len(data))) @@ -233,12 +230,6 @@ func UnmarshalTransactionFromBinary(data []byte) (Transaction, error) { return nil, fmt.Errorf("short input: %v", len(data)) } switch data[0] { - case BlobTxType: - t := &SignedBlobTx{} - if err := DecodeSSZ(data[1:], t); err != nil { - return nil, err - } - return t, nil case AccessListTxType: s := rlp.NewStream(bytes.NewReader(data[1:]), uint64(len(data)-1)) t := &AccessListTx{} @@ -260,6 +251,13 @@ func UnmarshalTransactionFromBinary(data []byte) (Transaction, error) { return nil, err } return t, nil + case BlobTxType: + s := rlp.NewStream(bytes.NewReader(data[1:]), uint64(len(data)-1)) + t := &BlobTx{} + if err := t.DecodeRLP(s); err != nil { + return nil, err + } + return t, nil default: if data[0] >= 0x80 { // Tx is type legacy which is RLP encoded @@ -277,8 +275,9 @@ func UnmarshalWrappedTransactionFromBinary(data []byte) (Transaction, error) { if data[0] != BlobTxType { return UnmarshalTransactionFromBinary(data) } + s := rlp.NewStream(bytes.NewReader(data[1:]), uint64(len(data)-1)) t := &BlobTxWrapper{} - if err := DecodeSSZ(data[1:], t); err != nil { + if err := t.DecodeRLP(s); err != nil { return nil, err } return t, nil diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index ff40d296848..032b6482d90 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/holiman/uint256" - . "github.com/protolambda/ztyp/view" "github.com/valyala/fastjson" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -69,6 +68,9 @@ func (tx LegacyTx) MarshalJSON() ([]byte, error) { enc.V = (*hexutil.Big)(tx.V.ToBig()) enc.R = (*hexutil.Big)(tx.R.ToBig()) enc.S = (*hexutil.Big)(tx.S.ToBig()) + if tx.Protected() { + enc.ChainID = (*hexutil.Big)(tx.GetChainID().ToBig()) + } return json.Marshal(&enc) } @@ -132,37 +134,34 @@ func (tx DepositTx) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } -func toSignedBlobTxJSON(tx *SignedBlobTx) *txJSON { +func toBlobTxJSON(tx *BlobTx) *txJSON { var enc txJSON // These are set for all tx types. enc.Hash = tx.Hash() enc.Type = hexutil.Uint64(tx.Type()) - enc.ChainID = (*hexutil.Big)(tx.GetChainID().ToBig()) - accessList := tx.GetAccessList() - enc.AccessList = &accessList - nonce := tx.GetNonce() - enc.Nonce = (*hexutil.Uint64)(&nonce) - gas := tx.GetGas() - enc.Gas = (*hexutil.Uint64)(&gas) - enc.FeeCap = (*hexutil.Big)(tx.GetFeeCap().ToBig()) - enc.Tip = (*hexutil.Big)(tx.GetTip().ToBig()) - enc.Value = (*hexutil.Big)(tx.GetValue().ToBig()) - enc.Data = (*hexutility.Bytes)(&tx.Message.Data) - enc.To = tx.GetTo() - enc.V = (*hexutil.Big)(tx.Signature.GetV().ToBig()) - enc.R = (*hexutil.Big)(tx.Signature.GetR().ToBig()) - enc.S = (*hexutil.Big)(tx.Signature.GetS().ToBig()) - enc.MaxFeePerDataGas = (*hexutil.Big)(tx.GetMaxFeePerDataGas().ToBig()) + enc.ChainID = (*hexutil.Big)(tx.ChainID.ToBig()) + enc.AccessList = &tx.AccessList + enc.Nonce = (*hexutil.Uint64)(&tx.Nonce) + enc.Gas = (*hexutil.Uint64)(&tx.Gas) + enc.FeeCap = (*hexutil.Big)(tx.FeeCap.ToBig()) + enc.Tip = (*hexutil.Big)(tx.Tip.ToBig()) + enc.Value = (*hexutil.Big)(tx.Value.ToBig()) + enc.Data = (*hexutility.Bytes)(&tx.Data) + enc.To = tx.To + enc.V = (*hexutil.Big)(tx.V.ToBig()) + enc.R = (*hexutil.Big)(tx.R.ToBig()) + enc.S = (*hexutil.Big)(tx.S.ToBig()) + enc.MaxFeePerDataGas = (*hexutil.Big)(tx.MaxFeePerDataGas.ToBig()) enc.BlobVersionedHashes = tx.GetDataHashes() return &enc } -func (tx SignedBlobTx) MarshalJSON() ([]byte, error) { - return json.Marshal(toSignedBlobTxJSON(&tx)) +func (tx BlobTx) MarshalJSON() ([]byte, error) { + return json.Marshal(toBlobTxJSON(&tx)) } func (tx BlobTxWrapper) MarshalJSON() ([]byte, error) { - enc := toSignedBlobTxJSON(&tx.Tx) + enc := toBlobTxJSON(&tx.Tx) enc.Blobs = tx.Blobs enc.Commitments = tx.Commitments @@ -502,129 +501,131 @@ func (tx *DepositTx) UnmarshalJSON(input []byte) error { } func UnmarshalBlobTxJSON(input []byte) (Transaction, error) { - var dec txJSON - if err := json.Unmarshal(input, &dec); err != nil { - return nil, err - } - tx := SignedBlobTx{} - if dec.AccessList != nil { - tx.Message.AccessList = AccessListView(*dec.AccessList) - } else { - tx.Message.AccessList = AccessListView([]types2.AccessTuple{}) - } - if dec.ChainID == nil { - return nil, errors.New("missing required field 'chainId' in transaction") - } - chainID, overflow := uint256.FromBig(dec.ChainID.ToInt()) - if overflow { - return nil, errors.New("'chainId' in transaction does not fit in 256 bits") - } - tx.Message.ChainID = Uint256View(*chainID) - if dec.To != nil { - address := AddressSSZ(*dec.To) - tx.Message.To = AddressOptionalSSZ{Address: &address} - } - if dec.Nonce == nil { - return nil, errors.New("missing required field 'nonce' in transaction") - } - tx.Message.Nonce = Uint64View(uint64(*dec.Nonce)) - tip, overflow := uint256.FromBig(dec.Tip.ToInt()) - if overflow { - return nil, errors.New("'tip' in transaction does not fit in 256 bits") - } - tx.Message.GasTipCap = Uint256View(*tip) - feeCap, overflow := uint256.FromBig(dec.FeeCap.ToInt()) - if overflow { - return nil, errors.New("'feeCap' in transaction does not fit in 256 bits") - } - tx.Message.GasFeeCap = Uint256View(*feeCap) - if dec.Gas == nil { - return nil, errors.New("missing required field 'gas' in transaction") - } - tx.Message.Gas = Uint64View(uint64(*dec.Gas)) - if dec.Value == nil { - return nil, errors.New("missing required field 'value' in transaction") - } - value, overflow := uint256.FromBig(dec.Value.ToInt()) - if overflow { - return nil, errors.New("'value' in transaction does not fit in 256 bits") - } - tx.Message.Value = Uint256View(*value) - if dec.Data == nil { - return nil, errors.New("missing required field 'input' in transaction") - } - tx.Message.Data = TxDataView(*dec.Data) - - if dec.MaxFeePerDataGas == nil { - return nil, errors.New("missing required field 'maxFeePerDataGas' in transaction") - } - maxFeePerDataGas, overflow := uint256.FromBig(dec.MaxFeePerDataGas.ToInt()) - if overflow { - return nil, errors.New("'maxFeePerDataGas' in transaction does not fit in 256 bits") - } - tx.Message.MaxFeePerDataGas = Uint256View(*maxFeePerDataGas) - - if dec.BlobVersionedHashes != nil { - tx.Message.BlobVersionedHashes = VersionedHashesView(dec.BlobVersionedHashes) - } else { - tx.Message.BlobVersionedHashes = VersionedHashesView([]libcommon.Hash{}) - } - - if dec.V == nil { - return nil, errors.New("missing required field 'v' in transaction") - } - var v uint256.Int - overflow = v.SetFromBig(dec.V.ToInt()) - if overflow { - return nil, fmt.Errorf("dec.V higher than 2^256-1") - } - if v.Uint64() > 255 { - return nil, fmt.Errorf("dev.V higher than 2^8 - 1") - } - - tx.Signature.V = Uint8View(v.Uint64()) - - if dec.R == nil { - return nil, errors.New("missing required field 'r' in transaction") - } - var r uint256.Int - overflow = r.SetFromBig(dec.R.ToInt()) - if overflow { - return nil, fmt.Errorf("dec.R higher than 2^256-1") - } - tx.Signature.R = Uint256View(r) - - if dec.S == nil { - return nil, errors.New("missing required field 's' in transaction") - } - var s uint256.Int - overflow = s.SetFromBig(dec.S.ToInt()) - if overflow { - return nil, errors.New("'s' in transaction does not fit in 256 bits") - } - tx.Signature.S = Uint256View(s) - - withSignature := !v.IsZero() || !r.IsZero() || !s.IsZero() - if withSignature { - if err := sanityCheckSignature(&v, &r, &s, false); err != nil { - return nil, err - } - } - - if len(dec.Blobs) == 0 { - // if no blobs are specified in the json we assume it is an unwrapped blob tx - return &tx, nil - } - - btx := BlobTxWrapper{ - Tx: tx, - Commitments: dec.Commitments, - Blobs: dec.Blobs, - Proofs: dec.Proofs, - } - err := btx.ValidateBlobTransactionWrapper() - if err != nil { - return nil, err - } - return &btx, nil + // var dec txJSON + // if err := json.Unmarshal(input, &dec); err != nil { + // return nil, err + // } + // tx := SignedBlobTx{} + // if dec.AccessList != nil { + // tx.Message.AccessList = AccessListView(*dec.AccessList) + // } else { + // tx.Message.AccessList = AccessListView([]types2.AccessTuple{}) + // } + // if dec.ChainID == nil { + // return nil, errors.New("missing required field 'chainId' in transaction") + // } + // chainID, overflow := uint256.FromBig(dec.ChainID.ToInt()) + // if overflow { + // return nil, errors.New("'chainId' in transaction does not fit in 256 bits") + // } + // tx.Message.ChainID = Uint256View(*chainID) + // if dec.To != nil { + // address := AddressSSZ(*dec.To) + // tx.Message.To = AddressOptionalSSZ{Address: &address} + // } + // if dec.Nonce == nil { + // return nil, errors.New("missing required field 'nonce' in transaction") + // } + // tx.Message.Nonce = Uint64View(uint64(*dec.Nonce)) + // tip, overflow := uint256.FromBig(dec.Tip.ToInt()) + // if overflow { + // return nil, errors.New("'tip' in transaction does not fit in 256 bits") + // } + // tx.Message.GasTipCap = Uint256View(*tip) + // feeCap, overflow := uint256.FromBig(dec.FeeCap.ToInt()) + // if overflow { + // return nil, errors.New("'feeCap' in transaction does not fit in 256 bits") + // } + // tx.Message.GasFeeCap = Uint256View(*feeCap) + // if dec.Gas == nil { + // return nil, errors.New("missing required field 'gas' in transaction") + // } + // tx.Message.Gas = Uint64View(uint64(*dec.Gas)) + // if dec.Value == nil { + // return nil, errors.New("missing required field 'value' in transaction") + // } + // value, overflow := uint256.FromBig(dec.Value.ToInt()) + // if overflow { + // return nil, errors.New("'value' in transaction does not fit in 256 bits") + // } + // tx.Message.Value = Uint256View(*value) + // if dec.Data == nil { + // return nil, errors.New("missing required field 'input' in transaction") + // } + // tx.Message.Data = TxDataView(*dec.Data) + + // if dec.MaxFeePerDataGas == nil { + // return nil, errors.New("missing required field 'maxFeePerDataGas' in transaction") + // } + // maxFeePerDataGas, overflow := uint256.FromBig(dec.MaxFeePerDataGas.ToInt()) + // if overflow { + // return nil, errors.New("'maxFeePerDataGas' in transaction does not fit in 256 bits") + // } + // tx.Message.MaxFeePerDataGas = Uint256View(*maxFeePerDataGas) + + // if dec.BlobVersionedHashes != nil { + // tx.Message.BlobVersionedHashes = VersionedHashesView(dec.BlobVersionedHashes) + // } else { + // tx.Message.BlobVersionedHashes = VersionedHashesView([]libcommon.Hash{}) + // } + + // if dec.V == nil { + // return nil, errors.New("missing required field 'v' in transaction") + // } + // var v uint256.Int + // overflow = v.SetFromBig(dec.V.ToInt()) + // if overflow { + // return nil, fmt.Errorf("dec.V higher than 2^256-1") + // } + // if v.Uint64() > 255 { + // return nil, fmt.Errorf("dev.V higher than 2^8 - 1") + // } + + // tx.Signature.V = Uint8View(v.Uint64()) + + // if dec.R == nil { + // return nil, errors.New("missing required field 'r' in transaction") + // } + // var r uint256.Int + // overflow = r.SetFromBig(dec.R.ToInt()) + // if overflow { + // return nil, fmt.Errorf("dec.R higher than 2^256-1") + // } + // tx.Signature.R = Uint256View(r) + + // if dec.S == nil { + // return nil, errors.New("missing required field 's' in transaction") + // } + // var s uint256.Int + // overflow = s.SetFromBig(dec.S.ToInt()) + // if overflow { + // return nil, errors.New("'s' in transaction does not fit in 256 bits") + // } + // tx.Signature.S = Uint256View(s) + + // withSignature := !v.IsZero() || !r.IsZero() || !s.IsZero() + // if withSignature { + // if err := sanityCheckSignature(&v, &r, &s, false); err != nil { + // return nil, err + // } + // } + + // if len(dec.Blobs) == 0 { + // // if no blobs are specified in the json we assume it is an unwrapped blob tx + // return &tx, nil + // } + + // btx := BlobTxWrapper{ + // Tx: tx, + // Commitments: dec.Commitments, + // Blobs: dec.Blobs, + // Proofs: dec.Proofs, + // } + // err := btx.ValidateBlobTransactionWrapper() + // if err != nil { + // return nil, err + // } + // return &btx, nil + + return nil, nil } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 0ccf0b39cb9..2a178048ba5 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -34,7 +34,7 @@ import ( var ErrInvalidChainId = errors.New("invalid chain id for signer") // MakeSigner returns a Signer based on the given chain config and block number. -func MakeSigner(config *chain.Config, blockNumber uint64) *Signer { +func MakeSigner(config *chain.Config, blockNumber uint64, blockTime uint64) *Signer { var signer Signer var chainId uint256.Int if config.ChainID != nil { @@ -45,16 +45,23 @@ func MakeSigner(config *chain.Config, blockNumber uint64) *Signer { } signer.unprotected = true switch { - case config.IsLondon(blockNumber): + case config.IsCancun(blockTime): // All transaction types are still supported signer.protected = true - signer.accesslist = true - signer.dynamicfee = true + signer.accessList = true + signer.dynamicFee = true + signer.blob = true + signer.chainID.Set(&chainId) + signer.chainIDMul.Mul(&chainId, u256.Num2) + case config.IsLondon(blockNumber): + signer.protected = true + signer.accessList = true + signer.dynamicFee = true signer.chainID.Set(&chainId) signer.chainIDMul.Mul(&chainId, u256.Num2) case config.IsBerlin(blockNumber): signer.protected = true - signer.accesslist = true + signer.accessList = true signer.chainID.Set(&chainId) signer.chainIDMul.Mul(&chainId, u256.Num2) case config.IsSpuriousDragon(blockNumber): @@ -64,14 +71,14 @@ func MakeSigner(config *chain.Config, blockNumber uint64) *Signer { case config.IsHomestead(blockNumber): default: // Only allow malleable transactions in Frontier - signer.maleable = true + signer.malleable = true } return &signer } func MakeFrontierSigner() *Signer { var signer Signer - signer.maleable = true + signer.malleable = true signer.unprotected = true return &signer } @@ -93,11 +100,14 @@ func LatestSigner(config *chain.Config) *Signer { signer.chainID.Set(chainId) signer.chainIDMul.Mul(chainId, u256.Num2) if config.ChainID != nil { + if config.CancunTime != nil { + signer.blob = true + } if config.LondonBlock != nil { - signer.dynamicfee = true + signer.dynamicFee = true } if config.BerlinBlock != nil { - signer.accesslist = true + signer.accessList = true } if config.SpuriousDragonBlock != nil { signer.protected = true @@ -126,8 +136,9 @@ func LatestSignerForChainID(chainID *big.Int) *Signer { signer.chainID.Set(chainId) signer.chainIDMul.Mul(chainId, u256.Num2) signer.protected = true - signer.accesslist = true - signer.dynamicfee = true + signer.accessList = true + signer.dynamicFee = true + signer.blob = true return &signer } @@ -169,15 +180,17 @@ func MustSignNewTx(prv *ecdsa.PrivateKey, s Signer, tx Transaction) Transaction // new protocol rules. type Signer struct { chainID, chainIDMul uint256.Int - maleable bool // Whether this signer should allow legacy transactions with malleability + malleable bool // Whether this signer should allow legacy transactions with malleability unprotected bool // Whether this signer should allow legacy transactions without chainId protection protected bool // Whether this signer should allow transactions with replay protection via chainId - accesslist bool // Whether this signer should allow transactions with access list, superseeds protected - dynamicfee bool // Whether this signer should allow transactions with basefee and tip (instead of gasprice), superseeds accesslist + accessList bool // Whether this signer should allow transactions with access list, supersedes protected + dynamicFee bool // Whether this signer should allow transactions with base fee and tip (instead of gasprice), supersedes accessList + blob bool // Whether this signer should allow blob transactions } func (sg Signer) String() string { - return fmt.Sprintf("Signer[chainId=%s,malleable=%t,unprotected=%t,protected=%t,accesslist=%t,dynamicfee=%t", &sg.chainID, sg.maleable, sg.unprotected, sg.protected, sg.accesslist, sg.dynamicfee) + return fmt.Sprintf("Signer[chainId=%s,malleable=%t,unprotected=%t,protected=%t,accessList=%t,dynamicFee=%t,blob=%t", + &sg.chainID, sg.malleable, sg.unprotected, sg.protected, sg.accessList, sg.dynamicFee, sg.blob) } // Sender returns the sender address of the transaction. @@ -213,8 +226,8 @@ func (sg Signer) SenderWithContext(context *secp256k1.Context, tx Transaction) ( } R, S = &t.R, &t.S case *AccessListTx: - if !sg.accesslist { - return libcommon.Address{}, fmt.Errorf("accesslist tx is not supported by signer %s", sg) + if !sg.accessList { + return libcommon.Address{}, fmt.Errorf("accessList tx is not supported by signer %s", sg) } if t.ChainID == nil { if !sg.chainID.IsZero() { @@ -228,8 +241,8 @@ func (sg Signer) SenderWithContext(context *secp256k1.Context, tx Transaction) ( V.Add(&t.V, u256.Num27) R, S = &t.R, &t.S case *DynamicFeeTransaction: - if !sg.dynamicfee { - return libcommon.Address{}, fmt.Errorf("dynamicfee tx is not supported by signer %s", sg) + if !sg.dynamicFee { + return libcommon.Address{}, fmt.Errorf("dynamicFee tx is not supported by signer %s", sg) } if t.ChainID == nil { if !sg.chainID.IsZero() { @@ -242,10 +255,25 @@ func (sg Signer) SenderWithContext(context *secp256k1.Context, tx Transaction) ( // id, add 27 to become equivalent to unprotected Homestead signatures. V.Add(&t.V, u256.Num27) R, S = &t.R, &t.S + case *BlobTx: + if !sg.blob { + return libcommon.Address{}, fmt.Errorf("blob tx is not supported by signer %s", sg) + } + if t.ChainID == nil { + if !sg.chainID.IsZero() { + return libcommon.Address{}, ErrInvalidChainId + } + } else if !t.ChainID.Eq(&sg.chainID) { + return libcommon.Address{}, ErrInvalidChainId + } + // ACL, DynamicFee, and blob txs are defined to use 0 and 1 as their recovery + // id, add 27 to become equivalent to unprotected Homestead signatures. + V.Add(&t.V, u256.Num27) + R, S = &t.R, &t.S default: return libcommon.Address{}, ErrTxTypeNotSupported } - return recoverPlain(context, tx.SigningHash(signChainID), R, S, &V, !sg.maleable) + return recoverPlain(context, tx.SigningHash(signChainID), R, S, &V, !sg.malleable) } // SignatureValues returns the raw R, S, V values corresponding to the @@ -276,6 +304,13 @@ func (sg Signer) SignatureValues(tx Transaction, sig []byte) (R, S, V *uint256.I R, S, V = decodeSignature(sig) case *DepositTx: return nil, nil, nil, fmt.Errorf("deposits do not have a signature") + case *BlobTx: + // Check that chain ID of tx matches the signer. We also accept ID zero here, + // because it indicates that the chain ID was not specified in the tx. + if t.ChainID != nil && !t.ChainID.IsZero() && !t.ChainID.Eq(&sg.chainID) { + return nil, nil, nil, ErrInvalidChainId + } + R, S, V = decodeSignature(sig) default: return nil, nil, nil, ErrTxTypeNotSupported } @@ -289,11 +324,12 @@ func (sg Signer) ChainID() *uint256.Int { // Equal returns true if the given signer is the same as the receiver. func (sg Signer) Equal(other Signer) bool { return sg.chainID.Eq(&other.chainID) && - sg.maleable == other.maleable && + sg.malleable == other.malleable && sg.unprotected == other.unprotected && sg.protected == other.protected && - sg.accesslist == other.accesslist && - sg.dynamicfee == other.dynamicfee + sg.accessList == other.accessList && + sg.dynamicFee == other.dynamicFee && + sg.blob == other.blob } func decodeSignature(sig []byte) (r, s, v *uint256.Int) { diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 81ba8cba1cb..838471f1ff2 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "math/big" + "math/rand" "reflect" "testing" "time" @@ -84,15 +85,15 @@ var ( dynFeeTx = &DynamicFeeTransaction{ CommonTx: CommonTx{ - ChainID: u256.Num1, - Nonce: 3, - To: &testAddr, - Value: uint256.NewInt(10), - Gas: 25000, - Data: common.FromHex("5544"), + Nonce: 3, + To: &testAddr, + Value: uint256.NewInt(10), + Gas: 25000, + Data: common.FromHex("5544"), }, - Tip: uint256.NewInt(1), - FeeCap: uint256.NewInt(1), + ChainID: u256.Num1, + Tip: uint256.NewInt(1), + FeeCap: uint256.NewInt(1), } signedDynFeeTx, _ = dynFeeTx.WithSignature( @@ -578,3 +579,120 @@ func assertEqual(orig Transaction, cpy Transaction) error { } return nil } + +const N = 50 + +var dummyBlobTxs = [N]*BlobTx{} +var addr [20]byte + +func randIntInRange(min, max int) int { + return (rand.Intn(max-min) + min) +} + +func randAddr() *libcommon.Address { + var a libcommon.Address + for j := 0; j < 20; j++ { + a[j] = byte(rand.Intn(255)) + } + return &a +} + +func randHash() libcommon.Hash { + var h libcommon.Hash + for i := 0; i < 32; i++ { + h[i] = byte(rand.Intn(255)) + } + return h +} + +func randHashes(n int) []libcommon.Hash { + h := make([]libcommon.Hash, n) + for i := 0; i < n; i++ { + h[i] = randHash() + } + return h +} + +func randAccessList() types2.AccessList { + size := randIntInRange(4, 10) + var result types2.AccessList + for i := 0; i < size; i++ { + var tup types2.AccessTuple + + tup.Address = *randAddr() + tup.StorageKeys = append(tup.StorageKeys, randHash()) + result = append(result, tup) + } + return result +} + +func randData() []byte { + data := make([]byte, 0, (1 << 16)) + for j := 0; j < rand.Intn(1<<16); j++ { + data = append(data, byte(rand.Intn(255))) + } + return data +} + +func newRandBlobTx() *BlobTx { + stx := &BlobTx{DynamicFeeTransaction: DynamicFeeTransaction{ + CommonTx: CommonTx{ + Nonce: rand.Uint64(), + Gas: rand.Uint64(), + To: randAddr(), + Value: uint256.NewInt(rand.Uint64()), + Data: randData(), + V: *uint256.NewInt(uint64(rand.Intn(2))), + R: *uint256.NewInt(rand.Uint64()), + S: *uint256.NewInt(rand.Uint64()), + }, + ChainID: uint256.NewInt(rand.Uint64()), + Tip: uint256.NewInt(rand.Uint64()), + FeeCap: uint256.NewInt(rand.Uint64()), + AccessList: randAccessList(), + }, + MaxFeePerDataGas: uint256.NewInt(rand.Uint64()), + BlobVersionedHashes: randHashes(randIntInRange(5, 10)), + } + return stx +} + +func printSTX(stx *BlobTx) { + fmt.Println("--BlobTx") + fmt.Printf("ChainID: %v\n", stx.ChainID) + fmt.Printf("Nonce: %v\n", stx.Nonce) + fmt.Printf("MaxPriorityFeePerGas: %v\n", stx.Tip) + fmt.Printf("MaxFeePerGas: %v\n", stx.FeeCap) + fmt.Printf("Gas: %v\n", stx.Gas) + fmt.Printf("To: %v\n", stx.To) + fmt.Printf("Value: %v\n", stx.Value) + fmt.Printf("Data: %v\n", stx.Data) + fmt.Printf("AccessList: %v\n", stx.AccessList) + fmt.Printf("MaxFeePerDataGas: %v\n", stx.MaxFeePerDataGas) + fmt.Printf("BlobVersionedHashes: %v\n", stx.BlobVersionedHashes) + fmt.Printf("YParity: %v\n", stx.V) + fmt.Printf("R: %v\n", stx.R) + fmt.Printf("S: %v\n", stx.S) + fmt.Println("-----") + fmt.Println() +} + +func populateBlobTxs() { + for i := 0; i < N; i++ { + dummyBlobTxs[i] = newRandBlobTx() + } +} + +func TestBlobTxEncodeDecode(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + populateBlobTxs() + for i := 0; i < N; i++ { + // printSTX(dummyBlobTxs[i]) + + tx, err := encodeDecodeBinary(dummyBlobTxs[i]) + if err != nil { + t.Fatal(err) + } + assertEqual(dummyBlobTxs[i], tx) + } +} diff --git a/core/vm/evmtypes/evmtypes.go b/core/vm/evmtypes/evmtypes.go index d6f79233510..81bf57949fe 100644 --- a/core/vm/evmtypes/evmtypes.go +++ b/core/vm/evmtypes/evmtypes.go @@ -6,7 +6,7 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/chain" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" types2 "github.com/ledgerwatch/erigon-lib/types" "github.com/ledgerwatch/erigon/core/types" @@ -24,15 +24,15 @@ type BlockContext struct { GetHash GetHashFunc // Block information - Coinbase libcommon.Address // Provides information for COINBASE - GasLimit uint64 // Provides information for GASLIMIT - MaxGasLimit bool // Use GasLimit override for 2^256-1 (to be compatible with OpenEthereum's trace_call) - BlockNumber uint64 // Provides information for NUMBER - Time uint64 // Provides information for TIME - Difficulty *big.Int // Provides information for DIFFICULTY - BaseFee *uint256.Int // Provides information for BASEFEE - PrevRanDao *libcommon.Hash // Provides information for PREVRANDAO - ExcessDataGas *uint64 // Provides information for handling data blobs + Coinbase common.Address // Provides information for COINBASE + GasLimit uint64 // Provides information for GASLIMIT + MaxGasLimit bool // Use GasLimit override for 2^256-1 (to be compatible with OpenEthereum's trace_call) + BlockNumber uint64 // Provides information for NUMBER + Time uint64 // Provides information for TIME + Difficulty *big.Int // Provides information for DIFFICULTY + BaseFee *uint256.Int // Provides information for BASEFEE + PrevRanDao *common.Hash // Provides information for PREVRANDAO + ExcessDataGas *uint64 // Provides information for handling data blobs // L1CostFunc returns the L1 cost of the rollup message, the function may be nil, or return nil L1CostFunc types.L1CostFunc @@ -42,70 +42,70 @@ type BlockContext struct { // All fields can change between transactions. type TxContext struct { // Message information - TxHash libcommon.Hash - Origin libcommon.Address // Provides information for ORIGIN - GasPrice *uint256.Int // Provides information for GASPRICE - DataHashes []libcommon.Hash // Provides versioned data hashes for DATAHASH + TxHash common.Hash + Origin common.Address // Provides information for ORIGIN + GasPrice *uint256.Int // Provides information for GASPRICE + DataHashes []common.Hash // Provides versioned data hashes for DATAHASH } type ( // CanTransferFunc is the signature of a transfer guard function - CanTransferFunc func(IntraBlockState, libcommon.Address, *uint256.Int) bool + CanTransferFunc func(IntraBlockState, common.Address, *uint256.Int) bool // TransferFunc is the signature of a transfer function - TransferFunc func(IntraBlockState, libcommon.Address, libcommon.Address, *uint256.Int, bool) + TransferFunc func(IntraBlockState, common.Address, common.Address, *uint256.Int, bool) // GetHashFunc returns the nth block hash in the blockchain // and is used by the BLOCKHASH EVM op code. - GetHashFunc func(uint64) libcommon.Hash + GetHashFunc func(uint64) common.Hash ) // IntraBlockState is an EVM database for full state querying. type IntraBlockState interface { - CreateAccount(libcommon.Address, bool) + CreateAccount(common.Address, bool) - SubBalance(libcommon.Address, *uint256.Int) - AddBalance(libcommon.Address, *uint256.Int) - GetBalance(libcommon.Address) *uint256.Int + SubBalance(common.Address, *uint256.Int) + AddBalance(common.Address, *uint256.Int) + GetBalance(common.Address) *uint256.Int - GetNonce(libcommon.Address) uint64 - SetNonce(libcommon.Address, uint64) + GetNonce(common.Address) uint64 + SetNonce(common.Address, uint64) - GetCodeHash(libcommon.Address) libcommon.Hash - GetCode(libcommon.Address) []byte - SetCode(libcommon.Address, []byte) - GetCodeSize(libcommon.Address) int + GetCodeHash(common.Address) common.Hash + GetCode(common.Address) []byte + SetCode(common.Address, []byte) + GetCodeSize(common.Address) int AddRefund(uint64) SubRefund(uint64) GetRefund() uint64 - GetCommittedState(libcommon.Address, *libcommon.Hash, *uint256.Int) - GetState(address libcommon.Address, slot *libcommon.Hash, outValue *uint256.Int) - SetState(libcommon.Address, *libcommon.Hash, uint256.Int) + GetCommittedState(common.Address, *common.Hash, *uint256.Int) + GetState(address common.Address, slot *common.Hash, outValue *uint256.Int) + SetState(common.Address, *common.Hash, uint256.Int) - GetTransientState(addr libcommon.Address, key libcommon.Hash) uint256.Int - SetTransientState(addr libcommon.Address, key libcommon.Hash, value uint256.Int) + GetTransientState(addr common.Address, key common.Hash) uint256.Int + SetTransientState(addr common.Address, key common.Hash, value uint256.Int) - Selfdestruct(libcommon.Address) bool - HasSelfdestructed(libcommon.Address) bool + Selfdestruct(common.Address) bool + HasSelfdestructed(common.Address) bool // Exist reports whether the given account exists in state. // Notably this should also return true for suicided accounts. - Exist(libcommon.Address) bool + Exist(common.Address) bool // Empty returns whether the given account is empty. Empty // is defined according to EIP161 (balance = nonce = code = 0). - Empty(libcommon.Address) bool + Empty(common.Address) bool - Prepare(rules *chain.Rules, sender, coinbase libcommon.Address, dest *libcommon.Address, - precompiles []libcommon.Address, txAccesses types2.AccessList) + Prepare(rules *chain.Rules, sender, coinbase common.Address, dest *common.Address, + precompiles []common.Address, txAccesses types2.AccessList) - AddressInAccessList(addr libcommon.Address) bool - SlotInAccessList(addr libcommon.Address, slot libcommon.Hash) (addressOk bool, slotOk bool) + AddressInAccessList(addr common.Address) bool + SlotInAccessList(addr common.Address, slot common.Hash) (addressOk bool, slotOk bool) // AddAddressToAccessList adds the given address to the access list. This operation is safe to perform // even if the feature/fork is not active yet - AddAddressToAccessList(addr libcommon.Address) + AddAddressToAccessList(addr common.Address) // AddSlotToAccessList adds the given (address,slot) to the access list. This operation is safe to perform // even if the feature/fork is not active yet - AddSlotToAccessList(addr libcommon.Address, slot libcommon.Hash) + AddSlotToAccessList(addr common.Address, slot common.Hash) RevertToSnapshot(int) Snapshot() int diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 60e94e40e5a..a48eb6c3c4b 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -20,6 +20,7 @@ import ( "errors" "github.com/holiman/uint256" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/math" @@ -346,7 +347,7 @@ func gasCreate2Eip3860(_ VMInterpreter, contract *Contract, stack *stack.Stack, } func gasExpFrontier(_ VMInterpreter, contract *Contract, stack *stack.Stack, mem *Memory, memorySize uint64) (uint64, error) { - expByteLen := uint64((stack.Data[stack.Len()-2].BitLen() + 7) / 8) + expByteLen := uint64(libcommon.BitLenToByteLen(stack.Data[stack.Len()-2].BitLen())) var ( gas = expByteLen * params.ExpByteFrontier // no overflow check required. Max is 256 * ExpByte gas @@ -359,7 +360,7 @@ func gasExpFrontier(_ VMInterpreter, contract *Contract, stack *stack.Stack, mem } func gasExpEIP160(_ VMInterpreter, contract *Contract, stack *stack.Stack, mem *Memory, memorySize uint64) (uint64, error) { - expByteLen := uint64((stack.Data[stack.Len()-2].BitLen() + 7) / 8) + expByteLen := uint64(libcommon.BitLenToByteLen(stack.Data[stack.Len()-2].BitLen())) var ( gas = expByteLen * params.ExpByteEIP160 // no overflow check required. Max is 256 * ExpByte gas diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 0e11e5defda..7e3300a37ea 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -17,6 +17,7 @@ package vm import ( + "context" "errors" "math" "strconv" @@ -24,12 +25,15 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv/memdb" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/vm/evmtypes" "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/turbo/rpchelper" ) func TestMemoryGasCost(t *testing.T) { @@ -133,14 +137,20 @@ var createGasTests = []struct { } func TestCreateGas(t *testing.T) { + _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) for i, tt := range createGasTests { address := libcommon.BytesToAddress([]byte("contract")) - _, tx := memdb.NewTestTx(t) - s := state.New(state.NewPlainStateReader(tx)) + tx, _ := db.BeginRw(context.Background()) + defer tx.Rollback() + + stateReader := rpchelper.NewLatestStateReader(tx) + stateWriter := rpchelper.NewLatestStateWriter(tx, 0) + + s := state.New(stateReader) s.CreateAccount(address, true) s.SetCode(address, hexutil.MustDecode(tt.code)) - _ = s.CommitBlock(params.TestChainConfig.Rules(0, 0), state.NewPlainStateWriter(tx, tx, 0)) + _ = s.CommitBlock(params.TestChainConfig.Rules(0, 0), stateWriter) vmctx := evmtypes.BlockContext{ CanTransfer: func(evmtypes.IntraBlockState, libcommon.Address, *uint256.Int) bool { return true }, @@ -161,5 +171,6 @@ func TestCreateGas(t *testing.T) { if gasUsed := startGas - gas; gasUsed != tt.gasUsed { t.Errorf("test %d: gas used mismatch: have %v, want %v", i, gasUsed, tt.gasUsed) } + tx.Rollback() } } diff --git a/crypto/ecies/ecies.go b/crypto/ecies/ecies.go index 64b5a99d03a..c5050095315 100644 --- a/crypto/ecies/ecies.go +++ b/crypto/ecies/ecies.go @@ -40,6 +40,8 @@ import ( "hash" "io" "math/big" + + libcommon "github.com/ledgerwatch/erigon-lib/common" ) var ( @@ -114,7 +116,7 @@ func GenerateKey(rand io.Reader, curve elliptic.Curve, params *ECIESParams) (prv // MaxSharedKeyLength returns the maximum length of the shared key the // public key can produce. func MaxSharedKeyLength(pub *PublicKey) int { - return (pub.Curve.Params().BitSize + 7) / 8 + return libcommon.BitLenToByteLen(pub.Curve.Params().BitSize) } // ECDH key agreement method used to establish secret keys for encryption. diff --git a/docker-compose.yml b/docker-compose.yml index bf5982c9b4a..37c20a28fbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: <<: *default-erigon-service entrypoint: rpcdaemon command: | - ${RPCDAEMON_FLAGS-} --http.addr=0.0.0.0 --http.vhosts=* --http.corsdomain=* --ws + ${RPCDAEMON_FLAGS-} --http.addr=0.0.0.0 --http.vhosts=any --http.corsdomain=* --ws --private.api.addr=erigon:9090 --txpool.api.addr=txpool:9094 --datadir=/home/erigon/.local/share/erigon ports: [ "8545:8545" ] diff --git a/docs/evm_semantics.md b/docs/evm_semantics.md index 38a4e363a5e..bc280498848 100644 --- a/docs/evm_semantics.md +++ b/docs/evm_semantics.md @@ -1,10 +1,10 @@ Corresponding code is in the folder `semantics`. ## EVM without opcodes (Ether transfers only) -We start with looking at a very restricted version of EVM, having no opcodes. That means only Ether transfers are possible. +We start by looking at a very restricted version of EVM, having no opcodes. That means only Ether transfers are possible. Even that seemingly simple case already has a relatively complex semantics. -First, we would like to define what kind semantics we are looking for. Ethereum is a state machine, which has a global state, and +First, we would like to define what kind of semantics we are looking for. Ethereum is a state machine, which has a global state, and transactions that trigger some state transition. There are also some state transitions that happen at the end of each block. These are related to miner and ommer rewards. The transition from one state to another is deterministic. Given the initial state, some extra environment (recent header hashes, current block number, timestamp of the current block), and the transaction object, there is only @@ -22,7 +22,7 @@ the initial state, environment, and the transaction object. So they look more li `EXPRESSION(STATE_init, env, tx, STATE_end) == true?` -Moreover, this representation allows for some non determinsm, which means that there could be some extra "oracle" input that helps the +Moreover, this representation allows for some non determinism, which means that there could be some extra "oracle" input that helps the evaluation: `EXPRESSION(STATE_init, env, tx, STATE_end, ORACLE_input) == true?` diff --git a/docs/examples/single-process.md b/docs/examples/single-process.md index 94a108f081c..6465128fba7 100644 --- a/docs/examples/single-process.md +++ b/docs/examples/single-process.md @@ -18,6 +18,7 @@ How to run Erigon in a single process (all parts of the system run as one). --ws \ --http.api=eth,debug,net,trace,web3,erigon \ --log.dir.path=/desired/path/to/logs + --log.dir.prefix=filename ``` ## Notes diff --git a/docs/merry-go-round-sync.md b/docs/merry-go-round-sync.md index c4b4cb8ee1a..ebc04708a82 100644 --- a/docs/merry-go-round-sync.md +++ b/docs/merry-go-round-sync.md @@ -20,7 +20,7 @@ always, as shown on the picture below. ![cycles-and-ticks](mgr-sync-1.png) If chain reorgs occur, and the timings of recent Ethereum blocks change as a result, we can accept these rules to prevent -the reorgs to be used to disrupt the sync. Imaginge the tick started at block A (height H), and then due to reorg, block A +the reorgs to be used to disrupt the sync. Imagine the tick started at block A (height H), and then due to reorg, block A was replaced by block B (also height H). * If timestamp(B) < timestamp(A), the tick does not shorten, but proceeds until timestamp(A) + tick_duration. @@ -29,7 +29,7 @@ was replaced by block B (also height H). As one would guess, we try to distribute the entire Ethereum state into as many pieces as many ticks there are in one cycle. Each piece would be exchanged over the duration of one tick. Obviously, we would like to make the distribution as even as possible. - Therefore, there is still a concern about situations when the blocks are coming in quick sucession, and the ticks corresponding + Therefore, there is still a concern about situations when the blocks are coming in quick succession, and the ticks corresponding to those blocks would largely overlap. ## Sync schedule @@ -37,7 +37,7 @@ was replaced by block B (also height H). When we split the entire Ethereum state into pieces and plan to exchange each piece during one tick, we are creating a sync schedule. Sync schedule is a mapping from the tick number (which can be derived from the block number) to the piece of state. These pieces need to be efficient to extract from the State Database (for a seeder), and add to the State Database (for a leecher). -Probably the most convinient way of specifying such a piece of state is a pair of bounds - lower bound and upper bound. +Probably the most convenient way of specifying such a piece of state is a pair of bounds - lower bound and upper bound. Each of the bounds would correspond to either Keccak256 hash of an address, or to a combination of Keccak256 hash of an address, and Keccak256 hash of a storage location in some contract. In other words, there could be four types of specification for a piece of state: @@ -51,5 +51,5 @@ In the last type, addresses `address1` and `address2` may mean the same address. ## How will seeders produce sync schedule -Seeders should have the ability to generate the sync schedule by the virtue of having the entire Ethreum state available. No +Seeders should have the ability to generate the sync schedule by the virtue of having the entire Ethereum state available. No extra coordination should be necessary. diff --git a/eth/backend.go b/eth/backend.go index 227c2b28aa6..ae8273853db 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -37,6 +37,7 @@ import ( "github.com/ledgerwatch/erigon/cl/phase1/execution_client" "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/ethdb/prune" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/turbo/snapshotsync/snap" "github.com/holiman/uint256" @@ -63,7 +64,7 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon-lib/kv/remotedbserver" libstate "github.com/ledgerwatch/erigon-lib/state" - txpool2 "github.com/ledgerwatch/erigon-lib/txpool" + "github.com/ledgerwatch/erigon-lib/txpool" "github.com/ledgerwatch/erigon-lib/txpool/txpooluitl" types2 "github.com/ledgerwatch/erigon-lib/types" @@ -105,7 +106,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/engineapi" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/shards" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" stages2 "github.com/ledgerwatch/erigon/turbo/stages" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" ) @@ -138,7 +138,7 @@ type Ethereum struct { seqRPCService *rpc.Client historicalRPCService *rpc.Client miningRPC txpool_proto.MiningServer - stateChangesClient txpool2.StateChangesClient + stateChangesClient txpool.StateChangesClient miningSealingQuit chan struct{} pendingBlocks chan *types.Block @@ -163,18 +163,18 @@ type Ethereum struct { waitForStageLoopStop chan struct{} waitForMiningStop chan struct{} - txPool2DB kv.RwDB - txPool2 *txpool2.TxPool - newTxs2 chan types2.Announcements - txPool2Fetch *txpool2.Fetch - txPool2Send *txpool2.Send - txPool2GrpcServer txpool_proto.TxpoolServer + txPoolDB kv.RwDB + txPool *txpool.TxPool + newTxs chan types2.Announcements + txPoolFetch *txpool.Fetch + txPoolSend *txpool.Send + txPoolGrpcServer txpool_proto.TxpoolServer notifyMiningAboutNewTxs chan struct{} forkValidator *engineapi.ForkValidator downloader *downloader3.Downloader agg *libstate.AggregatorV3 - blockSnapshots *snapshotsync.RoSnapshots + blockSnapshots *freezeblocks.RoSnapshots blockReader services.FullBlockReader blockWriter *blockio.BlockWriter kvRPC *remotedbserver.KvServer @@ -281,7 +281,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere genesisSpec = nil } var genesisErr error - chainConfig, genesis, genesisErr = core.WriteGenesisBlock(tx, genesisSpec, config.OverrideShanghaiTime, tmpdir, logger, blockWriter) + chainConfig, genesis, genesisErr = core.WriteGenesisBlock(tx, genesisSpec, config.OverrideShanghaiTime, tmpdir, logger) if _, ok := genesisErr.(*chain.ConfigCompatError); genesisErr != nil && !ok { return genesisErr } @@ -421,7 +421,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere notifications *shards.Notifications) error { // Needs its own notifications to not update RPC daemon and txpool about pending blocks stateSync, err := stages2.NewInMemoryExecution(backend.sentryCtx, backend.chainDB, config, backend.sentriesClient, - dirs, notifications, allSnapshots, backend.agg, log.New() /* logging will be discarded */) + dirs, notifications, blockReader, blockWriter, backend.agg, log.New() /* logging will be discarded */) if err != nil { return err } @@ -511,16 +511,16 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere var miningRPC txpool_proto.MiningServer stateDiffClient := direct.NewStateDiffClientDirect(kvRPC) if config.DeprecatedTxPool.Disable { - backend.txPool2GrpcServer = &txpool2.GrpcDisabled{} + backend.txPoolGrpcServer = &txpool.GrpcDisabled{} } else { //cacheConfig := kvcache.DefaultCoherentCacheConfig //cacheConfig.MetricsLabel = "txpool" - backend.newTxs2 = make(chan types2.Announcements, 1024) + backend.newTxs = make(chan types2.Announcements, 1024) //defer close(newTxs) config.TxPool.Optimism = chainConfig.Optimism != nil - backend.txPool2DB, backend.txPool2, backend.txPool2Fetch, backend.txPool2Send, backend.txPool2GrpcServer, err = txpooluitl.AllComponents( - ctx, config.TxPool, kvcache.NewDummy(), backend.newTxs2, backend.chainDB, backend.sentriesClient.Sentries(), stateDiffClient, logger, + backend.txPoolDB, backend.txPool, backend.txPoolFetch, backend.txPoolSend, backend.txPoolGrpcServer, err = txpooluitl.AllComponents( + ctx, config.TxPool, kvcache.NewDummy(), backend.newTxs, backend.chainDB, backend.sentriesClient.Sentries(), stateDiffClient, logger, ) if err != nil { return nil, err @@ -539,8 +539,8 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere // proof-of-work mining mining := stagedsync.New( stagedsync.MiningStages(backend.sentryCtx, - stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miner, *backend.chainConfig, backend.engine, backend.txPool2, backend.txPool2DB, nil, tmpdir, backend.blockReader), - stagedsync.StageMiningExecCfg(backend.chainDB, miner, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, nil, 0, backend.txPool2, backend.txPool2DB, false, blockReader), + stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miner, *backend.chainConfig, backend.engine, backend.txPoolDB, nil, tmpdir, backend.blockReader), + stagedsync.StageMiningExecCfg(backend.chainDB, miner, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, nil, 0, backend.txPool, backend.txPoolDB, false, blockReader), stagedsync.StageHashStateCfg(backend.chainDB, dirs, config.HistoryV3), stagedsync.StageTrieCfg(backend.chainDB, false, true, true, tmpdir, blockReader, nil, config.HistoryV3, backend.agg), stagedsync.StageMiningFinishCfg(backend.chainDB, *backend.chainConfig, backend.engine, miner, backend.miningSealingQuit, backend.blockReader), @@ -561,8 +561,8 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere miningStatePos.MiningConfig.GasLimit = *param.GasLimit proposingSync := stagedsync.New( stagedsync.MiningStages(backend.sentryCtx, - stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miningStatePos, *backend.chainConfig, backend.engine, backend.txPool2, backend.txPool2DB, param, tmpdir, backend.blockReader), - stagedsync.StageMiningExecCfg(backend.chainDB, miningStatePos, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, interrupt, param.PayloadId, backend.txPool2, backend.txPool2DB, param.NoTxPool, blockReader), + stagedsync.StageMiningCreateBlockCfg(backend.chainDB, miningStatePos, *backend.chainConfig, backend.engine, backend.txPoolDB, param, tmpdir, backend.blockReader), + stagedsync.StageMiningExecCfg(backend.chainDB, miningStatePos, backend.notifications.Events, *backend.chainConfig, backend.engine, &vm.Config{}, tmpdir, interrupt, param.PayloadId, backend.txPool, backend.txPoolDB, param.NoTxPool, blockReader), stagedsync.StageHashStateCfg(backend.chainDB, dirs, config.HistoryV3), stagedsync.StageTrieCfg(backend.chainDB, false, true, true, tmpdir, blockReader, nil, config.HistoryV3, backend.agg), stagedsync.StageMiningFinishCfg(backend.chainDB, *backend.chainConfig, backend.engine, miningStatePos, backend.miningSealingQuit, backend.blockReader), @@ -592,7 +592,7 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere backend.privateAPI, err = privateapi.StartGrpc( kvRPC, ethBackendRPC, - backend.txPool2GrpcServer, + backend.txPoolGrpcServer, miningRPC, stack.Config().PrivateApiAddr, stack.Config().PrivateApiRateLimit, @@ -666,15 +666,15 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere }() if !config.DeprecatedTxPool.Disable { - backend.txPool2Fetch.ConnectCore() - backend.txPool2Fetch.ConnectSentries() - var newTxsBroadcaster *txpool2.NewSlotsStreams - if casted, ok := backend.txPool2GrpcServer.(*txpool2.GrpcServer); ok { + backend.txPoolFetch.ConnectCore() + backend.txPoolFetch.ConnectSentries() + var newTxsBroadcaster *txpool.NewSlotsStreams + if casted, ok := backend.txPoolGrpcServer.(*txpool.GrpcServer); ok { newTxsBroadcaster = casted.NewSlotsStreams } - go txpool2.MainLoop(backend.sentryCtx, - backend.txPool2DB, backend.chainDB, - backend.txPool2, backend.newTxs2, backend.txPool2Send, newTxsBroadcaster, + go txpool.MainLoop(backend.sentryCtx, + backend.txPoolDB, backend.chainDB, + backend.txPool, backend.newTxs, backend.txPoolSend, newTxsBroadcaster, func() { select { case backend.notifyMiningAboutNewTxs <- struct{}{}: @@ -725,8 +725,9 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere } backend.ethBackendRPC, backend.miningRPC, backend.stateChangesClient = ethBackendRPC, miningRPC, stateDiffClient + blockRetire := freezeblocks.NewBlockRetire(1, dirs, blockReader, blockWriter, backend.chainDB, backend.notifications.Events, logger) - backend.syncStages = stages2.NewDefaultStages(backend.sentryCtx, backend.chainDB, stack.Config().P2P, config, backend.sentriesClient, backend.notifications, backend.downloaderClient, blockReader, backend.agg, backend.forkValidator, logger) + backend.syncStages = stages2.NewDefaultStages(backend.sentryCtx, backend.chainDB, stack.Config().P2P, config, backend.sentriesClient, backend.notifications, backend.downloaderClient, blockReader, blockRetire, backend.agg, backend.forkValidator, logger) backend.syncUnwindOrder = stagedsync.DefaultUnwindOrder backend.syncPruneOrder = stagedsync.DefaultPruneOrder backend.stagedSync = stagedsync.New(backend.syncStages, backend.syncUnwindOrder, backend.syncPruneOrder, logger) @@ -775,7 +776,7 @@ func (s *Ethereum) Init(stack *node.Node, config *ethconfig.Config) error { // start HTTP API httpRpcCfg := stack.Config().Http ethRpcClient, txPoolRpcClient, miningRpcClient, stateCache, ff, err := cli.EmbeddedServices(ctx, chainKv, httpRpcCfg.StateCache, blockReader, ethBackendRPC, - s.txPool2GrpcServer, miningRPC, stateDiffClient, s.logger) + s.txPoolGrpcServer, miningRPC, stateDiffClient, s.logger) if err != nil { return err } @@ -1050,13 +1051,13 @@ func (s *Ethereum) setUpSnapDownloader(ctx context.Context, downloaderCfg *downl return err } -func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig ethconfig.Snapshot, histV3 bool, logger log.Logger) (services.FullBlockReader, *blockio.BlockWriter, *snapshotsync.RoSnapshots, *libstate.AggregatorV3, error) { - allSnapshots := snapshotsync.NewRoSnapshots(snConfig, dirs.Snap, logger) +func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConfig ethconfig.BlocksFreezing, histV3 bool, logger log.Logger) (services.FullBlockReader, *blockio.BlockWriter, *freezeblocks.RoSnapshots, *libstate.AggregatorV3, error) { + allSnapshots := freezeblocks.NewRoSnapshots(snConfig, dirs.Snap, logger) var err error if !snConfig.NoDownloader { allSnapshots.OptimisticalyReopenWithDB(db) } - blockReader := snapshotsync.NewBlockReader(allSnapshots) + blockReader := freezeblocks.NewBlockReader(allSnapshots) blockWriter := blockio.NewBlockWriter(histV3) dir.MustExist(dirs.SnapHistory) @@ -1109,7 +1110,7 @@ func (s *Ethereum) Start() error { time.Sleep(10 * time.Millisecond) // just to reduce logs order confusion hook := stages2.NewHook(s.sentryCtx, s.notifications, s.stagedSync, s.blockReader, s.chainConfig, s.logger, s.sentriesClient.UpdateHead) - go stages2.StageLoop(s.sentryCtx, s.chainDB, s.stagedSync, s.sentriesClient.Hd, s.waitForStageLoopStop, s.config.Sync.LoopThrottle, s.logger, s.blockSnapshots, hook) + go stages2.StageLoop(s.sentryCtx, s.chainDB, s.stagedSync, s.sentriesClient.Hd, s.waitForStageLoopStop, s.config.Sync.LoopThrottle, s.logger, s.blockReader, hook) return nil } @@ -1147,8 +1148,8 @@ func (s *Ethereum) Stop() error { for _, sentryServer := range s.sentryServers { sentryServer.Close() } - if s.txPool2DB != nil { - s.txPool2DB.Close() + if s.txPoolDB != nil { + s.txPoolDB.Close() } if s.agg != nil { s.agg.Close() diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index d83ae67a563..337dc8e0c24 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -96,7 +96,7 @@ var Defaults = Config{ RPCTxFeeCap: 1, // 1 ether ImportMode: false, - Snapshot: Snapshot{ + Snapshot: BlocksFreezing{ Enabled: false, KeepBlocks: false, Produce: true, @@ -130,7 +130,7 @@ func init() { //go:generate gencodec -dir . -type Config -formats toml -out gen_config.go -type Snapshot struct { +type BlocksFreezing struct { Enabled bool KeepBlocks bool // produce new snapshots of blocks but don't remove blocks from DB Produce bool // produce new snapshots @@ -139,7 +139,7 @@ type Snapshot struct { DownloaderAddr string } -func (s Snapshot) String() string { +func (s BlocksFreezing) String() string { var out []string if s.Enabled { out = append(out, "--snapshots=true") @@ -158,8 +158,8 @@ var ( FlagSnapStop = "snap.stop" ) -func NewSnapCfg(enabled, keepBlocks, produce bool) Snapshot { - return Snapshot{Enabled: enabled, KeepBlocks: keepBlocks, Produce: produce} +func NewSnapCfg(enabled, keepBlocks, produce bool) BlocksFreezing { + return BlocksFreezing{Enabled: enabled, KeepBlocks: keepBlocks, Produce: produce} } // Config contains configuration options for ETH protocol. @@ -186,7 +186,7 @@ type Config struct { BadBlockHash common.Hash // hash of the block marked as bad - Snapshot Snapshot + Snapshot BlocksFreezing Downloader *downloadercfg.Cfg Dirs datadir.Dirs diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index ff557ae08d7..5d9db0e26cb 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -27,7 +27,7 @@ func (c Config) MarshalTOML() (interface{}, error) { BatchSize datasize.ByteSize ImportMode bool BadBlockHash libcommon.Hash - Snapshot Snapshot + Snapshot BlocksFreezing BlockDownloaderWindow int ExternalSnapshotDownloaderAddr string Whitelist map[uint64]libcommon.Hash `toml:"-"` @@ -78,7 +78,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { BatchSize *datasize.ByteSize ImportMode *bool BadBlockHash *libcommon.Hash - Snapshot *Snapshot + Snapshot *BlocksFreezing BlockDownloaderWindow *int ExternalSnapshotDownloaderAddr *string Whitelist map[uint64]libcommon.Hash `toml:"-"` diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 47957ff0f6a..b1ffdba2f8e 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -118,11 +118,10 @@ func newTestBackend(t *testing.T) *testBackend { t.Error(err) } // Construct testing chain - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Error(err) } - br, _ := m.NewBlocksIO() - return &testBackend{db: m.DB, cfg: params.TestChainConfig, blockReader: br} + return &testBackend{db: m.DB, cfg: params.TestChainConfig, blockReader: m.BlockReader} } func (b *testBackend) CurrentHeader() *types.Header { diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index a4428a91ba6..63395c18eee 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -90,10 +90,9 @@ func TestGetBlockReceipts(t *testing.T) { hashes []libcommon.Hash receipts []rlp.RawValue ) - br, _ := m.NewBlocksIO() err := m.DB.View(m.Ctx, func(tx kv.Tx) error { for i := uint64(0); i <= rawdb.ReadCurrentHeader(tx).Number.Uint64(); i++ { - block, err := br.BlockByNumber(m.Ctx, tx, i) + block, err := m.BlockReader.BlockByNumber(m.Ctx, tx, i) require.NoError(t, err) hashes = append(hashes, block.Hash()) @@ -138,7 +137,7 @@ func mockWithGenerator(t *testing.T, blocks int, generator func(int, *core.Block }, testKey, false) if blocks > 0 { chain, _ := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator, true) - err := m.InsertChain(chain) + err := m.InsertChain(chain, nil) require.NoError(t, err) } return m diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 6d8144de992..fa396a1e206 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -206,12 +206,12 @@ func (tp TransactionsPacket) EncodeRLP(w io.Writer) error { txLen = t.EncodingSize() } if txLen >= 56 { - txsLen += (bits.Len(uint(txLen)) + 7) / 8 + txsLen += libcommon.BitLenToByteLen(bits.Len(uint(txLen))) } txsLen += txLen } if txsLen >= 56 { - encodingSize += (bits.Len(uint(txsLen)) + 7) / 8 + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(txsLen))) } encodingSize += txsLen // encode Transactions @@ -328,7 +328,7 @@ func (nbp NewBlockPacket) EncodeRLP(w io.Writer) error { encodingSize++ blockLen := nbp.Block.EncodingSize() if blockLen >= 56 { - encodingSize += (bits.Len(uint(blockLen)) + 7) / 8 + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(blockLen))) } encodingSize += blockLen // size of TD @@ -337,7 +337,7 @@ func (nbp NewBlockPacket) EncodeRLP(w io.Writer) error { if nbp.TD != nil { tdBitLen = nbp.TD.BitLen() if tdBitLen >= 8 { - tdLen = (tdBitLen + 7) / 8 + tdLen = libcommon.BitLenToByteLen(tdBitLen) } } encodingSize += tdLen @@ -527,12 +527,12 @@ func (ptp PooledTransactionsPacket) EncodeRLP(w io.Writer) error { txLen = t.EncodingSize() } if txLen >= 56 { - txsLen += (bits.Len(uint(txLen)) + 7) / 8 + txsLen += libcommon.BitLenToByteLen(bits.Len(uint(txLen))) } txsLen += txLen } if txsLen >= 56 { - encodingSize += (bits.Len(uint(txsLen)) + 7) / 8 + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(txsLen))) } encodingSize += txsLen // encode Transactions @@ -607,12 +607,12 @@ func (ptp66 PooledTransactionsPacket66) EncodeRLP(w io.Writer) error { txLen = t.EncodingSize() } if txLen >= 56 { - txsLen += (bits.Len(uint(txLen)) + 7) / 8 + txsLen += libcommon.BitLenToByteLen(bits.Len(uint(txLen))) } txsLen += txLen } if txsLen >= 56 { - encodingSize += (bits.Len(uint(txsLen)) + 7) / 8 + encodingSize += libcommon.BitLenToByteLen(bits.Len(uint(txsLen))) } encodingSize += txsLen var b [33]byte diff --git a/eth/stagedsync/README.md b/eth/stagedsync/README.md index e2c0ba436a2..1a3c290dc45 100644 --- a/eth/stagedsync/README.md +++ b/eth/stagedsync/README.md @@ -2,7 +2,7 @@ Staged Sync is a version of [Go-Ethereum](https://github.com/ethereum/go-ethereum)'s Full Sync that was rearchitected for better performance. -It is I/O intensive and even though we have a goal on being able to sync the node on an HDD, we still recommend using fast SSDs. +It is I/O intensive and even though we have a goal of being able to sync the node on an HDD, we still recommend using fast SSDs. Staged Sync, as its name suggests, consists of 10 stages that are executed in order, one after another. @@ -14,7 +14,7 @@ The first stage (downloading headers) sets the local HEAD block. Each stage is executed in order and a stage N does not stop until the local head is reached for it. -That mean, that in the ideal scenario (no network interruptions, the app isn't restarted, etc), for the full initial sync, each stage will be executed exactly once. +That means, that in the ideal scenario (no network interruptions, the app isn't restarted, etc), for the full initial sync, each stage will be executed exactly once. After the last stage is finished, the process starts from the beginning, by looking for the new headers to download. @@ -65,7 +65,7 @@ In the Proof-of-Stake world staged sync becomes somewhat more complicated, as th ## Stages (for the up to date list see [`stages.go`](/eth/stagedsync/stages/stages.go) and [`stagebuilder.go`](/eth/stagedsync/stagebuilder.go)): -Each stage consists of 2 functions `ExecFunc` that progesses the stage forward and `UnwindFunc` that unwinds the stage backwards. +Each stage consists of 2 functions `ExecFunc` that progresses the stage forward and `UnwindFunc` that unwinds the stage backwards. Most of the stages can work offline though it isn't implemented in the current version. @@ -136,7 +136,7 @@ This stage build the Merkle trie and checks the root hash for the current state. It also builds Intermediate Hashes along the way and stores them into the database. -If there were no intermediate hashes stored before (that could happend during the first initial sync), it builds the full Merkle Trie and its root hash. +If there were no intermediate hashes stored before (that could happen during the first initial sync), it builds the full Merkle Trie and its root hash. If there are intermediate hashes in the database, it uses the block history to figure out which ones are outdated and which ones are still up to date. Then it builds a partial Merkle trie using the up-to-date hashes and only rebuilding the outdated ones. diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 8dcb4c66932..f2596b488fc 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -27,7 +27,6 @@ import ( libstate "github.com/ledgerwatch/erigon-lib/state" state2 "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/common/math" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/log/v3" "github.com/torquem-ch/mdbx-go/mdbx" "golang.org/x/sync/errgroup" @@ -49,8 +48,8 @@ var ExecStepsInDB = metrics.NewCounter(`exec_steps_in_db`) //nolint var ExecRepeats = metrics.NewCounter(`exec_repeats`) //nolint var ExecTriggers = metrics.NewCounter(`exec_triggers`) //nolint -func NewProgress(prevOutputBlockNum, commitThreshold uint64, workersCount int, logPrefix string) *Progress { - return &Progress{prevTime: time.Now(), prevOutputBlockNum: prevOutputBlockNum, commitThreshold: commitThreshold, workersCount: workersCount, logPrefix: logPrefix} +func NewProgress(prevOutputBlockNum, commitThreshold uint64, workersCount int, logPrefix string, logger log.Logger) *Progress { + return &Progress{prevTime: time.Now(), prevOutputBlockNum: prevOutputBlockNum, commitThreshold: commitThreshold, workersCount: workersCount, logPrefix: logPrefix, logger: logger} } type Progress struct { @@ -62,6 +61,7 @@ type Progress struct { workersCount int logPrefix string + logger log.Logger } func (p *Progress) Log(rs *state.StateV3, in *exec22.QueueWithRetry, rws *exec22.ResultsQueue, doneCount, inputBlockNum, outputBlockNum, outTxNum, repeatCount uint64, idxStepsAmountInDB float64) { @@ -77,7 +77,7 @@ func (p *Progress) Log(rs *state.StateV3, in *exec22.QueueWithRetry, rws *exec22 if doneCount > p.prevCount { repeatRatio = 100.0 * float64(repeatCount-p.prevRepeatCount) / float64(doneCount-p.prevCount) } - log.Info(fmt.Sprintf("[%s] Transaction replay", p.logPrefix), + p.logger.Info(fmt.Sprintf("[%s] Transaction replay", p.logPrefix), //"workers", workerCount, "blk", outputBlockNum, //"blk/s", fmt.Sprintf("%.1f", speedBlock), @@ -149,13 +149,13 @@ func ExecV3(ctx context.Context, parallel bool, logPrefix string, maxBlockNum uint64, logger log.Logger, + initialCycle bool, ) error { batchSize := cfg.batchSize chainDb := cfg.db blockReader := cfg.blockReader agg, engine := cfg.agg, cfg.engine chainConfig, genesis := cfg.chainConfig, cfg.genesis - blockSnapshots := blockReader.Snapshots().(*snapshotsync.RoSnapshots) useExternalTx := applyTx != nil if !useExternalTx && !parallel { @@ -250,7 +250,7 @@ func ExecV3(ctx context.Context, applyWorker.DiscardReadList() commitThreshold := batchSize.Bytes() - progress := NewProgress(block, commitThreshold, workerCount, execStage.LogPrefix()) + progress := NewProgress(block, commitThreshold, workerCount, execStage.LogPrefix(), logger) logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() pruneEvery := time.NewTicker(2 * time.Second) @@ -470,7 +470,7 @@ func ExecV3(ctx context.Context, }) } - if block < blockSnapshots.BlocksAvailable() { + if block < cfg.blockReader.FrozenBlocks() { agg.KeepInDB(0) defer agg.KeepInDB(ethconfig.HistoryV3AggregationStep) } @@ -503,6 +503,8 @@ func ExecV3(ctx context.Context, slowDownLimit := time.NewTicker(time.Second) defer slowDownLimit.Stop() + stateStream := !initialCycle && cfg.stateStream && maxBlockNum-block < stateStreamLimit + var b *types.Block var blockNum uint64 var err error @@ -520,7 +522,7 @@ Loop: txs := b.Transactions() header := b.HeaderNoCopy() skipAnalysis := core.SkipAnalysis(chainConfig, blockNum) - signer := *types.MakeSigner(chainConfig, blockNum) + signer := *types.MakeSigner(chainConfig, blockNum, header.Time) f := core.GetHashFn(header, getHeaderFunc) getHashFnMute := &sync.Mutex{} @@ -560,6 +562,14 @@ Loop: } } }() + } else { + if !initialCycle && stateStream { + txs, err := blockReader.RawTransactions(context.Background(), applyTx, b.NumberU64(), b.NumberU64()) + if err != nil { + return err + } + cfg.accumulator.StartChange(b.NumberU64(), b.Hash(), txs, false) + } } rules := chainConfig.Rules(blockNum, b.Time()) @@ -698,7 +708,7 @@ Loop: } } - if blockSnapshots.Cfg().Produce { + if cfg.blockReader.FreezingCfg().Produce { agg.BuildFilesInBackground(outputTxNum.Load()) } select { @@ -726,7 +736,7 @@ Loop: } } - if blockSnapshots.Cfg().Produce { + if cfg.blockReader.FreezingCfg().Produce { agg.BuildFilesInBackground(outputTxNum.Load()) } @@ -1037,7 +1047,7 @@ func reconstituteStep(last bool, txs := b.Transactions() header := b.HeaderNoCopy() skipAnalysis := core.SkipAnalysis(chainConfig, bn) - signer := *types.MakeSigner(chainConfig, bn) + signer := *types.MakeSigner(chainConfig, bn, header.Time) f := core.GetHashFn(header, getHeaderFunc) getHashFnMute := &sync.Mutex{} @@ -1326,8 +1336,6 @@ func ReconstituteState(ctx context.Context, s *StageState, dirs datadir.Dirs, wo chainConfig *chain.Config, genesis *types.Genesis) (err error) { startTime := time.Now() defer agg.EnableMadvNormal().DisableReadAhead() - blockSnapshots := blockReader.Snapshots().(*snapshotsync.RoSnapshots) - defer blockSnapshots.EnableReadAhead().DisableReadAhead() // force merge snapshots before reconstitution, to allign domains progress // un-finished merge can happen at "kill -9" during merge diff --git a/eth/stagedsync/stage_bodies.go b/eth/stagedsync/stage_bodies.go index 23976169473..08d64adf233 100644 --- a/eth/stagedsync/stage_bodies.go +++ b/eth/stagedsync/stage_bodies.go @@ -59,8 +59,8 @@ func BodiesForward( logger log.Logger, ) error { var doUpdate bool - if cfg.blockReader != nil && cfg.blockReader.Snapshots() != nil && s.BlockNumber < cfg.blockReader.Snapshots().BlocksAvailable() { - s.BlockNumber = cfg.blockReader.Snapshots().BlocksAvailable() + if s.BlockNumber < cfg.blockReader.FrozenBlocks() { + s.BlockNumber = cfg.blockReader.FrozenBlocks() doUpdate = true } @@ -201,7 +201,7 @@ func BodiesForward( } // Check existence before write - because WriteRawBody isn't idempotent (it allocates new sequence range for transactions on every call) - ok, err := cfg.blockWriter.WriteRawBodyIfNotExists(tx, header.Hash(), blockHeight, rawBody) + ok, err := rawdb.WriteRawBodyIfNotExists(tx, header.Hash(), blockHeight, rawBody) if err != nil { return false, fmt.Errorf("WriteRawBodyIfNotExists: %w", err) } diff --git a/eth/stagedsync/stage_bodies_test.go b/eth/stagedsync/stage_bodies_test.go index 8d94efd9427..b02c2f1dd35 100644 --- a/eth/stagedsync/stage_bodies_test.go +++ b/eth/stagedsync/stage_bodies_test.go @@ -24,9 +24,9 @@ func TestBodiesUnwind(t *testing.T) { tx, err := db.BeginRw(m.Ctx) require.NoError(err) defer tx.Rollback() - _, bw := m.NewBlocksIO() + _, bw := m.BlocksIO() - txn := &types.DynamicFeeTransaction{Tip: u256.N1, FeeCap: u256.N1, CommonTx: types.CommonTx{ChainID: u256.N1, Value: u256.N1, Gas: 1, Nonce: 1}} + txn := &types.DynamicFeeTransaction{Tip: u256.N1, FeeCap: u256.N1, ChainID: u256.N1, CommonTx: types.CommonTx{Value: u256.N1, Gas: 1, Nonce: 1}} buf := bytes.NewBuffer(nil) err = txn.MarshalBinary(buf) require.NoError(err) @@ -40,11 +40,11 @@ func TestBodiesUnwind(t *testing.T) { for i := uint64(1); i <= 10; i++ { h.Number = big.NewInt(int64(i)) hash := h.Hash() - err = bw.WriteHeader(tx, h) + err = rawdb.WriteHeader(tx, h) require.NoError(err) err = rawdb.WriteCanonicalHash(tx, hash, i) require.NoError(err) - _, err = bw.WriteRawBodyIfNotExists(tx, hash, i, b) + _, err = rawdb.WriteRawBodyIfNotExists(tx, hash, i, b) require.NoError(err) } @@ -78,7 +78,7 @@ func TestBodiesUnwind(t *testing.T) { } } { - _, err = bw.WriteRawBodyIfNotExists(tx, libcommon.Hash{11}, 11, b) + _, err = rawdb.WriteRawBodyIfNotExists(tx, libcommon.Hash{11}, 11, b) require.NoError(err) err = rawdb.WriteCanonicalHash(tx, libcommon.Hash{11}, 11) require.NoError(err) diff --git a/eth/stagedsync/stage_call_traces_test.go b/eth/stagedsync/stage_call_traces_test.go index 457675b5258..0248d65ae47 100644 --- a/eth/stagedsync/stage_call_traces_test.go +++ b/eth/stagedsync/stage_call_traces_test.go @@ -6,13 +6,13 @@ import ( "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" - "github.com/ledgerwatch/erigon-lib/kv/memdb" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/log/v3" ) @@ -34,44 +34,51 @@ func genTestCallTraceSet(t *testing.T, tx kv.RwTx, to uint64) { func TestCallTrace(t *testing.T) { logger := log.New() - ctx, assert := context.Background(), assert.New(t) - _, tx := memdb.NewTestTx(t) + ctx, require := context.Background(), require.New(t) + histV3, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + if histV3 { + t.Skip() + } + tx, err := db.BeginRw(context.Background()) + require.NoError(err) + defer tx.Rollback() + genTestCallTraceSet(t, tx, 30) addr := [20]byte{} addr[19] = byte(1) froms := func() *roaring64.Bitmap { b, err := bitmapdb.Get64(tx, kv.CallFromIndex, addr[:], 0, 30) - assert.NoError(err) + require.NoError(err) return b } tos := func() *roaring64.Bitmap { b, err := bitmapdb.Get64(tx, kv.CallToIndex, addr[:], 0, 30) - assert.NoError(err) + require.NoError(err) return b } - err := stages.SaveStageProgress(tx, stages.Execution, 30) - assert.NoError(err) + err = stages.SaveStageProgress(tx, stages.Execution, 30) + require.NoError(err) // forward 0->20 err = promoteCallTraces("test", tx, 0, 20, 0, time.Nanosecond, ctx.Done(), "", logger) - assert.NoError(err) - assert.Equal([]uint64{6, 16}, froms().ToArray()) - assert.Equal([]uint64{1, 11}, tos().ToArray()) + require.NoError(err) + require.Equal([]uint64{6, 16}, froms().ToArray()) + require.Equal([]uint64{1, 11}, tos().ToArray()) // unwind 20->10 err = DoUnwindCallTraces("test", tx, 20, 10, ctx, "", logger) - assert.NoError(err) - assert.Equal([]uint64{6}, froms().ToArray()) - assert.Equal([]uint64{1}, tos().ToArray()) + require.NoError(err) + require.Equal([]uint64{6}, froms().ToArray()) + require.Equal([]uint64{1}, tos().ToArray()) // forward 10->30 err = promoteCallTraces("test", tx, 10, 30, 0, time.Nanosecond, ctx.Done(), "", logger) - assert.NoError(err) - assert.Equal([]uint64{6, 16, 26}, froms().ToArray()) - assert.Equal([]uint64{1, 11, 21}, tos().ToArray()) + require.NoError(err) + require.Equal([]uint64{6, 16, 26}, froms().ToArray()) + require.Equal([]uint64{1, 11, 21}, tos().ToArray()) // prune 0 -> 10 err = pruneCallTraces(tx, "test", 10, ctx, "", logger) - assert.NoError(err) + require.NoError(err) } diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index 912404b1a02..ac7bdf92ef2 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -275,9 +275,9 @@ func ExecBlockV3(s *StageState, u Unwinder, tx kv.RwTx, toBlock uint64, ctx cont if to > s.BlockNumber+16 { logger.Info(fmt.Sprintf("[%s] Blocks execution", logPrefix), "from", s.BlockNumber, "to", to) } - parallel := initialCycle && tx == nil + parallel := tx == nil if err := ExecV3(ctx, s, u, workersCount, cfg, tx, parallel, logPrefix, - to, logger); err != nil { + to, logger, initialCycle); err != nil { return fmt.Errorf("ExecV3: %w", err) } return nil diff --git a/eth/stagedsync/stage_hashstate.go b/eth/stagedsync/stage_hashstate.go index 78d1ab8db2c..64231c2179c 100644 --- a/eth/stagedsync/stage_hashstate.go +++ b/eth/stagedsync/stage_hashstate.go @@ -20,7 +20,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/log/v3" "golang.org/x/sync/errgroup" @@ -566,7 +565,7 @@ func (p *Promoter) PromoteOnHistoryV3(logPrefix string, from, to uint64, storage defer collector.Close() if storage { - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -612,7 +611,7 @@ func (p *Promoter) PromoteOnHistoryV3(logPrefix string, from, to uint64, storage codeCollector := etl.NewCollector(logPrefix, p.dirs.Tmp, etl.NewSortableBuffer(etl.BufferOptimalSize), p.logger) defer codeCollector.Close() - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -731,7 +730,7 @@ func (p *Promoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo uint acc := accounts.NewAccount() if codes { - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -770,7 +769,7 @@ func (p *Promoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo uint } if storage { - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -800,7 +799,7 @@ func (p *Promoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo uint return collector.Load(p.tx, kv.HashedStorage, etl.IdentityLoadFunc, etl.TransformArgs{Quit: p.ctx.Done()}) } - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } diff --git a/eth/stagedsync/stage_hashstate_test.go b/eth/stagedsync/stage_hashstate_test.go index 823ed10a543..81d5c12206a 100644 --- a/eth/stagedsync/stage_hashstate_test.go +++ b/eth/stagedsync/stage_hashstate_test.go @@ -8,6 +8,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/stretchr/testify/require" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -15,6 +16,9 @@ import ( ) func TestPromoteHashedStateClearState(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } logger := log.New() dirs := datadir.New(t.TempDir()) historyV3 := false @@ -33,6 +37,9 @@ func TestPromoteHashedStateClearState(t *testing.T) { } func TestPromoteHashedStateIncremental(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } logger := log.New() dirs := datadir.New(t.TempDir()) historyV3 := false @@ -60,6 +67,9 @@ func TestPromoteHashedStateIncremental(t *testing.T) { } func TestPromoteHashedStateIncrementalMixed(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } logger := log.New() dirs := datadir.New(t.TempDir()) historyV3 := false @@ -78,6 +88,9 @@ func TestPromoteHashedStateIncrementalMixed(t *testing.T) { } func TestUnwindHashed(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } logger := log.New() dirs := datadir.New(t.TempDir()) historyV3 := false @@ -102,6 +115,9 @@ func TestUnwindHashed(t *testing.T) { } func TestPromoteIncrementallyShutdown(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } historyV3 := false tt := []struct { @@ -134,6 +150,9 @@ func TestPromoteIncrementallyShutdown(t *testing.T) { } func TestPromoteHashedStateCleanlyShutdown(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } logger := log.New() historyV3 := false @@ -170,6 +189,9 @@ func TestPromoteHashedStateCleanlyShutdown(t *testing.T) { } func TestUnwindHashStateShutdown(t *testing.T) { + if ethconfig.EnableHistoryV4InTest { + t.Skip() + } logger := log.New() historyV3 := false tt := []struct { diff --git a/eth/stagedsync/stage_headers.go b/eth/stagedsync/stage_headers.go index 5a44870aa94..d20acef1954 100644 --- a/eth/stagedsync/stage_headers.go +++ b/eth/stagedsync/stage_headers.go @@ -106,8 +106,8 @@ func SpawnStageHeaders( } defer tx.Rollback() } - if initialCycle && cfg.blockReader != nil && cfg.blockReader.Snapshots() != nil && cfg.blockReader.Snapshots().Cfg().Enabled { - if err := cfg.hd.AddHeadersFromSnapshot(tx, cfg.blockReader.Snapshots().BlocksAvailable(), cfg.blockReader); err != nil { + if initialCycle && cfg.blockReader.FreezingCfg().Enabled { + if err := cfg.hd.AddHeadersFromSnapshot(tx, cfg.blockReader); err != nil { return err } } @@ -178,7 +178,7 @@ func HeadersPOS( interrupt, requestId, requestWithStatus := cfg.hd.BeaconRequestList.WaitForRequest(syncing, test) cfg.hd.SetHeaderReader(&ChainReaderImpl{config: &cfg.chainConfig, tx: tx, blockReader: cfg.blockReader}) - headerInserter := headerdownload.NewHeaderInserter(s.LogPrefix(), nil, s.BlockNumber, cfg.blockReader, cfg.blockWriter) + headerInserter := headerdownload.NewHeaderInserter(s.LogPrefix(), nil, s.BlockNumber, cfg.blockReader) interrupted, err := handleInterrupt(interrupt, cfg, tx, headerInserter, useExternalTx, logger) if err != nil { @@ -811,7 +811,7 @@ func HeadersPOW( if localTd == nil { return fmt.Errorf("localTD is nil: %d, %x", headerProgress, hash) } - headerInserter := headerdownload.NewHeaderInserter(logPrefix, localTd, headerProgress, cfg.blockReader, cfg.blockWriter) + headerInserter := headerdownload.NewHeaderInserter(logPrefix, localTd, headerProgress, cfg.blockReader) cfg.hd.SetHeaderReader(&ChainReaderImpl{config: &cfg.chainConfig, tx: tx, blockReader: cfg.blockReader}) stopped := false diff --git a/eth/stagedsync/stage_interhashes.go b/eth/stagedsync/stage_interhashes.go index 6c5bcf0420e..a6ce5085b20 100644 --- a/eth/stagedsync/stage_interhashes.go +++ b/eth/stagedsync/stage_interhashes.go @@ -17,7 +17,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" "github.com/ledgerwatch/erigon-lib/state" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" @@ -119,7 +118,7 @@ func SpawnIntermediateHashesStage(s *StageState, u Unwinder, tx kv.RwTx, cfg Tri return trie.EmptyRoot, err } } else { - if root, err = incrementIntermediateHashes(logPrefix, s, tx, to, cfg, expectedRootHash, quit, logger); err != nil { + if root, err = IncrementIntermediateHashes(logPrefix, s, tx, to, cfg, expectedRootHash, quit, logger); err != nil { return trie.EmptyRoot, err } } @@ -219,7 +218,7 @@ func (p *HashPromoter) PromoteOnHistoryV3(logPrefix string, from, to uint64, sto if storage { compositeKey := make([]byte, length.Hash+length.Hash) - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -248,7 +247,7 @@ func (p *HashPromoter) PromoteOnHistoryV3(logPrefix string, from, to uint64, sto return nil } - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -371,7 +370,7 @@ func (p *HashPromoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo var deletedAccounts [][]byte if storage { - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.StorageHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -404,7 +403,7 @@ func (p *HashPromoter) UnwindOnHistoryV3(logPrefix string, unwindFrom, unwindTo return nil } - it, err := p.tx.(kv.TemporalTx).HistoryRange(temporal.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) + it, err := p.tx.(kv.TemporalTx).HistoryRange(kv.AccountsHistory, int(txnFrom), int(txnTo), order.Asc, kv.Unlim) if err != nil { return err } @@ -549,7 +548,7 @@ func (p *HashPromoter) Unwind(logPrefix string, s *StageState, u *UnwindState, s return nil } -func incrementIntermediateHashes(logPrefix string, s *StageState, db kv.RwTx, to uint64, cfg TrieCfg, expectedRootHash libcommon.Hash, quit <-chan struct{}, logger log.Logger) (libcommon.Hash, error) { +func IncrementIntermediateHashes(logPrefix string, s *StageState, db kv.RwTx, to uint64, cfg TrieCfg, expectedRootHash libcommon.Hash, quit <-chan struct{}, logger log.Logger) (libcommon.Hash, error) { p := NewHashPromoter(db, cfg.tmpDir, quit, logPrefix, logger) rl := trie.NewRetainList(0) if cfg.historyV3 { diff --git a/eth/stagedsync/stage_interhashes_test.go b/eth/stagedsync/stage_interhashes_test.go index 10c389b02d1..4eb31cad287 100644 --- a/eth/stagedsync/stage_interhashes_test.go +++ b/eth/stagedsync/stage_interhashes_test.go @@ -1,4 +1,4 @@ -package stagedsync +package stagedsync_test import ( "context" @@ -11,12 +11,13 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/memdb" "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon/eth/stagedsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/dbutils" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/params" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/erigon/turbo/trie" "github.com/ledgerwatch/log/v3" @@ -79,9 +80,9 @@ func TestAccountAndStorageTrie(t *testing.T) { // ---------------------------------------------------------------- historyV3 := false - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) - cfg := StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil) - _, err := RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, log.New()) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) + cfg := stagedsync.StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil) + _, err := stagedsync.RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, log.New()) assert.Nil(t, err) // ---------------------------------------------------------------- @@ -149,9 +150,9 @@ func TestAccountAndStorageTrie(t *testing.T) { err = tx.Put(kv.AccountChangeSet, hexutility.EncodeTs(1), newAddress[:]) assert.Nil(t, err) - var s StageState + var s stagedsync.StageState s.BlockNumber = 0 - _, err = incrementIntermediateHashes("IH", &s, tx, 1 /* to */, cfg, libcommon.Hash{} /* expectedRootHash */, nil /* quit */, log.New()) + _, err = stagedsync.IncrementIntermediateHashes("IH", &s, tx, 1 /* to */, cfg, libcommon.Hash{} /* expectedRootHash */, nil /* quit */, log.New()) assert.Nil(t, err) accountTrieB := make(map[string][]byte) @@ -201,8 +202,8 @@ func TestAccountTrieAroundExtensionNode(t *testing.T) { hash6 := libcommon.HexToHash("0x3100000000000000000000000000000000000000000000000000000000000000") assert.Nil(t, tx.Put(kv.HashedAccounts, hash6[:], encoded)) - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) - _, err := RegenerateIntermediateHashes("IH", tx, StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil), libcommon.Hash{} /* expectedRootHash */, ctx, log.New()) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) + _, err := stagedsync.RegenerateIntermediateHashes("IH", tx, stagedsync.StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil), libcommon.Hash{} /* expectedRootHash */, ctx, log.New()) assert.Nil(t, err) accountTrie := make(map[string][]byte) @@ -264,9 +265,9 @@ func TestStorageDeletion(t *testing.T) { // Populate account & storage trie DB tables // ---------------------------------------------------------------- historyV3 := false - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) - cfg := StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil) - _, err = RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, log.New()) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) + cfg := stagedsync.StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil) + _, err = stagedsync.RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, log.New()) assert.Nil(t, err) // ---------------------------------------------------------------- @@ -299,9 +300,9 @@ func TestStorageDeletion(t *testing.T) { err = tx.Put(kv.StorageChangeSet, append(hexutility.EncodeTs(1), dbutils.PlainGenerateStoragePrefix(address[:], incarnation)...), plainLocation3[:]) assert.Nil(t, err) - var s StageState + var s stagedsync.StageState s.BlockNumber = 0 - _, err = incrementIntermediateHashes("IH", &s, tx, 1 /* to */, cfg, libcommon.Hash{} /* expectedRootHash */, nil /* quit */, log.New()) + _, err = stagedsync.IncrementIntermediateHashes("IH", &s, tx, 1 /* to */, cfg, libcommon.Hash{} /* expectedRootHash */, nil /* quit */, log.New()) assert.Nil(t, err) storageTrieB := make(map[string][]byte) @@ -383,10 +384,10 @@ func TestHiveTrieRoot(t *testing.T) { common.FromHex("02081bc16d674ec80000"))) historyV3 := false - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) - cfg := StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) + cfg := stagedsync.StageTrieCfg(db, false, true, false, t.TempDir(), blockReader, nil, historyV3, nil) logger := log.New() - _, err := RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, logger) + _, err := stagedsync.RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, logger) require.Nil(t, err) // Now add a new account @@ -397,12 +398,12 @@ func TestHiveTrieRoot(t *testing.T) { require.Nil(t, tx.Put(kv.HashedAccounts, newHash[:], common.FromHex("02081bc16d674ec80000"))) require.Nil(t, tx.Put(kv.AccountChangeSet, hexutility.EncodeTs(1), newAddress[:])) - var s StageState + var s stagedsync.StageState s.BlockNumber = 0 - incrementalRoot, err := incrementIntermediateHashes("IH", &s, tx, 1 /* to */, cfg, libcommon.Hash{} /* expectedRootHash */, nil /* quit */, logger) + incrementalRoot, err := stagedsync.IncrementIntermediateHashes("IH", &s, tx, 1 /* to */, cfg, libcommon.Hash{} /* expectedRootHash */, nil /* quit */, logger) require.Nil(t, err) - regeneratedRoot, err := RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, logger) + regeneratedRoot, err := stagedsync.RegenerateIntermediateHashes("IH", tx, cfg, libcommon.Hash{} /* expectedRootHash */, ctx, logger) require.Nil(t, err) assert.Equal(t, regeneratedRoot, incrementalRoot) diff --git a/eth/stagedsync/stage_mining_create_block.go b/eth/stagedsync/stage_mining_create_block.go index 0b9e84e9dff..d173e00b6ab 100644 --- a/eth/stagedsync/stage_mining_create_block.go +++ b/eth/stagedsync/stage_mining_create_block.go @@ -15,8 +15,6 @@ import ( "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/txpool" - "github.com/ledgerwatch/erigon/common/debug" "github.com/ledgerwatch/erigon/consensus" "github.com/ledgerwatch/erigon/core" @@ -72,21 +70,19 @@ type MiningCreateBlockCfg struct { miner MiningState chainConfig chain.Config engine consensus.Engine - txPool2 *txpool.TxPool - txPool2DB kv.RoDB + txPoolDB kv.RoDB tmpdir string blockBuilderParameters *core.BlockBuilderParameters blockReader services.FullBlockReader } -func StageMiningCreateBlockCfg(db kv.RwDB, miner MiningState, chainConfig chain.Config, engine consensus.Engine, txPool2 *txpool.TxPool, txPool2DB kv.RoDB, blockBuilderParameters *core.BlockBuilderParameters, tmpdir string, blockReader services.FullBlockReader) MiningCreateBlockCfg { +func StageMiningCreateBlockCfg(db kv.RwDB, miner MiningState, chainConfig chain.Config, engine consensus.Engine, txPoolDB kv.RoDB, blockBuilderParameters *core.BlockBuilderParameters, tmpdir string, blockReader services.FullBlockReader) MiningCreateBlockCfg { return MiningCreateBlockCfg{ db: db, miner: miner, chainConfig: chainConfig, engine: engine, - txPool2: txPool2, - txPool2DB: txPool2DB, + txPoolDB: txPoolDB, tmpdir: tmpdir, blockBuilderParameters: blockBuilderParameters, blockReader: blockReader, @@ -184,19 +180,6 @@ func SpawnMiningCreateBlockStage(s *StageState, tx kv.RwTx, cfg MiningCreateBloc return } - type envT struct { - signer *types.Signer - ancestors mapset.Set // ancestor set (used for checking uncle parent validity) - family mapset.Set // family set (used for checking uncle invalidity) - uncles mapset.Set // uncle set - } - env := &envT{ - signer: types.MakeSigner(&cfg.chainConfig, blockNum), - ancestors: mapset.NewSet(), - family: mapset.NewSet(), - uncles: mapset.NewSet(), - } - // re-written miner/worker.go:commitNewWork var timestamp uint64 if cfg.blockBuilderParameters == nil { @@ -213,6 +196,19 @@ func SpawnMiningCreateBlockStage(s *StageState, tx kv.RwTx, cfg MiningCreateBloc if cfg.chainConfig.IsOptimism() { targetGasLimit = cfg.blockBuilderParameters.GasLimit } + type envT struct { + signer *types.Signer + ancestors mapset.Set // ancestor set (used for checking uncle parent validity) + family mapset.Set // family set (used for checking uncle invalidity) + uncles mapset.Set // uncle set + } + env := &envT{ + signer: types.MakeSigner(&cfg.chainConfig, blockNum, timestamp), + ancestors: mapset.NewSet(), + family: mapset.NewSet(), + uncles: mapset.NewSet(), + } + header := core.MakeEmptyHeader(parent, &cfg.chainConfig, timestamp, targetGasLimit) header.Coinbase = coinbase header.Extra = cfg.miner.MiningConfig.ExtraData diff --git a/eth/stagedsync/stage_mining_exec.go b/eth/stagedsync/stage_mining_exec.go index f08b6351c9a..4ac44955437 100644 --- a/eth/stagedsync/stage_mining_exec.go +++ b/eth/stagedsync/stage_mining_exec.go @@ -23,7 +23,6 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/memdb" - "github.com/ledgerwatch/erigon-lib/txpool" types2 "github.com/ledgerwatch/erigon-lib/types" "github.com/ledgerwatch/erigon/consensus" @@ -51,17 +50,22 @@ type MiningExecCfg struct { tmpdir string interrupt *int32 payloadId uint64 - txPool2 *txpool.TxPool + txPool2 TxPoolForMining txPool2DB kv.RoDB noTxPool bool } +type TxPoolForMining interface { + YieldBest(n uint16, txs *types2.TxsRlp, tx kv.Tx, onTopOf, availableGas uint64, toSkip mapset.Set[[32]byte]) (bool, int, error) +} + func StageMiningExecCfg( db kv.RwDB, miningState MiningState, notifier ChainEventNotifier, chainConfig chain.Config, engine consensus.Engine, vmConfig *vm.Config, tmpdir string, interrupt *int32, payloadId uint64, - txPool2 *txpool.TxPool, txPool2DB kv.RoDB, noTxPool bool, + txPool2 TxPoolForMining, txPool2DB kv.RoDB, + noTxPool bool, blockReader services.FullBlockReader, ) MiningExecCfg { return MiningExecCfg{ @@ -375,7 +379,7 @@ func addTransactionsToMiningBlock(logPrefix string, current *MiningBlock, chainC header := current.Header tcount := 0 gasPool := new(core.GasPool).AddGas(header.GasLimit - header.GasUsed) - signer := types.MakeSigner(&chainConfig, header.Number.Uint64()) + signer := types.MakeSigner(&chainConfig, header.Number.Uint64(), header.Time) var coalescedLogs types.Logs noop := state.NewNoopWriter() diff --git a/eth/stagedsync/stage_senders.go b/eth/stagedsync/stage_senders.go index fae2c617b05..fa01abe9103 100644 --- a/eth/stagedsync/stage_senders.go +++ b/eth/stagedsync/stage_senders.go @@ -16,7 +16,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/secp256k1" @@ -27,7 +26,6 @@ import ( "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/ethdb/prune" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" ) @@ -42,13 +40,11 @@ type SendersCfg struct { tmpdir string prune prune.Mode chainConfig *chain.Config - blockRetire *snapshotsync.BlockRetire hd *headerdownload.HeaderDownload blockReader services.FullBlockReader - blockWriter *blockio.BlockWriter } -func StageSendersCfg(db kv.RwDB, chainCfg *chain.Config, badBlockHalt bool, tmpdir string, prune prune.Mode, br *snapshotsync.BlockRetire, blockWriter *blockio.BlockWriter, blockReader services.FullBlockReader, hd *headerdownload.HeaderDownload) SendersCfg { +func StageSendersCfg(db kv.RwDB, chainCfg *chain.Config, badBlockHalt bool, tmpdir string, prune prune.Mode, blockReader services.FullBlockReader, hd *headerdownload.HeaderDownload) SendersCfg { const sendersBatchSize = 10000 const sendersBlockSize = 4096 @@ -63,17 +59,15 @@ func StageSendersCfg(db kv.RwDB, chainCfg *chain.Config, badBlockHalt bool, tmpd tmpdir: tmpdir, chainConfig: chainCfg, prune: prune, - blockRetire: br, hd: hd, blockReader: blockReader, - blockWriter: blockWriter, } } func SpawnRecoverSendersStage(cfg SendersCfg, s *StageState, u Unwinder, tx kv.RwTx, toBlock uint64, ctx context.Context, logger log.Logger) error { - if cfg.blockRetire != nil && cfg.blockRetire.Snapshots() != nil && cfg.blockRetire.Snapshots().Cfg().Enabled && s.BlockNumber < cfg.blockRetire.Snapshots().BlocksAvailable() { - s.BlockNumber = cfg.blockRetire.Snapshots().BlocksAvailable() + if cfg.blockReader.FreezingCfg().Enabled && s.BlockNumber < cfg.blockReader.FrozenBlocks() { + s.BlockNumber = cfg.blockReader.FrozenBlocks() } quitCh := ctx.Done() @@ -312,7 +306,8 @@ func recoverSenders(ctx context.Context, logPrefix string, cryptoContext *secp25 } body := job.body - signer := types.MakeSigner(config, job.blockNumber) + blockTime := uint64(0) // TODO(yperbasis) proper timestamp + signer := types.MakeSigner(config, job.blockNumber, blockTime) job.senders = make([]byte, len(body.Transactions)*length.Addr) for i, tx := range body.Transactions { from, err := signer.SenderWithContext(cryptoContext, tx) @@ -367,8 +362,7 @@ func PruneSendersStage(s *PruneState, tx kv.RwTx, cfg SendersCfg, ctx context.Co } defer tx.Rollback() } - sn := cfg.blockRetire.Snapshots() - if sn.Cfg().Enabled { + if cfg.blockReader.FreezingCfg().Enabled { // noop. in this case senders will be deleted by BlockRetire.PruneAncientBlocks after data-freezing. } else if cfg.prune.TxIndex.Enabled() { to := cfg.prune.TxIndex.PruneTo(s.ForwardProgress) diff --git a/eth/stagedsync/stage_senders_test.go b/eth/stagedsync/stage_senders_test.go index 7a8cb4fd5c3..b0c6be16f7c 100644 --- a/eth/stagedsync/stage_senders_test.go +++ b/eth/stagedsync/stage_senders_test.go @@ -17,12 +17,10 @@ import ( "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/ethdb/prune" "github.com/ledgerwatch/erigon/params" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/log/v3" ) func TestSenders(t *testing.T) { - logger := log.New() require := require.New(t) m := stages2.Mock(t) @@ -30,7 +28,7 @@ func TestSenders(t *testing.T) { tx, err := db.BeginRw(m.Ctx) require.NoError(err) defer tx.Rollback() - br, bw := m.NewBlocksIO() + br := m.BlockReader var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddr := crypto.PubkeyToAddress(testKey.PublicKey) @@ -42,11 +40,11 @@ func TestSenders(t *testing.T) { } // prepare tx so it works with our test - signer1 := types.MakeSigner(params.TestChainConfig, params.TestChainConfig.BerlinBlock.Uint64()) + signer1 := types.MakeSigner(params.TestChainConfig, params.TestChainConfig.BerlinBlock.Uint64(), 0) header := &types.Header{Number: libcommon.Big1} hash := header.Hash() - require.NoError(bw.WriteHeader(tx, header)) - require.NoError(bw.WriteBody(tx, hash, 1, &types.Body{ + require.NoError(rawdb.WriteHeader(tx, header)) + require.NoError(rawdb.WriteBody(tx, hash, 1, &types.Body{ Transactions: []types.Transaction{ mustSign(&types.AccessListTx{ LegacyTx: types.LegacyTx{ @@ -74,11 +72,11 @@ func TestSenders(t *testing.T) { })) require.NoError(rawdb.WriteCanonicalHash(tx, hash, 1)) - signer2 := types.MakeSigner(params.TestChainConfig, params.TestChainConfig.BerlinBlock.Uint64()) + signer2 := types.MakeSigner(params.TestChainConfig, params.TestChainConfig.BerlinBlock.Uint64(), 0) header.Number = libcommon.Big2 hash = header.Hash() - require.NoError(bw.WriteHeader(tx, header)) - require.NoError(bw.WriteBody(tx, hash, 2, &types.Body{ + require.NoError(rawdb.WriteHeader(tx, header)) + require.NoError(rawdb.WriteBody(tx, hash, 2, &types.Body{ Transactions: []types.Transaction{ mustSign(&types.AccessListTx{ LegacyTx: types.LegacyTx{ @@ -120,8 +118,8 @@ func TestSenders(t *testing.T) { header.Number = libcommon.Big3 hash = header.Hash() - require.NoError(bw.WriteHeader(tx, header)) - err = bw.WriteBody(tx, hash, 3, &types.Body{ + require.NoError(rawdb.WriteHeader(tx, header)) + err = rawdb.WriteBody(tx, hash, 3, &types.Body{ Transactions: []types.Transaction{}, Uncles: []*types.Header{{GasLimit: 3}}, }) require.NoError(err) @@ -130,8 +128,7 @@ func TestSenders(t *testing.T) { require.NoError(stages.SaveStageProgress(tx, stages.Bodies, 3)) - blockRetire := snapshotsync.NewBlockRetire(1, "", br, bw, db, nil, nil, logger) - cfg := stagedsync.StageSendersCfg(db, params.TestChainConfig, false, "", prune.Mode{}, blockRetire, bw, br, nil) + cfg := stagedsync.StageSendersCfg(db, params.TestChainConfig, false, "", prune.Mode{}, br, nil) err = stagedsync.SpawnRecoverSendersStage(cfg, &stagedsync.StageState{ID: stages.Senders}, nil, tx, 3, m.Ctx, log.New()) require.NoError(err) diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 3a210fe7f69..7d140b93f7c 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -5,18 +5,13 @@ import ( "encoding/binary" "fmt" "math/big" - "path/filepath" - "runtime" + "reflect" "time" - "github.com/holiman/uint256" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon-lib/chain" - libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" - "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/etl" proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" "github.com/ledgerwatch/erigon-lib/kv" @@ -30,7 +25,6 @@ import ( "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/snapshotsync" - "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapcfg" ) type SnapshotsCfg struct { @@ -38,10 +32,10 @@ type SnapshotsCfg struct { chainConfig chain.Config dirs datadir.Dirs - blockRetire *snapshotsync.BlockRetire + blockRetire services.BlockRetire snapshotDownloader proto_downloader.DownloaderClient blockReader services.FullBlockReader - dbEventNotifier snapshotsync.DBEventNotifier + dbEventNotifier services.DBEventNotifier historyV3 bool agg *state.AggregatorV3 @@ -49,9 +43,9 @@ type SnapshotsCfg struct { func StageSnapshotsCfg(db kv.RwDB, chainConfig chain.Config, dirs datadir.Dirs, - blockRetire *snapshotsync.BlockRetire, + blockRetire services.BlockRetire, snapshotDownloader proto_downloader.DownloaderClient, - blockReader services.FullBlockReader, dbEventNotifier snapshotsync.DBEventNotifier, + blockReader services.FullBlockReader, dbEventNotifier services.DBEventNotifier, historyV3 bool, agg *state.AggregatorV3, ) SnapshotsCfg { return SnapshotsCfg{ @@ -111,50 +105,24 @@ func SpawnStageSnapshots( } func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.RwTx, cfg SnapshotsCfg, initialCycle bool, logger log.Logger) error { - if !initialCycle || cfg.blockReader == nil { + if !initialCycle { return nil } - snapshots := cfg.blockReader.Snapshots().(*snapshotsync.RoSnapshots) - if snapshots == nil || !snapshots.Cfg().Enabled { + if !cfg.blockReader.FreezingCfg().Enabled { return nil } - if err := WaitForDownloader(s, ctx, cfg, tx); err != nil { + if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { return err } - snapshots.LogStat() cfg.agg.LogStats(tx, func(endTxNumMinimax uint64) uint64 { _, histBlockNumProgress, _ := rawdbv3.TxNums.FindBlockNum(tx, endTxNumMinimax) return histBlockNumProgress }) - // Create .idx files - if snapshots.IndicesMax() < snapshots.SegmentsMax() { - if !snapshots.Cfg().Produce && snapshots.IndicesMax() == 0 { - return fmt.Errorf("please remove --snap.stop, erigon can't work without creating basic indices") - } - if snapshots.Cfg().Produce { - if !snapshots.SegmentsReady() { - return fmt.Errorf("not all snapshot segments are available") - } - - // wait for Downloader service to download all expected snapshots - if snapshots.IndicesMax() < snapshots.SegmentsMax() { - chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID) - indexWorkers := estimate.IndexSnapshot.Workers() - if err := snapshotsync.BuildMissedIndices(s.LogPrefix(), ctx, cfg.dirs, *chainID, indexWorkers, logger); err != nil { - return fmt.Errorf("BuildMissedIndices: %w", err) - } - } - - if err := snapshots.ReopenFolder(); err != nil { - return err - } - if cfg.dbEventNotifier != nil { - cfg.dbEventNotifier.OnNewSnapshot() - } - } + if err := cfg.blockRetire.BuildMissedIndicesIfNeed(ctx, s.LogPrefix(), cfg.dbEventNotifier, &cfg.chainConfig); err != nil { + return err } if cfg.historyV3 { @@ -169,25 +137,22 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R } } - blocksAvailable := snapshots.BlocksAvailable() - if s.BlockNumber < blocksAvailable { // allow genesis - if err := s.Update(tx, blocksAvailable); err != nil { + frozenBlocks := cfg.blockReader.FrozenBlocks() + if s.BlockNumber < frozenBlocks { // allow genesis + if err := s.Update(tx, frozenBlocks); err != nil { return err } - s.BlockNumber = blocksAvailable + s.BlockNumber = frozenBlocks } - if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, snapshots, cfg.blockReader, cfg.agg, logger); err != nil { + if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.blockReader, cfg.agg, logger); err != nil { return err } return nil } -func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, - dirs datadir.Dirs, sn *snapshotsync.RoSnapshots, - blockReader services.FullBlockReader, agg *state.AggregatorV3, - logger log.Logger) error { - blocksAvailable := sn.BlocksAvailable() +func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs datadir.Dirs, blockReader services.FullBlockReader, agg *state.AggregatorV3, logger log.Logger) error { + blocksAvailable := blockReader.FrozenBlocks() logEvery := time.NewTicker(logInterval) defer logEvery.Stop() // updating the progress of further stages (but only forward) that are contained inside of snapshots @@ -213,7 +178,7 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, // for now easier just store them in db td := big.NewInt(0) blockNumBytes := make([]byte, 8) - if err := snapshotsync.ForEachHeader(ctx, sn, func(header *types.Header) error { + if err := blockReader.HeadersRange(ctx, func(header *types.Header) error { blockNum, blockHash := header.Number.Uint64(), header.Hash() td.Add(td, header.Difficulty) @@ -232,7 +197,7 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, case <-ctx.Done(): return ctx.Err() case <-logEvery.C: - logger.Info(fmt.Sprintf("[%s] Total difficulty index: %dk/%dk", logPrefix, header.Number.Uint64()/1000, sn.BlocksAvailable()/1000)) + logger.Info(fmt.Sprintf("[%s] Total difficulty index: %dk/%dk", logPrefix, header.Number.Uint64()/1000, blockReader.FrozenBlocks()/1000)) default: } return nil @@ -277,7 +242,7 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, case <-ctx.Done(): return ctx.Err() case <-logEvery.C: - logger.Info(fmt.Sprintf("[%s] MaxTxNums index: %dk/%dk", logPrefix, blockNum/1000, sn.BlocksAvailable()/1000)) + logger.Info(fmt.Sprintf("[%s] MaxTxNums index: %dk/%dk", logPrefix, blockNum/1000, blockReader.FrozenBlocks()/1000)) default: } maxTxNum := baseTxNum + txAmount - 1 @@ -289,8 +254,8 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, }); err != nil { return fmt.Errorf("build txNum => blockNum mapping: %w", err) } - if blockReader.Snapshots().BlocksAvailable() > 0 { - if err := rawdb.AppendCanonicalTxNums(tx, blockReader.Snapshots().BlocksAvailable()+1); err != nil { + if blockReader.FrozenBlocks() > 0 { + if err := rawdb.AppendCanonicalTxNums(tx, blockReader.FrozenBlocks()+1); err != nil { return err } } else { @@ -299,7 +264,7 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, } } } - if err := rawdb.WriteSnapshots(tx, sn.Files(), agg.Files()); err != nil { + if err := rawdb.WriteSnapshots(tx, blockReader.FrozenFiles(), agg.Files()); err != nil { return err } } @@ -307,180 +272,6 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, return nil } -// WaitForDownloader - wait for Downloader service to download all expected snapshots -// for MVP we sync with Downloader only once, in future will send new snapshots also -func WaitForDownloader(s *StageState, ctx context.Context, cfg SnapshotsCfg, tx kv.RwTx) error { - snapshots := cfg.blockReader.Snapshots().(*snapshotsync.RoSnapshots) - if snapshots.Cfg().NoDownloader { - if err := snapshots.ReopenFolder(); err != nil { - return err - } - if cfg.dbEventNotifier != nil { // can notify right here, even that write txn is not commit - cfg.dbEventNotifier.OnNewSnapshot() - } - return nil - } - // Original intent of snInDB was to contain the file names of the snapshot files for the very first run of the Erigon instance - // Then, we would insist to only download such files, and no others (whitelist) - // However, at some point later, the code was incorrectly changed to update this record in each iteration of the stage loop (function WriteSnapshots) - // And so this list cannot be relied upon as the whitelist, because it also includes all the files created by the node itself - // Not sure what to do it is so far, but the temporary solution is to instead use it as a blacklist (existingFilesMap) - snInDB, snHistInDB, err := rawdb.ReadSnapshots(tx) - if err != nil { - return err - } - dbEmpty := len(snInDB) == 0 - var missingSnapshots []snapshotsync.Range - var existingFiles []snaptype.FileInfo - if !dbEmpty { - existingFiles, missingSnapshots, err = snapshotsync.Segments(snapshots.Dir()) - if err != nil { - return err - } - } - existingFilesMap := map[string]struct{}{} - for _, existingFile := range existingFiles { - _, fname := filepath.Split(existingFile.Path) - existingFilesMap[fname] = struct{}{} - } - if len(missingSnapshots) > 0 { - log.Warn(fmt.Sprintf("[%s] downloading missing snapshots", s.LogPrefix())) - } - - // send all hashes to the Downloader service - preverifiedBlockSnapshots := snapcfg.KnownCfg(cfg.chainConfig.ChainName, snInDB, snHistInDB).Preverified - downloadRequest := make([]snapshotsync.DownloadRequest, 0, len(preverifiedBlockSnapshots)+len(missingSnapshots)) - // build all download requests - // builds preverified snapshots request - for _, p := range preverifiedBlockSnapshots { - if _, exists := existingFilesMap[p.Name]; !exists { // Not to download existing files "behind the scenes" - downloadRequest = append(downloadRequest, snapshotsync.NewDownloadRequest(nil, p.Name, p.Hash)) - } - } - if cfg.historyV3 { - preverifiedHistorySnapshots := snapcfg.KnownCfg(cfg.chainConfig.ChainName, snInDB, snHistInDB).PreverifiedHistory - for _, p := range preverifiedHistorySnapshots { - downloadRequest = append(downloadRequest, snapshotsync.NewDownloadRequest(nil, p.Name, p.Hash)) - } - } - - // builds missing snapshots request - for i := range missingSnapshots { - downloadRequest = append(downloadRequest, snapshotsync.NewDownloadRequest(&missingSnapshots[i], "", "")) - } - - log.Info(fmt.Sprintf("[%s] Fetching torrent files metadata", s.LogPrefix())) - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if err := snapshotsync.RequestSnapshotsDownload(ctx, downloadRequest, cfg.snapshotDownloader); err != nil { - log.Error(fmt.Sprintf("[%s] call downloader", s.LogPrefix()), "err", err) - time.Sleep(10 * time.Second) - continue - } - break - } - downloadStartTime := time.Now() - logEvery := time.NewTicker(logInterval) - defer logEvery.Stop() - var m runtime.MemStats - - // Check once without delay, for faster erigon re-start - stats, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}) - if err == nil && stats.Completed { - goto Finish - } - - // Print download progress until all segments are available -Loop: - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-logEvery.C: - if stats, err := cfg.snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil { - log.Warn("Error while waiting for snapshots progress", "err", err) - } else if stats.Completed { - if !snapshots.Cfg().Verify { // will verify after loop - if _, err := cfg.snapshotDownloader.Verify(ctx, &proto_downloader.VerifyRequest{}); err != nil { - return err - } - } - log.Info(fmt.Sprintf("[%s] download finished", s.LogPrefix()), "time", time.Since(downloadStartTime).String()) - break Loop - } else { - if stats.MetadataReady < stats.FilesTotal { - log.Info(fmt.Sprintf("[%s] Waiting for torrents metadata: %d/%d", s.LogPrefix(), stats.MetadataReady, stats.FilesTotal)) - continue - } - dbg.ReadMemStats(&m) - downloadTimeLeft := calculateTime(stats.BytesTotal-stats.BytesCompleted, stats.DownloadRate) - log.Info(fmt.Sprintf("[%s] download", s.LogPrefix()), - "progress", fmt.Sprintf("%.2f%% %s/%s", stats.Progress, libcommon.ByteCount(stats.BytesCompleted), libcommon.ByteCount(stats.BytesTotal)), - "download-time-left", downloadTimeLeft, - "total-download-time", time.Since(downloadStartTime).Round(time.Second).String(), - "download", libcommon.ByteCount(stats.DownloadRate)+"/s", - "upload", libcommon.ByteCount(stats.UploadRate)+"/s", - ) - log.Info(fmt.Sprintf("[%s] download", s.LogPrefix()), - "peers", stats.PeersUnique, - "connections", stats.ConnectionsTotal, - "files", stats.FilesTotal, - "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys), - ) - } - } - } - -Finish: - if snapshots.Cfg().Verify { - if _, err := cfg.snapshotDownloader.Verify(ctx, &proto_downloader.VerifyRequest{}); err != nil { - return err - } - } - - if err := snapshots.ReopenFolder(); err != nil { - return err - } - if err := cfg.agg.OpenFolder(); err != nil { - return err - } - - if err := rawdb.WriteSnapshots(tx, snapshots.Files(), cfg.agg.Files()); err != nil { - return err - } - if cfg.dbEventNotifier != nil { // can notify right here, even that write txn is not commit - cfg.dbEventNotifier.OnNewSnapshot() - } - - firstNonGenesis, err := rawdbv3.SecondKey(tx, kv.Headers) - if err != nil { - return err - } - if firstNonGenesis != nil { - firstNonGenesisBlockNumber := binary.BigEndian.Uint64(firstNonGenesis) - if snapshots.SegmentsMax()+1 < firstNonGenesisBlockNumber { - log.Warn(fmt.Sprintf("[%s] Some blocks are not in snapshots and not in db", s.LogPrefix()), "max_in_snapshots", snapshots.SegmentsMax(), "min_in_db", firstNonGenesisBlockNumber) - } - } - return nil -} - -func calculateTime(amountLeft, rate uint64) string { - if rate == 0 { - return "999hrs:99m" - } - timeLeftInSeconds := amountLeft / rate - - hours := timeLeftInSeconds / 3600 - minutes := (timeLeftInSeconds / 60) % 60 - - return fmt.Sprintf("%dhrs:%dm", hours, minutes) -} - /* ====== PRUNING ====== */ // snapshots pruning sections works more as a retiring of blocks // retiring blocks means moving block data from db into snapshots @@ -494,22 +285,28 @@ func SnapshotsPrune(s *PruneState, initialCycle bool, cfg SnapshotsCfg, ctx cont defer tx.Rollback() } - br := cfg.blockRetire - sn := br.Snapshots() - if sn.Cfg().Enabled { - if err := br.PruneAncientBlocks(tx, 100); err != nil { + freezingCfg := cfg.blockReader.FreezingCfg() + if freezingCfg.Enabled { + if err := cfg.blockRetire.PruneAncientBlocks(tx, 100); err != nil { return err } } - if sn.Cfg().Enabled && sn.Cfg().Produce { + if freezingCfg.Enabled && freezingCfg.Produce { //TODO: initialSync maybe save files progress here - if cfg.agg.NeedSaveFilesListInDB() || br.NeedSaveFilesListInDB() { - if err := rawdb.WriteSnapshots(tx, br.Snapshots().Files(), cfg.agg.Files()); err != nil { + if cfg.blockRetire.HasNewFrozenFiles() || cfg.agg.HasNewFrozenFiles() { + if err := rawdb.WriteSnapshots(tx, cfg.blockReader.FrozenFiles(), cfg.agg.Files()); err != nil { return err } } - br.RetireBlocksInBackground(ctx, s.ForwardProgress, log.LvlDebug) + cfg.blockRetire.RetireBlocksInBackground(ctx, s.ForwardProgress, log.LvlDebug, func(downloadRequest []services.DownloadRequest) error { + if cfg.snapshotDownloader != nil && !reflect.ValueOf(cfg.snapshotDownloader).IsNil() { + if err := snapshotsync.RequestSnapshotsDownload(ctx, downloadRequest, cfg.snapshotDownloader); err != nil { + return err + } + } + return nil + }) //cfg.agg.BuildFilesInBackground() } diff --git a/eth/stagedsync/stage_txlookup.go b/eth/stagedsync/stage_txlookup.go index 64bc3cd5278..53f6fed30c0 100644 --- a/eth/stagedsync/stage_txlookup.go +++ b/eth/stagedsync/stage_txlookup.go @@ -18,7 +18,6 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/ethdb/prune" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" ) type TxLookupCfg struct { @@ -77,11 +76,10 @@ func SpawnTxLookup(s *StageState, tx kv.RwTx, toBlock uint64, cfg TxLookupCfg, c } } - snapshots := cfg.blockReader.Snapshots() - if snapshots.Cfg().Enabled { - if snapshots.BlocksAvailable() > startBlock { + if cfg.blockReader.FreezingCfg().Enabled { + if cfg.blockReader.FrozenBlocks() > startBlock { // Snapshot .idx files already have TxLookup index - then no reason iterate over them here - startBlock = snapshots.BlocksAvailable() + startBlock = cfg.blockReader.FrozenBlocks() if err = s.UpdatePrune(tx, startBlock); err != nil { // prune func of this stage will use this value to prevent all ancient blocks traversal return err } @@ -187,9 +185,8 @@ func UnwindTxLookup(u *UnwindState, s *StageState, tx kv.RwTx, cfg TxLookupCfg, // end key needs to be s.BlockNumber + 1 and not s.BlockNumber, because // the keys in BlockBody table always have hash after the block number blockFrom, blockTo := u.UnwindPoint+1, s.BlockNumber+1 - snapshots := cfg.blockReader.Snapshots() - if snapshots.Cfg().Enabled { - smallestInDB := snapshots.BlocksAvailable() + if cfg.blockReader.FreezingCfg().Enabled { + smallestInDB := cfg.blockReader.FrozenBlocks() blockFrom, blockTo = cmp.Max(blockFrom, smallestInDB), cmp.Max(blockTo, smallestInDB) } // etl.Transform uses ExtractEndKey as exclusive bound, therefore blockTo + 1 @@ -227,12 +224,11 @@ func PruneTxLookup(s *PruneState, tx kv.RwTx, cfg TxLookupCfg, ctx context.Conte var pruneBor bool // Forward stage doesn't write anything before PruneTo point - blockSnapshots := cfg.blockReader.Snapshots() if cfg.prune.TxIndex.Enabled() { blockTo = cfg.prune.TxIndex.PruneTo(s.ForwardProgress) pruneBor = true - } else if blockSnapshots != nil && blockSnapshots.Cfg().Enabled { - blockTo = snapshotsync.CanDeleteTo(s.ForwardProgress, blockSnapshots) + } else if cfg.blockReader.FreezingCfg().Enabled { + blockTo = cfg.blockReader.CanPruneTo(s.ForwardProgress) } // can't prune much here: because tx_lookup index has crypto-hashed-keys, and 1 block producing hundreds of deletes blockTo = cmp.Min(blockTo, blockFrom+10) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 4e4f707ff29..68fed2e1251 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -132,7 +132,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } // Configure a blockchain with the given prestate var ( - signer = types.MakeSigner(test.Genesis.Config, uint64(test.Context.Number)) + signer = types.MakeSigner(test.Genesis.Config, uint64(test.Context.Number), uint64(test.Context.Time)) origin, _ = signer.Sender(tx) txContext = evmtypes.TxContext{ Origin: origin, @@ -236,7 +236,7 @@ func benchTracer(b *testing.B, tracerName string, test *callTracerTest) { if err != nil { b.Fatalf("failed to parse testcase input: %v", err) } - signer := types.MakeSigner(test.Genesis.Config, uint64(test.Context.Number)) + signer := types.MakeSigner(test.Genesis.Config, uint64(test.Context.Number), uint64(test.Context.Time)) rules := &chain.Rules{} msg, err := tx.AsMessage(*signer, nil, rules) if err != nil { diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 7eba7a124b3..2405221c9a2 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -97,7 +97,7 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { } // Configure a blockchain with the given prestate var ( - signer = types.MakeSigner(test.Genesis.Config, uint64(test.Context.Number)) + signer = types.MakeSigner(test.Genesis.Config, uint64(test.Context.Number), uint64(test.Context.Time)) origin, _ = signer.Sender(tx) txContext = evmtypes.TxContext{ Origin: origin, diff --git a/ethdb/Readme.md b/ethdb/Readme.md index 84364396ebf..162eefae459 100644 --- a/ethdb/Readme.md +++ b/ethdb/Readme.md @@ -4,7 +4,7 @@ Words "KV" and "DB" have special meaning here: - KV - key-value-style API to access data: let developer manage transactions, stateful cursors. - DB - object-oriented-style API to access data: Get/Put/Delete/WalkOverTable/MultiPut, managing transactions internally. -So, DB abstraction fits 95% times and leads to more maintainable code - because it's looks stateless. +So, DB abstraction fits 95% times and leads to more maintainable code - because it looks stateless. About "key-value-style": Modern key-value databases don't provide Get/Put/Delete methods, because it's very hard-drive-unfriendly - it pushes developers do random-disk-access which is [order of magnitude slower than sequential read](https://www.seagate.com/sg/en/tech-insights/lies-damn-lies-and-ssd-benchmark-master-ti/). @@ -72,14 +72,14 @@ About "key-value-style": Modern key-value databases don't provide Get/Put/Delete - MultipleDatabases, Customization: `NewMDBX().Path(path).WithBucketsConfig(config).Open()` -- 1 Transaction object can be used only withing 1 goroutine. +- 1 Transaction object can be used only within 1 goroutine. - Only 1 write transaction can be active at a time (other will wait). - Unlimited read transactions can be active concurrently (not blocked by write transaction). - Methods db.Update, db.View - can be used to open and close short transaction. - Methods Begin/Commit/Rollback - for long transaction. -- it's safe to call .Rollback() after .Commit(), multiple rollbacks are also safe. Common transaction patter: +- it's safe to call .Rollback() after .Commit(), multiple rollbacks are also safe. Common transaction pattern: ``` tx, err := db.Begin(true, ethdb.RW) @@ -127,7 +127,7 @@ for k, v, err := c.First(); k != nil; k, v, err = c.Next() { - method Begin DOESN'T create new TxDb object, it means this object can be passed into other objects by pointer, and high-level app code can start/commit transactions when it needs without re-creating all objects which holds TxDb pointer. -- This is reason why txDb.CommitAndBegin() method works: inside it creating new transaction object, pinter to TxDb stays valid. +- This is the reason why txDb.CommitAndBegin() method works: inside it creating new transaction object, pinter to TxDb stays valid. ## How to dump/load table diff --git a/ethdb/kv_util.go b/ethdb/kv_util.go index 430d9d73860..796ad8fe2a3 100644 --- a/ethdb/kv_util.go +++ b/ethdb/kv_util.go @@ -3,6 +3,7 @@ package ethdb import ( "bytes" + libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" ) @@ -29,7 +30,7 @@ func Walk(c kv.Cursor, startkey []byte, fixedbits int, walker func(k, v []byte) } func Bytesmask(fixedbits int) (fixedbytes int, mask byte) { - fixedbytes = (fixedbits + 7) / 8 + fixedbytes = libcommon.BitLenToByteLen(fixedbits) shiftbits := fixedbits & 7 mask = byte(0xff) if shiftbits != 0 { diff --git a/go.mod b/go.mod index 208f4762a74..515cdb2f189 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/ledgerwatch/erigon go 1.19 //fork with minor protobuf file changes and txpool support -replace github.com/ledgerwatch/erigon-lib v0.0.0-20230607134918-6106e5abdf0b => github.com/testinprod-io/erigon-lib v0.0.0-20230719062643-cbe732675966 +replace github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6 => github.com/testinprod-io/erigon-lib v0.0.0-20230720073253-a5298b697ab7 //for local dev: //replace github.com/ledgerwatch/erigon-lib v0.0.0-20230423044930-fc9dd74e6407 => ../erigon-lib require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230607134918-6106e5abdf0b + github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6 github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475 github.com/ledgerwatch/log/v3 v3.8.0 github.com/ledgerwatch/secp256k1 v1.0.0 @@ -19,8 +19,8 @@ require ( require ( gfx.cafe/util/go/generic v0.0.0-20230502013805-237fcc25d586 - github.com/99designs/gqlgen v0.17.31 - github.com/Giulio2002/bls v0.0.0-20230507111335-fa36c339a11f + github.com/99designs/gqlgen v0.17.33 + github.com/Giulio2002/bls v0.0.0-20230611172327-c0b9800e7b57 github.com/RoaringBitmap/roaring v1.2.3 github.com/VictoriaMetrics/fastcache v1.12.1 github.com/VictoriaMetrics/metrics v1.23.1 @@ -53,15 +53,16 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d - github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/hashicorp/golang-lru/arc/v2 v2.0.3 + github.com/hashicorp/golang-lru/v2 v2.0.3 github.com/holiman/uint256 v1.2.2 github.com/huandu/xstrings v1.4.0 - github.com/huin/goupnp v1.1.0 + github.com/huin/goupnp v1.2.0 github.com/jackpal/go-nat-pmp v1.0.2 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/kevinburke/go-bindata v3.21.0+incompatible - github.com/libp2p/go-libp2p v0.27.0 + github.com/libp2p/go-libp2p v0.28.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/maticnetwork/crand v1.0.2 github.com/maticnetwork/polyproto v0.0.2 @@ -69,9 +70,9 @@ require ( github.com/nxadm/tail v1.4.9-0.20211216163028-4472660a31a6 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pelletier/go-toml v1.9.5 - github.com/pelletier/go-toml/v2 v2.0.7 + github.com/pelletier/go-toml/v2 v2.0.8 github.com/pion/randutil v0.1.0 - github.com/pion/stun v0.4.0 + github.com/pion/stun v0.6.0 github.com/prometheus/client_golang v1.15.1 github.com/prometheus/common v0.42.0 github.com/protolambda/ztyp v0.2.2 @@ -87,26 +88,26 @@ require ( github.com/tidwall/btree v1.6.0 github.com/ugorji/go/codec v1.1.13 github.com/ugorji/go/codec/codecgen v1.1.13 - github.com/urfave/cli/v2 v2.25.3 + github.com/urfave/cli/v2 v2.25.6 github.com/valyala/fastjson v1.6.4 - github.com/vektah/gqlparser/v2 v2.5.1 + github.com/vektah/gqlparser/v2 v2.5.3 github.com/xsleonard/go-merkle v1.1.0 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.10.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/net v0.10.0 - golang.org/x/sync v0.2.0 - golang.org/x/sys v0.8.0 + golang.org/x/net v0.11.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.9.0 golang.org/x/time v0.3.0 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.56.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.30.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.22.1 - pgregory.net/rapid v0.5.7 + modernc.org/sqlite v1.23.1 + pgregory.net/rapid v1.0.0 ) require ( @@ -129,7 +130,7 @@ require ( github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 // indirect github.com/anacrolix/utp v0.1.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/benbjohnson/immutable v0.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.5.0 // indirect @@ -141,7 +142,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -150,7 +151,7 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect @@ -160,7 +161,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b // indirect + github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/google/uuid v1.3.0 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20220405231054-a1ae3e4bba26 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -168,8 +169,8 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -179,20 +180,20 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-mplex v0.7.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-nat v0.1.0 // indirect + github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.2.0 // indirect + github.com/libp2p/go-reuseport v0.3.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect github.com/lispad/go-generics-tools v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.53 // indirect + github.com/miekg/dns v1.1.54 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -204,14 +205,14 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multicodec v0.8.1 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multihash v0.2.2 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.9.2 // indirect + github.com/onsi/ginkgo/v2 v2.9.7 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/pion/datachannel v1.5.2 // indirect - github.com/pion/dtls/v2 v2.2.4 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect github.com/pion/ice/v2 v2.2.6 // indirect github.com/pion/interceptor v0.1.11 // indirect github.com/pion/logging v0.2.2 // indirect @@ -222,20 +223,19 @@ require ( github.com/pion/sdp/v3 v3.0.5 // indirect github.com/pion/srtp/v2 v2.0.9 // indirect github.com/pion/transport v0.13.1 // indirect - github.com/pion/transport/v2 v2.0.0 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect github.com/pion/turn/v2 v2.0.8 // indirect - github.com/pion/udp v0.1.4 // indirect github.com/pion/webrtc/v3 v3.1.42 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.3.2 // indirect github.com/quic-go/qtls-go1-20 v0.2.2 // indirect github.com/quic-go/quic-go v0.33.0 // indirect - github.com/quic-go/webtransport-go v0.5.2 // indirect + github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect @@ -253,16 +253,16 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/dig v1.16.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.19.2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/text v0.9.0 // indirect - golang.org/x/tools v0.8.0 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + golang.org/x/text v0.10.0 // indirect + golang.org/x/tools v0.9.3 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - lukechampine.com/blake3 v1.1.7 // indirect + lukechampine.com/blake3 v1.2.1 // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect @@ -272,7 +272,6 @@ require ( modernc.org/opt v0.1.3 // indirect modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.1.0 // indirect - nhooyr.io/websocket v1.8.7 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index a01583e7f90..dc7fd92222d 100644 --- a/go.sum +++ b/go.sum @@ -15,11 +15,11 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG gfx.cafe/util/go/generic v0.0.0-20230502013805-237fcc25d586 h1:dlvliDuuuI3E+HtVeZVQgKuGcf0fGNNNadt04fgTyX8= gfx.cafe/util/go/generic v0.0.0-20230502013805-237fcc25d586/go.mod h1:WvSX4JsCRBuIXj0FRBFX9YLg+2SoL3w8Ww19uZO9yNE= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158= -github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= +github.com/99designs/gqlgen v0.17.33 h1:VTUpAtElDszatPSe26N0SD0deJCSxb7TZLlUb6JnVRY= +github.com/99designs/gqlgen v0.17.33/go.mod h1:ygDK+m8zGpoQuSh8xoq80UfisR5JTZr7mN57qXlSIZs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Giulio2002/bls v0.0.0-20230507111335-fa36c339a11f h1:7H8fhLJwlYAzzVUE1bQsQZjJBg8Rw+x5IZyb3i7PfZw= -github.com/Giulio2002/bls v0.0.0-20230507111335-fa36c339a11f/go.mod h1:o6qWofeW8A1XImbo3eHbC/wXnw/dasu0YuHEtdrjYzw= +github.com/Giulio2002/bls v0.0.0-20230611172327-c0b9800e7b57 h1:583GFQgWYOAz3dKqHqARVY3KkgebRcJtU4tzy+87gzc= +github.com/Giulio2002/bls v0.0.0-20230611172327-c0b9800e7b57/go.mod h1:vwm1rY/WKYdwv5Ii5US2bZ3MQVcHadnev+1Ml2QYWFk= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= @@ -29,7 +29,6 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0= @@ -108,8 +107,9 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw= github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= @@ -170,9 +170,9 @@ github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsP github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E= @@ -225,10 +225,6 @@ github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c/go.mod h1:Q0X6pkwTILD github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 h1:I8QswD9gf3VEpr7bpepKKOm7ChxFITIG+oc1I5/S0no= github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35/go.mod h1:DMDd04jjQgdynaAwbEgiRERIGpC8fDjx0+y06an7Psg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -246,19 +242,12 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -267,12 +256,6 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -300,7 +283,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -335,13 +317,12 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b h1:Qcx5LM0fSiks9uCyFZwDBUasd3lxd1RM0GYpL+Li5o4= -github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -353,7 +334,6 @@ github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORR github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -365,8 +345,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/arc/v2 v2.0.3 h1:D+r4C25CbvVaMiyerWsrcvfzQLwDwHFFb4PzgwhWqBU= +github.com/hashicorp/golang-lru/arc/v2 v2.0.3/go.mod h1:e1kvlTaZVi6wntRwqfHWdL5ZXhrHUwl04M9LSwK6vQE= +github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= +github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= @@ -379,10 +361,8 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.1.0 h1:gEe0Dp/lZmPZiDFzJJaOfUpOvv2MKUkoBX8lDrn9vKU= -github.com/huin/goupnp v1.1.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= +github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20220405231054-a1ae3e4bba26 h1:UT3hQ6+5hwqUT83cKhKlY5I0W/kqsl6lpn3iFb3Gtqs= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20220405231054-a1ae3e4bba26/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -415,15 +395,11 @@ github.com/kevinburke/go-bindata v3.21.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= -github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -449,16 +425,14 @@ github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf github.com/ledgerwatch/secp256k1 v1.0.0/go.mod h1:SPmqJFciiF/Q0mPt2jVs2dTr/1TZBTIA+kPMmKgBAak= github.com/ledgerwatch/trackerslist v1.1.0 h1:eKhgeURD9x/J3qzMnL6C0e0cLy6Ld7Ck/VR/yF+7cZQ= github.com/ledgerwatch/trackerslist v1.1.0/go.mod h1:wWU/V810cpsEl//o49ebwAWf0BL0WOJiu/577L4IVok= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.0 h1:QbhrTuB0ln9j9op6yAOR0o+cx/qa9NyNZ5ov0Tql8ZU= -github.com/libp2p/go-libp2p v0.27.0/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.28.0 h1:zO8cY98nJiPzZpFv5w5gqqb8aVzt4ukQ0nVOSaaKhJ8= +github.com/libp2p/go-libp2p v0.28.0/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= @@ -468,14 +442,12 @@ github.com/libp2p/go-mplex v0.7.0 h1:BDhFZdlk5tbr0oyFq/xv/NPGfjbnrsDam1EvutpBDbY github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= -github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= -github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= -github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= -github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= +github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= +github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lispad/go-generics-tools v1.1.0 h1:mbSgcxdFVmpoyso1X/MJHXbSbSL3dD+qhRryyxk+/XY= @@ -492,20 +464,19 @@ github.com/maticnetwork/polyproto v0.0.2 h1:cPxuxbIDItdwGnucc3lZB58U8Zfe1mH73PWT github.com/maticnetwork/polyproto v0.0.2/go.mod h1:e1mU2EXSwEpn5jM7GfNwu3AupsV6WAGoPFFfswXOF0o= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= -github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= -github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -514,8 +485,8 @@ github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdn github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= @@ -548,11 +519,11 @@ github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/e github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= -github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.2 h1:Uu7LWs/PmWby1gkj1S1DXx3zyd3aVabA4FiMKn/2tAc= +github.com/multiformats/go-multihash v0.2.2/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -571,13 +542,13 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -587,16 +558,16 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= -github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= github.com/pion/datachannel v1.5.2/go.mod h1:FTGQWaHrdCwIJ1rw6xBIfZVkslikjShim5yr05XFuCQ= github.com/pion/dtls/v2 v2.1.3/go.mod h1:o6+WvyLDAlXF7YiPB/RlskRoeK+/JtuaZa5emwQcWus= github.com/pion/dtls/v2 v2.1.5/go.mod h1:BqCE7xPZbPSubGasRoDFJeTsyJtdD1FanJYL0JGheqY= -github.com/pion/dtls/v2 v2.2.4 h1:YSfYwDQgrxMYXLBc/m7PFY5BVtWlNm/DN4qoU2CbcWg= -github.com/pion/dtls/v2 v2.2.4/go.mod h1:WGKfxqhrddne4Kg3p11FUMJrynkOY4lb25zHNO49wuw= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/ice/v2 v2.2.6 h1:R/vaLlI1J2gCx141L5PEwtuGAGcyS6e7E0hDeJFq5Ig= github.com/pion/ice/v2 v2.2.6/go.mod h1:SWuHiOGP17lGromHTFadUe1EuPgFh/oCU6FCMZHooVE= github.com/pion/interceptor v0.1.11 h1:00U6OlqxA3FFB50HSg25J/8cWi7P6FbSzw4eFn24Bvs= @@ -619,20 +590,18 @@ github.com/pion/sdp/v3 v3.0.5/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0 github.com/pion/srtp/v2 v2.0.9 h1:JJq3jClmDFBPX/F5roEb0U19jSU7eUhyDqR/NZ34EKQ= github.com/pion/srtp/v2 v2.0.9/go.mod h1:5TtM9yw6lsH0ppNCehB/EjEUli7VkUgKSPJqWVqbhQ4= github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA= -github.com/pion/stun v0.4.0 h1:vgRrbBE2htWHy7l3Zsxckk7rkjnjOsSM7PHZnBwo8rk= -github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= +github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= +github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A= github.com/pion/transport v0.13.0/go.mod h1:yxm9uXpK9bpBBWkITk13cLo1y5/ur5VQpG22ny6EP7g= github.com/pion/transport v0.13.1 h1:/UH5yLeQtwm2VZIPjxwnNFxjS4DFhyLfS4GlfuKUzfA= github.com/pion/transport v0.13.1/go.mod h1:EBxbqzyv+ZrmDb82XswEE0BjfQFtuw1Nu6sjnjWCsGg= -github.com/pion/transport/v2 v2.0.0 h1:bsMYyqHCbkvHwj+eNCFBuxtlKndKfyGI2vaQmM3fIE4= -github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/turn/v2 v2.0.8 h1:KEstL92OUN3k5k8qxsXHpr7WWfrdp7iJZHx99ud8muw= github.com/pion/turn/v2 v2.0.8/go.mod h1:+y7xl719J8bAEVpSXBXvTxStjJv3hbz9YFflvkpcGPw= github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= -github.com/pion/udp v0.1.4 h1:OowsTmu1Od3sD6i3fQUJxJn2fEvJO6L1TidgadtbTI8= -github.com/pion/udp v0.1.4/go.mod h1:G8LDo56HsFwC24LIcnT4YIDU5qcB6NepqqjP0keL2us= github.com/pion/webrtc/v3 v3.1.42 h1:wJEQFIXVanptnQcHOLTuIo4AtGB2+mG2x4OhIhnITOA= github.com/pion/webrtc/v3 v3.1.42/go.mod h1:ffD9DulDrPxyWvDPUIPAOSAWx9GUlOExiJPf7cCcMLA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -656,8 +625,8 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -688,8 +657,8 @@ github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8G github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= -github.com/quic-go/webtransport-go v0.5.2 h1:GA6Bl6oZY+g/flt00Pnu0XtivSD8vukOu3lYhJjnGEk= -github.com/quic-go/webtransport-go v0.5.2/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= +github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= +github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -712,8 +681,8 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shirou/gopsutil/v3 v3.23.4 h1:hZwmDxZs7Ewt75DV81r4pFMqbq+di2cbt9FsQBqLD2o= github.com/shirou/gopsutil/v3 v3.23.4/go.mod h1:ZcGxyfzAMRevhUR2+cfhXDH6gQdFYE/t8j1nsU4mPI8= github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= @@ -771,15 +740,15 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/supranational/blst v0.3.10/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/testinprod-io/erigon-lib v0.0.0-20230719062643-cbe732675966 h1:GfqqivZvfCrp344/Evz61cEHOIQHLutSKp19ZEbgqWI= -github.com/testinprod-io/erigon-lib v0.0.0-20230719062643-cbe732675966/go.mod h1:C7uEtUp6w0jbMUVXvOYlfkaGFM7PBIN5ILk7qaHPffk= +github.com/testinprod-io/erigon-lib v0.0.0-20230720073253-a5298b697ab7 h1:uqAcaW1+3QPUUufgvWNJrzz/Wou5I0knRUIGAK0J1kM= +github.com/testinprod-io/erigon-lib v0.0.0-20230720073253-a5298b697ab7/go.mod h1:qqbAdf5MngJaJPvflLOkVuJT08em4UGqnWDYGIJI5fw= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= @@ -793,24 +762,22 @@ github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYm github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/torquem-ch/mdbx-go v0.27.10 h1:iwb8Wn9gse4MEYIltAna+pxMPCY7hA1/5LLN/Qrcsx0= github.com/torquem-ch/mdbx-go v0.27.10/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.1.13/go.mod h1:jxau1n+/wyTGLQoCkjok9r5zFa/FxT6eI5HiHKQszjc= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.1.13 h1:013LbFhocBoIqgHeIHKlV4JWYhqogATYWZhIcH0WHn4= github.com/ugorji/go/codec v1.1.13/go.mod h1:oNVt3Dq+FO91WNQ/9JnHKQP2QJxTzoN7wCBFCq1OeuU= github.com/ugorji/go/codec/codecgen v1.1.13 h1:rGpZ4Q63VcWA3DMBbIHvg+SQweUkfXBBa/f9X0W+tFg= github.com/ugorji/go/codec/codecgen v1.1.13/go.mod h1:EhCxlc7Crov+HLygD4+hBCitXNrrGKRrRWj+pRsyJGg= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= -github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= +github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= -github.com/vektah/gqlparser/v2 v2.5.1 h1:ZGu+bquAY23jsxDRcYpWjttRZrUz07LbiY77gUOHcr4= -github.com/vektah/gqlparser/v2 v2.5.1/go.mod h1:mPgqFBu/woKTVYWyNk8cO3kh4S/f4aRFZrvOnp3hmCs= +github.com/vektah/gqlparser/v2 v2.5.3 h1:goUwv4+blhtwR3GwefadPVI4ubYc/WZSypljWMQa6IE= +github.com/vektah/gqlparser/v2 v2.5.3/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= @@ -836,10 +803,10 @@ go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRM go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8= -go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -867,9 +834,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220516162934-403b01795ae8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= @@ -885,12 +852,12 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -917,11 +884,11 @@ golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -936,8 +903,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -946,9 +914,7 @@ golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -958,7 +924,6 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -981,37 +946,31 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1032,8 +991,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1055,8 +1015,8 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1066,8 +1026,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE= +google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1110,8 +1070,8 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= @@ -1128,18 +1088,16 @@ modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.22.1 h1:P2+Dhp5FR1RlVRkQ3dDfCiv3Ok8XPxqpe70IjYVA9oE= -modernc.org/sqlite v1.22.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= -nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -pgregory.net/rapid v0.5.7 h1:p7/XbOgyFY1I/3Q12UTXfos70VZTcgc3WeoyiEru5cs= -pgregory.net/rapid v0.5.7/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +pgregory.net/rapid v1.0.0 h1:iQaM2w5PZ6xvt6x7hbd7tiDS+nk7YPp5uCaEba+T/F4= +pgregory.net/rapid v1.0.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml b/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml index c8185cf2814..9a9543261f0 100644 --- a/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml +++ b/k8s/google-kubernetes-engine/statefulset-erigon-patch.yaml @@ -24,7 +24,7 @@ - '--http.addr=0.0.0.0' - '--http.api=eth,erigon,web3,net,debug,ots,trace,txpool' - '--http.corsdomain=*' - - '--http.vhosts=*' + - '--http.vhosts=any' - '--log.console.verbosity=1' - '--log.json' - '--metrics' diff --git a/migrations/txs_begin_end_test.go b/migrations/txs_begin_end_test.go index 0094a3c3a3b..0ed8819ae9b 100644 --- a/migrations/txs_begin_end_test.go +++ b/migrations/txs_begin_end_test.go @@ -24,7 +24,7 @@ import ( func TestTxsBeginEnd(t *testing.T) { require, tmpDir, db := require.New(t), t.TempDir(), memdb.NewTestDB(t) - txn := &types.DynamicFeeTransaction{Tip: u256.N1, FeeCap: u256.N1, CommonTx: types.CommonTx{ChainID: u256.N1, Value: u256.N1, Gas: 1, Nonce: 1}} + txn := &types.DynamicFeeTransaction{Tip: u256.N1, FeeCap: u256.N1, ChainID: u256.N1, CommonTx: types.CommonTx{Value: u256.N1, Gas: 1, Nonce: 1}} buf := bytes.NewBuffer(nil) err := txn.MarshalBinary(buf) require.NoError(err) diff --git a/migrations/txs_v3_test.go b/migrations/txs_v3_test.go index 729827ad1eb..50ac319f12a 100644 --- a/migrations/txs_v3_test.go +++ b/migrations/txs_v3_test.go @@ -26,7 +26,7 @@ import ( func TestTxsV3(t *testing.T) { require, tmpDir, db := require.New(t), t.TempDir(), memdb.NewTestDB(t) - txn := &types.DynamicFeeTransaction{Tip: u256.N1, FeeCap: u256.N1, CommonTx: types.CommonTx{ChainID: u256.N1, Value: u256.N1, Gas: 1, Nonce: 1}} + txn := &types.DynamicFeeTransaction{Tip: u256.N1, FeeCap: u256.N1, ChainID: u256.N1, CommonTx: types.CommonTx{Value: u256.N1, Gas: 1, Nonce: 1}} buf := bytes.NewBuffer(nil) err := txn.MarshalBinary(buf) require.NoError(err) diff --git a/node/rpcstack.go b/node/rpcstack.go index 3dbd4a306b1..2959aca0b74 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -402,7 +402,11 @@ func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.next.ServeHTTP(w, r) return } - if _, exist := h.vhosts[host]; exist { + if _, exist := h.vhosts["any"]; exist { + h.next.ServeHTTP(w, r) + return + } + if _, exist := h.vhosts[strings.ToLower(host)]; exist { h.next.ServeHTTP(w, r) return } diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 8aa76c85ba1..0280ab84e41 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -50,7 +50,7 @@ func TestCorsHandler(t *testing.T) { assert.Equal(t, "", resp2.Header.Get("Access-Control-Allow-Origin")) } -// TestVhosts makes sure vhosts are properly handled on the http server. +// TestVhosts makes sure vhosts is properly handled on the http server. func TestVhosts(t *testing.T) { srv := createAndStartServer(t, &httpConfig{Vhosts: []string{"test"}}, false, &wsConfig{}) defer srv.stop() @@ -65,6 +65,21 @@ func TestVhosts(t *testing.T) { assert.Equal(t, resp2.StatusCode, http.StatusForbidden) } +// TestVhostsAny makes sure vhosts any is properly handled on the http server. +func TestVhostsAny(t *testing.T) { + srv := createAndStartServer(t, &httpConfig{Vhosts: []string{"any"}}, false, &wsConfig{}) + defer srv.stop() + url := "http://" + srv.listenAddr() + + resp := rpcRequest(t, url, "host", "test") + defer resp.Body.Close() + assert.Equal(t, resp.StatusCode, http.StatusOK) + + resp2 := rpcRequest(t, url, "host", "bad") + defer resp2.Body.Close() + assert.Equal(t, resp.StatusCode, http.StatusOK) +} + type originTest struct { spec string expOk []string diff --git a/params/protocol_params.go b/params/protocol_params.go index 95f965ab7a0..e83fe585f65 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -174,11 +174,11 @@ const ( // stuff from EIP-4844 FieldElementsPerBlob = 4096 // each field element is 32 bytes - MaxDataGasPerBlock = 1 << 19 - DataGasPerBlob = 1 << 17 - TargetDataGasPerBlock uint64 = 1 << 18 + MaxDataGasPerBlock uint64 = 0xC0000 + TargetDataGasPerBlock uint64 = 0x60000 + DataGasPerBlob uint64 = 0x20000 MinDataGasPrice = 1 - DataGasPriceUpdateFraction = 2225652 + DataGasPriceUpdateFraction = 3338477 MaxBlobsPerBlock = MaxDataGasPerBlock / DataGasPerBlob BlobVerificationGas uint64 = 1800000 diff --git a/params/version.go b/params/version.go index 2fe964c7e2d..19486e0c094 100644 --- a/params/version.go +++ b/params/version.go @@ -32,10 +32,10 @@ var ( // Version is the version of upstream erigon // see https://calver.org const ( - VersionMajor = 2 // Major version component of the current release - VersionMinor = 45 // Minor version component of the current release - VersionMicro = 1 // Patch version component of the current release - VersionModifier = "stable" // Modifier component of the current release + VersionMajor = 2 // Major version component of the current release + VersionMinor = 46 // Minor version component of the current release + VersionMicro = 0 // Patch version component of the current release + VersionModifier = "dev" // Modifier component of the current release VersionKeyCreated = "ErigonVersionCreated" VersionKeyFinished = "ErigonVersionFinished" ) diff --git a/rlp/encode.go b/rlp/encode.go index afa5281d274..f43b71670d8 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -26,6 +26,8 @@ import ( "sync" "github.com/holiman/uint256" + + libcommon "github.com/ledgerwatch/erigon-lib/common" ) // https://github.com/ethereum/wiki/wiki/RLP @@ -747,7 +749,7 @@ func putint(b []byte, i uint64) (size int) { // intsize computes the minimum number of bytes required to store i. func intsize(i uint64) (size int) { - return (bits.Len64(i) + 7) / 8 + return libcommon.BitLenToByteLen(bits.Len64(i)) } func IntLenExcludingHead(i uint64) int { @@ -762,7 +764,7 @@ func BigIntLenExcludingHead(i *big.Int) int { if bitLen < 8 { return 0 } - return (bitLen + 7) / 8 + return libcommon.BitLenToByteLen(bitLen) } func Uint256LenExcludingHead(i *uint256.Int) int { @@ -770,7 +772,7 @@ func Uint256LenExcludingHead(i *uint256.Int) int { if bitLen < 8 { return 0 } - return (bitLen + 7) / 8 + return libcommon.BitLenToByteLen(bitLen) } // precondition: len(buffer) >= 9 @@ -803,7 +805,7 @@ func EncodeBigInt(i *big.Int, w io.Writer, buffer []byte) error { return err } - size := (bitLen + 7) / 8 + size := libcommon.BitLenToByteLen(bitLen) buffer[0] = 0x80 + byte(size) i.FillBytes(buffer[1 : 1+size]) _, err := w.Write(buffer[:1+size]) @@ -840,7 +842,7 @@ func EncodeString(s []byte, w io.Writer, buffer []byte) error { func EncodeStringSizePrefix(size int, w io.Writer, buffer []byte) error { if size >= 56 { - beSize := (bits.Len(uint(size)) + 7) / 8 + beSize := libcommon.BitLenToByteLen(bits.Len(uint(size))) binary.BigEndian.PutUint64(buffer[1:], uint64(size)) buffer[8-beSize] = byte(beSize) + 183 if _, err := w.Write(buffer[8-beSize : 9]); err != nil { diff --git a/rpc/client.go b/rpc/client.go index e60c99db416..06b0bb6eac3 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -198,7 +198,7 @@ func newClient(initctx context.Context, connect reconnectFunc, logger log.Logger if err != nil { return nil, err } - c := initClient(conn, randomIDGenerator(), new(serviceRegistry), logger) + c := initClient(conn, randomIDGenerator(), &serviceRegistry{logger: logger}, logger) c.reconnectFunc = connect return c, nil } diff --git a/rpc/handler.go b/rpc/handler.go index b1da7fa2301..679aa2c8e38 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -134,7 +134,7 @@ func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg * if conn.remoteAddr() != "" { h.logger = h.logger.New("conn", conn.remoteAddr()) } - h.unsubscribeCb = newCallback(reflect.Value{}, reflect.ValueOf(h.unsubscribe), "unsubscribe") + h.unsubscribeCb = newCallback(reflect.Value{}, reflect.ValueOf(h.unsubscribe), "unsubscribe", h.logger) return h } diff --git a/rpc/server.go b/rpc/server.go index b0805702ca1..25603cb5f7c 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -59,7 +59,7 @@ type Server struct { // NewServer creates a new server instance with no registered handlers. func NewServer(batchConcurrency uint, traceRequests, disableStreaming bool, logger log.Logger) *Server { - server := &Server{idgen: randomIDGenerator(), codecs: mapset.NewSet(), run: 1, batchConcurrency: batchConcurrency, + server := &Server{services: serviceRegistry{logger: logger}, idgen: randomIDGenerator(), codecs: mapset.NewSet(), run: 1, batchConcurrency: batchConcurrency, disableStreaming: disableStreaming, traceRequests: traceRequests, logger: logger} // Register the default service providing meta information about the RPC service such // as the services and methods it offers. diff --git a/rpc/service.go b/rpc/service.go index e04225f82a9..c2252194584 100644 --- a/rpc/service.go +++ b/rpc/service.go @@ -41,6 +41,7 @@ var ( type serviceRegistry struct { mu sync.Mutex services map[string]service + logger log.Logger } // service represents a registered object. @@ -59,6 +60,7 @@ type callback struct { errPos int // err return idx, of -1 when method cannot return error isSubscribe bool // true if this is a subscription callback streamable bool // support JSON streaming (more efficient for large responses) + logger log.Logger } func (r *serviceRegistry) registerName(name string, rcvr interface{}) error { @@ -66,7 +68,7 @@ func (r *serviceRegistry) registerName(name string, rcvr interface{}) error { if name == "" { return fmt.Errorf("no service name for type %s", rcvrVal.Type().String()) } - callbacks := suitableCallbacks(rcvrVal) + callbacks := suitableCallbacks(rcvrVal, r.logger) if len(callbacks) == 0 { return fmt.Errorf("service %T doesn't have any suitable methods/subscriptions to expose", rcvr) } @@ -116,7 +118,7 @@ func (r *serviceRegistry) subscription(service, name string) *callback { // suitableCallbacks iterates over the methods of the given type. It determines if a method // satisfies the criteria for a RPC callback or a subscription callback and adds it to the // collection of callbacks. See server documentation for a summary of these criteria. -func suitableCallbacks(receiver reflect.Value) map[string]*callback { +func suitableCallbacks(receiver reflect.Value, logger log.Logger) map[string]*callback { typ := receiver.Type() callbacks := make(map[string]*callback) for m := 0; m < typ.NumMethod(); m++ { @@ -125,7 +127,7 @@ func suitableCallbacks(receiver reflect.Value) map[string]*callback { continue // method not exported } name := formatName(method.Name) - cb := newCallback(receiver, method.Func, name) + cb := newCallback(receiver, method.Func, name, logger) if cb == nil { continue // function invalid } @@ -136,9 +138,9 @@ func suitableCallbacks(receiver reflect.Value) map[string]*callback { // newCallback turns fn (a function) into a callback object. It returns nil if the function // is unsuitable as an RPC callback. -func newCallback(receiver, fn reflect.Value, name string) *callback { +func newCallback(receiver, fn reflect.Value, name string, logger log.Logger) *callback { fntype := fn.Type() - c := &callback{fn: fn, rcvr: receiver, errPos: -1, isSubscribe: isPubSub(fntype)} + c := &callback{fn: fn, rcvr: receiver, errPos: -1, isSubscribe: isPubSub(fntype), logger: logger} // Determine parameter types. They must all be exported or builtin types. c.makeArgTypes() @@ -149,7 +151,7 @@ func newCallback(receiver, fn reflect.Value, name string) *callback { outs[i] = fntype.Out(i) } if len(outs) > 2 { - log.Warn(fmt.Sprintf("Cannot register RPC callback [%s] - maximum 2 return values are allowed, got %d", name, len(outs))) + logger.Warn(fmt.Sprintf("Cannot register RPC callback [%s] - maximum 2 return values are allowed, got %d", name, len(outs))) return nil } // If an error is returned, it must be the last returned value. @@ -158,7 +160,7 @@ func newCallback(receiver, fn reflect.Value, name string) *callback { c.errPos = 0 case len(outs) == 2: if isErrorType(outs[0]) || !isErrorType(outs[1]) { - log.Warn(fmt.Sprintf("Cannot register RPC callback [%s] - error must the last return value", name)) + logger.Warn(fmt.Sprintf("Cannot register RPC callback [%s] - error must the last return value", name)) return nil } c.errPos = 1 @@ -214,7 +216,7 @@ func (c *callback) call(ctx context.Context, method string, args []reflect.Value // Catch panic while running the callback. defer func() { if err := recover(); err != nil { - log.Error("RPC method " + method + " crashed: " + fmt.Sprintf("%v\n%s", err, dbg.Stack())) + c.logger.Error("RPC method " + method + " crashed: " + fmt.Sprintf("%v\n%s", err, dbg.Stack())) errRes = errors.New("method handler crashed") } }() diff --git a/spectest/case.go b/spectest/case.go index 2940701ab24..b541abe83b1 100644 --- a/spectest/case.go +++ b/spectest/case.go @@ -1,6 +1,7 @@ package spectest import ( + "github.com/ledgerwatch/erigon/cl/transition/machine" "io/fs" "os" "strings" @@ -16,6 +17,8 @@ type TestCase struct { HandlerName string SuiteName string CaseName string + + Machine machine.Interface } func (t *TestCase) Version() clparams.StateVersion { diff --git a/spectest/suite.go b/spectest/suite.go index 757fc868dc1..fe10c65f3cb 100644 --- a/spectest/suite.go +++ b/spectest/suite.go @@ -1,6 +1,7 @@ package spectest import ( + "github.com/ledgerwatch/erigon/cl/transition/machine" "io/fs" "path/filepath" "testing" @@ -9,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func RunCases(t *testing.T, app Appendix, root fs.FS) { +func RunCases(t *testing.T, app Appendix, machineImpl machine.Interface, root fs.FS) { cases, err := ReadTestCases(root) require.Nil(t, err, "reading cases") // prepare for gore..... @@ -59,6 +60,7 @@ func RunCases(t *testing.T, app Appendix, root fs.FS) { value.SuiteName, value.CaseName, )) + value.Machine = machineImpl require.NoError(t, err) err = handler.Run(t, subfs, value) require.NoError(t, err) diff --git a/spectest/util.go b/spectest/util.go index 10e2207b3c9..dd74ce12a5c 100644 --- a/spectest/util.go +++ b/spectest/util.go @@ -2,16 +2,17 @@ package spectest import ( "fmt" - "github.com/ledgerwatch/erigon/cl/phase1/core/state" "io/fs" "os" + "gopkg.in/yaml.v3" + "github.com/ledgerwatch/erigon-lib/types/ssz" "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/utils" - "gopkg.in/yaml.v3" ) func ReadMeta(root fs.FS, name string, obj any) error { diff --git a/tests/automated-testing/docker-compose.yml b/tests/automated-testing/docker-compose.yml index 23a46c47a80..ed740ff2744 100644 --- a/tests/automated-testing/docker-compose.yml +++ b/tests/automated-testing/docker-compose.yml @@ -35,7 +35,7 @@ services: image: testinprod/op-erigon:$ERIGON_TAG entrypoint: rpcdaemon command: | - --private.api.addr=erigon:9090 --http.api=admin,eth,erigon,web3,net,debug,trace,txpool,parity --http.addr=0.0.0.0 --http.vhosts=* --http.corsdomain=* --http.port=8545 --graphql --log.dir.path=/logs/node1 + --private.api.addr=erigon:9090 --http.api=admin,eth,erigon,web3,net,debug,trace,txpool,parity --http.addr=0.0.0.0 --http.vhosts=any --http.corsdomain=* --http.port=8545 --graphql --log.dir.path=/logs/node1 volumes: - ./logdir:/logs user: ${DOCKER_UID}:${DOCKER_GID} @@ -47,7 +47,7 @@ services: image: testinprod/op-erigon:$ERIGON_TAG entrypoint: rpcdaemon command: | - --private.api.addr=erigon-node2:9090 --http.api=admin,eth,erigon,web3,net,debug,trace,txpool,parity --http.addr=0.0.0.0 --http.vhosts=* --http.corsdomain=* --http.port=8545 --log.dir.path=/logs/node2 + --private.api.addr=erigon-node2:9090 --http.api=admin,eth,erigon,web3,net,debug,trace,txpool,parity --http.addr=0.0.0.0 --http.vhosts=any --http.corsdomain=* --http.port=8545 --log.dir.path=/logs/node2 volumes: - ./logdir:/logs user: ${DOCKER_UID}:${DOCKER_GID} diff --git a/tests/automated-testing/run.sh b/tests/automated-testing/run.sh index bd15e4cda54..a0ac39b9f76 100755 --- a/tests/automated-testing/run.sh +++ b/tests/automated-testing/run.sh @@ -3,8 +3,8 @@ function stopContainers () { # stop containers echo "stopping containers..." - docker-compose --profile=first down -v --remove-orphans - docker-compose --profile=second down -v --remove-orphans + docker compose --profile=first down -v --remove-orphans + docker compose --profile=second down -v --remove-orphans } ORIGINAL_DIR=$(pwd) @@ -46,7 +46,7 @@ docker compose pull # run node 1 echo "starting node 1..." -docker-compose --profile=first up -d --force-recreate --remove-orphans +docker compose --profile=first up -d --force-recreate --remove-orphans # wait for node 1 to start up echo "waiting for node 1 to start up..." @@ -55,7 +55,7 @@ sleep 10 # run node 2 echo "starting node 2..." export ENODE=$(./scripts/enode.sh) -docker-compose --profile=second up -d --force-recreate --remove-orphans +docker compose --profile=second up -d --force-recreate --remove-orphans # wait for node 2 to start up echo "waiting for node 2 to start up..." diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 2dae0b93af7..2cdbdd10c48 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -27,12 +27,15 @@ import ( "testing" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/common/math" @@ -43,7 +46,6 @@ import ( "github.com/ledgerwatch/erigon/eth/ethconsensusconfig" "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/stages" - "github.com/ledgerwatch/log/v3" ) // A BlockTest checks handling of entire blocks. @@ -114,7 +116,7 @@ func (bt *BlockTest) Run(t *testing.T, _ bool) error { engine := ethconsensusconfig.CreateConsensusEngineBareBones(config, log.New()) m := stages.MockWithGenesisEngine(t, bt.genesis(config), engine, false) - bt.br, _ = m.NewBlocksIO() + bt.br = m.BlockReader // import pre accounts & construct test genesis block & state root if m.Genesis.Hash() != bt.json.Genesis.Hash { return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", m.Genesis.Hash().Bytes()[:6], bt.json.Genesis.Hash[:6]) @@ -123,22 +125,23 @@ func (bt *BlockTest) Run(t *testing.T, _ bool) error { return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", m.Genesis.Root().Bytes()[:6], bt.json.Genesis.StateRoot[:6]) } - validBlocks, err := bt.insertBlocks(m) + tx, err := m.DB.BeginRw(m.Ctx) if err != nil { return err } + defer tx.Rollback() - tx, err1 := m.DB.BeginRo(context.Background()) - if err1 != nil { - return fmt.Errorf("blockTest create tx: %w", err1) + validBlocks, err := bt.insertBlocks(m, tx) + if err != nil { + return err } - defer tx.Rollback() + cmlast := rawdb.ReadHeadBlockHash(tx) if libcommon.Hash(bt.json.BestBlock) != cmlast { return fmt.Errorf("last block hash validation mismatch: want: %x, have: %x", bt.json.BestBlock, cmlast) } newDB := state.New(m.NewStateReader(tx)) - if err = bt.validatePostState(newDB); err != nil { + if err := bt.validatePostState(newDB); err != nil { return fmt.Errorf("post state validation failed: %w", err) } return bt.validateImportedHeaders(tx, validBlocks, m) @@ -174,7 +177,7 @@ See https://github.com/ethereum/tests/wiki/Blockchain-Tests-II expected we are expected to ignore it and continue processing and then validate the post state. */ -func (bt *BlockTest) insertBlocks(m *stages.MockSentry) ([]btBlock, error) { +func (bt *BlockTest) insertBlocks(m *stages.MockSentry, tx kv.RwTx) ([]btBlock, error) { validBlocks := make([]btBlock, 0) // insert the test blocks, which will execute all transaction for bi, b := range bt.json.Blocks { @@ -188,7 +191,8 @@ func (bt *BlockTest) insertBlocks(m *stages.MockSentry) ([]btBlock, error) { } // RLP decoding worked, try to insert into chain: chain := &core.ChainPack{Blocks: []*types.Block{cb}, Headers: []*types.Header{cb.Header()}, TopBlock: cb} - err1 := m.InsertChain(chain) + + err1 := m.InsertChain(chain, tx) if err1 != nil { if b.BlockHeader == nil { continue // OK - block is supposed to be invalid, continue with next block @@ -196,17 +200,12 @@ func (bt *BlockTest) insertBlocks(m *stages.MockSentry) ([]btBlock, error) { return nil, fmt.Errorf("block #%v insertion into chain failed: %w", cb.Number(), err1) } } else if b.BlockHeader == nil { - if err := m.DB.View(context.Background(), func(tx kv.Tx) error { - canonical, cErr := bt.br.CanonicalHash(context.Background(), tx, cb.NumberU64()) - if cErr != nil { - return cErr - } - if canonical == cb.Hash() { - return fmt.Errorf("block (index %d) insertion should have failed due to: %v", bi, b.ExpectException) - } - return nil - }); err != nil { - return nil, err + canonical, cErr := bt.br.CanonicalHash(context.Background(), tx, cb.NumberU64()) + if cErr != nil { + return nil, cErr + } + if canonical == cb.Hash() { + return nil, fmt.Errorf("block (index %d) insertion should have failed due to: %v", bi, b.ExpectException) } } if b.BlockHeader == nil { @@ -250,7 +249,7 @@ func validateHeader(h *btHeader, h2 *types.Header) error { return fmt.Errorf("receipt hash: want: %x have: %x", h.ReceiptTrie, h2.ReceiptHash) } if h.TransactionsTrie != h2.TxHash { - return fmt.Errorf("tx hash: want: %x have: %x", h.TransactionsTrie, h2.TxHash) + return fmt.Errorf("txn hash: want: %x have: %x", h.TransactionsTrie, h2.TxHash) } if h.StateRoot != h2.Root { return fmt.Errorf("state hash: want: %x have: %x", h.StateRoot, h2.Root) @@ -305,7 +304,6 @@ func (bt *BlockTest) validatePostState(statedb *state.IntraBlockState) error { } func (bt *BlockTest) validateImportedHeaders(tx kv.Tx, validBlocks []btBlock, m *stages.MockSentry) error { - br, _ := m.NewBlocksIO() // to get constant lookup when verifying block headers by hash (some tests have many blocks) bmap := make(map[libcommon.Hash]btBlock, len(bt.json.Blocks)) for _, b := range validBlocks { @@ -316,7 +314,7 @@ func (bt *BlockTest) validateImportedHeaders(tx kv.Tx, validBlocks []btBlock, m // block-by-block, so we can only validate imported headers after // all blocks have been processed by BlockChain, as they may not // be part of the longest chain until last block is imported. - for b, _ := br.CurrentBlock(tx); b != nil && b.NumberU64() != 0; { + for b, _ := m.BlockReader.CurrentBlock(tx); b != nil && b.NumberU64() != 0; { if err := validateHeader(bmap[b.Hash()].BlockHeader, b.Header()); err != nil { return fmt.Errorf("imported block header validation failed: %w", err) } @@ -324,7 +322,7 @@ func (bt *BlockTest) validateImportedHeaders(tx kv.Tx, validBlocks []btBlock, m if number == nil { break } - b, _, _ = br.BlockWithSenders(m.Ctx, tx, b.ParentHash(), *number) + b, _, _ = m.BlockReader.BlockWithSenders(m.Ctx, tx, b.ParentHash(), *number) } return nil } diff --git a/tests/state_test.go b/tests/state_test.go index a8c79b2c441..1cbda422936 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -27,10 +27,12 @@ import ( "runtime" "testing" - "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/log/v3" + + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/eth/tracers/logger" - "github.com/ledgerwatch/log/v3" ) func TestState(t *testing.T) { @@ -51,7 +53,7 @@ func TestState(t *testing.T) { st.skipLoad(`.*vmPerformance/loop.*`) st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { - db := memdb.NewTestDB(t) + _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) for _, subtest := range test.Subtests() { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index daf1da53d82..cd04b0563b2 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -45,6 +45,7 @@ import ( "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" + "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/trie" ) @@ -194,12 +195,7 @@ func (t *StateTest) RunNoVerify(tx kv.RwTx, subtest StateSubtest, vmconfig vm.Co return nil, libcommon.Hash{}, UnsupportedForkError{subtest.Fork} } - var r state.StateReader - if ethconfig.EnableHistoryV4InTest { - panic("implement me") - } else { - r = state.NewPlainStateReader(tx) - } + r := rpchelper.NewLatestStateReader(tx) statedb := state.New(r) var w state.StateWriter @@ -228,7 +224,7 @@ func (t *StateTest) RunNoVerify(tx kv.RwTx, subtest StateSubtest, vmconfig vm.Co if err != nil { return nil, libcommon.Hash{}, err } - msg, err = txn.AsMessage(*types.MakeSigner(config, 0), baseFee, config.Rules(0, 0)) + msg, err = txn.AsMessage(*types.MakeSigner(config, 0, 0), baseFee, config.Rules(0, 0)) if err != nil { return nil, libcommon.Hash{}, err } @@ -311,12 +307,7 @@ func (t *StateTest) RunNoVerify(tx kv.RwTx, subtest StateSubtest, vmconfig vm.Co } func MakePreState(rules *chain.Rules, tx kv.RwTx, accounts types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { - var r state.StateReader - if ethconfig.EnableHistoryV4InTest { - panic("implement me") - } else { - r = state.NewPlainStateReader(tx) - } + r := rpchelper.NewLatestStateReader(tx) statedb := state.New(r) for addr, a := range accounts { statedb.SetCode(addr, a.Code) @@ -406,13 +397,13 @@ func toMessage(tx stTransactionMarshaling, ps stPostState, baseFee *big.Int) (co // Get values specific to this post state. if ps.Indexes.Data > len(tx.Data) { - return nil, fmt.Errorf("tx data index %d out of bounds", ps.Indexes.Data) + return nil, fmt.Errorf("txn data index %d out of bounds", ps.Indexes.Data) } if ps.Indexes.Value > len(tx.Value) { - return nil, fmt.Errorf("tx value index %d out of bounds", ps.Indexes.Value) + return nil, fmt.Errorf("txn value index %d out of bounds", ps.Indexes.Value) } if ps.Indexes.Gas > len(tx.GasLimit) { - return nil, fmt.Errorf("tx gas limit index %d out of bounds", ps.Indexes.Gas) + return nil, fmt.Errorf("txn gas limit index %d out of bounds", ps.Indexes.Gas) } dataHex := tx.Data[ps.Indexes.Data] valueHex := tx.Value[ps.Indexes.Value] @@ -422,17 +413,17 @@ func toMessage(tx stTransactionMarshaling, ps stPostState, baseFee *big.Int) (co if valueHex != "0x" { va, ok := math.ParseBig256(valueHex) if !ok { - return nil, fmt.Errorf("invalid tx value %q", valueHex) + return nil, fmt.Errorf("invalid txn value %q", valueHex) } v, overflow := uint256.FromBig(va) if overflow { - return nil, fmt.Errorf("invalid tx value (overflowed) %q", valueHex) + return nil, fmt.Errorf("invalid txn value (overflowed) %q", valueHex) } value = v } data, err := hex.DecodeString(strings.TrimPrefix(dataHex, "0x")) if err != nil { - return nil, fmt.Errorf("invalid tx data %q", dataHex) + return nil, fmt.Errorf("invalid txn data %q", dataHex) } var accessList types2.AccessList if tx.AccessLists != nil && tx.AccessLists[ps.Indexes.Data] != nil { diff --git a/tests/statedb_chain_test.go b/tests/statedb_chain_test.go index feb4439c987..edb2932dbfa 100644 --- a/tests/statedb_chain_test.go +++ b/tests/statedb_chain_test.go @@ -98,7 +98,7 @@ func TestSelfDestructReceive(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - tx, err := m.DB.BeginRo(context.Background()) + tx, err := m.DB.BeginRw(context.Background()) if err != nil { panic(err) } @@ -113,23 +113,18 @@ func TestSelfDestructReceive(t *testing.T) { } // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), tx); err != nil { t.Fatal(err) } // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), tx); err != nil { t.Fatal(err) } // If we got this far, the newly created blockchain (with empty trie cache) loaded trie from the database // and that means that the state of the accounts written in the first block was correct. // This test checks that the storage root of the account is properly set to the root of the empty tree - tx, err = m.DB.BeginRo(context.Background()) - if err != nil { - panic(err) - } - defer tx.Rollback() st = state.New(m.NewStateReader(tx)) if !st.Exist(address) { t.Error("expected account to exist") diff --git a/tests/statedb_insert_chain_transaction_test.go b/tests/statedb_insert_chain_transaction_test.go index 87182adad5f..8c1ef4349e3 100644 --- a/tests/statedb_insert_chain_transaction_test.go +++ b/tests/statedb_insert_chain_transaction_test.go @@ -30,7 +30,7 @@ func TestInsertIncorrectStateRootDifferentAccounts(t *testing.T) { fromKey := data.keys[0] to := libcommon.Address{1} - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -54,12 +54,12 @@ func TestInsertIncorrectStateRootDifferentAccounts(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[0].Transactions(), chain.Blocks[0].Uncles(), chain.Receipts[0], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // insert a correct block - m, chain, err = genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err = genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(data.addresses[1], to, uint256.NewInt(5000)), data.keys[1], @@ -69,14 +69,14 @@ func TestInsertIncorrectStateRootDifferentAccounts(t *testing.T) { t.Fatal(err) } - if err = m.InsertChain(chain); err != nil { - t.Fatal(err) - } - - tx, err := m.DB.BeginRo(context.Background()) + tx, err := m.DB.BeginRw(context.Background()) require.NoError(t, err) defer tx.Rollback() + if err = m.InsertChain(chain, tx); err != nil { + t.Fatal(err) + } + st := state.New(m.NewStateReader(tx)) if !st.Exist(to) { t.Error("expected account to exist") @@ -99,7 +99,7 @@ func TestInsertIncorrectStateRootSameAccount(t *testing.T) { fromKey := data.keys[0] to := libcommon.Address{1} - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -122,12 +122,12 @@ func TestInsertIncorrectStateRootSameAccount(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[0].Transactions(), chain.Blocks[0].Uncles(), chain.Receipts[0], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // insert a correct block - m, chain, err = genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err = genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(5000)), fromKey, @@ -137,7 +137,7 @@ func TestInsertIncorrectStateRootSameAccount(t *testing.T) { t.Fatal(err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatal(err) } @@ -164,7 +164,7 @@ func TestInsertIncorrectStateRootSameAccountSameAmount(t *testing.T) { fromKey := data.keys[0] to := libcommon.Address{1} - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -184,12 +184,12 @@ func TestInsertIncorrectStateRootSameAccountSameAmount(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[0].Transactions(), chain.Blocks[0].Uncles(), chain.Receipts[0], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // insert a correct block - m, chain, err = genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err = genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -199,7 +199,7 @@ func TestInsertIncorrectStateRootSameAccountSameAmount(t *testing.T) { t.Fatal(err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatal(err) } @@ -226,7 +226,7 @@ func TestInsertIncorrectStateRootAllFundsRoot(t *testing.T) { fromKey := data.keys[0] to := libcommon.Address{1} - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -246,12 +246,12 @@ func TestInsertIncorrectStateRootAllFundsRoot(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[0].Transactions(), chain.Blocks[0].Uncles(), chain.Receipts[0], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // insert a correct block - m, chain, err = genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err = genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -261,7 +261,7 @@ func TestInsertIncorrectStateRootAllFundsRoot(t *testing.T) { t.Fatal(err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatal(err) } @@ -288,7 +288,7 @@ func TestInsertIncorrectStateRootAllFunds(t *testing.T) { fromKey := data.keys[0] to := libcommon.Address{1} - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(3000)), fromKey, @@ -308,12 +308,12 @@ func TestInsertIncorrectStateRootAllFunds(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[0].Transactions(), chain.Blocks[0].Uncles(), chain.Receipts[0], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // insert a correct block - m, chain, err = genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err = genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(1000)), fromKey, @@ -323,7 +323,7 @@ func TestInsertIncorrectStateRootAllFunds(t *testing.T) { t.Fatal(err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatal(err) } @@ -353,7 +353,7 @@ func TestAccountDeployIncorrectRoot(t *testing.T) { var contractAddress libcommon.Address eipContract := new(contracts.Testcontract) - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(10)), fromKey, @@ -368,7 +368,7 @@ func TestAccountDeployIncorrectRoot(t *testing.T) { } // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -390,7 +390,7 @@ func TestAccountDeployIncorrectRoot(t *testing.T) { incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} // BLOCK 2 - INCORRECT - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } @@ -408,7 +408,7 @@ func TestAccountDeployIncorrectRoot(t *testing.T) { require.NoError(t, err) // BLOCK 2 - CORRECT - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } @@ -435,7 +435,7 @@ func TestAccountCreateIncorrectRoot(t *testing.T) { var contractAddress libcommon.Address eipContract := new(contracts.Testcontract) - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(10)), fromKey, @@ -454,7 +454,7 @@ func TestAccountCreateIncorrectRoot(t *testing.T) { } // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -473,7 +473,7 @@ func TestAccountCreateIncorrectRoot(t *testing.T) { require.NoError(t, err) // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } err = m.DB.View(context.Background(), func(tx kv.Tx) error { @@ -496,12 +496,12 @@ func TestAccountCreateIncorrectRoot(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[2].Transactions(), chain.Blocks[2].Uncles(), chain.Receipts[2], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // BLOCK 3 - if err = m.InsertChain(chain.Slice(2, 3)); err != nil { + if err = m.InsertChain(chain.Slice(2, 3), nil); err != nil { t.Fatal(err) } } @@ -515,7 +515,7 @@ func TestAccountUpdateIncorrectRoot(t *testing.T) { var contractAddress libcommon.Address eipContract := new(contracts.Testcontract) - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(10)), fromKey, @@ -538,7 +538,7 @@ func TestAccountUpdateIncorrectRoot(t *testing.T) { } // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -557,7 +557,7 @@ func TestAccountUpdateIncorrectRoot(t *testing.T) { require.NoError(t, err) // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } @@ -575,7 +575,7 @@ func TestAccountUpdateIncorrectRoot(t *testing.T) { require.NoError(t, err) // BLOCK 3 - if err = m.InsertChain(chain.Slice(2, 3)); err != nil { + if err = m.InsertChain(chain.Slice(2, 3), nil); err != nil { t.Fatal(err) } @@ -585,12 +585,12 @@ func TestAccountUpdateIncorrectRoot(t *testing.T) { incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[3].Transactions(), chain.Blocks[3].Uncles(), chain.Receipts[3], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // BLOCK 4 - if err = m.InsertChain(chain.Slice(3, 4)); err != nil { + if err = m.InsertChain(chain.Slice(3, 4), nil); err != nil { t.Fatal(err) } } @@ -604,7 +604,7 @@ func TestAccountDeleteIncorrectRoot(t *testing.T) { var contractAddress libcommon.Address eipContract := new(contracts.Testcontract) - m, chain, err := genBlocks(t, data.genesisSpec, map[int]tx{ + m, chain, err := genBlocks(t, data.genesisSpec, map[int]txn{ 0: { getBlockTx(from, to, uint256.NewInt(10)), fromKey, @@ -627,7 +627,7 @@ func TestAccountDeleteIncorrectRoot(t *testing.T) { } // BLOCK 1 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatal(err) } @@ -645,7 +645,7 @@ func TestAccountDeleteIncorrectRoot(t *testing.T) { require.NoError(t, err) // BLOCK 2 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), nil); err != nil { t.Fatal(err) } @@ -663,7 +663,7 @@ func TestAccountDeleteIncorrectRoot(t *testing.T) { require.NoError(t, err) // BLOCK 3 - if err = m.InsertChain(chain.Slice(2, 3)); err != nil { + if err = m.InsertChain(chain.Slice(2, 3), nil); err != nil { t.Fatal(err) } @@ -672,12 +672,12 @@ func TestAccountDeleteIncorrectRoot(t *testing.T) { incorrectHeader.Root = chain.Headers[1].Root incorrectBlock := types.NewBlock(&incorrectHeader, chain.Blocks[3].Transactions(), chain.Blocks[3].Uncles(), chain.Receipts[3], nil) incorrectChain := &core.ChainPack{Blocks: []*types.Block{incorrectBlock}, Headers: []*types.Header{&incorrectHeader}, TopBlock: incorrectBlock} - if err = m.InsertChain(incorrectChain); err == nil { + if err = m.InsertChain(incorrectChain, nil); err == nil { t.Fatal("should fail") } // BLOCK 4 - if err = m.InsertChain(chain.Slice(3, 4)); err != nil { + if err = m.InsertChain(chain.Slice(3, 4), nil); err != nil { t.Fatal(err) } } @@ -733,12 +733,12 @@ func getGenesis(funds ...*big.Int) initialData { } } -type tx struct { +type txn struct { txFn blockTx key *ecdsa.PrivateKey } -func genBlocks(t *testing.T, gspec *types.Genesis, txs map[int]tx) (*stages.MockSentry, *core.ChainPack, error) { +func genBlocks(t *testing.T, gspec *types.Genesis, txs map[int]txn) (*stages.MockSentry, *core.ChainPack, error) { key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") m := stages.MockWithGenesis(t, gspec, key, false) diff --git a/turbo/adapter/ethapi/api.go b/turbo/adapter/ethapi/api.go index ff2c0e3b96b..4879aa988a7 100644 --- a/turbo/adapter/ethapi/api.go +++ b/turbo/adapter/ethapi/api.go @@ -479,21 +479,18 @@ func newRPCTransaction(tx types.Transaction, blockHash libcommon.Hash, blockNumb result.V = (*hexutil.Big)(libcommon.Big0) result.R = (*hexutil.Big)(libcommon.Big0) result.S = (*hexutil.Big)(libcommon.Big0) - // case *types.SignedBlobTx: // TODO - case *types.SignedBlobTx: - chainId.Set(t.GetChainID()) + case *types.BlobTx: + chainId.Set(t.ChainID) result.ChainID = (*hexutil.Big)(chainId.ToBig()) - result.Tip = (*hexutil.Big)(t.GetTip().ToBig()) - result.FeeCap = (*hexutil.Big)(t.GetFeeCap().ToBig()) - v, r, s := t.RawSignatureValues() - result.V = (*hexutil.Big)(v.ToBig()) - result.R = (*hexutil.Big)(r.ToBig()) - result.S = (*hexutil.Big)(s.ToBig()) - al := t.GetAccessList() - result.Accesses = &al + result.Tip = (*hexutil.Big)(t.Tip.ToBig()) + result.FeeCap = (*hexutil.Big)(t.FeeCap.ToBig()) + result.V = (*hexutil.Big)(t.V.ToBig()) + result.R = (*hexutil.Big)(t.R.ToBig()) + result.S = (*hexutil.Big)(t.S.ToBig()) + result.Accesses = &t.AccessList // if the transaction has been mined, compute the effective gas price result.GasPrice = computeGasPrice(tx, blockHash, baseFee) - result.MaxFeePerDataGas = (*hexutil.Big)(t.GetMaxFeePerDataGas().ToBig()) + result.MaxFeePerDataGas = (*hexutil.Big)(t.MaxFeePerDataGas.ToBig()) result.BlobVersionedHashes = t.GetDataHashes() } signer := types.LatestSignerForChainID(chainId.ToBig()) diff --git a/turbo/app/import_cmd.go b/turbo/app/import_cmd.go index 4a201a125c2..986b1a873bd 100644 --- a/turbo/app/import_cmd.go +++ b/turbo/app/import_cmd.go @@ -13,7 +13,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/log/v3" "github.com/urfave/cli/v2" @@ -223,7 +222,7 @@ func InsertChain(ethereum *eth.Ethereum, chain *core.ChainPack, logger log.Logge blockReader, _ := ethereum.BlockIO() hook := stages.NewHook(ethereum.SentryCtx(), ethereum.Notifications(), ethereum.StagedSync(), blockReader, ethereum.ChainConfig(), logger, sentryControlServer.UpdateHead) - _, err := stages.StageLoopStep(ethereum.SentryCtx(), ethereum.ChainDB(), ethereum.StagedSync(), initialCycle, logger, blockReader.Snapshots().(*snapshotsync.RoSnapshots), hook) + err := stages.StageLoopStep(ethereum.SentryCtx(), ethereum.ChainDB(), nil, ethereum.StagedSync(), initialCycle, logger, blockReader, hook) if err != nil { return err } diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 408a956a72c..afded3d5709 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -34,7 +34,7 @@ import ( "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/logging" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" "github.com/urfave/cli/v2" ) @@ -265,13 +265,13 @@ func doIndicesCommand(cliCtx *cli.Context) error { } cfg := ethconfig.NewSnapCfg(true, true, false) - allSnapshots := snapshotsync.NewRoSnapshots(cfg, dirs.Snap, logger) + allSnapshots := freezeblocks.NewRoSnapshots(cfg, dirs.Snap, logger) if err := allSnapshots.ReopenFolder(); err != nil { return err } allSnapshots.LogStat() indexWorkers := estimate.IndexSnapshot.Workers() - if err := snapshotsync.BuildMissedIndices("Indexing", ctx, dirs, *chainID, indexWorkers, logger); err != nil { + if err := freezeblocks.BuildMissedIndices("Indexing", ctx, dirs, *chainID, indexWorkers, logger); err != nil { return err } agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, chainDB, logger) @@ -411,14 +411,14 @@ func doRetireCommand(cliCtx *cli.Context) error { defer db.Close() cfg := ethconfig.NewSnapCfg(true, true, true) - snapshots := snapshotsync.NewRoSnapshots(cfg, dirs.Snap, logger) + snapshots := freezeblocks.NewRoSnapshots(cfg, dirs.Snap, logger) if err := snapshots.ReopenFolder(); err != nil { return err } - blockReader := snapshotsync.NewBlockReader(snapshots) + blockReader := freezeblocks.NewBlockReader(snapshots) blockWriter := blockio.NewBlockWriter(fromdb.HistV3(db)) - br := snapshotsync.NewBlockRetire(estimate.CompressSnapshot.Workers(), dirs.Tmp, blockReader, blockWriter, db, nil, nil, logger) + br := freezeblocks.NewBlockRetire(estimate.CompressSnapshot.Workers(), dirs, blockReader, blockWriter, db, nil, logger) agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db, logger) if err != nil { return err @@ -436,7 +436,7 @@ func doRetireCommand(cliCtx *cli.Context) error { forwardProgress, err = stages.GetStageProgress(tx, stages.Senders) return err }) - from2, to2, ok := snapshotsync.CanRetire(forwardProgress, br.Snapshots()) + from2, to2, ok := freezeblocks.CanRetire(forwardProgress, blockReader.FrozenBlocks()) if ok { from, to, every = from2, to2, to2-from2 } @@ -444,11 +444,11 @@ func doRetireCommand(cliCtx *cli.Context) error { logger.Info("Params", "from", from, "to", to, "every", every) for i := from; i < to; i += every { - if err := br.RetireBlocks(ctx, i, i+every, log.LvlInfo); err != nil { + if err := br.RetireBlocks(ctx, i, i+every, log.LvlInfo, nil); err != nil { panic(err) } if err := db.UpdateNosync(ctx, func(tx kv.RwTx) error { - if err := rawdb.WriteSnapshots(tx, br.Snapshots().Files(), agg.Files()); err != nil { + if err := rawdb.WriteSnapshots(tx, blockReader.FrozenFiles(), agg.Files()); err != nil { return err } for j := 0; j < 10_000; j++ { // prune happens by small steps, so need many runs diff --git a/turbo/logging/flags.go b/turbo/logging/flags.go index bf070e2fe1e..b7e9e56727a 100644 --- a/turbo/logging/flags.go +++ b/turbo/logging/flags.go @@ -38,6 +38,11 @@ var ( Usage: "Path to store user and error logs to disk", } + LogDirPrefixFlag = cli.StringFlag{ + Name: "log.dir.prefix", + Usage: "The file name prefix for logs stored to disk", + } + LogDirVerbosityFlag = cli.StringFlag{ Name: "log.dir.verbosity", Usage: "Set the log verbosity for logs stored to disk", @@ -52,5 +57,6 @@ var Flags = []cli.Flag{ &LogVerbosityFlag, &LogConsoleVerbosityFlag, &LogDirPathFlag, + &LogDirPrefixFlag, &LogDirVerbosityFlag, } diff --git a/turbo/logging/logging.go b/turbo/logging/logging.go index 588f6c86284..fb9e467e7e8 100644 --- a/turbo/logging/logging.go +++ b/turbo/logging/logging.go @@ -52,6 +52,11 @@ func SetupLoggerCtx(filePrefix string, ctx *cli.Context, rootHandler bool) log.L } else { logger = log.New() } + + if logDirPrefix := ctx.String(LogDirPrefixFlag.Name); len(logDirPrefix) > 0 { + filePrefix = logDirPrefix + } + initSeparatedLogging(logger, filePrefix, dirPath, consoleLevel, dirLevel, consoleJson, dirJson) return logger } @@ -100,6 +105,11 @@ func SetupLoggerCmd(filePrefix string, cmd *cobra.Command) log.Logger { dirPath = filepath.Join(datadir, "logs") } } + + if logDirPrefix := cmd.Flags().Lookup(LogDirPrefixFlag.Name).Value.String(); len(logDirPrefix) > 0 { + filePrefix = logDirPrefix + } + initSeparatedLogging(log.Root(), filePrefix, dirPath, consoleLevel, dirLevel, consoleJson, dirJson) return log.Root() } @@ -110,6 +120,7 @@ func SetupLogger(filePrefix string) log.Logger { var logConsoleVerbosity = flag.String(LogConsoleVerbosityFlag.Name, "", LogConsoleVerbosityFlag.Usage) var logDirVerbosity = flag.String(LogDirVerbosityFlag.Name, "", LogDirVerbosityFlag.Usage) var logDirPath = flag.String(LogDirPathFlag.Name, "", LogDirPathFlag.Usage) + var logDirPrefix = flag.String(LogDirPrefixFlag.Name, "", LogDirPrefixFlag.Usage) var logVerbosity = flag.String(LogVerbosityFlag.Name, "", LogVerbosityFlag.Usage) var logConsoleJson = flag.Bool(LogConsoleJsonFlag.Name, false, LogConsoleJsonFlag.Usage) var logJson = flag.Bool(LogJsonFlag.Name, false, LogJsonFlag.Usage) @@ -133,6 +144,10 @@ func SetupLogger(filePrefix string) log.Logger { dirLevel = log.LvlInfo } + if logDirPrefix != nil && len(*logDirPrefix) > 0 { + filePrefix = *logDirPrefix + } + initSeparatedLogging(log.Root(), filePrefix, *logDirPath, consoleLevel, dirLevel, consoleJson, *dirJson) return log.Root() } diff --git a/turbo/node/node.go b/turbo/node/node.go index b4d9bc51580..cb3eddb108e 100644 --- a/turbo/node/node.go +++ b/turbo/node/node.go @@ -24,7 +24,7 @@ type ErigonNode struct { // Serve runs the node and blocks the execution. It returns when the node is existed. func (eri *ErigonNode) Serve() error { - defer eri.stack.Close() + defer eri.Close() eri.run() @@ -33,6 +33,10 @@ func (eri *ErigonNode) Serve() error { return nil } +func (eri *ErigonNode) Close() { + eri.stack.Close() +} + func (eri *ErigonNode) run() { node.StartNode(eri.stack) // we don't have accounts locally and we don't do mining diff --git a/turbo/rpchelper/helper.go b/turbo/rpchelper/helper.go index 418a5dccffd..46122d762e0 100644 --- a/turbo/rpchelper/helper.go +++ b/turbo/rpchelper/helper.go @@ -11,6 +11,7 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/systemcontracts" + "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/rpc" ) @@ -129,3 +130,17 @@ func CreateHistoryStateReader(tx kv.Tx, blockNumber uint64, txnIndex int, histor r.SetTxNum(uint64(int(minTxNum) + txnIndex + 1)) return r, nil } + +func NewLatestStateReader(tx kv.Getter) state.StateReader { + if ethconfig.EnableHistoryV4InTest { + panic("implement me") + //b.pendingReader = state.NewReaderV4(b.pendingReaderTx.(kv.TemporalTx)) + } + return state.NewPlainStateReader(tx) +} +func NewLatestStateWriter(tx kv.RwTx, blockNum uint64) state.StateWriter { + if ethconfig.EnableHistoryV4InTest { + panic("implement me") + } + return state.NewPlainStateWriter(tx, tx, blockNum) +} diff --git a/turbo/rpchelper/subscription.go b/turbo/rpchelper/subscription.go index 7207872570d..e7eb12cbd5f 100644 --- a/turbo/rpchelper/subscription.go +++ b/turbo/rpchelper/subscription.go @@ -1,8 +1,8 @@ package rpchelper import ( - "context" "sync" + "sync/atomic" ) // a simple interface for subscriptions for rpc helper @@ -12,12 +12,8 @@ type Sub[T any] interface { } type chan_sub[T any] struct { - ch chan T - - closed chan struct{} - - ctx context.Context - cn context.CancelFunc + ch chan T + closed atomic.Bool } // buffered channel @@ -28,38 +24,23 @@ func newChanSub[T any](size int) *chan_sub[T] { } o := &chan_sub[T]{} o.ch = make(chan T, size) - o.closed = make(chan struct{}) - o.ctx, o.cn = context.WithCancel(context.Background()) return o } func (s *chan_sub[T]) Send(x T) { + if s.closed.Load() { + return + } + select { - // if the output buffer is empty, send case s.ch <- x: - // if sub is canceled, dispose message - case <-s.ctx.Done(): - // the sub is overloaded, dispose message - default: + default: // the sub is overloaded, dispose message } } func (s *chan_sub[T]) Close() { - select { - case <-s.ctx.Done(): + if swapped := s.closed.CompareAndSwap(false, true); !swapped { return - default: - } - // its possible for multiple goroutines to get to this point, if Close is called twice at the same time - // close the context - allows any sends to exit - s.cn() - select { - case s.closed <- struct{}{}: - // but it is not possible for multiple goroutines to get to this point - // drain the channel - for range s.ch { - } - close(s.ch) - default: } + close(s.ch) } func NewSyncMap[K comparable, T any]() *SyncMap[K, T] { @@ -117,8 +98,7 @@ func (m *SyncMap[K, T]) Range(fn func(k K, v T) error) error { return nil } -func (m *SyncMap[K, T]) Delete(k K) (T, bool) { - var t T +func (m *SyncMap[K, T]) Delete(k K) (t T, deleted bool) { m.mu.Lock() defer m.mu.Unlock() val, ok := m.m[k] diff --git a/turbo/services/interfaces.go b/turbo/services/interfaces.go index a7e0a1b4736..681d6d05e49 100644 --- a/turbo/services/interfaces.go +++ b/turbo/services/interfaces.go @@ -3,11 +3,13 @@ package services import ( "context" + "github.com/ledgerwatch/erigon-lib/chain" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/rlp" + "github.com/ledgerwatch/log/v3" ) type All struct { @@ -26,6 +28,9 @@ type HeaderReader interface { HeaderByNumber(ctx context.Context, tx kv.Getter, blockNum uint64) (*types.Header, error) HeaderByHash(ctx context.Context, tx kv.Getter, hash common.Hash) (*types.Header, error) ReadAncestor(db kv.Getter, hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) + + //TODO: change it to `iter` + HeadersRange(ctx context.Context, walker func(header *types.Header) error) error } type CanonicalReader interface { @@ -63,12 +68,26 @@ type FullBlockReader interface { TxnReader CanonicalReader + FrozenBlocks() uint64 + FrozenFiles() (list []string) + FreezingCfg() ethconfig.BlocksFreezing + CanPruneTo(currentBlockInDB uint64) (canPruneBlocksTo uint64) + Snapshots() BlockSnapshots } type BlockSnapshots interface { - Cfg() ethconfig.Snapshot - BlocksAvailable() uint64 + ReopenFolder() error + SegmentsMax() uint64 + ScanDir() (map[string]struct{}, []*Range, error) +} + +// BlockRetire - freezing blocks: moving old data from DB to snapshot files +type BlockRetire interface { + PruneAncientBlocks(tx kv.RwTx, limit int) error + RetireBlocksInBackground(ctx context.Context, maxBlockNumInDB uint64, lvl log.Lvl, seedNewSnapshots func(downloadRequest []DownloadRequest) error) + HasNewFrozenFiles() bool + BuildMissedIndicesIfNeed(ctx context.Context, logPrefix string, notifier DBEventNotifier, cc *chain.Config) error } /* @@ -86,3 +105,21 @@ type BlockWriter interface { WriteBody(tx kv.RwTx, hash libcommon.Hash, number uint64, body *types.Body) error } */ + +type DBEventNotifier interface { + OnNewSnapshot() +} + +type DownloadRequest struct { + Ranges *Range + Path string + TorrentHash string +} + +func NewDownloadRequest(ranges *Range, path string, torrentHash string) DownloadRequest { + return DownloadRequest{Ranges: ranges, Path: path, TorrentHash: torrentHash} +} + +type Range struct { + From, To uint64 +} diff --git a/turbo/snapshotsync/block_reader.go b/turbo/snapshotsync/freezeblocks/block_reader.go similarity index 94% rename from turbo/snapshotsync/block_reader.go rename to turbo/snapshotsync/freezeblocks/block_reader.go index ec4fab4e6bc..1a353f984d0 100644 --- a/turbo/snapshotsync/block_reader.go +++ b/turbo/snapshotsync/freezeblocks/block_reader.go @@ -1,9 +1,8 @@ -package snapshotsync +package freezeblocks import ( "bytes" "context" - "encoding/binary" "fmt" "github.com/ledgerwatch/erigon-lib/common" @@ -12,6 +11,7 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces/remote" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/recsplit" + "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/types" @@ -23,6 +23,9 @@ type RemoteBlockReader struct { client remote.ETHBACKENDClient } +func (r *RemoteBlockReader) CanPruneTo(uint64) uint64 { + panic("not implemented") +} func (r *RemoteBlockReader) CurrentBlock(db kv.Tx) (*types.Block, error) { headHash := rawdb.ReadHeadBlockHash(db) headNumber := rawdb.ReadHeaderNumber(db, headHash) @@ -38,6 +41,9 @@ func (r *RemoteBlockReader) RawTransactions(ctx context.Context, tx kv.Getter, f func (r *RemoteBlockReader) ReadAncestor(db kv.Getter, hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { panic("not implemented") } +func (r *RemoteBlockReader) HeadersRange(ctx context.Context, walker func(header *types.Header) error) error { + panic("not implemented") +} func (r *RemoteBlockReader) BadHeaderNumber(ctx context.Context, tx kv.Getter, hash common.Hash) (blockHeight *uint64, err error) { return rawdb.ReadBadHeaderNumber(tx, hash) @@ -74,7 +80,10 @@ func (r *RemoteBlockReader) HeaderByNumber(ctx context.Context, tx kv.Getter, bl return block.Header(), nil } -func (r *RemoteBlockReader) Snapshots() services.BlockSnapshots { panic("not implemented") } +func (r *RemoteBlockReader) Snapshots() services.BlockSnapshots { panic("not implemented") } +func (r *RemoteBlockReader) FrozenBlocks() uint64 { panic("not supported") } +func (r *RemoteBlockReader) FrozenFiles() (list []string) { panic("not supported") } +func (r *RemoteBlockReader) FreezingCfg() ethconfig.BlocksFreezing { panic("not supported") } func (r *RemoteBlockReader) HeaderByHash(ctx context.Context, tx kv.Getter, hash common.Hash) (*types.Header, error) { blockNum := rawdb.ReadHeaderNumber(tx, hash) @@ -206,7 +215,17 @@ func NewBlockReader(snapshots services.BlockSnapshots) *BlockReader { return &BlockReader{sn: snapshots.(*RoSnapshots), TransactionsV3: true} } -func (r *BlockReader) Snapshots() services.BlockSnapshots { return r.sn } +func (r *BlockReader) CanPruneTo(currentBlockInDB uint64) uint64 { + return CanDeleteTo(currentBlockInDB, r.sn.BlocksAvailable()) +} +func (r *BlockReader) Snapshots() services.BlockSnapshots { return r.sn } +func (r *BlockReader) FrozenBlocks() uint64 { return r.sn.BlocksAvailable() } +func (r *BlockReader) FrozenFiles() []string { return r.sn.Files() } +func (r *BlockReader) FreezingCfg() ethconfig.BlocksFreezing { return r.sn.Cfg() } + +func (r *BlockReader) HeadersRange(ctx context.Context, walker func(header *types.Header) error) error { + return ForEachHeader(ctx, r.sn, walker) +} func (r *BlockReader) HeaderByNumber(ctx context.Context, tx kv.Getter, blockHeight uint64) (h *types.Header, err error) { blockHash, err := rawdb.ReadCanonicalHash(tx, blockHeight) @@ -685,29 +704,14 @@ func (r *BlockReader) txnByHash(txnHash common.Hash, segments []*TxnSegment, buf // TxnByIdxInBlock - doesn't include system-transactions in the begin/end of block // return nil if 0 < i < body.TxAmount -func (r *BlockReader) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, i int) (txn types.Transaction, err error) { +func (r *BlockReader) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, txIdxInBlock int) (txn types.Transaction, err error) { blocksAvailable := r.sn.BlocksAvailable() if blocksAvailable == 0 || blockNum > blocksAvailable { canonicalHash, err := rawdb.ReadCanonicalHash(tx, blockNum) if err != nil { return nil, err } - var k [8 + 32]byte - binary.BigEndian.PutUint64(k[:], blockNum) - copy(k[8:], canonicalHash[:]) - b, err := rawdb.ReadBodyForStorageByKey(tx, k[:]) - if err != nil { - return nil, err - } - if b == nil { - return nil, nil - } - - txn, err = rawdb.CanonicalTxnByID(tx, b.BaseTxId+1+uint64(i)) - if err != nil { - return nil, err - } - return txn, nil + return rawdb.TxnByIdxInBlock(tx, canonicalHash, blockNum, txIdxInBlock) } view := r.sn.View() @@ -727,7 +731,7 @@ func (r *BlockReader) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, blockNu } // if block has no transactions, or requested txNum out of non-system transactions length - if b.TxAmount == 2 || i == -1 || i >= int(b.TxAmount-2) { + if b.TxAmount == 2 || txIdxInBlock == -1 || txIdxInBlock >= int(b.TxAmount-2) { return nil, nil } @@ -736,7 +740,7 @@ func (r *BlockReader) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, blockNu return } // +1 because block has system-txn in the beginning of block - return r.txnByID(b.BaseTxId+1+uint64(i), txnSeg, nil) + return r.txnByID(b.BaseTxId+1+uint64(txIdxInBlock), txnSeg, nil) } // TxnLookup - find blockNumber and txnID by txnHash diff --git a/turbo/snapshotsync/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go similarity index 92% rename from turbo/snapshotsync/block_snapshots.go rename to turbo/snapshotsync/freezeblocks/block_snapshots.go index b70346b15d7..ba5b48e5c9b 100644 --- a/turbo/snapshotsync/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -1,4 +1,4 @@ -package snapshotsync +package freezeblocks import ( "bytes" @@ -27,10 +27,8 @@ import ( dir2 "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/compress" - "github.com/ledgerwatch/erigon-lib/downloader/downloadergrpc" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/etl" - proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/recsplit" types2 "github.com/ledgerwatch/erigon-lib/types" @@ -41,6 +39,7 @@ import ( "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/crypto/cryptopool" "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" @@ -51,12 +50,6 @@ import ( "golang.org/x/sync/errgroup" ) -type DownloadRequest struct { - ranges *Range - path string - torrentHash string -} - type HeaderSegment struct { seg *compress.Decompressor // value: first_byte_of_header_hash + header_rlp idxHeaderHash *recsplit.Index // header_hash -> headers_segment_offset @@ -337,7 +330,7 @@ type RoSnapshots struct { dir string segmentsMax atomic.Uint64 // all types of .seg files are available - up to this number idxMax atomic.Uint64 // all types of .idx files are available - up to this number - cfg ethconfig.Snapshot + cfg ethconfig.BlocksFreezing logger log.Logger } @@ -346,17 +339,17 @@ type RoSnapshots struct { // - all snapshots of given blocks range must exist - to make this blocks range available // - gaps are not allowed // - segment have [from:to) semantic -func NewRoSnapshots(cfg ethconfig.Snapshot, snapDir string, logger log.Logger) *RoSnapshots { +func NewRoSnapshots(cfg ethconfig.BlocksFreezing, snapDir string, logger log.Logger) *RoSnapshots { return &RoSnapshots{dir: snapDir, cfg: cfg, Headers: &headerSegments{}, Bodies: &bodySegments{}, Txs: &txnSegments{}, logger: logger} } -func (s *RoSnapshots) Cfg() ethconfig.Snapshot { return s.cfg } -func (s *RoSnapshots) Dir() string { return s.dir } -func (s *RoSnapshots) SegmentsReady() bool { return s.segmentsReady.Load() } -func (s *RoSnapshots) IndicesReady() bool { return s.indicesReady.Load() } -func (s *RoSnapshots) IndicesMax() uint64 { return s.idxMax.Load() } -func (s *RoSnapshots) SegmentsMax() uint64 { return s.segmentsMax.Load() } -func (s *RoSnapshots) BlocksAvailable() uint64 { return cmp.Min(s.segmentsMax.Load(), s.idxMax.Load()) } +func (s *RoSnapshots) Cfg() ethconfig.BlocksFreezing { return s.cfg } +func (s *RoSnapshots) Dir() string { return s.dir } +func (s *RoSnapshots) SegmentsReady() bool { return s.segmentsReady.Load() } +func (s *RoSnapshots) IndicesReady() bool { return s.indicesReady.Load() } +func (s *RoSnapshots) IndicesMax() uint64 { return s.idxMax.Load() } +func (s *RoSnapshots) SegmentsMax() uint64 { return s.segmentsMax.Load() } +func (s *RoSnapshots) BlocksAvailable() uint64 { return cmp.Min(s.segmentsMax.Load(), s.idxMax.Load()) } func (s *RoSnapshots) LogStat() { var m runtime.MemStats dbg.ReadMemStats(&m) @@ -366,6 +359,23 @@ func (s *RoSnapshots) LogStat() { "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) } +func (s *RoSnapshots) ScanDir() (map[string]struct{}, []*services.Range, error) { + existingFiles, missingSnapshots, err := Segments(s.dir) + if err != nil { + return nil, nil, err + } + existingFilesMap := map[string]struct{}{} + for _, existingFile := range existingFiles { + _, fname := filepath.Split(existingFile.Path) + existingFilesMap[fname] = struct{}{} + } + + res := make([]*services.Range, 0, len(missingSnapshots)) + for _, sn := range missingSnapshots { + res = append(res, &services.Range{From: sn.from, To: sn.to}) + } + return existingFilesMap, res, nil +} func (s *RoSnapshots) EnsureExpectedBlocksAreAvailable(cfg *snapcfg.Cfg) error { if s.BlocksAvailable() < cfg.ExpectBlocks { return fmt.Errorf("app must wait until all expected snapshots are available. Expected: %d, Available: %d", cfg.ExpectBlocks, s.BlocksAvailable()) @@ -1005,26 +1015,26 @@ type BlockRetire struct { tmpDir string db kv.RoDB - downloader proto_downloader.DownloaderClient - notifier DBEventNotifier + notifier services.DBEventNotifier logger log.Logger blockReader services.FullBlockReader blockWriter *blockio.BlockWriter + dirs datadir.Dirs } -func NewBlockRetire(workers int, tmpDir string, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter, db kv.RoDB, downloader proto_downloader.DownloaderClient, notifier DBEventNotifier, logger log.Logger) *BlockRetire { - return &BlockRetire{workers: workers, tmpDir: tmpDir, blockReader: blockReader, blockWriter: blockWriter, db: db, downloader: downloader, notifier: notifier, logger: logger} +func NewBlockRetire(workers int, dirs datadir.Dirs, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter, db kv.RoDB, notifier services.DBEventNotifier, logger log.Logger) *BlockRetire { + return &BlockRetire{workers: workers, tmpDir: dirs.Tmp, dirs: dirs, blockReader: blockReader, blockWriter: blockWriter, db: db, notifier: notifier, logger: logger} } -func (br *BlockRetire) Snapshots() *RoSnapshots { return br.blockReader.Snapshots().(*RoSnapshots) } -func (br *BlockRetire) NeedSaveFilesListInDB() bool { +func (br *BlockRetire) snapshots() *RoSnapshots { return br.blockReader.Snapshots().(*RoSnapshots) } +func (br *BlockRetire) HasNewFrozenFiles() bool { return br.needSaveFilesListInDB.CompareAndSwap(true, false) } -func CanRetire(curBlockNum uint64, snapshots *RoSnapshots) (blockFrom, blockTo uint64, can bool) { +func CanRetire(curBlockNum uint64, blocksInSnapshots uint64) (blockFrom, blockTo uint64, can bool) { if curBlockNum <= params.FullImmutabilityThreshold { return } - blockFrom = snapshots.BlocksAvailable() + 1 + blockFrom = blocksInSnapshots + 1 return canRetire(blockFrom, curBlockNum-params.FullImmutabilityThreshold) } func canRetire(from, to uint64) (blockFrom, blockTo uint64, can bool) { @@ -1057,20 +1067,20 @@ func canRetire(from, to uint64) (blockFrom, blockTo uint64, can bool) { } return blockFrom, blockTo, blockTo-blockFrom >= 1_000 } -func CanDeleteTo(curBlockNum uint64, snapshots services.BlockSnapshots) (blockTo uint64) { +func CanDeleteTo(curBlockNum uint64, blocksInSnapshots uint64) (blockTo uint64) { if curBlockNum+999 < params.FullImmutabilityThreshold { // To prevent overflow of uint64 below - return snapshots.BlocksAvailable() + 1 + return blocksInSnapshots + 1 } hardLimit := (curBlockNum/1_000)*1_000 - params.FullImmutabilityThreshold - return cmp.Min(hardLimit, snapshots.BlocksAvailable()+1) + return cmp.Min(hardLimit, blocksInSnapshots+1) } -func (br *BlockRetire) RetireBlocks(ctx context.Context, blockFrom, blockTo uint64, lvl log.Lvl) error { +func (br *BlockRetire) RetireBlocks(ctx context.Context, blockFrom, blockTo uint64, lvl log.Lvl, seedNewSnapshots func(downloadRequest []services.DownloadRequest) error) error { chainConfig := fromdb.ChainConfig(br.db) chainID, _ := uint256.FromBig(chainConfig.ChainID) - downloader, notifier, logger, blockReader, tmpDir, db, workers := br.downloader, br.notifier, br.logger, br.blockReader, br.tmpDir, br.db, br.workers + notifier, logger, blockReader, tmpDir, db, workers := br.notifier, br.logger, br.blockReader, br.tmpDir, br.db, br.workers logger.Log(lvl, "[snapshots] Retire Blocks", "range", fmt.Sprintf("%dk-%dk", blockFrom/1000, blockTo/1000)) - snapshots := blockReader.Snapshots().(*RoSnapshots) + snapshots := br.snapshots() firstTxNum := blockReader.(*BlockReader).FirstTxNumNotInSnapshots() // in future we will do it in background @@ -1101,36 +1111,35 @@ func (br *BlockRetire) RetireBlocks(ctx context.Context, blockFrom, blockTo uint notifier.OnNewSnapshot() } - if downloader != nil && !reflect.ValueOf(downloader).IsNil() { - downloadRequest := make([]DownloadRequest, 0, len(rangesToMerge)) - for i := range rangesToMerge { - downloadRequest = append(downloadRequest, NewDownloadRequest(&rangesToMerge[i], "", "")) - } + downloadRequest := make([]services.DownloadRequest, 0, len(rangesToMerge)) + for i := range rangesToMerge { + r := &services.Range{From: rangesToMerge[i].from, To: rangesToMerge[i].to} + downloadRequest = append(downloadRequest, services.NewDownloadRequest(r, "", "")) + } - if err := RequestSnapshotsDownload(ctx, downloadRequest, downloader); err != nil { + if seedNewSnapshots != nil { + if err := seedNewSnapshots(downloadRequest); err != nil { return err } } return nil } - func (br *BlockRetire) PruneAncientBlocks(tx kv.RwTx, limit int) error { - blockSnapshots := br.blockReader.Snapshots().(*RoSnapshots) - if blockSnapshots.cfg.KeepBlocks { + if br.blockReader.FreezingCfg().KeepBlocks { return nil } currentProgress, err := stages.GetStageProgress(tx, stages.Senders) if err != nil { return err } - canDeleteTo := CanDeleteTo(currentProgress, blockSnapshots) + canDeleteTo := CanDeleteTo(currentProgress, br.blockReader.FrozenBlocks()) if err := br.blockWriter.PruneBlocks(context.Background(), tx, canDeleteTo, limit); err != nil { return nil } return nil } -func (br *BlockRetire) RetireBlocksInBackground(ctx context.Context, forwardProgress uint64, lvl log.Lvl) { +func (br *BlockRetire) RetireBlocksInBackground(ctx context.Context, forwardProgress uint64, lvl log.Lvl, seedNewSnapshots func(downloadRequest []services.DownloadRequest) error) { ok := br.working.CompareAndSwap(false, true) if !ok { // go-routine is still working @@ -1139,20 +1148,51 @@ func (br *BlockRetire) RetireBlocksInBackground(ctx context.Context, forwardProg go func() { defer br.working.Store(false) - blockFrom, blockTo, ok := CanRetire(forwardProgress, br.Snapshots()) + blockFrom, blockTo, ok := CanRetire(forwardProgress, br.blockReader.FrozenBlocks()) if !ok { return } - err := br.RetireBlocks(ctx, blockFrom, blockTo, lvl) + err := br.RetireBlocks(ctx, blockFrom, blockTo, lvl, seedNewSnapshots) if err != nil { br.logger.Warn("[snapshots] retire blocks", "err", err, "fromBlock", blockFrom, "toBlock", blockTo) } }() } +func (br *BlockRetire) BuildMissedIndicesIfNeed(ctx context.Context, logPrefix string, notifier services.DBEventNotifier, cc *chain.Config) error { + snapshots := br.snapshots() + snapshots.LogStat() -type DBEventNotifier interface { - OnNewSnapshot() + // Create .idx files + if snapshots.IndicesMax() >= snapshots.SegmentsMax() { + return nil + } + + if !snapshots.Cfg().Produce && snapshots.IndicesMax() == 0 { + return fmt.Errorf("please remove --snap.stop, erigon can't work without creating basic indices") + } + if snapshots.Cfg().Produce { + if !snapshots.SegmentsReady() { + return fmt.Errorf("not all snapshot segments are available") + } + + // wait for Downloader service to download all expected snapshots + if snapshots.IndicesMax() < snapshots.SegmentsMax() { + chainID, _ := uint256.FromBig(cc.ChainID) + indexWorkers := estimate.IndexSnapshot.Workers() + if err := BuildMissedIndices(logPrefix, ctx, br.dirs, *chainID, indexWorkers, br.logger); err != nil { + return fmt.Errorf("BuildMissedIndices: %w", err) + } + } + + if err := snapshots.ReopenFolder(); err != nil { + return err + } + if notifier != nil { + notifier.OnNewSnapshot() + } + } + return nil } func DumpBlocks(ctx context.Context, blockFrom, blockTo, blocksPerFile uint64, tmpDir, snapDir string, firstTxNum uint64, chainDB kv.RoDB, workers int, lvl log.Lvl, logger log.Logger, blockReader services.FullBlockReader) error { @@ -1930,11 +1970,11 @@ type Merger struct { compressWorkers int tmpDir string chainID uint256.Int - notifier DBEventNotifier + notifier services.DBEventNotifier logger log.Logger } -func NewMerger(tmpDir string, compressWorkers int, lvl log.Lvl, chainID uint256.Int, notifier DBEventNotifier, logger log.Logger) *Merger { +func NewMerger(tmpDir string, compressWorkers int, lvl log.Lvl, chainID uint256.Int, notifier services.DBEventNotifier, logger log.Logger) *Merger { return &Merger{tmpDir: tmpDir, compressWorkers: compressWorkers, lvl: lvl, chainID: chainID, notifier: notifier, logger: logger} } @@ -1942,7 +1982,8 @@ type Range struct { from, to uint64 } -func (r Range) String() string { return fmt.Sprintf("%dk-%dk", r.from/1000, r.to/1000) } +func (r Range) From() uint64 { return r.from } +func (r Range) To() uint64 { return r.to } func (*Merger) FindMergeRanges(currentRanges []Range) (toMerge []Range) { for i := len(currentRanges) - 1; i > 0; i-- { @@ -2151,49 +2192,3 @@ func (m *Merger) removeOldFiles(toDel []string, snapDir string) { _ = os.Remove(f) } } - -func NewDownloadRequest(ranges *Range, path string, torrentHash string) DownloadRequest { - return DownloadRequest{ - ranges: ranges, - path: path, - torrentHash: torrentHash, - } -} - -// RequestSnapshotsDownload - builds the snapshots download request and downloads them -func RequestSnapshotsDownload(ctx context.Context, downloadRequest []DownloadRequest, downloader proto_downloader.DownloaderClient) error { - // start seed large .seg of large size - req := BuildProtoRequest(downloadRequest) - if _, err := downloader.Download(ctx, req); err != nil { - return err - } - return nil -} - -func BuildProtoRequest(downloadRequest []DownloadRequest) *proto_downloader.DownloadRequest { - req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, 0, len(snaptype.AllSnapshotTypes))} - for _, r := range downloadRequest { - if r.path != "" { - if r.torrentHash != "" { - req.Items = append(req.Items, &proto_downloader.DownloadItem{ - TorrentHash: downloadergrpc.String2Proto(r.torrentHash), - Path: r.path, - }) - } else { - req.Items = append(req.Items, &proto_downloader.DownloadItem{ - Path: r.path, - }) - } - } else { - if r.ranges.to-r.ranges.from != snaptype.Erigon2SegmentSize { - continue - } - for _, t := range snaptype.AllSnapshotTypes { - req.Items = append(req.Items, &proto_downloader.DownloadItem{ - Path: snaptype.SegmentFileName(r.ranges.from, r.ranges.to, t), - }) - } - } - } - return req -} diff --git a/turbo/snapshotsync/block_snapshots_test.go b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go similarity index 98% rename from turbo/snapshotsync/block_snapshots_test.go rename to turbo/snapshotsync/freezeblocks/block_snapshots_test.go index fdee9a10163..fcdde33ed35 100644 --- a/turbo/snapshotsync/block_snapshots_test.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go @@ -1,4 +1,4 @@ -package snapshotsync +package freezeblocks import ( "context" @@ -71,7 +71,7 @@ func TestMergeSnapshots(t *testing.T) { for i := uint64(500_000); i < 500_000+N*100_000; i += 100_000 { createFile(i, i+100_000) } - s := NewRoSnapshots(ethconfig.Snapshot{Enabled: true}, dir, logger) + s := NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) defer s.Close() require.NoError(s.ReopenFolder()) { @@ -129,7 +129,7 @@ func TestOpenAllSnapshot(t *testing.T) { dir, require := t.TempDir(), require.New(t) chainSnapshotCfg := snapcfg.KnownCfg(networkname.MainnetChainName, nil, nil) chainSnapshotCfg.ExpectBlocks = math.MaxUint64 - cfg := ethconfig.Snapshot{Enabled: true} + cfg := ethconfig.BlocksFreezing{Enabled: true} createFile := func(from, to uint64, name snaptype.Type) { createTestSegmentFile(t, from, to, name, dir, logger) } s := NewRoSnapshots(cfg, dir, logger) defer s.Close() diff --git a/turbo/snapshotsync/dump_test.go b/turbo/snapshotsync/freezeblocks/dump_test.go similarity index 89% rename from turbo/snapshotsync/dump_test.go rename to turbo/snapshotsync/freezeblocks/dump_test.go index 80b19988c2c..6da633aa6ea 100644 --- a/turbo/snapshotsync/dump_test.go +++ b/turbo/snapshotsync/freezeblocks/dump_test.go @@ -1,4 +1,4 @@ -package snapshotsync_test +package freezeblocks_test import ( "math/big" @@ -13,7 +13,7 @@ import ( "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/turbo/stages" "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/require" @@ -31,7 +31,7 @@ func TestDump(t *testing.T) { var systemTxs int var nonceList []uint64 - cnt, err := snapshotsync.DumpTxs(m.Ctx, m.DB, 0, 10, m.ChainConfig, 1, log.LvlInfo, log.New(), func(v []byte) error { + cnt, err := freezeblocks.DumpTxs(m.Ctx, m.DB, 0, 10, m.ChainConfig, 1, log.LvlInfo, log.New(), func(v []byte) error { if v == nil { systemTxs++ } else { @@ -56,7 +56,7 @@ func TestDump(t *testing.T) { var systemTxs int var nonceList []uint64 - cnt, err := snapshotsync.DumpTxs(m.Ctx, m.DB, 2, 5, m.ChainConfig, 1, log.LvlInfo, log.New(), func(v []byte) error { + cnt, err := freezeblocks.DumpTxs(m.Ctx, m.DB, 2, 5, m.ChainConfig, 1, log.LvlInfo, log.New(), func(v []byte) error { if v == nil { systemTxs++ } else { @@ -75,7 +75,7 @@ func TestDump(t *testing.T) { t.Run("headers", func(t *testing.T) { require := require.New(t) var nonceList []uint64 - err := snapshotsync.DumpHeaders(m.Ctx, m.DB, 0, 10, 1, log.LvlInfo, log.New(), func(v []byte) error { + err := freezeblocks.DumpHeaders(m.Ctx, m.DB, 0, 10, 1, log.LvlInfo, log.New(), func(v []byte) error { h := types.Header{} if err := rlp.DecodeBytes(v[1:], &h); err != nil { return err @@ -89,7 +89,7 @@ func TestDump(t *testing.T) { t.Run("headers_not_from_zero", func(t *testing.T) { require := require.New(t) var nonceList []uint64 - err := snapshotsync.DumpHeaders(m.Ctx, m.DB, 2, 5, 1, log.LvlInfo, log.New(), func(v []byte) error { + err := freezeblocks.DumpHeaders(m.Ctx, m.DB, 2, 5, 1, log.LvlInfo, log.New(), func(v []byte) error { h := types.Header{} if err := rlp.DecodeBytes(v[1:], &h); err != nil { return err @@ -106,7 +106,7 @@ func TestDump(t *testing.T) { txsAmount := uint64(0) var baseIdList []uint64 firstTxNum := uint64(0) - err := snapshotsync.DumpBodies(m.Ctx, m.DB, 0, 2, firstTxNum, 1, log.LvlInfo, log.New(), func(v []byte) error { + err := freezeblocks.DumpBodies(m.Ctx, m.DB, 0, 2, firstTxNum, 1, log.LvlInfo, log.New(), func(v []byte) error { i++ body := &types.BodyForStorage{} require.NoError(rlp.DecodeBytes(v, body)) @@ -122,7 +122,7 @@ func TestDump(t *testing.T) { firstTxNum += txsAmount i = 0 baseIdList = baseIdList[:0] - err = snapshotsync.DumpBodies(m.Ctx, m.DB, 2, 10, firstTxNum, 1, log.LvlInfo, log.New(), func(v []byte) error { + err = freezeblocks.DumpBodies(m.Ctx, m.DB, 2, 10, firstTxNum, 1, log.LvlInfo, log.New(), func(v []byte) error { i++ body := &types.BodyForStorage{} require.NoError(rlp.DecodeBytes(v, body)) @@ -140,7 +140,7 @@ func TestDump(t *testing.T) { i := 0 var baseIdList []uint64 firstTxNum := uint64(1000) - err := snapshotsync.DumpBodies(m.Ctx, m.DB, 2, 5, firstTxNum, 1, log.LvlInfo, log.New(), func(v []byte) error { + err := freezeblocks.DumpBodies(m.Ctx, m.DB, 2, 5, firstTxNum, 1, log.LvlInfo, log.New(), func(v []byte) error { i++ body := &types.BodyForStorage{} require.NoError(rlp.DecodeBytes(v, body)) @@ -178,7 +178,7 @@ func createDumpTestKV(t *testing.T, chainSize int) *stages.MockSentry { t.Error(err) } // Construct testing chain - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Error(err) } diff --git a/turbo/snapshotsync/snap/flags.go b/turbo/snapshotsync/snap/flags.go index e65a279ce9e..741e3223549 100644 --- a/turbo/snapshotsync/snap/flags.go +++ b/turbo/snapshotsync/snap/flags.go @@ -13,7 +13,7 @@ func Enabled(tx kv.Getter) (bool, error) { return kv.GetBool(tx, kv.DatabaseInfo, blockSnapshotEnabledKey) } -func EnsureNotChanged(tx kv.GetPut, cfg ethconfig.Snapshot) (bool, bool, error) { +func EnsureNotChanged(tx kv.GetPut, cfg ethconfig.BlocksFreezing) (bool, bool, error) { ok, v, err := kv.EnsureNotChangedBool(tx, kv.DatabaseInfo, blockSnapshotEnabledKey, cfg.Enabled) if err != nil { return false, false, err @@ -25,7 +25,7 @@ func EnsureNotChanged(tx kv.GetPut, cfg ethconfig.Snapshot) (bool, bool, error) } // ForceSetFlags - if you know what you are doing -func ForceSetFlags(tx kv.GetPut, cfg ethconfig.Snapshot) error { +func ForceSetFlags(tx kv.GetPut, cfg ethconfig.BlocksFreezing) error { if cfg.Enabled { if err := tx.Put(kv.DatabaseInfo, blockSnapshotEnabledKey, []byte{1}); err != nil { return err diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go new file mode 100644 index 00000000000..df3c252fbdd --- /dev/null +++ b/turbo/snapshotsync/snapshotsync.go @@ -0,0 +1,233 @@ +package snapshotsync + +import ( + "context" + "encoding/binary" + "fmt" + "runtime" + "time" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/erigon-lib/downloader/downloadergrpc" + "github.com/ledgerwatch/erigon-lib/downloader/snaptype" + proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" + "github.com/ledgerwatch/erigon-lib/state" + + "github.com/ledgerwatch/erigon/core/rawdb" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapcfg" + "github.com/ledgerwatch/log/v3" +) + +func BuildProtoRequest(downloadRequest []services.DownloadRequest) *proto_downloader.DownloadRequest { + req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, 0, len(snaptype.AllSnapshotTypes))} + for _, r := range downloadRequest { + if r.Path != "" { + if r.TorrentHash != "" { + req.Items = append(req.Items, &proto_downloader.DownloadItem{ + TorrentHash: downloadergrpc.String2Proto(r.TorrentHash), + Path: r.Path, + }) + } else { + req.Items = append(req.Items, &proto_downloader.DownloadItem{ + Path: r.Path, + }) + } + } else { + if r.Ranges.To-r.Ranges.From != snaptype.Erigon2SegmentSize { + continue + } + for _, t := range snaptype.AllSnapshotTypes { + req.Items = append(req.Items, &proto_downloader.DownloadItem{ + Path: snaptype.SegmentFileName(r.Ranges.From, r.Ranges.To, t), + }) + } + } + } + return req +} + +// RequestSnapshotsDownload - builds the snapshots download request and downloads them +func RequestSnapshotsDownload(ctx context.Context, downloadRequest []services.DownloadRequest, downloader proto_downloader.DownloaderClient) error { + // start seed large .seg of large size + req := BuildProtoRequest(downloadRequest) + if _, err := downloader.Download(ctx, req); err != nil { + return err + } + return nil +} + +// WaitForDownloader - wait for Downloader service to download all expected snapshots +// for MVP we sync with Downloader only once, in future will send new snapshots also +func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, agg *state.AggregatorV3, tx kv.RwTx, blockReader services.FullBlockReader, notifier services.DBEventNotifier, cc *chain.Config, snapshotDownloader proto_downloader.DownloaderClient) error { + snapshots := blockReader.Snapshots() + if blockReader.FreezingCfg().NoDownloader { + if err := snapshots.ReopenFolder(); err != nil { + return err + } + if notifier != nil { // can notify right here, even that write txn is not commit + notifier.OnNewSnapshot() + } + return nil + } + + // Original intent of snInDB was to contain the file names of the snapshot files for the very first run of the Erigon instance + // Then, we would insist to only download such files, and no others (whitelist) + // However, at some point later, the code was incorrectly changed to update this record in each iteration of the stage loop (function WriteSnapshots) + // And so this list cannot be relied upon as the whitelist, because it also includes all the files created by the node itself + // Not sure what to do it is so far, but the temporary solution is to instead use it as a blacklist (existingFilesMap) + snInDB, snHistInDB, err := rawdb.ReadSnapshots(tx) + if err != nil { + return err + } + dbEmpty := len(snInDB) == 0 + var existingFilesMap map[string]struct{} + var missingSnapshots []*services.Range + if !dbEmpty { + existingFilesMap, missingSnapshots, err = snapshots.ScanDir() + if err != nil { + return err + } + } + if len(missingSnapshots) > 0 { + log.Warn(fmt.Sprintf("[%s] downloading missing snapshots", logPrefix)) + } + + // send all hashes to the Downloader service + preverifiedBlockSnapshots := snapcfg.KnownCfg(cc.ChainName, snInDB, snHistInDB).Preverified + downloadRequest := make([]services.DownloadRequest, 0, len(preverifiedBlockSnapshots)+len(missingSnapshots)) + // build all download requests + // builds preverified snapshots request + for _, p := range preverifiedBlockSnapshots { + if _, exists := existingFilesMap[p.Name]; !exists { // Not to download existing files "behind the scenes" + downloadRequest = append(downloadRequest, services.NewDownloadRequest(nil, p.Name, p.Hash)) + } + } + if histV3 { + preverifiedHistorySnapshots := snapcfg.KnownCfg(cc.ChainName, snInDB, snHistInDB).PreverifiedHistory + for _, p := range preverifiedHistorySnapshots { + downloadRequest = append(downloadRequest, services.NewDownloadRequest(nil, p.Name, p.Hash)) + } + } + + // builds missing snapshots request + for _, r := range missingSnapshots { + downloadRequest = append(downloadRequest, services.NewDownloadRequest(r, "", "")) + } + + log.Info(fmt.Sprintf("[%s] Fetching torrent files metadata", logPrefix)) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err := RequestSnapshotsDownload(ctx, downloadRequest, snapshotDownloader); err != nil { + log.Error(fmt.Sprintf("[%s] call downloader", logPrefix), "err", err) + time.Sleep(10 * time.Second) + continue + } + break + } + downloadStartTime := time.Now() + const logInterval = 20 * time.Second + logEvery := time.NewTicker(logInterval) + defer logEvery.Stop() + var m runtime.MemStats + + // Check once without delay, for faster erigon re-start + stats, err := snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}) + if err == nil && stats.Completed { + goto Finish + } + + // Print download progress until all segments are available +Loop: + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-logEvery.C: + if stats, err := snapshotDownloader.Stats(ctx, &proto_downloader.StatsRequest{}); err != nil { + log.Warn("Error while waiting for snapshots progress", "err", err) + } else if stats.Completed { + if !blockReader.FreezingCfg().Verify { // will verify after loop + if _, err := snapshotDownloader.Verify(ctx, &proto_downloader.VerifyRequest{}); err != nil { + return err + } + } + log.Info(fmt.Sprintf("[%s] download finished", logPrefix), "time", time.Since(downloadStartTime).String()) + break Loop + } else { + if stats.MetadataReady < stats.FilesTotal { + log.Info(fmt.Sprintf("[%s] Waiting for torrents metadata: %d/%d", logPrefix, stats.MetadataReady, stats.FilesTotal)) + continue + } + dbg.ReadMemStats(&m) + downloadTimeLeft := calculateTime(stats.BytesTotal-stats.BytesCompleted, stats.DownloadRate) + log.Info(fmt.Sprintf("[%s] download", logPrefix), + "progress", fmt.Sprintf("%.2f%% %s/%s", stats.Progress, common.ByteCount(stats.BytesCompleted), common.ByteCount(stats.BytesTotal)), + "download-time-left", downloadTimeLeft, + "total-download-time", time.Since(downloadStartTime).Round(time.Second).String(), + "download", common.ByteCount(stats.DownloadRate)+"/s", + "upload", common.ByteCount(stats.UploadRate)+"/s", + ) + log.Info(fmt.Sprintf("[%s] download", logPrefix), + "peers", stats.PeersUnique, + "connections", stats.ConnectionsTotal, + "files", stats.FilesTotal, + "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys), + ) + } + } + } + +Finish: + if blockReader.FreezingCfg().Verify { + if _, err := snapshotDownloader.Verify(ctx, &proto_downloader.VerifyRequest{}); err != nil { + return err + } + } + + if err := snapshots.ReopenFolder(); err != nil { + return err + } + if err := agg.OpenFolder(); err != nil { + return err + } + + if err := rawdb.WriteSnapshots(tx, blockReader.FrozenFiles(), agg.Files()); err != nil { + return err + } + if notifier != nil { // can notify right here, even that write txn is not commit + notifier.OnNewSnapshot() + } + + firstNonGenesis, err := rawdbv3.SecondKey(tx, kv.Headers) + if err != nil { + return err + } + if firstNonGenesis != nil { + firstNonGenesisBlockNumber := binary.BigEndian.Uint64(firstNonGenesis) + if snapshots.SegmentsMax()+1 < firstNonGenesisBlockNumber { + log.Warn(fmt.Sprintf("[%s] Some blocks are not in snapshots and not in db", logPrefix), "max_in_snapshots", snapshots.SegmentsMax(), "min_in_db", firstNonGenesisBlockNumber) + } + } + return nil +} + +func calculateTime(amountLeft, rate uint64) string { + if rate == 0 { + return "999hrs:99m" + } + timeLeftInSeconds := amountLeft / rate + + hours := timeLeftInSeconds / 3600 + minutes := (timeLeftInSeconds / 60) % 60 + + return fmt.Sprintf("%dhrs:%dm", hours, minutes) +} diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index 9d8d6f95978..ac9dccd9508 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -26,6 +26,10 @@ import ( "testing" "github.com/holiman/uint256" + "github.com/ledgerwatch/log/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ledgerwatch/erigon-lib/chain" chain2 "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -33,10 +37,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" types2 "github.com/ledgerwatch/erigon-lib/types" - "github.com/ledgerwatch/erigon/turbo/services" - "github.com/ledgerwatch/log/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/common/u256" @@ -49,6 +49,7 @@ import ( "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/ethdb/prune" "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/stages" ) @@ -79,7 +80,7 @@ func newCanonical(t *testing.T, n int) *stages.MockSentry { // Full block-chain requested chain := makeBlockChain(m.Genesis, n, m, canonicalSeed) - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatal(err) } return m @@ -90,16 +91,15 @@ func testFork(t *testing.T, m *stages.MockSentry, i, n int, comparator func(td1, // Copy old chain up to #i into a new db canonicalMock := newCanonical(t, i) var err error - blockReader, _ := m.NewBlocksIO() ctx := context.Background() // Assert the chains have the same header/block at #i var hash1, hash2 libcommon.Hash err = m.DB.View(m.Ctx, func(tx kv.Tx) error { - if hash1, err = blockReader.CanonicalHash(m.Ctx, tx, uint64(i)); err != nil { + if hash1, err = m.BlockReader.CanonicalHash(m.Ctx, tx, uint64(i)); err != nil { t.Fatalf("Failed to read canonical hash: %v", err) } - if block1, _, _ := blockReader.BlockWithSenders(ctx, tx, hash1, uint64(i)); block1 == nil { + if block1, _, _ := m.BlockReader.BlockWithSenders(ctx, tx, hash1, uint64(i)); block1 == nil { t.Fatalf("Did not find canonical block") } return nil @@ -107,10 +107,10 @@ func testFork(t *testing.T, m *stages.MockSentry, i, n int, comparator func(td1, require.NoError(t, err) canonicalMock.DB.View(ctx, func(tx kv.Tx) error { - if hash2, err = blockReader.CanonicalHash(m.Ctx, tx, uint64(i)); err != nil { + if hash2, err = m.BlockReader.CanonicalHash(m.Ctx, tx, uint64(i)); err != nil { t.Fatalf("Failed to read canonical hash: %v", err) } - if block2, _, _ := blockReader.BlockWithSenders(ctx, tx, hash2, uint64(i)); block2 == nil { + if block2, _, _ := m.BlockReader.BlockWithSenders(ctx, tx, hash2, uint64(i)); block2 == nil { t.Fatalf("Did not find canonical block 2") } return nil @@ -126,7 +126,7 @@ func testFork(t *testing.T, m *stages.MockSentry, i, n int, comparator func(td1, var currentBlockB *types.Block err = canonicalMock.DB.View(context.Background(), func(tx kv.Tx) error { - currentBlockB, err = blockReader.CurrentBlock(tx) + currentBlockB, err = m.BlockReader.CurrentBlock(tx) return err }) require.NoError(t, err) @@ -134,7 +134,7 @@ func testFork(t *testing.T, m *stages.MockSentry, i, n int, comparator func(td1, blockChainB = makeBlockChain(currentBlockB, n, canonicalMock, forkSeed) err = m.DB.View(context.Background(), func(tx kv.Tx) error { - currentBlock, err := blockReader.CurrentBlock(tx) + currentBlock, err := m.BlockReader.CurrentBlock(tx) if err != nil { return err } @@ -146,13 +146,13 @@ func testFork(t *testing.T, m *stages.MockSentry, i, n int, comparator func(td1, }) require.NoError(t, err) - if err = m.InsertChain(blockChainB); err != nil { + if err = m.InsertChain(blockChainB, nil); err != nil { t.Fatalf("failed to insert forking chain: %v", err) } currentBlockHash := blockChainB.TopBlock.Hash() err = m.DB.View(context.Background(), func(tx kv.Tx) error { number := rawdb.ReadHeaderNumber(tx, currentBlockHash) - currentBlock, _, _ := blockReader.BlockWithSenders(ctx, tx, currentBlockHash, *number) + currentBlock, _, _ := m.BlockReader.BlockWithSenders(ctx, tx, currentBlockHash, *number) tdPost, err = rawdb.ReadTd(tx, currentBlockHash, currentBlock.NumberU64()) if err != nil { t.Fatalf("Failed to read TD for current header: %v", err) @@ -162,7 +162,7 @@ func testFork(t *testing.T, m *stages.MockSentry, i, n int, comparator func(td1, require.NoError(t, err) // Sanity check that the forked chain can be imported into the original - if err := canonicalMock.InsertChain(blockChainB); err != nil { + if err := canonicalMock.InsertChain(blockChainB, nil); err != nil { t.Fatalf("failed to import forked block chain: %v", err) } // Compare the total difficulties of the chains @@ -173,8 +173,8 @@ func TestLastBlock(t *testing.T) { m := newCanonical(t, 0) var err error - chain := makeBlockChain(current(m), 1, m, 0) - if err = m.InsertChain(chain); err != nil { + chain := makeBlockChain(current(m, nil), 1, m, 0) + if err = m.InsertChain(chain, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } @@ -272,10 +272,10 @@ func testBrokenChain(t *testing.T) { m := newCanonical(t, 10) // Create a forked chain, and try to insert with a missing link - chain := makeBlockChain(current(m), 5, m, forkSeed) + chain := makeBlockChain(current(m, nil), 5, m, forkSeed) brokenChain := chain.Slice(1, chain.Length()) - if err := m.InsertChain(brokenChain); err == nil { + if err := m.InsertChain(brokenChain, nil); err == nil { t.Errorf("broken block chain not reported") } } @@ -311,34 +311,38 @@ func testReorg(t *testing.T, first, second []int64, td int64) { require := require.New(t) // Create a pristine chain and database m := newCanonical(t, 0) - br, _ := m.NewBlocksIO() // Insert an easy and a difficult chain afterwards - easyChain, err := core.GenerateChain(m.ChainConfig, current(m), m.Engine, m.DB, len(first), func(i int, b *core.BlockGen) { + easyChain, err := core.GenerateChain(m.ChainConfig, current(m, nil), m.Engine, m.DB, len(first), func(i int, b *core.BlockGen) { b.OffsetTime(first[i]) }, false /* intermediateHashes */) if err != nil { t.Fatalf("generate chain: %v", err) } - diffChain, err := core.GenerateChain(m.ChainConfig, current(m), m.Engine, m.DB, len(second), func(i int, b *core.BlockGen) { + diffChain, err := core.GenerateChain(m.ChainConfig, current(m, nil), m.Engine, m.DB, len(second), func(i int, b *core.BlockGen) { b.OffsetTime(second[i]) }, false /* intermediateHashes */) if err != nil { t.Fatalf("generate chain: %v", err) } - if err = m.InsertChain(easyChain); err != nil { + + tx, err := m.DB.BeginRw(m.Ctx) + if err != nil { + fmt.Printf("beginro error: %v\n", err) + return + } + defer tx.Rollback() + + if err = m.InsertChain(easyChain, tx); err != nil { t.Fatalf("failed to insert easy chain: %v", err) } - if err = m.InsertChain(diffChain); err != nil { + if err = m.InsertChain(diffChain, tx); err != nil { t.Fatalf("failed to insert difficult chain: %v", err) } - tx, err := m.DB.BeginRo(context.Background()) - require.NoError(err) - defer tx.Rollback() // Check that the chain is valid number and link wise - prev, err := br.CurrentBlock(tx) + prev, err := m.BlockReader.CurrentBlock(tx) require.NoError(err) - block, err := br.BlockByNumber(m.Ctx, tx, rawdb.ReadCurrentHeader(tx).Number.Uint64()-1) + block, err := m.BlockReader.BlockByNumber(m.Ctx, tx, rawdb.ReadCurrentHeader(tx).Number.Uint64()-1) if err != nil { t.Fatal(err) } @@ -347,7 +351,7 @@ func testReorg(t *testing.T, first, second []int64, td int64) { t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash()) } prev = block - block, err = br.BlockByNumber(m.Ctx, tx, block.NumberU64()-1) + block, err = m.BlockReader.BlockByNumber(m.Ctx, tx, block.NumberU64()-1) if err != nil { t.Fatal(err) } @@ -371,12 +375,12 @@ func testBadHashes(t *testing.T) { var err error // Create a chain, ban a hash and try to import - chain := makeBlockChain(current(m), 3, m, 10) + chain := makeBlockChain(current(m, nil), 3, m, 10) core.BadHashes[chain.Headers[2].Hash()] = true defer func() { delete(core.BadHashes, chain.Headers[2].Hash()) }() - err = m.InsertChain(chain) + err = m.InsertChain(chain, nil) if !errors.Is(err, core.ErrBlacklistedHash) { t.Errorf("error mismatch: have: %v, want: %v", err, core.ErrBlacklistedHash) } @@ -445,7 +449,7 @@ func TestChainTxReorgs(t *testing.T) { t.Fatalf("generate chain: %v", err) } // Import the chain. This runs all block validation rules. - if err1 := m.InsertChain(chain); err1 != nil { + if err1 := m.InsertChain(chain, nil); err1 != nil { t.Fatalf("failed to insert original chain: %v", err1) } @@ -472,21 +476,20 @@ func TestChainTxReorgs(t *testing.T) { t.Fatalf("generate chain: %v", err) } - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } tx, err := m.DB.BeginRo(context.Background()) require.NoError(t, err) defer tx.Rollback() - br, _ := m.NewBlocksIO() // removed tx txs := types.Transactions{pastDrop, freshDrop} for i, txn := range txs { if bn, _ := rawdb.ReadTxLookupEntry(tx, txn.Hash()); bn != nil { t.Errorf("drop %d: tx %v found while shouldn't have been", i, txn) } - if rcpt, _, _, _, _ := readReceipt(m.ChainConfig, tx, txn.Hash(), br); rcpt != nil { + if rcpt, _, _, _, _ := readReceipt(m.ChainConfig, tx, txn.Hash(), m.BlockReader); rcpt != nil { t.Errorf("drop %d: receipt %v found while shouldn't have been", i, rcpt) } } @@ -494,14 +497,14 @@ func TestChainTxReorgs(t *testing.T) { // added tx txs = types.Transactions{pastAdd, freshAdd, futureAdd} for i, txn := range txs { - _, found, err := br.TxnLookup(m.Ctx, tx, txn.Hash()) + _, found, err := m.BlockReader.TxnLookup(m.Ctx, tx, txn.Hash()) require.NoError(t, err) require.True(t, found) if m.HistoryV3 { // m.HistoryV3 doesn't store } else { - if rcpt, _, _, _, _ := readReceipt(m.ChainConfig, tx, txn.Hash(), br); rcpt == nil { + if rcpt, _, _, _, _ := readReceipt(m.ChainConfig, tx, txn.Hash(), m.BlockReader); rcpt == nil { t.Errorf("add %d: expected receipt to be found", i) } } @@ -515,7 +518,7 @@ func TestChainTxReorgs(t *testing.T) { if m.HistoryV3 { // m.HistoryV3 doesn't store } else { - if rcpt, _, _, _, _ := readReceipt(m.ChainConfig, tx, txn.Hash(), br); rcpt == nil { + if rcpt, _, _, _, _ := readReceipt(m.ChainConfig, tx, txn.Hash(), m.BlockReader); rcpt == nil { t.Errorf("share %d: expected receipt to be found", i) } } @@ -561,17 +564,16 @@ func TestCanonicalBlockRetrieval(t *testing.T) { if err2 != nil { t.Fatalf("generate chain: %v", err2) } - err := m.InsertChain(chain) + err := m.InsertChain(chain, nil) require.NoError(t, err) tx, err := m.DB.BeginRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() - br, _ := m.NewBlocksIO() for _, block := range chain.Blocks { // try to retrieve a block by its canonical hash and see if the block data can be retrieved. - ch, err := br.CanonicalHash(m.Ctx, tx, block.NumberU64()) + ch, err := m.BlockReader.CanonicalHash(m.Ctx, tx, block.NumberU64()) require.NoError(t, err) if err != nil { panic(err) @@ -583,7 +585,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { t.Errorf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex()) return } - fb, _ := br.Header(m.Ctx, tx, ch, block.NumberU64()) + fb, _ := m.BlockReader.Header(m.Ctx, tx, ch, block.NumberU64()) if fb == nil { t.Errorf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex()) return @@ -655,17 +657,16 @@ func TestEIP155Transition(t *testing.T) { t.Fatalf("generate chain: %v", chainErr) } - if chainErr = m.InsertChain(chain); chainErr != nil { + if chainErr = m.InsertChain(chain, nil); chainErr != nil { t.Fatal(chainErr) } - br, _ := m.NewBlocksIO() if err := m.DB.View(context.Background(), func(tx kv.Tx) error { - block, _ := br.BlockByNumber(m.Ctx, tx, 1) + block, _ := m.BlockReader.BlockByNumber(m.Ctx, tx, 1) if block.Transactions()[0].Protected() { t.Error("Expected block[0].txs[0] to not be replay protected") } - block, _ = br.BlockByNumber(m.Ctx, tx, 3) + block, _ = m.BlockReader.BlockByNumber(m.Ctx, tx, 3) if block.Transactions()[0].Protected() { t.Error("Expected block[3].txs[0] to not be replay protected") } @@ -696,7 +697,7 @@ func TestEIP155Transition(t *testing.T) { if chainErr != nil { t.Fatalf("generate blocks: %v", chainErr) } - if err := m.InsertChain(chain); err == nil { + if err := m.InsertChain(chain, nil); err == nil { t.Errorf("expected error") } } @@ -780,7 +781,7 @@ func doModesTest(t *testing.T, pm prune.Mode) error { return fmt.Errorf("generate blocks: %w", err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { return err } @@ -835,7 +836,7 @@ func doModesTest(t *testing.T, pm prune.Mode) error { } else { if pm.History.Enabled() { afterPrune := uint64(0) - err := tx.ForEach(kv.AccountsHistory, nil, func(k, _ []byte) error { + err := tx.ForEach(kv.E2AccountsHistory, nil, func(k, _ []byte) error { n := binary.BigEndian.Uint64(k[length.Addr:]) require.Greater(n, pm.History.PruneTo(head)) afterPrune++ @@ -844,16 +845,14 @@ func doModesTest(t *testing.T, pm prune.Mode) error { require.Greater(afterPrune, uint64(0)) assert.NoError(t, err) } else { - found, err := bitmapdb.Get64(tx, kv.AccountsHistory, address[:], 0, 1024) + found, err := bitmapdb.Get64(tx, kv.E2AccountsHistory, address[:], 0, 1024) require.NoError(err) require.Equal(uint64(0), found.Minimum()) } } - br, _ := m.NewBlocksIO() - if pm.TxIndex.Enabled() { - b, err := br.BlockByNumber(m.Ctx, tx, 1) + b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 1) require.NoError(err) for _, txn := range b.Transactions() { found, err := rawdb.ReadTxLookupEntry(tx, txn.Hash()) @@ -861,10 +860,10 @@ func doModesTest(t *testing.T, pm prune.Mode) error { require.Nil(found) } } else { - b, err := br.BlockByNumber(m.Ctx, tx, 1) + b, err := m.BlockReader.BlockByNumber(m.Ctx, tx, 1) require.NoError(err) for _, txn := range b.Transactions() { - foundBlockNum, found, err := br.TxnLookup(context.Background(), tx, txn.Hash()) + foundBlockNum, found, err := m.BlockReader.TxnLookup(context.Background(), tx, txn.Hash()) require.NoError(err) require.True(found) require.Equal(uint64(1), foundBlockNum) @@ -980,42 +979,37 @@ func TestEIP161AccountRemoval(t *testing.T) { if err != nil { t.Fatalf("generate blocks: %v", err) } + tx, err := m.DB.BeginRw(m.Ctx) + if err != nil { + fmt.Printf("beginro error: %v\n", err) + return + } + defer tx.Rollback() + // account must exist pre eip 161 - if err = m.InsertChain(chain.Slice(0, 1)); err != nil { + if err = m.InsertChain(chain.Slice(0, 1), tx); err != nil { t.Fatal(err) } - err = m.DB.View(context.Background(), func(tx kv.Tx) error { - if st := state.New(m.NewStateReader(tx)); !st.Exist(theAddr) { - t.Error("expected account to exist") - } - return nil - }) - require.NoError(t, err) + if st := state.New(m.NewStateReader(tx)); !st.Exist(theAddr) { + t.Error("expected account to exist") + } // account needs to be deleted post eip 161 - if err = m.InsertChain(chain.Slice(1, 2)); err != nil { + if err = m.InsertChain(chain.Slice(1, 2), tx); err != nil { t.Fatal(err) } - err = m.DB.View(m.Ctx, func(tx kv.Tx) error { - if st := state.New(m.NewStateReader(tx)); st.Exist(theAddr) { - t.Error("account should not exist") - } - return nil - }) - require.NoError(t, err) + if st := state.New(m.NewStateReader(tx)); st.Exist(theAddr) { + t.Error("account should not exist") + } // account mustn't be created post eip 161 - if err = m.InsertChain(chain.Slice(2, 3)); err != nil { + if err = m.InsertChain(chain.Slice(2, 3), tx); err != nil { t.Fatal(err) } - err = m.DB.View(m.Ctx, func(tx kv.Tx) error { - if st := state.New(m.NewStateReader(tx)); st.Exist(theAddr) { - t.Error("account should not exist") - } - return nil - }) + if st := state.New(m.NewStateReader(tx)); st.Exist(theAddr) { + t.Error("account should not exist") + } require.NoError(t, err) - } func TestDoubleAccountRemoval(t *testing.T) { @@ -1063,24 +1057,20 @@ func TestDoubleAccountRemoval(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - err = m.InsertChain(chain) - assert.NoError(t, err) - - err = m.DB.View(m.Ctx, func(tx kv.Tx) error { - st := state.New(m.NewStateReader(tx)) - assert.NoError(t, err) - assert.False(t, st.Exist(theAddr), "Contract should've been removed") - return nil - }) - assert.NoError(t, err) - - tx, err := m.DB.BeginRo(m.Ctx) + tx, err := m.DB.BeginRw(m.Ctx) if err != nil { - t.Fatalf("read only db tx to read state: %v", err) + fmt.Printf("beginro error: %v\n", err) + return } defer tx.Rollback() + err = m.InsertChain(chain, tx) + assert.NoError(t, err) + + st := state.New(m.NewStateReader(tx)) + assert.NoError(t, err) + assert.False(t, st.Exist(theAddr), "Contract should've been removed") - st := state.New(m.NewHistoryStateReader(1, tx)) + st = state.New(m.NewHistoryStateReader(1, tx)) assert.NoError(t, err) assert.False(t, st.Exist(theAddr), "Contract should not exist at block #0") @@ -1124,16 +1114,15 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } forks[i] = fork.Slice(i, i+1) } - br, _ := m.NewBlocksIO() // Import the canonical and fork chain side by side, verifying the current block // and current header consistency for i := 0; i < chain.Length(); i++ { - if err := m2.InsertChain(chain.Slice(i, i+1)); err != nil { + if err := m2.InsertChain(chain.Slice(i, i+1), nil); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", i, err) } if err := m2.DB.View(m2.Ctx, func(tx kv.Tx) error { - b, err := br.CurrentBlock(tx) + b, err := m.BlockReader.CurrentBlock(tx) if err != nil { return err } @@ -1141,10 +1130,10 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { if b.Hash() != h.Hash() { t.Errorf("block %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, b.Number(), b.Hash().Bytes()[:4], h.Number, h.Hash().Bytes()[:4]) } - if err := m2.InsertChain(forks[i]); err != nil { + if err := m2.InsertChain(forks[i], nil); err != nil { t.Fatalf(" fork %d: failed to insert into chain: %v", i, err) } - b, err = br.CurrentBlock(tx) + b, err = m.BlockReader.CurrentBlock(tx) if err != nil { return err } @@ -1195,20 +1184,20 @@ func TestLargeReorgTrieGC(t *testing.T) { } // Import the shared chain and the original canonical one - if err := m2.InsertChain(shared); err != nil { + if err := m2.InsertChain(shared, nil); err != nil { t.Fatalf("failed to insert shared chain: %v", err) } - if err := m2.InsertChain(original); err != nil { + if err := m2.InsertChain(original, nil); err != nil { t.Fatalf("failed to insert original chain: %v", err) } // Import the competitor chain without exceeding the canonical's TD and ensure // we have not processed any of the blocks (protection against malicious blocks) - if err := m2.InsertChain(competitor.Slice(0, competitor.Length()-2)); err != nil { + if err := m2.InsertChain(competitor.Slice(0, competitor.Length()-2), nil); err != nil { t.Fatalf("failed to insert competitor chain: %v", err) } // Import the head of the competitor chain, triggering the reorg and ensure we // successfully reprocess all the stashed away blocks. - if err := m2.InsertChain(competitor.Slice(competitor.Length()-2, competitor.Length())); err != nil { + if err := m2.InsertChain(competitor.Slice(competitor.Length()-2, competitor.Length()), nil); err != nil { t.Fatalf("failed to finalize competitor chain: %v", err) } } @@ -1249,18 +1238,17 @@ func TestLowDiffLongChain(t *testing.T) { // Import the canonical chain m2 := stages.Mock(t) - if err := m2.InsertChain(chain); err != nil { + if err := m2.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } // And now import the fork - if err := m2.InsertChain(fork); err != nil { + if err := m2.InsertChain(fork, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } - br, _ := m.NewBlocksIO() if err := m2.DB.View(context.Background(), func(tx kv.Tx) error { - head, err := br.CurrentBlock(tx) + head, err := m.BlockReader.CurrentBlock(tx) if err != nil { return err } @@ -1271,12 +1259,12 @@ func TestLowDiffLongChain(t *testing.T) { // Sanity check that all the canonical numbers are present header := rawdb.ReadCurrentHeader(tx) for number := head.NumberU64(); number > 0; number-- { - hh, _ := br.HeaderByNumber(m.Ctx, tx, number) + hh, _ := m.BlockReader.HeaderByNumber(m.Ctx, tx, number) if hash := hh.Hash(); hash != header.Hash() { t.Fatalf("header %d: canonical hash mismatch: have %x, want %x", number, hash, header.Hash()) } - header, _ = br.Header(m.Ctx, tx, header.ParentHash, number-1) + header, _ = m.BlockReader.Header(m.Ctx, tx, header.ParentHash, number-1) } return nil }); err != nil { @@ -1348,7 +1336,7 @@ func TestDeleteCreateRevert(t *testing.T) { t.Fatalf("generate blocks: %v", err) } - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } } @@ -1453,7 +1441,7 @@ func TestDeleteRecreateSlots(t *testing.T) { t.Fatalf("generate blocks: %v", err) } // Import the canonical chain - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } err = m.DB.View(m.Ctx, func(tx kv.Tx) error { @@ -1571,7 +1559,7 @@ func TestCVE2020_26265(t *testing.T) { t.Fatalf("generate blocks: %v", err) } // Import the canonical chain - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } err = m.DB.View(m.Ctx, func(tx kv.Tx) error { @@ -1638,7 +1626,7 @@ func TestDeleteRecreateAccount(t *testing.T) { t.Fatalf("generate blocks: %v", err) } // Import the canonical chain - if err := m.InsertChain(chain); err != nil { + if err := m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } err = m.DB.View(m.Ctx, func(tx kv.Tx) error { @@ -1818,7 +1806,7 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { } for i := range chain.Blocks { blockNum := i + 1 - if err := m.InsertChain(chain.Slice(i, i+1)); err != nil { + if err := m.InsertChain(chain.Slice(i, i+1), nil); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", i, err) } err = m.DB.View(m.Ctx, func(tx kv.Tx) error { @@ -1961,7 +1949,7 @@ func TestInitThenFailCreateContract(t *testing.T) { // First block tries to create, but fails { block := chain.Blocks[0] - if err := m.InsertChain(chain.Slice(0, 1)); err != nil { + if err := m.InsertChain(chain.Slice(0, 1), nil); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) } statedb = state.New(m.NewHistoryStateReader(1, tx)) @@ -1971,7 +1959,7 @@ func TestInitThenFailCreateContract(t *testing.T) { } // Import the rest of the blocks for i, block := range chain.Blocks[1:] { - if err := m.InsertChain(chain.Slice(1+i, 2+i)); err != nil { + if err := m.InsertChain(chain.Slice(1+i, 2+i), nil); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) } } @@ -2045,7 +2033,7 @@ func TestEIP2718Transition(t *testing.T) { // Import the canonical chain - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } @@ -2055,8 +2043,7 @@ func TestEIP2718Transition(t *testing.T) { } defer tx.Rollback() - br, _ := m.NewBlocksIO() - block, _ := br.BlockByNumber(m.Ctx, tx, 1) + block, _ := m.BlockReader.BlockByNumber(m.Ctx, tx, 1) // Expected gas is intrinsic + 2 * pc + hot load + cold load, since only one load is in the access list expected := params.TxGas + params.TxAccessListAddressGas + params.TxAccessListStorageKeyGas + @@ -2128,12 +2115,12 @@ func TestEIP1559Transition(t *testing.T) { chainID.SetFromBig(gspec.Config.ChainID) var tx types.Transaction = &types.DynamicFeeTransaction{ CommonTx: types.CommonTx{ - ChainID: &chainID, - Nonce: 0, - To: &aa, - Gas: 30000, - Data: []byte{}, + Nonce: 0, + To: &aa, + Gas: 30000, + Data: []byte{}, }, + ChainID: &chainID, FeeCap: new(uint256.Int).Mul(new(uint256.Int).SetUint64(5), new(uint256.Int).SetUint64(params.GWei)), Tip: u256.Num2, AccessList: accesses, @@ -2148,7 +2135,12 @@ func TestEIP1559Transition(t *testing.T) { } // Import the canonical chain - if err = m.InsertChain(chain); err != nil { + if err := m.DB.Update(m.Ctx, func(tx kv.RwTx) error { + if err = m.InsertChain(chain, tx); err != nil { + t.Fatalf("failed to insert into chain: %v", err) + } + return nil + }); err != nil { t.Fatalf("failed to insert into chain: %v", err) } @@ -2196,7 +2188,7 @@ func TestEIP1559Transition(t *testing.T) { t.Fatalf("generate chain: %v", err) } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { t.Fatalf("failed to insert into chain: %v", err) } @@ -2226,14 +2218,20 @@ func TestEIP1559Transition(t *testing.T) { require.NoError(t, err) } -func current(m *stages.MockSentry) *types.Block { +func current(m *stages.MockSentry, tx kv.Tx) *types.Block { + if tx != nil { + b, err := m.BlockReader.CurrentBlock(tx) + if err != nil { + panic(err) + } + return b + } tx, err := m.DB.BeginRo(context.Background()) if err != nil { panic(err) } defer tx.Rollback() - br, _ := m.NewBlocksIO() - b, err := br.CurrentBlock(tx) + b, err := m.BlockReader.CurrentBlock(tx) if err != nil { panic(err) } diff --git a/turbo/stages/bodydownload/body_test.go b/turbo/stages/bodydownload/body_test.go index 89ef6e4cfc4..72bf9823b26 100644 --- a/turbo/stages/bodydownload/body_test.go +++ b/turbo/stages/bodydownload/body_test.go @@ -15,8 +15,7 @@ func TestCreateBodyDownload(t *testing.T) { tx, err := m.DB.BeginRo(m.Ctx) require.NoError(t, err) defer tx.Rollback() - blockReader, _ := m.NewBlocksIO() - bd := bodydownload.NewBodyDownload(ethash.NewFaker(), 100, blockReader) + bd := bodydownload.NewBodyDownload(ethash.NewFaker(), 100, m.BlockReader) if _, _, _, _, err := bd.UpdateFromDb(tx); err != nil { t.Fatalf("update from db: %v", err) } diff --git a/turbo/stages/chain_makers_test.go b/turbo/stages/chain_makers_test.go index e1c14713acd..e2b64262f64 100644 --- a/turbo/stages/chain_makers_test.go +++ b/turbo/stages/chain_makers_test.go @@ -92,22 +92,22 @@ func TestGenerateChain(t *testing.T) { fmt.Printf("generate chain: %v\n", err) } - // Import the chain. This runs all block validation rules. - if err := m.InsertChain(chain); err != nil { - fmt.Printf("insert error%v\n", err) - return - } - - tx, err := m.DB.BeginRo(m.Ctx) + tx, err := m.DB.BeginRw(m.Ctx) if err != nil { fmt.Printf("beginro error: %v\n", err) return } defer tx.Rollback() + // Import the chain. This runs all block validation rules. + if err := m.InsertChain(chain, tx); err != nil { + fmt.Printf("insert error%v\n", err) + return + } + st := state.New(m.NewStateReader(tx)) - if big.NewInt(5).Cmp(current(m).Number()) != 0 { - t.Errorf("wrong block number: %d", current(m).Number()) + if big.NewInt(5).Cmp(current(m, tx).Number()) != 0 { + t.Errorf("wrong block number: %d", current(m, tx).Number()) } if !uint256.NewInt(989000).Eq(st.GetBalance(addr1)) { t.Errorf("wrong balance of addr1: %s", st.GetBalance(addr1)) diff --git a/turbo/stages/genesis_test.go b/turbo/stages/genesis_test.go index b35ad10fc5f..c18496488f5 100644 --- a/turbo/stages/genesis_test.go +++ b/turbo/stages/genesis_test.go @@ -33,7 +33,7 @@ import ( "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/turbo/stages" "github.com/ledgerwatch/log/v3" ) @@ -123,7 +123,7 @@ func TestSetupGenesis(t *testing.T) { if err != nil { return nil, nil, err } - if err = m.InsertChain(chain); err != nil { + if err = m.InsertChain(chain, nil); err != nil { return nil, nil, err } // This should return a compatibility error. @@ -143,8 +143,8 @@ func TestSetupGenesis(t *testing.T) { for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { - _, db, _ := temporal.NewTestDB(t, context.Background(), datadir.New(tmpdir), nil, log.New()) - blockReader := snapshotsync.NewBlockReader(snapshotsync.NewRoSnapshots(ethconfig.Snapshot{Enabled: false}, "", log.New())) + _, db, _ := temporal.NewTestDB(t, datadir.New(tmpdir), nil) + blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, "", log.New())) config, genesis, err := test.fn(db) // Check the return values. if !reflect.DeepEqual(err, test.wantErr) { diff --git a/turbo/stages/headerdownload/header_algo_test.go b/turbo/stages/headerdownload/header_algo_test.go index 1f1874b76c5..f4dbea44f50 100644 --- a/turbo/stages/headerdownload/header_algo_test.go +++ b/turbo/stages/headerdownload/header_algo_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/crypto" @@ -17,7 +17,6 @@ import ( ) func TestInserter1(t *testing.T) { - m := stages.Mock(t) funds := big.NewInt(1000000000) key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address := crypto.PubkeyToAddress(key.PublicKey) @@ -28,8 +27,8 @@ func TestInserter1(t *testing.T) { address: {Balance: funds}, }, } - db := memdb.NewTestDB(t) - defer db.Close() + m := stages.MockWithGenesis(t, gspec, key, false) + db := m.DB _, genesis, err := core.CommitGenesisBlock(db, gspec, "", m.Log) if err != nil { t.Fatal(err) @@ -39,8 +38,8 @@ func TestInserter1(t *testing.T) { t.Fatal(err) } defer tx.Rollback() - br, bw := m.NewBlocksIO() - hi := headerdownload.NewHeaderInserter("headers", big.NewInt(0), 0, br, bw) + br := m.BlockReader + hi := headerdownload.NewHeaderInserter("headers", big.NewInt(0), 0, br) h1 := types.Header{ Number: big.NewInt(1), Difficulty: big.NewInt(10), diff --git a/turbo/stages/headerdownload/header_algos.go b/turbo/stages/headerdownload/header_algos.go index 2981ce174b7..15c0f54540d 100644 --- a/turbo/stages/headerdownload/header_algos.go +++ b/turbo/stages/headerdownload/header_algos.go @@ -17,7 +17,6 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" "github.com/ledgerwatch/erigon/dataflow" @@ -303,7 +302,7 @@ func (hd *HeaderDownload) logAnchorState() { sort.Strings(ss) hd.logger.Debug("[downloader] Queue sizes", "anchors", hd.anchorTree.Len(), "links", hd.linkQueue.Len(), "persisted", hd.persistedLinkQueue.Len()) for _, s := range ss { - log.Debug(s) + hd.logger.Debug(s) } } @@ -476,7 +475,7 @@ func (hd *HeaderDownload) UpdateStats(req *HeaderRequest, skeleton bool, peer [6 } } } - //log.Debug("Header request sent", "req", fmt.Sprintf("%+v", req), "peer", fmt.Sprintf("%x", peer)[:8]) + //hd.logger.Debug("Header request sent", "req", fmt.Sprintf("%+v", req), "peer", fmt.Sprintf("%x", peer)[:8]) } func (hd *HeaderDownload) UpdateRetryTime(req *HeaderRequest, currentTime time.Time, timeout time.Duration) { @@ -570,7 +569,7 @@ func (hd *HeaderDownload) InsertHeader(hf FeedHeaderFunc, terminalTotalDifficult if terminalTotalDifficulty != nil { if td.Cmp(terminalTotalDifficulty) >= 0 { hd.highestInDb = link.blockHeight - log.Info(POSPandaBanner) + hd.logger.Info(POSPandaBanner) dataflow.HeaderDownloadStates.AddChange(link.blockHeight, dataflow.HeaderInserted) return true, true, 0, lastTime, nil } @@ -906,11 +905,11 @@ func (hi *HeaderInserter) FeedHeaderPoW(db kv.StatelessRwTx, headerReader servic // This makes sure we end up choosing the chain with the max total difficulty hi.localTd.Set(td) } - if err = hi.headerWriter.WriteTd(db, hash, blockHeight, td); err != nil { + if err = rawdb.WriteTd(db, hash, blockHeight, td); err != nil { return nil, fmt.Errorf("[%s] failed to WriteTd: %w", hi.logPrefix, err) } // skipIndexing=true - because next stages will build indices in-batch (for example StageBlockHash) - if err = hi.headerWriter.WriteHeaderRaw(db, blockHeight, hash, headerRaw, true); err != nil { + if err = rawdb.WriteHeaderRaw(db, blockHeight, hash, headerRaw, true); err != nil { return nil, fmt.Errorf("[%s] failed to WriteTd: %w", hi.logPrefix, err) } @@ -927,10 +926,10 @@ func (hi *HeaderInserter) FeedHeaderPoS(db kv.RwTx, header *types.Header, hash l return fmt.Errorf("[%s] parent's total difficulty not found with hash %x and height %d for header %x %d: %v", hi.logPrefix, header.ParentHash, blockHeight-1, hash, blockHeight, err) } td := new(big.Int).Add(parentTd, header.Difficulty) - if err = hi.headerWriter.WriteHeader(db, header); err != nil { + if err = rawdb.WriteHeader(db, header); err != nil { return fmt.Errorf("[%s] failed to WriteHeader: %w", hi.logPrefix, err) } - if err = hi.headerWriter.WriteTd(db, hash, blockHeight, td); err != nil { + if err = rawdb.WriteTd(db, hash, blockHeight, td); err != nil { return fmt.Errorf("[%s] failed to WriteTd: %w", hi.logPrefix, err) } @@ -1224,7 +1223,9 @@ func (hd *HeaderDownload) AddMinedHeader(header *types.Header) error { return nil } -func (hd *HeaderDownload) AddHeadersFromSnapshot(tx kv.Tx, n uint64, r services.FullBlockReader) error { +func (hd *HeaderDownload) AddHeadersFromSnapshot(tx kv.Tx, r services.FullBlockReader) error { + n := r.FrozenBlocks() + hd.lock.Lock() defer hd.lock.Unlock() for i := n; i > 0 && hd.persistedLinkQueue.Len() < hd.persistedLinkLimit; i-- { diff --git a/turbo/stages/headerdownload/header_data_struct.go b/turbo/stages/headerdownload/header_data_struct.go index 4fa2cbf0fd8..120b32d7922 100644 --- a/turbo/stages/headerdownload/header_data_struct.go +++ b/turbo/stages/headerdownload/header_data_struct.go @@ -11,8 +11,6 @@ import ( "github.com/hashicorp/golang-lru/v2" "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/etl" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" - "github.com/ledgerwatch/erigon/consensus" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/rlp" @@ -385,16 +383,14 @@ type HeaderInserter struct { highestTimestamp uint64 canonicalCache *lru.Cache[uint64, common.Hash] headerReader services.HeaderAndCanonicalReader - headerWriter *blockio.BlockWriter } -func NewHeaderInserter(logPrefix string, localTd *big.Int, headerProgress uint64, headerReader services.HeaderAndCanonicalReader, headerWriter *blockio.BlockWriter) *HeaderInserter { +func NewHeaderInserter(logPrefix string, localTd *big.Int, headerProgress uint64, headerReader services.HeaderAndCanonicalReader) *HeaderInserter { hi := &HeaderInserter{ logPrefix: logPrefix, localTd: localTd, unwindPoint: headerProgress, headerReader: headerReader, - headerWriter: headerWriter, } hi.canonicalCache, _ = lru.New[uint64, common.Hash](1000) return hi diff --git a/turbo/stages/mock_sentry.go b/turbo/stages/mock_sentry.go index c099ac3dec9..e7a30513e66 100644 --- a/turbo/stages/mock_sentry.go +++ b/turbo/stages/mock_sentry.go @@ -11,8 +11,6 @@ import ( "github.com/c2h5oh/datasize" "github.com/holiman/uint256" - "github.com/ledgerwatch/erigon/core/rawdb/blockio" - "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/log/v3" "google.golang.org/protobuf/types/known/emptypb" @@ -38,6 +36,7 @@ import ( "github.com/ledgerwatch/erigon/consensus/ethash" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" + "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" @@ -53,8 +52,9 @@ import ( "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/engineapi" "github.com/ledgerwatch/erigon/turbo/rpchelper" + "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/shards" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" + "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon/turbo/stages/bodydownload" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" "github.com/ledgerwatch/erigon/turbo/trie" @@ -100,7 +100,8 @@ type MockSentry struct { HistoryV3 bool agg *libstate.AggregatorV3 - BlockSnapshots *snapshotsync.RoSnapshots + BlockSnapshots *freezeblocks.RoSnapshots + BlockReader services.FullBlockReader } func (ms *MockSentry) Close() { @@ -233,11 +234,11 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK logger := log.New() ctx, ctxCancel := context.WithCancel(context.Background()) - histV3, db, agg := temporal.NewTestDB(tb, ctx, dirs, gspec, logger) + histV3, db, agg := temporal.NewTestDB(tb, dirs, nil) cfg.HistoryV3 = histV3 erigonGrpcServeer := remotedbserver.NewKvServer(ctx, db, nil, nil, logger) - allSnapshots := snapshotsync.NewRoSnapshots(ethconfig.Defaults.Snapshot, dirs.Snap, logger) + allSnapshots := freezeblocks.NewRoSnapshots(ethconfig.Defaults.Snapshot, dirs.Snap, logger) mock := &MockSentry{ Ctx: ctx, cancel: ctxCancel, DB: db, agg: agg, tb: tb, @@ -256,12 +257,13 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK }, PeerId: gointerfaces.ConvertHashToH512([64]byte{0x12, 0x34, 0x50}), // "12345" BlockSnapshots: allSnapshots, + BlockReader: freezeblocks.NewBlockReader(allSnapshots), HistoryV3: cfg.HistoryV3, } if tb != nil { tb.Cleanup(mock.Close) } - blockReader, blockWriter := mock.NewBlocksIO() + blockWriter := blockio.NewBlockWriter(mock.HistoryV3) mock.Address = crypto.PubkeyToAddress(mock.Key.PublicKey) @@ -319,7 +321,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK inMemoryExecution := func(batch kv.RwTx, header *types.Header, body *types.RawBody, unwindPoint uint64, headersChain []*types.Header, bodiesChain []*types.RawBody, notifications *shards.Notifications) error { // Needs its own notifications to not update RPC daemon and txpool about pending blocks - stateSync, err := NewInMemoryExecution(ctx, mock.DB, ðconfig.Defaults, mock.sentriesClient, dirs, notifications, allSnapshots, agg, log.New() /* logging will be discarded */) + stateSync, err := NewInMemoryExecution(ctx, mock.DB, ðconfig.Defaults, mock.sentriesClient, dirs, notifications, mock.BlockReader, blockWriter, agg, log.New() /* logging will be discarded */) if err != nil { return err } @@ -337,8 +339,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK } return nil } - br, _ := mock.NewBlocksIO() - forkValidator := engineapi.NewForkValidator(1, inMemoryExecution, dirs.Tmp, br) + forkValidator := engineapi.NewForkValidator(1, inMemoryExecution, dirs.Tmp, mock.BlockReader) networkID := uint64(1) mock.sentriesClient, err = sentry.NewMultiClient( mock.DB, @@ -349,7 +350,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK networkID, sentries, cfg.Sync, - blockReader, + mock.BlockReader, false, forkValidator, cfg.DropUselessPeers, @@ -367,15 +368,15 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK var snapshotsDownloader proto_downloader.DownloaderClient - blockRetire := snapshotsync.NewBlockRetire(1, dirs.Tmp, blockReader, blockWriter, mock.DB, snapshotsDownloader, mock.Notifications.Events, logger) + blockRetire := freezeblocks.NewBlockRetire(1, dirs, mock.BlockReader, blockWriter, mock.DB, mock.Notifications.Events, logger) mock.Sync = stagedsync.New( stagedsync.DefaultStages(mock.Ctx, - stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, blockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg), - stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, blockReader, blockWriter, dirs.Tmp, mock.Notifications, engineapi.NewForkValidatorMock(1)), - stagedsync.StageCumulativeIndexCfg(mock.DB, blockReader), + stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg), + stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications, engineapi.NewForkValidatorMock(1)), + stagedsync.StageCumulativeIndexCfg(mock.DB, mock.BlockReader), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), - stagedsync.StageBodiesCfg(mock.DB, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, blockReader, cfg.HistoryV3, blockWriter), - stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, false, dirs.Tmp, prune, blockRetire, blockWriter, blockReader, mock.sentriesClient.Hd), + stagedsync.StageBodiesCfg(mock.DB, mock.sentriesClient.Bd, sendBodyRequest, penalize, blockPropagator, cfg.Sync.BodyDownloadTimeoutSeconds, *mock.ChainConfig, mock.BlockReader, cfg.HistoryV3, blockWriter), + stagedsync.StageSendersCfg(mock.DB, mock.ChainConfig, false, dirs.Tmp, prune, mock.BlockReader, mock.sentriesClient.Hd), stagedsync.StageExecuteBlocksCfg( mock.DB, prune, @@ -389,18 +390,18 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK /*stateStream=*/ false, /*exec22=*/ cfg.HistoryV3, dirs, - blockReader, + mock.BlockReader, mock.sentriesClient.Hd, mock.gspec, ethconfig.Defaults.Sync, mock.agg, ), stagedsync.StageHashStateCfg(mock.DB, mock.Dirs, cfg.HistoryV3), - stagedsync.StageTrieCfg(mock.DB, true, true, false, dirs.Tmp, blockReader, mock.sentriesClient.Hd, cfg.HistoryV3, mock.agg), + stagedsync.StageTrieCfg(mock.DB, true, true, false, dirs.Tmp, mock.BlockReader, mock.sentriesClient.Hd, cfg.HistoryV3, mock.agg), stagedsync.StageHistoryCfg(mock.DB, prune, dirs.Tmp), stagedsync.StageLogIndexCfg(mock.DB, prune, dirs.Tmp), stagedsync.StageCallTracesCfg(mock.DB, prune, 0, dirs.Tmp), - stagedsync.StageTxLookupCfg(mock.DB, prune, dirs.Tmp, mock.ChainConfig.Bor, blockReader), + stagedsync.StageTxLookupCfg(mock.DB, prune, dirs.Tmp, mock.ChainConfig.Bor, mock.BlockReader), stagedsync.StageFinishCfg(mock.DB, dirs.Tmp, forkValidator), !withPosDownloader), stagedsync.DefaultUnwindOrder, @@ -426,11 +427,11 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK mock.MinedBlocks = miner.MiningResultCh mock.MiningSync = stagedsync.New( stagedsync.MiningStages(mock.Ctx, - stagedsync.StageMiningCreateBlockCfg(mock.DB, miner, *mock.ChainConfig, mock.Engine, mock.TxPool, nil, nil, dirs.Tmp, blockReader), - stagedsync.StageMiningExecCfg(mock.DB, miner, nil, *mock.ChainConfig, mock.Engine, &vm.Config{}, dirs.Tmp, nil, 0, mock.TxPool, nil, false, blockReader), + stagedsync.StageMiningCreateBlockCfg(mock.DB, miner, *mock.ChainConfig, mock.Engine, nil, nil, dirs.Tmp, mock.BlockReader), + stagedsync.StageMiningExecCfg(mock.DB, miner, nil, *mock.ChainConfig, mock.Engine, &vm.Config{}, dirs.Tmp, nil, 0, mock.TxPool, nil, false, mock.BlockReader), stagedsync.StageHashStateCfg(mock.DB, dirs, cfg.HistoryV3), - stagedsync.StageTrieCfg(mock.DB, false, true, false, dirs.Tmp, blockReader, mock.sentriesClient.Hd, cfg.HistoryV3, mock.agg), - stagedsync.StageMiningFinishCfg(mock.DB, *mock.ChainConfig, mock.Engine, miner, miningCancel, blockReader), + stagedsync.StageTrieCfg(mock.DB, false, true, false, dirs.Tmp, mock.BlockReader, mock.sentriesClient.Hd, cfg.HistoryV3, mock.agg), + stagedsync.StageMiningFinishCfg(mock.DB, *mock.ChainConfig, mock.Engine, miner, miningCancel, mock.BlockReader), ), stagedsync.MiningUnwindOrder, stagedsync.MiningPruneOrder, @@ -523,7 +524,7 @@ func (ms *MockSentry) numberOfPoWBlocks(chain *core.ChainPack) int { return chain.NumberOfPoWBlocks() } -func (ms *MockSentry) insertPoWBlocks(chain *core.ChainPack) error { +func (ms *MockSentry) insertPoWBlocks(chain *core.ChainPack, tx kv.RwTx) error { n := ms.numberOfPoWBlocks(chain) if n == 0 { // No Proof-of-Work blocks @@ -584,9 +585,8 @@ func (ms *MockSentry) insertPoWBlocks(chain *core.ChainPack) error { ms.ReceiveWg.Add(1) } initialCycle := MockInsertAsInitialCycle - blockReader, _ := ms.NewBlocksIO() - hook := NewHook(ms.Ctx, ms.Notifications, ms.Sync, blockReader, ms.ChainConfig, ms.Log, ms.UpdateHead) - if _, err = StageLoopStep(ms.Ctx, ms.DB, ms.Sync, initialCycle, ms.Log, blockReader.Snapshots().(*snapshotsync.RoSnapshots), hook); err != nil { + hook := NewHook(ms.Ctx, ms.Notifications, ms.Sync, ms.BlockReader, ms.ChainConfig, ms.Log, ms.UpdateHead) + if err = StageLoopStep(ms.Ctx, ms.DB, tx, ms.Sync, initialCycle, ms.Log, ms.BlockReader, hook); err != nil { return err } if ms.TxPool != nil { @@ -595,7 +595,7 @@ func (ms *MockSentry) insertPoWBlocks(chain *core.ChainPack) error { return nil } -func (ms *MockSentry) insertPoSBlocks(chain *core.ChainPack) error { +func (ms *MockSentry) insertPoSBlocks(chain *core.ChainPack, tx kv.RwTx) error { n := ms.numberOfPoWBlocks(chain) if n >= chain.Length() { return nil @@ -608,14 +608,13 @@ func (ms *MockSentry) insertPoSBlocks(chain *core.ChainPack) error { ms.SendPayloadRequest(chain.Blocks[i]) } - initialCycle := false - blockReader, _ := ms.NewBlocksIO() - hook := NewHook(ms.Ctx, ms.Notifications, ms.Sync, blockReader, ms.ChainConfig, ms.Log, ms.UpdateHead) - headBlockHash, err := StageLoopStep(ms.Ctx, ms.DB, ms.Sync, initialCycle, ms.Log, blockReader.Snapshots().(*snapshotsync.RoSnapshots), hook) + initialCycle := MockInsertAsInitialCycle + hook := NewHook(ms.Ctx, ms.Notifications, ms.Sync, ms.BlockReader, ms.ChainConfig, ms.Log, ms.UpdateHead) + err := StageLoopStep(ms.Ctx, ms.DB, tx, ms.Sync, initialCycle, ms.Log, ms.BlockReader, hook) if err != nil { return err } - SendPayloadStatus(ms.HeaderDownload(), headBlockHash, err) + SendPayloadStatus(ms.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) ms.ReceivePayloadStatus() fc := engineapi.ForkChoiceMessage{ @@ -624,39 +623,46 @@ func (ms *MockSentry) insertPoSBlocks(chain *core.ChainPack) error { FinalizedBlockHash: chain.TopBlock.Hash(), } ms.SendForkChoiceRequest(&fc) - headBlockHash, err = StageLoopStep(ms.Ctx, ms.DB, ms.Sync, initialCycle, ms.Log, blockReader.Snapshots().(*snapshotsync.RoSnapshots), hook) + err = StageLoopStep(ms.Ctx, ms.DB, tx, ms.Sync, initialCycle, ms.Log, ms.BlockReader, hook) if err != nil { return err } - SendPayloadStatus(ms.HeaderDownload(), headBlockHash, err) + SendPayloadStatus(ms.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) ms.ReceivePayloadStatus() return nil } -func (ms *MockSentry) InsertChain(chain *core.ChainPack) error { - if err := ms.insertPoWBlocks(chain); err != nil { +func (ms *MockSentry) InsertChain(chain *core.ChainPack, tx kv.RwTx) error { + externalTx := tx != nil + if !externalTx { + var err error + tx, err = ms.DB.BeginRw(ms.Ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err := ms.insertPoWBlocks(chain, tx); err != nil { return err } - if err := ms.insertPoSBlocks(chain); err != nil { + if err := ms.insertPoSBlocks(chain, tx); err != nil { return err } + // Check if the latest header was imported or rolled back - if err := ms.DB.View(ms.Ctx, func(tx kv.Tx) error { - if rawdb.ReadHeader(tx, chain.TopBlock.Hash(), chain.TopBlock.NumberU64()) == nil { - return fmt.Errorf("did not import block %d %x", chain.TopBlock.NumberU64(), chain.TopBlock.Hash()) - } - execAt, err := stages.GetStageProgress(tx, stages.Execution) - if err != nil { - return err - } - if execAt == 0 { - return fmt.Errorf("sentryMock.InsertChain end up with Execution stage progress = 0") - } - return nil - }); err != nil { + if rawdb.ReadHeader(tx, chain.TopBlock.Hash(), chain.TopBlock.NumberU64()) == nil { + return fmt.Errorf("did not import block %d %x", chain.TopBlock.NumberU64(), chain.TopBlock.Hash()) + } + execAt, err := stages.GetStageProgress(tx, stages.Execution) + if err != nil { return err } + if execAt == 0 { + return fmt.Errorf("sentryMock.InsertChain end up with Execution stage progress = 0") + } + if ms.sentriesClient.Hd.IsBadHeader(chain.TopBlock.Hash()) { return fmt.Errorf("block %d %x was invalid", chain.TopBlock.NumberU64(), chain.TopBlock.Hash()) } @@ -674,6 +680,12 @@ func (ms *MockSentry) InsertChain(chain *core.ChainPack) error { // return err //} //} + + if !externalTx { + if err := tx.Commit(); err != nil { + return err + } + } return nil } @@ -730,6 +742,6 @@ func (ms *MockSentry) HistoryV3Components() *libstate.AggregatorV3 { return ms.agg } -func (ms *MockSentry) NewBlocksIO() (services.FullBlockReader, *blockio.BlockWriter) { - return snapshotsync.NewBlockReader(ms.BlockSnapshots), blockio.NewBlockWriter(ms.HistoryV3) +func (ms *MockSentry) BlocksIO() (services.FullBlockReader, *blockio.BlockWriter) { + return ms.BlockReader, blockio.NewBlockWriter(ms.HistoryV3) } diff --git a/turbo/stages/sentry_mock_test.go b/turbo/stages/sentry_mock_test.go index aa1cedf484c..b3db4a46b75 100644 --- a/turbo/stages/sentry_mock_test.go +++ b/turbo/stages/sentry_mock_test.go @@ -8,6 +8,7 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/gointerfaces/remote" "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" + "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -58,7 +59,7 @@ func TestHeaderStep(t *testing.T) { m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceed initialCycle := stages.MockInsertAsInitialCycle - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil); err != nil { + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, nil); err != nil { t.Fatal(err) } } @@ -96,7 +97,7 @@ func TestMineBlockWith1Tx(t *testing.T) { m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed initialCycle := stages.MockInsertAsInitialCycle - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, log.New(), m.BlockSnapshots, nil); err != nil { + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, log.New(), m.BlockReader, nil); err != nil { t.Fatal(err) } } @@ -163,8 +164,8 @@ func TestReorg(t *testing.T) { } m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed - initialCycle := true - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil); err != nil { + initialCycle := stages.MockInsertAsInitialCycle + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, nil); err != nil { t.Fatal(err) } @@ -217,7 +218,7 @@ func TestReorg(t *testing.T) { m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed initialCycle = false - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil); err != nil { + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, nil); err != nil { t.Fatal(err) } @@ -260,7 +261,7 @@ func TestReorg(t *testing.T) { m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed // This is unwind step - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil); err != nil { + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, nil); err != nil { t.Fatal(err) } @@ -296,8 +297,8 @@ func TestReorg(t *testing.T) { } m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed - initialCycle = false - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil); err != nil { + initialCycle = stages.MockInsertAsInitialCycle + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, nil); err != nil { t.Fatal(err) } } @@ -392,8 +393,8 @@ func TestAnchorReplace(t *testing.T) { m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed - initialCycle := true - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil); err != nil { + initialCycle := stages.MockInsertAsInitialCycle + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, nil); err != nil { t.Fatal(err) } } @@ -496,10 +497,9 @@ func TestAnchorReplace2(t *testing.T) { m.ReceiveWg.Wait() // Wait for all messages to be processed before we proceeed - br, _ := m.NewBlocksIO() - initialCycle := true - hook := stages.NewHook(m.Ctx, m.Notifications, m.Sync, br, m.ChainConfig, m.Log, m.UpdateHead) - if _, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, hook); err != nil { + initialCycle := stages.MockInsertAsInitialCycle + hook := stages.NewHook(m.Ctx, m.Notifications, m.Sync, m.BlockReader, m.ChainConfig, m.Log, m.UpdateHead) + if err := stages.StageLoopStep(m.Ctx, m.DB, nil, m.Sync, initialCycle, m.Log, m.BlockReader, hook); err != nil { t.Fatal(err) } } @@ -516,11 +516,14 @@ func TestForkchoiceToGenesis(t *testing.T) { m.SendForkChoiceRequest(&forkChoiceMessage) initialCycle := stages.MockInsertAsInitialCycle - headBlockHash, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + tx, err := m.DB.BeginRw(m.Ctx) + require.NoError(t, err) + defer tx.Rollback() + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) - assert.Equal(t, m.Genesis.Hash(), headBlockHash) + assert.Equal(t, m.Genesis.Hash(), rawdb.ReadHeadBlockHash(tx)) payloadStatus := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus.Status) @@ -537,10 +540,13 @@ func TestBogusForkchoice(t *testing.T) { } m.SendForkChoiceRequest(&forkChoiceMessage) + tx, err := m.DB.BeginRw(m.Ctx) + require.NoError(t, err) + defer tx.Rollback() initialCycle := stages.MockInsertAsInitialCycle - headBlockHash, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_SYNCING, payloadStatus.Status) @@ -553,9 +559,9 @@ func TestBogusForkchoice(t *testing.T) { } m.SendForkChoiceRequest(&forkChoiceMessage) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus.Status) @@ -572,10 +578,13 @@ func TestPoSDownloader(t *testing.T) { // Send a payload whose parent isn't downloaded yet m.SendPayloadRequest(chain.TopBlock) + tx, err := m.DB.BeginRw(m.Ctx) + require.NoError(t, err) + defer tx.Rollback() initialCycle := stages.MockInsertAsInitialCycle - headBlockHash, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_SYNCING, payloadStatus.Status) @@ -593,14 +602,14 @@ func TestPoSDownloader(t *testing.T) { m.ReceiveWg.Wait() // First cycle: save the downloaded header - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) // Second cycle: process the previous beacon request - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) // Point forkChoice to the head forkChoiceMessage := engineapi.ForkChoiceMessage{ @@ -609,14 +618,14 @@ func TestPoSDownloader(t *testing.T) { FinalizedBlockHash: chain.TopBlock.Hash(), } m.SendForkChoiceRequest(&forkChoiceMessage) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) - assert.Equal(t, chain.TopBlock.Hash(), headBlockHash) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) + assert.Equal(t, chain.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) payloadStatus = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus.Status) - assert.Equal(t, chain.TopBlock.Hash(), headBlockHash) + assert.Equal(t, chain.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) } // https://hackmd.io/GDc0maGsQeKfP8o2C7L52w @@ -640,10 +649,13 @@ func TestPoSSyncWithInvalidHeader(t *testing.T) { payloadMessage := types.NewBlockFromStorage(invalidTip.Hash(), invalidTip, chain.TopBlock.Transactions(), nil, nil) m.SendPayloadRequest(payloadMessage) + tx, err := m.DB.BeginRw(m.Ctx) + require.NoError(t, err) + defer tx.Rollback() initialCycle := stages.MockInsertAsInitialCycle - headBlockHash, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus1 := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_SYNCING, payloadStatus1.Status) @@ -660,9 +672,9 @@ func TestPoSSyncWithInvalidHeader(t *testing.T) { } m.ReceiveWg.Wait() - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) // Point forkChoice to the invalid tip forkChoiceMessage := engineapi.ForkChoiceMessage{ @@ -671,7 +683,7 @@ func TestPoSSyncWithInvalidHeader(t *testing.T) { FinalizedBlockHash: invalidTip.Hash(), } m.SendForkChoiceRequest(&forkChoiceMessage) - _, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(t, err) bad, lastValidHash := m.HeaderDownload().IsBadHeaderPoS(invalidTip.Hash()) @@ -726,10 +738,15 @@ func TestPOSWrongTrieRootReorgs(t *testing.T) { //------------------------------------------ m.SendPayloadRequest(chain0.TopBlock) + tx, err := m.DB.BeginRw(m.Ctx) + + require.NoError(err) + defer tx.Rollback() initialCycle := stages.MockInsertAsInitialCycle - headBlockHash, err := stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) + payloadStatus0 := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus0.Status) forkChoiceMessage := engineapi.ForkChoiceMessage{ @@ -738,19 +755,19 @@ func TestPOSWrongTrieRootReorgs(t *testing.T) { FinalizedBlockHash: chain0.TopBlock.Hash(), } m.SendForkChoiceRequest(&forkChoiceMessage) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) - assert.Equal(t, chain0.TopBlock.Hash(), headBlockHash) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) + assert.Equal(t, chain0.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) payloadStatus0 = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus0.Status) - assert.Equal(t, chain0.TopBlock.Hash(), headBlockHash) + assert.Equal(t, chain0.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) //------------------------------------------ m.SendPayloadRequest(chain1.TopBlock) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus1 := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus1.Status) forkChoiceMessage = engineapi.ForkChoiceMessage{ @@ -759,19 +776,19 @@ func TestPOSWrongTrieRootReorgs(t *testing.T) { FinalizedBlockHash: chain1.TopBlock.Hash(), } m.SendForkChoiceRequest(&forkChoiceMessage) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) - assert.Equal(t, chain1.TopBlock.Hash(), headBlockHash) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) + assert.Equal(t, chain1.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) payloadStatus1 = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus1.Status) - assert.Equal(t, chain1.TopBlock.Hash(), headBlockHash) + assert.Equal(t, chain1.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) //------------------------------------------ m.SendPayloadRequest(chain2.TopBlock) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus2 := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus2.Status) forkChoiceMessage = engineapi.ForkChoiceMessage{ @@ -780,26 +797,26 @@ func TestPOSWrongTrieRootReorgs(t *testing.T) { FinalizedBlockHash: chain2.TopBlock.Hash(), } m.SendForkChoiceRequest(&forkChoiceMessage) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) - assert.Equal(t, chain2.TopBlock.Hash(), headBlockHash) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) + assert.Equal(t, chain2.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) payloadStatus2 = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus2.Status) - assert.Equal(t, chain2.TopBlock.Hash(), headBlockHash) + assert.Equal(t, chain2.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) //------------------------------------------ preTop3 := chain3.Blocks[chain3.Length()-2] m.SendPayloadRequest(preTop3) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus3 := m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus3.Status) m.SendPayloadRequest(chain3.TopBlock) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) payloadStatus3 = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus3.Status) forkChoiceMessage = engineapi.ForkChoiceMessage{ @@ -808,11 +825,11 @@ func TestPOSWrongTrieRootReorgs(t *testing.T) { FinalizedBlockHash: chain3.TopBlock.Hash(), } m.SendForkChoiceRequest(&forkChoiceMessage) - headBlockHash, err = stages.StageLoopStep(m.Ctx, m.DB, m.Sync, initialCycle, m.Log, m.BlockSnapshots, nil) + err = stages.StageLoopStep(m.Ctx, m.DB, tx, m.Sync, initialCycle, m.Log, m.BlockReader, nil) require.NoError(err) - stages.SendPayloadStatus(m.HeaderDownload(), headBlockHash, err) - assert.Equal(t, chain3.TopBlock.Hash(), headBlockHash) + stages.SendPayloadStatus(m.HeaderDownload(), rawdb.ReadHeadBlockHash(tx), err) + assert.Equal(t, chain3.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) payloadStatus3 = m.ReceivePayloadStatus() assert.Equal(t, remote.EngineStatus_VALID, payloadStatus3.Status) - assert.Equal(t, chain3.TopBlock.Hash(), headBlockHash) + assert.Equal(t, chain3.TopBlock.Hash(), rawdb.ReadHeadBlockHash(tx)) } diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 54865393cbb..dfb43a9c746 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -32,7 +32,6 @@ import ( "github.com/ledgerwatch/erigon/p2p" "github.com/ledgerwatch/erigon/turbo/engineapi" "github.com/ledgerwatch/erigon/turbo/shards" - "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/erigon/turbo/stages/bodydownload" "github.com/ledgerwatch/erigon/turbo/stages/headerdownload" ) @@ -73,7 +72,7 @@ func StageLoop(ctx context.Context, waitForDone chan struct{}, loopMinTime time.Duration, logger log.Logger, - blockSnapshots *snapshotsync.RoSnapshots, + blockReader services.FullBlockReader, hook *Hook, ) { defer close(waitForDone) @@ -90,18 +89,20 @@ func StageLoop(ctx context.Context, } // Estimate the current top height seen from the peer - headBlockHash, err := StageLoopStep(ctx, db, sync, initialCycle, logger, blockSnapshots, hook) - - SendPayloadStatus(hd, headBlockHash, err) + err := StageLoopStep(ctx, db, nil, sync, initialCycle, logger, blockReader, hook) + db.View(ctx, func(tx kv.Tx) error { + SendPayloadStatus(hd, rawdb.ReadHeadBlockHash(tx), err) + return nil + }) if err != nil { if errors.Is(err, libcommon.ErrStopped) || errors.Is(err, context.Canceled) { return } - log.Error("Staged Sync", "err", err) + logger.Error("Staged Sync", "err", err) if recoveryErr := hd.RecoverFromDb(db); recoveryErr != nil { - log.Error("Failed to recover header sentriesClient", "err", recoveryErr) + logger.Error("Failed to recover header sentriesClient", "err", recoveryErr) } time.Sleep(500 * time.Millisecond) // just to avoid too much similar errors in logs continue @@ -112,7 +113,7 @@ func StageLoop(ctx context.Context, if loopMinTime != 0 { waitTime := loopMinTime - time.Since(start) - log.Info("Wait time until next loop", "for", waitTime) + logger.Info("Wait time until next loop", "for", waitTime) c := time.After(waitTime) select { case <-ctx.Done(): @@ -123,34 +124,26 @@ func StageLoop(ctx context.Context, } } -func StageLoopStep(ctx context.Context, db kv.RwDB, sync *stagedsync.Sync, initialCycle bool, logger log.Logger, blockSnapshots *snapshotsync.RoSnapshots, hook *Hook) (headBlockHash libcommon.Hash, err error) { +func StageLoopStep(ctx context.Context, db kv.RwDB, tx kv.RwTx, sync *stagedsync.Sync, initialCycle bool, logger log.Logger, blockReader services.FullBlockReader, hook *Hook) (err error) { defer func() { if rec := recover(); rec != nil { err = fmt.Errorf("%+v, trace: %s", rec, dbg.Stack()) } }() // avoid crash because Erigon's core does many things - var finishProgressBefore, headersProgressBefore uint64 - if err := db.View(ctx, func(tx kv.Tx) error { - if finishProgressBefore, err = stages.GetStageProgress(tx, stages.Finish); err != nil { - return err - } - if headersProgressBefore, err = stages.GetStageProgress(tx, stages.Headers); err != nil { - return err - } - return nil - }); err != nil { - return headBlockHash, err + externalTx := tx != nil + finishProgressBefore, headersProgressBefore, err := stagesHeadersAndFinish(db, tx) + if err != nil { + return err } // Sync from scratch must be able Commit partial progress // In all other cases - process blocks batch in 1 RwTx - blocksInSnapshots := uint64(0) - if blockSnapshots != nil { - blocksInSnapshots = blockSnapshots.BlocksAvailable() - } // 2 corner-cases: when sync with --snapshots=false and when executed only blocks from snapshots (in this case all stages progress is equal and > 0, but node is not synced) - isSynced := finishProgressBefore > 0 && finishProgressBefore > blocksInSnapshots && finishProgressBefore == headersProgressBefore + isSynced := finishProgressBefore > 0 && finishProgressBefore > blockReader.FrozenBlocks() && finishProgressBefore == headersProgressBefore canRunCycleInOneTransaction := isSynced + if externalTx { + canRunCycleInOneTransaction = true + } // Main steps: // - process new blocks @@ -159,57 +152,60 @@ func StageLoopStep(ctx context.Context, db kv.RwDB, sync *stagedsync.Sync, initi // - Send Notifications: about new blocks, new receipts, state changes, etc... // - Prune(limited time)+Commit(sync). Write to disk happening here. - var tx kv.RwTx // on this variable will run sync cycle. - if canRunCycleInOneTransaction { + if canRunCycleInOneTransaction && !externalTx { tx, err = db.BeginRwNosync(ctx) if err != nil { - return headBlockHash, err + return err } defer tx.Rollback() } if hook != nil { - if err = hook.BeforeRun(tx, canRunCycleInOneTransaction); err != nil { - return headBlockHash, err + if err = hook.BeforeRun(tx, isSynced); err != nil { + return err } } err = sync.Run(db, tx, initialCycle) if err != nil { - return headBlockHash, err + return err } logCtx := sync.PrintTimings() var tableSizes []interface{} var commitTime time.Duration - if canRunCycleInOneTransaction { + if canRunCycleInOneTransaction && !externalTx { tableSizes = stagedsync.PrintTables(db, tx) // Need to do this before commit to access tx commitStart := time.Now() errTx := tx.Commit() + tx = nil if errTx != nil { - return headBlockHash, errTx + return errTx } commitTime = time.Since(commitStart) } // -- send notifications START - var head uint64 - if err := db.View(ctx, func(tx kv.Tx) error { - headBlockHash = rawdb.ReadHeadBlockHash(tx) - if head, err = stages.GetStageProgress(tx, stages.Headers); err != nil { - return err - } + if externalTx { if hook != nil { if err = hook.AfterRun(tx, finishProgressBefore); err != nil { return err } } - return nil - }); err != nil { - return headBlockHash, err + } else { + if err := db.View(ctx, func(tx kv.Tx) error { + if hook != nil { + if err = hook.AfterRun(tx, finishProgressBefore); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } } - if canRunCycleInOneTransaction && (head != finishProgressBefore || commitTime > 500*time.Millisecond) { + if canRunCycleInOneTransaction && !externalTx && commitTime > 500*time.Millisecond { logger.Info("Commit cycle", "in", commitTime) } - if head != finishProgressBefore && len(logCtx) > 0 { // No printing of timings or table sizes if there were no progress + if len(logCtx) > 0 { // No printing of timings or table sizes if there were no progress logger.Info("Timings (slower than 50ms)", logCtx...) if len(tableSizes) > 0 { logger.Info("Tables", tableSizes...) @@ -218,11 +214,41 @@ func StageLoopStep(ctx context.Context, db kv.RwDB, sync *stagedsync.Sync, initi // -- send notifications END // -- Prune+commit(sync) - if err := db.Update(ctx, func(tx kv.RwTx) error { return sync.RunPrune(db, tx, initialCycle) }); err != nil { - return headBlockHash, err + if err := stageLoopStepPrune(ctx, db, tx, sync, initialCycle); err != nil { + return err } - return headBlockHash, nil + return nil +} +func stageLoopStepPrune(ctx context.Context, db kv.RwDB, tx kv.RwTx, sync *stagedsync.Sync, initialCycle bool) (err error) { + if tx != nil { + return sync.RunPrune(db, tx, initialCycle) + } + return db.Update(ctx, func(tx kv.RwTx) error { return sync.RunPrune(db, tx, initialCycle) }) +} + +func stagesHeadersAndFinish(db kv.RoDB, tx kv.Tx) (head, fin uint64, err error) { + if tx != nil { + if fin, err = stages.GetStageProgress(tx, stages.Finish); err != nil { + return head, fin, err + } + if head, err = stages.GetStageProgress(tx, stages.Headers); err != nil { + return head, fin, err + } + return head, fin, nil + } + if err := db.View(context.Background(), func(tx kv.Tx) error { + if fin, err = stages.GetStageProgress(tx, stages.Finish); err != nil { + return err + } + if head, err = stages.GetStageProgress(tx, stages.Headers); err != nil { + return err + } + return nil + }); err != nil { + return head, fin, err + } + return head, fin, nil } type Hook struct { @@ -238,9 +264,9 @@ type Hook struct { func NewHook(ctx context.Context, notifications *shards.Notifications, sync *stagedsync.Sync, blockReader services.FullBlockReader, chainConfig *chain.Config, logger log.Logger, updateHead func(ctx context.Context, headHeight uint64, headTime uint64, hash libcommon.Hash, td *uint256.Int)) *Hook { return &Hook{ctx: ctx, notifications: notifications, sync: sync, blockReader: blockReader, chainConfig: chainConfig, logger: logger, updateHead: updateHead} } -func (h *Hook) BeforeRun(tx kv.Tx, canRunCycleInOneTransaction bool) error { +func (h *Hook) BeforeRun(tx kv.Tx, inSync bool) error { notifications := h.notifications - if notifications != nil && notifications.Accumulator != nil && canRunCycleInOneTransaction { + if notifications != nil && notifications.Accumulator != nil && inSync { stateVersion, err := rawdb.GetStateVersion(tx) if err != nil { h.logger.Error("problem reading plain state version", "err", err) @@ -352,10 +378,10 @@ func StateStep(ctx context.Context, batch kv.RwTx, blockWriter *blockio.BlockWri currentHash := headersChain[i].Hash() // Prepare memory state for block execution Bd.AddToPrefetch(currentHeader, currentBody) - if err := blockWriter.WriteHeader(batch, currentHeader); err != nil { + if err := rawdb.WriteHeader(batch, currentHeader); err != nil { return err } - if err := blockWriter.WriteCanonicalHash(batch, currentHash, currentHeight); err != nil { + if err := rawdb.WriteCanonicalHash(batch, currentHash, currentHeight); err != nil { return err } } @@ -368,10 +394,10 @@ func StateStep(ctx context.Context, batch kv.RwTx, blockWriter *blockio.BlockWri height := header.Number.Uint64() hash := header.Hash() // Prepare memory state for block execution - if err = blockWriter.WriteHeader(batch, header); err != nil { + if err = rawdb.WriteHeader(batch, header); err != nil { return err } - if err = blockWriter.WriteCanonicalHash(batch, hash, height); err != nil { + if err = rawdb.WriteCanonicalHash(batch, hash, height); err != nil { return err } @@ -400,13 +426,13 @@ func NewDefaultStages(ctx context.Context, notifications *shards.Notifications, snapDownloader proto_downloader.DownloaderClient, blockReader services.FullBlockReader, + blockRetire services.BlockRetire, agg *state.AggregatorV3, forkValidator *engineapi.ForkValidator, logger log.Logger, ) []*stagedsync.Stage { dirs := cfg.Dirs blockWriter := blockio.NewBlockWriter(cfg.HistoryV3) - blockRetire := snapshotsync.NewBlockRetire(1, dirs.Tmp, blockReader, blockWriter, db, snapDownloader, notifications.Events, logger) // During Import we don't want other services like header requests, body requests etc. to be running. // Hence we run it in the test mode. @@ -418,7 +444,7 @@ func NewDefaultStages(ctx context.Context, stagedsync.StageCumulativeIndexCfg(db, blockReader), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageBodiesCfg(db, controlServer.Bd, controlServer.SendBodyRequest, controlServer.Penalize, controlServer.BroadcastNewBlock, cfg.Sync.BodyDownloadTimeoutSeconds, *controlServer.ChainConfig, blockReader, cfg.HistoryV3, blockWriter), - stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockRetire, blockWriter, blockReader, controlServer.Hd), + stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg( db, cfg.Prune, @@ -449,16 +475,15 @@ func NewDefaultStages(ctx context.Context, } func NewInMemoryExecution(ctx context.Context, db kv.RwDB, cfg *ethconfig.Config, controlServer *sentry.MultiClient, - dirs datadir.Dirs, notifications *shards.Notifications, snapshots *snapshotsync.RoSnapshots, agg *state.AggregatorV3, + dirs datadir.Dirs, notifications *shards.Notifications, blockReader services.FullBlockReader, blockWriter *blockio.BlockWriter, agg *state.AggregatorV3, logger log.Logger) (*stagedsync.Sync, error) { - blockReader, blockWriter := snapshotsync.NewBlockReader(snapshots), blockio.NewBlockWriter(cfg.HistoryV3) return stagedsync.New( stagedsync.StateStages(ctx, stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, false, blockReader, blockWriter, dirs.Tmp, nil, nil), stagedsync.StageBodiesCfg(db, controlServer.Bd, controlServer.SendBodyRequest, controlServer.Penalize, controlServer.BroadcastNewBlock, cfg.Sync.BodyDownloadTimeoutSeconds, *controlServer.ChainConfig, blockReader, cfg.HistoryV3, blockWriter), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), - stagedsync.StageSendersCfg(db, controlServer.ChainConfig, true, dirs.Tmp, cfg.Prune, nil, blockWriter, blockReader, controlServer.Hd), + stagedsync.StageSendersCfg(db, controlServer.ChainConfig, true, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg( db, cfg.Prune, diff --git a/turbo/transactions/tracing.go b/turbo/transactions/tracing.go index dd9704570ed..de49b4e0297 100644 --- a/turbo/transactions/tracing.go +++ b/turbo/transactions/tracing.go @@ -60,7 +60,7 @@ func ComputeTxEnv(ctx context.Context, engine consensus.EngineReader, block *typ blockContext.L1CostFunc = types.NewL1CostFunc(cfg, statedb) // Recompute transactions up to the target index. - signer := types.MakeSigner(cfg, block.NumberU64()) + signer := types.MakeSigner(cfg, block.NumberU64(), block.Time()) if historyV3 { rules := cfg.Rules(blockContext.BlockNumber, blockContext.Time) txn := block.Transactions()[txIndex]