feature/rpc-plugins

.github/workflows/go.yml

@@ -0,0 +1,64 @@
+name: Go
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Test
+      run: go test ./core ./miner/... ./internal/ethapi/... ./les/...
+
+    - name: Build
+      run: make geth
+
+  e2e:
+    name: End to End
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13
+      id: go
+
+    - name: Use Node.js 12.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12.x
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Build
+      run: make geth
+
+    - name: Check out the e2e code repo
+      uses: actions/checkout@v2
+      with:
+        repository: flashbots/mev-geth-demo
+        path: e2e
+
+    - run: cd e2e && yarn install
+    - run: |
+        cd e2e
+        GETH=`pwd`/../build/bin/geth ./run.sh &
+        sleep 15
+        yarn run demo-simple
+        yarn run demo-contract

cmd/geth/main.go

@@ -125,6 +125,7 @@ var (
 		utils.MinerExtraDataFlag,
 		utils.MinerRecommitIntervalFlag,
 		utils.MinerNoVerfiyFlag,
+		utils.MinerMaxMergedBundles,
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,

cmd/geth/usage.go

@@ -151,6 +151,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.GraphQLVirtualHostsFlag,
 			utils.RPCGlobalGasCapFlag,
 			utils.RPCGlobalTxFeeCapFlag,
+			utils.RPCPluginsFlag,
 			utils.AllowUnprotectedTxs,
 			utils.JSpathFlag,
 			utils.ExecFlag,
@@ -186,6 +187,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.MinerExtraDataFlag,
 			utils.MinerRecommitIntervalFlag,
 			utils.MinerNoVerfiyFlag,
+			utils.MinerMaxMergedBundles,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -460,6 +460,11 @@ var (
 		Usage: "Time interval to recreate the block being mined",
 		Value: ethconfig.Defaults.Miner.Recommit,
 	}
+	MinerMaxMergedBundles = cli.IntFlag{
+		Name:  "miner.maxmergedbundles",
+		Usage: "flashbots - The maximum amount of bundles to merge. The miner will run this many workers in parallel to calculate if the full block is more profitable with these additional bundles.",
+		Value: 3,
+	}
 	MinerNoVerfiyFlag = cli.BoolFlag{
 		Name:  "miner.noverify",
 		Usage: "Disable remote sealing verification",
@@ -498,6 +503,11 @@ var (
 		Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)",
 		Value: ethconfig.Defaults.RPCTxFeeCap,
 	}
+	RPCPluginsFlag = cli.StringFlag{
+		Name:  "rpc.plugins",
+		Usage: "List of rpc endpoint plugins to add",
+		Value: "",
+	}
 	// Logging and debug settings
 	EthStatsURLFlag = cli.StringFlag{
 		Name:  "ethstats",
@@ -1404,6 +1414,8 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
 	if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) {
 		log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!")
 	}
+
+	cfg.MaxMergedBundles = ctx.GlobalInt(MinerMaxMergedBundles.Name)
 }
 
 func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) {
@@ -1590,6 +1602,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(RPCGlobalTxFeeCapFlag.Name) {
 		cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name)
 	}
+	if ctx.GlobalIsSet(RPCPluginsFlag.Name) {
+		cfg.RPCPlugins = strings.Split(ctx.String(RPCPluginsFlag.Name), ",")
+	}
 	if ctx.GlobalIsSet(NoDiscoverFlag.Name) {
 		cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{}
 	} else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) {

core/tx_pool.go

@@ -242,11 +242,12 @@ type TxPool struct {
 	locals  *accountSet // Set of local transaction to exempt from eviction rules
 	journal *txJournal  // Journal of local transaction to back up to disk
 
-	pending map[common.Address]*txList   // All currently processable transactions
-	queue   map[common.Address]*txList   // Queued but non-processable transactions
-	beats   map[common.Address]time.Time // Last heartbeat from each known account
-	all     *txLookup                    // All transactions to allow lookups
-	priced  *txPricedList                // All transactions sorted by price
+	pending    map[common.Address]*txList   // All currently processable transactions
+	queue      map[common.Address]*txList   // Queued but non-processable transactions
+	beats      map[common.Address]time.Time // Last heartbeat from each known account
+	mevBundles []types.MevBundle
+	all        *txLookup     // All transactions to allow lookups
+	priced     *txPricedList // All transactions sorted by price
 
 	chainHeadCh     chan ChainHeadEvent
 	chainHeadSub    event.Subscription
@@ -542,6 +543,59 @@ func (pool *TxPool) Pending(enforceTips bool) (map[common.Address]types.Transact
 	return pending, nil
 }
 
+/// AllMevBundles returns all the MEV Bundles currently in the pool
+func (pool *TxPool) AllMevBundles() []types.MevBundle {
+	return pool.mevBundles
+}
+
+// MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp
+// also prunes bundles that are outdated
+func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.MevBundle, error) {
+	pool.mu.Lock()
+	defer pool.mu.Unlock()
+
+	// returned values
+	var ret []types.MevBundle
+	// rolled over values
+	var bundles []types.MevBundle
+
+	for _, bundle := range pool.mevBundles {
+		// Prune outdated bundles
+		if (bundle.MaxTimestamp != 0 && blockTimestamp > bundle.MaxTimestamp) || blockNumber.Cmp(bundle.BlockNumber) > 0 {
+			continue
+		}
+
+		// Roll over future bundles
+		if (bundle.MinTimestamp != 0 && blockTimestamp < bundle.MinTimestamp) || blockNumber.Cmp(bundle.BlockNumber) < 0 {
+			bundles = append(bundles, bundle)
+			continue
+		}
+
+		// return the ones which are in time
+		ret = append(ret, bundle)
+		// keep the bundles around internally until they need to be pruned
+		bundles = append(bundles, bundle)
+	}
+
+	pool.mevBundles = bundles
+	return ret, nil
+}
+
+// AddMevBundle adds a mev bundle to the pool
+func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp, maxTimestamp uint64, revertingTxHashes []common.Hash) error {
+	pool.mu.Lock()
+	defer pool.mu.Unlock()
+
+	pool.mevBundles = append(pool.mevBundles, types.MevBundle{
+		Txs:               txs,
+		BlockNumber:       blockNumber,
+		MinTimestamp:      minTimestamp,
+		MaxTimestamp:      maxTimestamp,
+		RevertingTxHashes: revertingTxHashes,
+	})
+	return nil
+}
+
 // Locals retrieves the accounts currently considered local by the pool.
 func (pool *TxPool) Locals() []common.Address {
 	pool.mu.Lock()

core/tx_pool_test.go

@@ -2541,3 +2541,13 @@ func BenchmarkInsertRemoteWithAllLocals(b *testing.B) {
 		pool.Stop()
 	}
 }
+
+func checkBundles(t *testing.T, pool *TxPool, block int64, timestamp uint64, expectedRes int, expectedRemaining int) {
+	res, _ := pool.MevBundles(big.NewInt(block), timestamp)
+	if len(res) != expectedRes {
+		t.Fatalf("expected returned bundles did not match got %d, expected %d", len(res), expectedRes)
+	}
+	if len(pool.mevBundles) != expectedRemaining {
+		t.Fatalf("expected remaining bundles did not match got %d, expected %d", len(pool.mevBundles), expectedRemaining)
+	}
+}

core/types/transaction.go

@@ -632,3 +632,11 @@ func (m Message) Nonce() uint64          { return m.nonce }
 func (m Message) Data() []byte           { return m.data }
 func (m Message) AccessList() AccessList { return m.accessList }
 func (m Message) IsFake() bool           { return m.isFake }
+
+type MevBundle struct {
+	Txs               Transactions
+	BlockNumber       *big.Int
+	MinTimestamp      uint64
+	MaxTimestamp      uint64
+	RevertingTxHashes []common.Hash
+}

eth/api_backend.go

@@ -234,6 +234,10 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction)
 	return b.eth.txPool.AddLocal(signedTx)
 }
 
+func (b *EthAPIBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error {
+	return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp, revertingTxHashes)
+}
+
 func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) {
 	pending, err := b.eth.txPool.Pending(false)
 	if err != nil {

eth/backend.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"plugin"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -254,6 +255,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 
 	// Register the backend on the node
 	stack.RegisterAPIs(eth.APIs())
+	stack.RegisterAPIs(loadRPCPlugins(config.RPCPlugins, eth))
 	stack.RegisterProtocols(eth.Protocols())
 	stack.RegisterLifecycle(eth)
 	// Check for unclean shutdown
@@ -272,6 +274,39 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	return eth, nil
 }
 
+func loadRPCPlugins(plugins []string, eth *Ethereum) []rpc.API {
+	apis := make([]rpc.API, 0)
+
+	for _, path := range plugins {
+		p, err := plugin.Open(path)
+		if err != nil {
+			// @TODO
+			continue
+		}
+
+		v, err := p.Lookup("Register")
+		if err != nil {
+			// @TODO
+			continue
+		}
+
+		api, ok := v.(Plugin)
+		if !ok {
+			// @TODO
+			continue
+		}
+
+		apis = append(apis, rpc.API{
+			Namespace: api.Namespace(),
+			Version:   api.Version(),
+			Service:   api.Service(eth),
+			Public:    true,
+		})
+	}
+
+	return apis
+}
+
 func makeExtraData(extra []byte) []byte {
 	if len(extra) == 0 {
 		// create default extradata
@@ -563,3 +598,10 @@ func (s *Ethereum) Stop() error {
 
 	return nil
 }
+
+// Plugin describes an RPC endpoint that can be added to the node as a plugin.
+type Plugin interface {
+	Namespace() string
+	Version() string
+	Service(ethereum *Ethereum) interface{}
+}

eth/ethconfig/config.go

@@ -192,6 +192,9 @@ type Config struct {
 	// send-transction variants. The unit is ether.
 	RPCTxFeeCap float64
 
+	// RPCPlugins are the paths to the custom RPC plugins.
+	RPCPlugins []string
+
 	// Checkpoint is a hardcoded checkpoint which can be nil.
 	Checkpoint *params.TrustedCheckpoint `toml:",omitempty"`
 

infra/Dockerfile.node

@@ -0,0 +1,23 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.15-alpine as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers git
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make geth
+
+# Pull Geth into a second stage deploy alpine container
+FROM alpine:latest
+
+ENV PYTHONUNBUFFERED=1
+RUN apk add --update --no-cache groff less python3 curl jq ca-certificates && ln -sf python3 /usr/bin/python
+RUN python3 -m ensurepip
+RUN pip3 install --no-cache --upgrade pip setuptools awscli
+
+COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
+
+COPY ./infra/start-mev-geth-node.sh /root/start-mev-geth-node.sh
+RUN chmod 755 /root/start-mev-geth-node.sh
+
+EXPOSE 8545 8546 30303 30303/udp
+ENTRYPOINT ["/root/start-mev-geth-node.sh"]

infra/Dockerfile.updater

@@ -0,0 +1,23 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.15-alpine as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers git
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make geth
+
+# Pull Geth into a second stage deploy alpine container
+FROM alpine:latest
+
+ENV PYTHONUNBUFFERED=1
+RUN apk add --update --no-cache groff less python3 curl jq ca-certificates && ln -sf python3 /usr/bin/python
+RUN python3 -m ensurepip
+RUN pip3 install --no-cache --upgrade pip setuptools awscli
+
+COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
+
+COPY ./infra/start-mev-geth-updater.sh /root/start-mev-geth-updater.sh
+RUN chmod 755 /root/start-mev-geth-updater.sh
+
+EXPOSE 8545 8546 30303 30303/udp
+ENTRYPOINT ["/root/start-mev-geth-updater.sh"]