feat(f3): prepare for f3 bootstrap

chain/lf3/config.go

@@ -8,41 +8,89 @@ import (
 
 	"github.com/filecoin-project/go-f3/gpbft"
 	"github.com/filecoin-project/go-f3/manifest"
+	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/build/buildconstants"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 type Config struct {
-	InitialManifest         *manifest.Manifest
+	// BaseNetworkName is the base from which dynamic network names are defined and is usually
+	// the name of the network defined by the static manifest. This must be set correctly or,
+	// e.g., pubsub topic filters won't work correctly.
+	BaseNetworkName gpbft.NetworkName
+	// StaticManifest this instance's default manifest absent any dynamic manifests. Also see
+	// PrioritizeStaticManifest.
+	StaticManifest *manifest.Manifest
+	// DynamicManifestProvider is the peer ID of the peer authorized to send us dynamic manifest
+	// updates. Dynamic manifest updates can be used for testing but will not be used to affect
+	// finality.
 	DynamicManifestProvider peer.ID
+	// PrioritizeStaticManifest means that, once we get within one finality of the static
+	// manifest's bootstrap epoch we'll switch to it and ignore any further dynamic manifest
+	// updates. This exists to enable bootstrapping F3.
+	PrioritizeStaticManifest bool
+	// TESTINGAllowDynamicFinalize allow dynamic manifests to finalize tipsets. DO NOT ENABLE
+	// THIS IN PRODUCTION!
+	AllowDynamicFinalize bool
 }
 
-func NewConfig(manifestProvider peer.ID, initialPowerTable cid.Cid) func(dtypes.NetworkName) *Config {
-	return func(nn dtypes.NetworkName) *Config {
-		m := manifest.LocalDevnetManifest()
-		m.NetworkName = gpbft.NetworkName(nn)
-		m.EC.Period = time.Duration(buildconstants.BlockDelaySecs) * time.Second
-		m.CatchUpAlignment = time.Duration(buildconstants.BlockDelaySecs) * time.Second / 2
-		if buildconstants.F3BootstrapEpoch < 0 {
-			// if unset, set to a sane default so we don't get scary logs and pause.
-			m.BootstrapEpoch = 2 * int64(policy.ChainFinality)
-			m.Pause = true
-		} else {
-			m.BootstrapEpoch = int64(buildconstants.F3BootstrapEpoch)
-		}
-		m.EC.Finality = int64(policy.ChainFinality)
-		m.CommitteeLookback = 5
-		m.InitialPowerTable = initialPowerTable
-		m.EC.Finalize = buildconstants.F3Consensus
+// NewManifest constructs a sane F3 manifest based on the passed parameters. This function does not
+// look at and/or depend on the nodes build params, etc.
+func NewManifest(
+	nn gpbft.NetworkName,
+	finality, bootstrapEpoch abi.ChainEpoch,
+	ecPeriod time.Duration,
+	initialPowerTable cid.Cid,
+) *manifest.Manifest {
+	return &manifest.Manifest{
+		ProtocolVersion:   manifest.VersionCapability,
+		BootstrapEpoch:    int64(bootstrapEpoch),
+		NetworkName:       nn,
+		InitialPowerTable: initialPowerTable,
+		CommitteeLookback: manifest.DefaultCommitteeLookback,
+		CatchUpAlignment:  ecPeriod / 2,
+		Gpbft:             manifest.DefaultGpbftConfig,
+		EC: manifest.EcConfig{
+			Period:                   ecPeriod,
+			Finality:                 int64(finality),
+			DelayMultiplier:          manifest.DefaultEcConfig.DelayMultiplier,
+			BaseDecisionBackoffTable: manifest.DefaultEcConfig.BaseDecisionBackoffTable,
+			HeadLookback:             0,
+			Finalize:                 true,
+		},
+		CertificateExchange: manifest.CxConfig{
+			ClientRequestTimeout: manifest.DefaultCxConfig.ClientRequestTimeout,
+			ServerRequestTimeout: manifest.DefaultCxConfig.ServerRequestTimeout,
+			MinimumPollInterval:  ecPeriod,
+			MaximumPollInterval:  4 * ecPeriod,
+		},
+	}
+}
 
-		// TODO: We're forcing this to start paused for now. We need to remove this for the final
-		// mainnet launch.
-		m.Pause = true
-		return &Config{
-			InitialManifest:         m,
-			DynamicManifestProvider: manifestProvider,
-		}
+// NewConfig creates a new F3 config based on the node's build parameters and the passed network
+// name.
+func NewConfig(nn dtypes.NetworkName) *Config {
+	// Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes,
+	// mainnet is called testnetnet in state.
+	if nn == "testnetnet" {
+		nn = "filecoin"
+	}
+	c := &Config{
+		BaseNetworkName:          gpbft.NetworkName(nn),
+		PrioritizeStaticManifest: true,
+		DynamicManifestProvider:  buildconstants.F3ManifestServerID,
+		AllowDynamicFinalize:     false,
+	}
+	if buildconstants.F3BootstrapEpoch >= 0 {
+		c.StaticManifest = NewManifest(
+			c.BaseNetworkName,
+			policy.ChainFinality,
+			buildconstants.F3BootstrapEpoch,
+			time.Duration(buildconstants.BlockDelaySecs)*time.Second,
+			buildconstants.F3InitialPowerTableCID,
+		)
 	}
+	return c
 }

chain/lf3/manifest.go

@@ -1,45 +1,91 @@
 package lf3
 
 import (
+	"context"
 	"fmt"
+	"strings"
 
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-f3/ec"
 	"github.com/filecoin-project/go-f3/manifest"
 
+	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/node/modules/helpers"
 )
 
+type headGetter store.ChainStore
+
+func (hg *headGetter) GetHead(context.Context) (ec.TipSet, error) {
+	head := (*store.ChainStore)(hg).GetHeaviestTipSet()
+	if head == nil {
+		return nil, xerrors.New("no heaviest tipset")
+	}
+	return &f3TipSet{TipSet: head}, nil
+}
+
 // Determines the max. number of configuration changes
 // that are allowed for the dynamic manifest.
 // If the manifest changes more than this number, the F3
 // message topic will be filtered
 var MaxDynamicManifestChangesAllowed = 1000
 
-func NewManifestProvider(config *Config, ps *pubsub.PubSub, mds dtypes.MetadataDS) (manifest.ManifestProvider, error) {
+func NewManifestProvider(mctx helpers.MetricsCtx, config *Config, cs *store.ChainStore, ps *pubsub.PubSub, mds dtypes.MetadataDS) (prov manifest.ManifestProvider, err error) {
 	if config.DynamicManifestProvider == "" {
-		return manifest.NewStaticManifestProvider(config.InitialManifest)
+		if config.StaticManifest == nil {
+			return manifest.NoopManifestProvider{}, nil
+		}
+		return manifest.NewStaticManifestProvider(config.StaticManifest)
+	}
+
+	opts := []manifest.DynamicManifestProviderOption{
+		manifest.DynamicManifestProviderWithDatastore(
+			namespace.Wrap(mds, datastore.NewKey("/f3-dynamic-manifest")),
+		),
 	}
 
-	primaryNetworkName := config.InitialManifest.NetworkName
+	if config.StaticManifest != nil {
+		opts = append(opts,
+			manifest.DynamicManifestProviderWithInitialManifest(config.StaticManifest),
+		)
+	}
+
+	if config.AllowDynamicFinalize {
+		log.Error("dynamic F3 manifests are allowed to finalize tipsets, do not enable this in production!")
+	}
+
+	networkNameBase := config.BaseNetworkName + "/"
 	filter := func(m *manifest.Manifest) error {
 		if m.EC.Finalize {
-			return fmt.Errorf("refusing dynamic manifest that finalizes tipsets")
+			if !config.AllowDynamicFinalize {
+				return fmt.Errorf("refusing dynamic manifest that finalizes tipsets")
+			}
+			log.Error("WARNING: loading a dynamic F3 manifest that will finalize new tipsets")
 		}
-		if m.NetworkName == primaryNetworkName {
+		if !strings.HasPrefix(string(m.NetworkName), string(networkNameBase)) {
 			return fmt.Errorf(
-				"refusing dynamic manifest with network name %q that clashes with initial manifest",
-				primaryNetworkName,
+				"refusing dynamic manifest with network name %q, must start with %q",
+				m.NetworkName,
+				networkNameBase,
 			)
 		}
 		return nil
 	}
-	ds := namespace.Wrap(mds, datastore.NewKey("/f3-dynamic-manifest"))
-	return manifest.NewDynamicManifestProvider(ps, config.DynamicManifestProvider,
-		manifest.DynamicManifestProviderWithInitialManifest(config.InitialManifest),
-		manifest.DynamicManifestProviderWithDatastore(ds),
+	opts = append(opts,
 		manifest.DynamicManifestProviderWithFilter(filter),
 	)
+
+	prov, err = manifest.NewDynamicManifestProvider(ps, config.DynamicManifestProvider, opts...)
+	if err != nil {
+		return nil, err
+	}
+	if config.PrioritizeStaticManifest && config.StaticManifest != nil {
+		prov, err = manifest.NewFusingManifestProvider(mctx,
+			(*headGetter)(cs), prov, config.StaticManifest)
+	}
+	return prov, err
 }

go.mod

@@ -43,7 +43,7 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.1
 	github.com/filecoin-project/go-commp-utils/v2 v2.1.0
 	github.com/filecoin-project/go-crypto v0.1.0
-	github.com/filecoin-project/go-f3 v0.4.0
+	github.com/filecoin-project/go-f3 v0.5.0
 	github.com/filecoin-project/go-fil-commcid v0.2.0
 	github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0
 	github.com/filecoin-project/go-jsonrpc v0.6.0

go.sum

@@ -273,8 +273,8 @@ github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0=
 github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI=
-github.com/filecoin-project/go-f3 v0.4.0 h1:3UUjFMmZYvytDZPI5oeMroaEGO691icQM/7XoioYVxg=
-github.com/filecoin-project/go-f3 v0.4.0/go.mod h1:QoxuoK4aktNZD1R/unlhNbhV6TnlNTAbA/QODCnAjak=
+github.com/filecoin-project/go-f3 v0.5.0 h1:pRw8A9moEXSxK8v4nmR/Hs09TO2Wrz9h/Al76yt+8jI=
+github.com/filecoin-project/go-f3 v0.5.0/go.mod h1:QoxuoK4aktNZD1R/unlhNbhV6TnlNTAbA/QODCnAjak=
 github.com/filecoin-project/go-fil-commcid v0.2.0 h1:B+5UX8XGgdg/XsdUpST4pEBviKkFOw+Fvl2bLhSKGpI=
 github.com/filecoin-project/go-fil-commcid v0.2.0/go.mod h1:8yigf3JDIil+/WpqR5zoKyP0jBPCOGtEqq/K1CcMy9Q=
 github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk=

itests/f3_test.go

@@ -5,24 +5,28 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/filecoin-project/go-f3/gpbft"
 	"github.com/filecoin-project/go-f3/manifest"
 	"github.com/filecoin-project/go-state-types/abi"
 
+	"github.com/filecoin-project/lotus/chain/lf3"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/itests/kit"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/modules"
 )
 
 const (
-	DefaultBootsrapEpoch = 20
-	DefaultFinality      = 5
+	DefaultBootstrapEpoch                   = 20
+	DefaultFinality                         = 5
+	BaseNetworkName       gpbft.NetworkName = "test"
 )
 
 type testEnv struct {
@@ -61,7 +65,7 @@ func TestF3_Rebootstrap(t *testing.T) {
 
 	cpy := *e.m
 	cpy.BootstrapEpoch = 25
-	cpy.NetworkName += "/1"
+	cpy.NetworkName = BaseNetworkName + "/2"
 	e.ms.UpdateManifest(&cpy)
 
 	newManifest := e.waitTillManifestChange(&cpy, 20*time.Second)
@@ -90,14 +94,43 @@ func TestF3_PauseAndRebootstrap(t *testing.T) {
 	e.waitTillF3Runs(30 * time.Second)
 
 	cpy := *e.m
-	cpy.NetworkName += "/1"
+	cpy.NetworkName = BaseNetworkName + "/2"
 	cpy.BootstrapEpoch = 25
 	e.ms.UpdateManifest(&cpy)
 
 	e.waitTillManifestChange(&cpy, 20*time.Second)
 	e.waitTillF3Rebootstrap(20 * time.Second)
 }
 
+// Tests that pause/resume and rebootstrapping F3 works
+func TestF3_Bootstrap(t *testing.T) {
+	kit.QuietMiningLogs()
+
+	var bootstrapEpoch abi.ChainEpoch = 50
+	blocktime := 100 * time.Millisecond
+	staticManif := lf3.NewManifest(BaseNetworkName, DefaultFinality, bootstrapEpoch, blocktime, cid.Undef)
+	dynamicManif := *staticManif
+	dynamicManif.BootstrapEpoch = 5
+	dynamicManif.EC.Finalize = false
+	dynamicManif.NetworkName = BaseNetworkName + "/1"
+
+	e := setupWithStaticManifest(t, staticManif, true)
+	e.ms.UpdateManifest(&dynamicManif)
+	e.waitTillManifestChange(&dynamicManif, 20*time.Second)
+	e.waitTillF3Instance(2, 20*time.Second)
+	e.waitTillManifestChange(staticManif, 20*time.Second)
+	e.waitTillF3Instance(2, 20*time.Second)
+
+	// Try to switch back, we should ignore the manifest update.
+	e.ms.UpdateManifest(&dynamicManif)
+	time.Sleep(time.Second)
+	for _, n := range e.minerFullNodes {
+		m, err := n.F3GetManifest(e.testCtx)
+		require.NoError(e.t, err)
+		require.True(t, m.Equal(staticManif))
+	}
+}
+
 func (e *testEnv) waitTillF3Rebootstrap(timeout time.Duration) {
 	e.waitFor(func(n *kit.TestFullNode) bool {
 		// the prev epoch yet, check if we already bootstrapped and from
@@ -176,9 +209,16 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio
 // and the second full-node is an observer that is not directly connected to
 // a miner. The last return value is the manifest sender for the network.
 func setup(t *testing.T, blocktime time.Duration) *testEnv {
+	manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef)
+	return setupWithStaticManifest(t, manif, false)
+}
+
+func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstrap bool) *testEnv {
 	ctx, stopServices := context.WithCancel(context.Background())
 	errgrp, ctx := errgroup.WithContext(ctx)
 
+	blocktime := manif.EC.Period
+
 	t.Cleanup(func() {
 		stopServices()
 		require.NoError(t, errgrp.Wait())
@@ -188,7 +228,15 @@ func setup(t *testing.T, blocktime time.Duration) *testEnv {
 	manifestServerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/udp/0/quic-v1"))
 	require.NoError(t, err)
 
-	f3NOpt := kit.F3Enabled(DefaultBootsrapEpoch, blocktime, DefaultFinality, manifestServerHost.ID())
+	cfg := &lf3.Config{
+		BaseNetworkName:          BaseNetworkName,
+		StaticManifest:           manif,
+		DynamicManifestProvider:  manifestServerHost.ID(),
+		PrioritizeStaticManifest: testBootstrap,
+		AllowDynamicFinalize:     !testBootstrap,
+	}
+
+	f3NOpt := kit.F3Enabled(cfg)
 	f3MOpt := kit.ConstructorOpts(node.Override(node.F3Participation, modules.F3Participation))
 
 	var (