Feat add preset to config of persist store

damocles-manager/cmd/damocles-manager/internal/util_storage.go

@@ -82,7 +82,7 @@ var utilStorageAttachCmd = &cli.Command{
 
 		scfg := objstore.DefaultConfig(abs, readOnly)
 		scfg.Name = name
-		scfg.Strict = strict
+		scfg.Strict = &strict
 
 		store, err := filestore.Open(scfg, false)
 		if err != nil {
@@ -440,6 +440,8 @@ var utilStorageListCmd = &cli.Command{
 			fmt.Printf("%s:\n", detail.Name)
 			fmt.Printf("\tPath: %s\n", detail.Path)
 			fmt.Printf("\tType: %s\n", detail.Type)
+			fmt.Printf("\tReadOnly: %t\n", detail.ReadOnly)
+			fmt.Printf("\tWeight: %d\n", detail.Weight)
 			fmt.Printf("\tTotal: %s\n", units.BytesSize(float64(detail.Total)))
 			fmt.Printf("\tFree: %s\n", units.BytesSize(float64(detail.Free)))
 			fmt.Printf("\tUsed: %s\n", units.BytesSize(float64(detail.Used)))

damocles-manager/core/types.go

@@ -324,9 +324,12 @@ type SectorAccessStores struct {
 }
 
 type StoreBasicInfo struct {
-	Name string
-	Path string
-	Meta map[string]string
+	Name     string
+	Path     string
+	Strict   bool
+	ReadOnly bool
+	Weight   uint
+	Meta     map[string]string
 }
 
 type StoreDetailedInfo struct {

damocles-manager/dep/sealer_constructor.go

@@ -468,7 +468,7 @@ func openObjStore(cfg objstore.Config, pluginName string, loadedPlugins *manager
 }
 
 func BuildPersistedFileStoreMgr(scfg *modules.SafeConfig, globalStore CommonMetaStore, loadedPlugins *managerplugin.LoadedPlugins) (PersistedObjectStoreManager, error) {
-	persistCfg := scfg.MustCommonConfig().PersistStores
+	persistCfg := scfg.MustCommonConfig().GetPersistStores()
 
 	stores := make([]objstore.Store, 0, len(persistCfg))
 	policy := map[string]objstore.StoreSelectPolicy{}
@@ -564,7 +564,7 @@ func BuildMarketAPIRelated(gctx GlobalContext, lc fx.Lifecycle, scfg *modules.Sa
 			Name:     pcfg.Name,
 			Path:     pcfg.Path,
 			Meta:     pcfg.Meta,
-			ReadOnly: pcfg.ReadOnly,
+			ReadOnly: &pcfg.ReadOnly,
 		}
 		// For compatibility with v0.5
 		if pcfg.PluginName == "" && pcfg.Plugin != "" {

damocles-manager/go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230613072356-8713ced20557
+	github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230726115052-54f1d983f762
 	github.com/ipfs-force-community/venus-cluster-assets v0.1.0
 	github.com/ipfs/go-cid v0.3.2
 	github.com/ipfs/go-datastore v0.6.0

damocles-manager/go.sum

@@ -587,8 +587,8 @@ github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lTo
 github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230613072356-8713ced20557 h1:/qqCFCmqbpqW8tFpv7xr1J5pIGL90Mo+SQcJ0Df1SY8=
-github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230613072356-8713ced20557/go.mod h1:me1u2cl7qdxBCZiVL0laDop8uBHDdUwlUNnQ7KkHF64=
+github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230726115052-54f1d983f762 h1:Xs5HZL8NSlIEIW5Ec83blA4rlRC7kW8nemx7Mgm1Zok=
+github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230726115052-54f1d983f762/go.mod h1:me1u2cl7qdxBCZiVL0laDop8uBHDdUwlUNnQ7KkHF64=
 github.com/ipfs-force-community/go-jsonrpc v0.1.7-0.20230220074347-8db78dbc20d4 h1:iu/3irYevdNpdc0B/gRi1vuS3+lRn+6Ro9G0FeBiAfE=
 github.com/ipfs-force-community/go-jsonrpc v0.1.7-0.20230220074347-8db78dbc20d4/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM=
 github.com/ipfs-force-community/venus-cluster-assets v0.1.0 h1:K/0+OV9Jm7HjSa7O9MAtgfLDIudQYZUTymhJsp8rGXg=

damocles-manager/modules/config.go

@@ -2,7 +2,9 @@ package modules
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -13,9 +15,12 @@ import (
 	"github.com/ipfs-force-community/damocles/damocles-manager/pkg/messager"
 
 	"github.com/ipfs-force-community/damocles/damocles-manager/pkg/confmgr"
+	"github.com/ipfs-force-community/damocles/damocles-manager/pkg/logging"
 	"github.com/ipfs-force-community/damocles/damocles-manager/pkg/objstore"
 )
 
+var log = logging.New("config")
+
 func init() {
 	fake, err := address.NewFromString("f1abjxfbp274xpdqcpuaykwkfb43omjotacm2p3za")
 	if err != nil {
@@ -154,6 +159,28 @@ type PersistStoreConfig struct {
 	PluginName string
 }
 
+type PieceStorePreset struct {
+	Meta     map[string]string
+	Strict   *bool
+	ReadOnly *bool
+	Weight   *uint
+
+	AllowMiners []abi.ActorID
+	DenyMiners  []abi.ActorID
+
+	// compatibility for storage.json with lotus
+	StorageConfigPath *string
+}
+
+type StoragePathConfig struct {
+	StoragePaths []LocalPath
+}
+
+type LocalPath struct {
+	Name string
+	Path string
+}
+
 type ProvingConfig struct {
 	// Maximum number of sector checks to run in parallel. (0 = unlimited)
 	//
@@ -190,15 +217,135 @@ func defaultProvingConfig() ProvingConfig {
 }
 
 type CommonConfig struct {
-	API           CommonAPIConfig
-	Plugins       *PluginConfig
-	PieceStores   []PieceStoreConfig
+	API              CommonAPIConfig
+	Plugins          *PluginConfig
+	PieceStores      []PieceStoreConfig
+	PieceStorePreset PieceStorePreset
+
+	// PersistStores should not be used directly, use GetPersistStores instead
 	PersistStores []PersistStoreConfig
 	MongoKVStore  *KVStoreMongoDBConfig // For compatibility with v0.5
 	DB            *DBConfig
 	Proving       ProvingConfig
 }
 
+func (c CommonConfig) GetPersistStores() []PersistStoreConfig {
+	// apply preset
+	preset := c.PieceStorePreset
+	ret := make([]PersistStoreConfig, 0, len(c.PersistStores))
+
+	// fill preset with default values if not set
+	if preset.Strict == nil {
+		preset.Strict = new(bool)
+		*preset.Strict = false
+	}
+	if preset.ReadOnly == nil {
+		preset.ReadOnly = new(bool)
+		*preset.ReadOnly = false
+	}
+	if preset.Weight == nil {
+		preset.Weight = new(uint)
+		*preset.Weight = 1
+	}
+	if preset.Meta == nil {
+		preset.Meta = make(map[string]string)
+	}
+	if preset.AllowMiners == nil {
+		preset.AllowMiners = make([]abi.ActorID, 0)
+	}
+	if preset.DenyMiners == nil {
+		preset.DenyMiners = make([]abi.ActorID, 0)
+	}
+
+	for i := range c.PersistStores {
+		ps := c.PersistStores[i]
+		if ps.Strict == nil {
+			ps.Strict = preset.Strict
+		}
+		if ps.ReadOnly == nil {
+			ps.ReadOnly = preset.ReadOnly
+		}
+		if ps.Weight == nil {
+			ps.Weight = preset.Weight
+		}
+		mergeMapInto[string, string](preset.Meta, ps.Meta)
+		mergeSliceInto[abi.ActorID](preset.AllowMiners, ps.AllowMiners)
+		mergeSliceInto[abi.ActorID](preset.DenyMiners, ps.DenyMiners)
+
+		ret = append(ret, ps)
+	}
+
+	if preset.StorageConfigPath != nil {
+		p := *preset.StorageConfigPath
+		if p != "" {
+			cfg := StoragePathConfig{}
+			file, err := os.Open(p)
+			if err != nil {
+				log.Errorf("open storage config file %s failed: %s", p, err)
+			} else {
+				defer file.Close()
+				err := json.NewDecoder(file).Decode(&cfg)
+				if err != nil {
+					log.Errorf("decode storage config file %s failed: %s", p, err)
+				} else {
+					for _, lp := range cfg.StoragePaths {
+						psc := PersistStoreConfig{
+							Config: objstore.Config{
+								Path:     lp.Path,
+								Meta:     preset.Meta,
+								Strict:   preset.Strict,
+								ReadOnly: preset.ReadOnly,
+								Weight:   preset.Weight,
+							},
+							StoreSelectPolicy: objstore.StoreSelectPolicy{
+								AllowMiners: preset.AllowMiners,
+								DenyMiners:  preset.DenyMiners,
+							},
+						}
+						if lp.Name != "" {
+							psc.Name = lp.Name
+						}
+						ret = append(ret, psc)
+					}
+					log.Infof("load storage config file %s success", p)
+				}
+			}
+		}
+	}
+	return ret
+}
+
+func mergeSliceInto[T comparable](from, into []T) []T {
+	if len(from) == 0 {
+		return into
+	}
+	has := make(map[T]struct{})
+	for _, m := range into {
+		has[m] = struct{}{}
+	}
+	for _, m := range from {
+		if _, ok := has[m]; !ok {
+			into = append(into, m)
+		}
+	}
+	return into
+}
+
+func mergeMapInto[T comparable, V any](from, into map[T]V) map[T]V {
+	if len(from) == 0 {
+		return into
+	}
+	if into == nil {
+		into = make(map[T]V)
+	}
+	for k, v := range from {
+		if _, ok := into[k]; !ok {
+			into[k] = v
+		}
+	}
+	return into
+}
+
 func exampleFilestoreConfig() objstore.Config {
 	cfg := objstore.DefaultConfig("{store_path}", false)
 	cfg.Name = "{store_name}"
@@ -225,6 +372,20 @@ func defaultCommonConfig(example bool) CommonConfig {
 			PluginName: "s3store",
 		})
 
+		cfg.PieceStorePreset = PieceStorePreset{
+			Meta:     map[string]string{"SomeKey": "SomeValue"},
+			Strict:   new(bool),
+			ReadOnly: new(bool),
+			Weight:   new(uint),
+
+			AllowMiners: []abi.ActorID{1, 2},
+			DenyMiners:  []abi.ActorID{3, 4},
+
+			StorageConfigPath: new(string),
+		}
+		*cfg.PieceStorePreset.Weight = 1
+		*cfg.PieceStorePreset.StorageConfigPath = "/optional/path/to/your/storage.json"
+
 		cfg.PersistStores = append(cfg.PersistStores, PersistStoreConfig{
 			Config: objstore.Config{
 				Name: exampleCfg.Name,

damocles-manager/modules/config_util_test.go

@@ -1,8 +1,11 @@
 package modules_test
 
 import (
+	"bytes"
+	"fmt"
 	"testing"
 
+	"github.com/BurntSushi/toml"
 	"github.com/filecoin-project/go-address"
 	"github.com/ipfs-force-community/damocles/damocles-manager/modules"
 	"github.com/ipfs-force-community/damocles/damocles-manager/testutil"
@@ -54,3 +57,22 @@ func TestMustAddressUnmarshalText(t *testing.T) {
 		require.Equal(t, tc.expected, actual.Std())
 	}
 }
+
+func TestStructWithNilField(t *testing.T) {
+	type A struct {
+		B *int
+		C *bool
+		D *string
+		E map[int]int
+	}
+
+	a := A{}
+
+	buf := bytes.Buffer{}
+	enc := toml.NewEncoder(&buf)
+	enc.Indent = ""
+	err := enc.Encode(a)
+	require.NoError(t, err)
+	fmt.Println(string(buf.Bytes()))
+
+}

damocles-manager/modules/sealer/sealer_cli.go

@@ -354,9 +354,12 @@ func (s *Sealer) StoreList(ctx context.Context) ([]core.StoreDetailedInfo, error
 
 func storeConfig2StoreBasic(ocfg *objstore.Config) core.StoreBasicInfo {
 	return core.StoreBasicInfo{
-		Name: ocfg.Name,
-		Path: ocfg.Path,
-		Meta: ocfg.Meta,
+		Name:     ocfg.Name,
+		Path:     ocfg.Path,
+		Meta:     ocfg.Meta,
+		Strict:   ocfg.GetStrict(),
+		ReadOnly: ocfg.GetReadOnly(),
+		Weight:   ocfg.GetWeight(),
 	}
 }
 

damocles-manager/pkg/objstore/filestore/store.go

@@ -137,7 +137,7 @@ func (s *Store) open(p string, r *readRange) (io.ReadCloser, error) {
 		}
 	}()
 
-	if s.cfg.Strict {
+	if s.cfg.GetStrict() {
 		stat, err := file.Stat()
 		if err != nil {
 			return nil, fmt.Errorf("obj %s: get stat: %w", p, err)
@@ -211,7 +211,7 @@ func (s *Store) Get(ctx context.Context, p string) (io.ReadCloser, error) {
 }
 
 func (s *Store) Del(ctx context.Context, p string) error {
-	if s.cfg.ReadOnly {
+	if s.cfg.GetReadOnly() {
 		return objstore.ErrReadOnlyStore
 	}
 
@@ -268,7 +268,7 @@ func (s *Store) getAbsPath(p string) (string, error) {
 }
 
 func (s *Store) Put(ctx context.Context, p string, r io.Reader) (int64, error) {
-	if s.cfg.ReadOnly {
+	if s.cfg.GetReadOnly() {
 		return 0, objstore.ErrReadOnlyStore
 	}
 

damocles-manager/pkg/objstore/mgr.go

@@ -208,11 +208,11 @@ func (m *StoreManager) ReserveSpace(ctx context.Context, sid abi.SectorID, size
 				}
 
 				// readonly, or not enough space
-				if info.Config.ReadOnly || info.Free < resSize+size {
+				if info.Config.GetReadOnly() || info.Free < resSize+size {
 					continue
 				}
 
-				weight := info.Config.Weight
+				weight := info.Config.GetWeight()
 				if weight == 0 {
 					weight = 1
 				}