From 592203f96a43ab193d0e9352d9280641d4a0fe1d Mon Sep 17 00:00:00 2001 From: tanlang Date: Wed, 26 Jul 2023 19:50:52 +0800 Subject: [PATCH 1/6] feat: change some property to pointer --- manager-plugin/objstore/objstore.go | 32 ++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/manager-plugin/objstore/objstore.go b/manager-plugin/objstore/objstore.go index eb7f8cd79..030e525ad 100644 --- a/manager-plugin/objstore/objstore.go +++ b/manager-plugin/objstore/objstore.go @@ -20,17 +20,39 @@ type Config struct { Name string Path string Meta map[string]string - Strict bool - ReadOnly bool - Weight uint + Strict *bool + ReadOnly *bool + Weight *uint +} + +func (c Config) GetStrict() bool { + if c.Strict == nil { + return false + } + return *c.Strict +} + +func (c Config) GetReadOnly() bool { + if c.ReadOnly == nil { + return false + } + return *c.ReadOnly +} + +func (c Config) GetWeight() uint { + if c.Weight == nil { + return 1 + } + return *c.Weight } func DefaultConfig(path string, readonly bool) Config { + one := uint(1) return Config{ Path: path, Meta: map[string]string{}, - ReadOnly: readonly, - Weight: 1, + ReadOnly: &readonly, + Weight: &one, } } From 6c047ecfeb8707cdfab9c3226a5c46490cfde903 Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 27 Jul 2023 10:37:39 +0800 Subject: [PATCH 2/6] feat: wrap a get method for some field which need preprocess --- .../damocles-manager/internal/util_storage.go | 4 +- damocles-manager/core/types.go | 9 +- damocles-manager/dep/sealer_constructor.go | 4 +- damocles-manager/go.mod | 2 +- damocles-manager/go.sum | 4 +- damocles-manager/modules/config.go | 98 ++++++++++++++++++- damocles-manager/modules/config_util_test.go | 22 +++++ damocles-manager/modules/sealer/sealer_cli.go | 9 +- .../pkg/objstore/filestore/store.go | 6 +- damocles-manager/pkg/objstore/mgr.go | 4 +- damocles-manager/pkg/objstore/mgr_test.go | 15 +-- damocles-manager/pkg/objstore/mock_store.go | 4 +- damocles-manager/pkg/piecestore/proxy.go | 4 +- damocles-manager/pkg/piecestore/proxy_test.go | 5 + manager-plugin/objstore/objstore.go | 14 ++- 15 files changed, 169 insertions(+), 35 deletions(-) diff --git a/damocles-manager/cmd/damocles-manager/internal/util_storage.go b/damocles-manager/cmd/damocles-manager/internal/util_storage.go index dbb172c2c..636441f6d 100644 --- a/damocles-manager/cmd/damocles-manager/internal/util_storage.go +++ b/damocles-manager/cmd/damocles-manager/internal/util_storage.go @@ -82,7 +82,7 @@ var utilStorageAttachCmd = &cli.Command{ scfg := objstore.DefaultConfig(abs, readOnly) scfg.Name = name - scfg.Strict = strict + scfg.Strict = &strict store, err := filestore.Open(scfg, false) if err != nil { @@ -440,6 +440,8 @@ var utilStorageListCmd = &cli.Command{ fmt.Printf("%s:\n", detail.Name) fmt.Printf("\tPath: %s\n", detail.Path) fmt.Printf("\tType: %s\n", detail.Type) + fmt.Printf("\tReadOnly: %t\n", detail.ReadOnly) + fmt.Printf("\tWeight: %d\n", detail.Weight) fmt.Printf("\tTotal: %s\n", units.BytesSize(float64(detail.Total))) fmt.Printf("\tFree: %s\n", units.BytesSize(float64(detail.Free))) fmt.Printf("\tUsed: %s\n", units.BytesSize(float64(detail.Used))) diff --git a/damocles-manager/core/types.go b/damocles-manager/core/types.go index 20d2b06d6..5a6bd61a3 100644 --- a/damocles-manager/core/types.go +++ b/damocles-manager/core/types.go @@ -324,9 +324,12 @@ type SectorAccessStores struct { } type StoreBasicInfo struct { - Name string - Path string - Meta map[string]string + Name string + Path string + Strict bool + ReadOnly bool + Weight uint + Meta map[string]string } type StoreDetailedInfo struct { diff --git a/damocles-manager/dep/sealer_constructor.go b/damocles-manager/dep/sealer_constructor.go index 31e3b0dd1..38ea035dc 100644 --- a/damocles-manager/dep/sealer_constructor.go +++ b/damocles-manager/dep/sealer_constructor.go @@ -468,7 +468,7 @@ func openObjStore(cfg objstore.Config, pluginName string, loadedPlugins *manager } func BuildPersistedFileStoreMgr(scfg *modules.SafeConfig, globalStore CommonMetaStore, loadedPlugins *managerplugin.LoadedPlugins) (PersistedObjectStoreManager, error) { - persistCfg := scfg.MustCommonConfig().PersistStores + persistCfg := scfg.MustCommonConfig().GetPersistStores() stores := make([]objstore.Store, 0, len(persistCfg)) policy := map[string]objstore.StoreSelectPolicy{} @@ -564,7 +564,7 @@ func BuildMarketAPIRelated(gctx GlobalContext, lc fx.Lifecycle, scfg *modules.Sa Name: pcfg.Name, Path: pcfg.Path, Meta: pcfg.Meta, - ReadOnly: pcfg.ReadOnly, + ReadOnly: &pcfg.ReadOnly, } // For compatibility with v0.5 if pcfg.PluginName == "" && pcfg.Plugin != "" { diff --git a/damocles-manager/go.mod b/damocles-manager/go.mod index d952226be..b48684a3e 100644 --- a/damocles-manager/go.mod +++ b/damocles-manager/go.mod @@ -25,7 +25,7 @@ require ( github.com/golang/mock v1.6.0 github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230613072356-8713ced20557 + github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230726115052-54f1d983f762 github.com/ipfs-force-community/venus-cluster-assets v0.1.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 diff --git a/damocles-manager/go.sum b/damocles-manager/go.sum index d5868db56..51d872475 100644 --- a/damocles-manager/go.sum +++ b/damocles-manager/go.sum @@ -587,8 +587,8 @@ github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lTo github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230613072356-8713ced20557 h1:/qqCFCmqbpqW8tFpv7xr1J5pIGL90Mo+SQcJ0Df1SY8= -github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230613072356-8713ced20557/go.mod h1:me1u2cl7qdxBCZiVL0laDop8uBHDdUwlUNnQ7KkHF64= +github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230726115052-54f1d983f762 h1:Xs5HZL8NSlIEIW5Ec83blA4rlRC7kW8nemx7Mgm1Zok= +github.com/ipfs-force-community/damocles/manager-plugin v0.0.0-20230726115052-54f1d983f762/go.mod h1:me1u2cl7qdxBCZiVL0laDop8uBHDdUwlUNnQ7KkHF64= github.com/ipfs-force-community/go-jsonrpc v0.1.7-0.20230220074347-8db78dbc20d4 h1:iu/3irYevdNpdc0B/gRi1vuS3+lRn+6Ro9G0FeBiAfE= github.com/ipfs-force-community/go-jsonrpc v0.1.7-0.20230220074347-8db78dbc20d4/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM= github.com/ipfs-force-community/venus-cluster-assets v0.1.0 h1:K/0+OV9Jm7HjSa7O9MAtgfLDIudQYZUTymhJsp8rGXg= diff --git a/damocles-manager/modules/config.go b/damocles-manager/modules/config.go index d921214ae..d4c943d31 100644 --- a/damocles-manager/modules/config.go +++ b/damocles-manager/modules/config.go @@ -154,6 +154,16 @@ type PersistStoreConfig struct { PluginName string } +type PieceStorePreset struct { + Meta map[string]string + Strict *bool + ReadOnly *bool + Weight *uint + + AllowMiners []abi.ActorID + DenyMiners []abi.ActorID +} + type ProvingConfig struct { // Maximum number of sector checks to run in parallel. (0 = unlimited) // @@ -190,15 +200,97 @@ func defaultProvingConfig() ProvingConfig { } type CommonConfig struct { - API CommonAPIConfig - Plugins *PluginConfig - PieceStores []PieceStoreConfig + API CommonAPIConfig + Plugins *PluginConfig + PieceStores []PieceStoreConfig + PieceStorePreset PieceStorePreset + + // PersistStores should not be used directly, use GetPersistStores instead PersistStores []PersistStoreConfig MongoKVStore *KVStoreMongoDBConfig // For compatibility with v0.5 DB *DBConfig Proving ProvingConfig } +func (c CommonConfig) GetPersistStores() []PersistStoreConfig { + // apply preset + preset := c.PieceStorePreset + ret := make([]PersistStoreConfig, 0, len(c.PersistStores)) + + // fill preset with default values if not set + if preset.Strict == nil { + preset.Strict = new(bool) + *preset.Strict = false + } + if preset.ReadOnly == nil { + preset.ReadOnly = new(bool) + *preset.ReadOnly = false + } + if preset.Weight == nil { + preset.Weight = new(uint) + *preset.Weight = 1 + } + if preset.Meta == nil { + preset.Meta = make(map[string]string) + } + if preset.AllowMiners == nil { + preset.AllowMiners = make([]abi.ActorID, 0) + } + if preset.DenyMiners == nil { + preset.DenyMiners = make([]abi.ActorID, 0) + } + + for i := range c.PersistStores { + ps := c.PersistStores[i] + if ps.Strict == nil { + ps.Strict = preset.Strict + } + if ps.ReadOnly == nil { + ps.ReadOnly = preset.ReadOnly + } + if ps.Weight == nil { + ps.Weight = preset.Weight + } + mergeMapInto[string, string](preset.Meta, ps.Meta) + mergeSliceInto[abi.ActorID](preset.AllowMiners, ps.AllowMiners) + mergeSliceInto[abi.ActorID](preset.DenyMiners, ps.DenyMiners) + + ret = append(ret, ps) + } + return ret +} + +func mergeSliceInto[T comparable](from, into []T) []T { + if len(from) == 0 { + return into + } + has := make(map[T]struct{}) + for _, m := range into { + has[m] = struct{}{} + } + for _, m := range from { + if _, ok := has[m]; !ok { + into = append(into, m) + } + } + return into +} + +func mergeMapInto[T comparable, V any](from, into map[T]V) map[T]V { + if len(from) == 0 { + return into + } + if into == nil { + into = make(map[T]V) + } + for k, v := range from { + if _, ok := into[k]; !ok { + into[k] = v + } + } + return into +} + func exampleFilestoreConfig() objstore.Config { cfg := objstore.DefaultConfig("{store_path}", false) cfg.Name = "{store_name}" diff --git a/damocles-manager/modules/config_util_test.go b/damocles-manager/modules/config_util_test.go index 22c9dcb5d..30ac9761c 100644 --- a/damocles-manager/modules/config_util_test.go +++ b/damocles-manager/modules/config_util_test.go @@ -1,8 +1,11 @@ package modules_test import ( + "bytes" + "fmt" "testing" + "github.com/BurntSushi/toml" "github.com/filecoin-project/go-address" "github.com/ipfs-force-community/damocles/damocles-manager/modules" "github.com/ipfs-force-community/damocles/damocles-manager/testutil" @@ -54,3 +57,22 @@ func TestMustAddressUnmarshalText(t *testing.T) { require.Equal(t, tc.expected, actual.Std()) } } + +func TestStructWithNilField(t *testing.T) { + type A struct { + B *int + C *bool + D *string + E map[int]int + } + + a := A{} + + buf := bytes.Buffer{} + enc := toml.NewEncoder(&buf) + enc.Indent = "" + err := enc.Encode(a) + require.NoError(t, err) + fmt.Println(string(buf.Bytes())) + +} diff --git a/damocles-manager/modules/sealer/sealer_cli.go b/damocles-manager/modules/sealer/sealer_cli.go index 85c1ff44c..c3d888403 100644 --- a/damocles-manager/modules/sealer/sealer_cli.go +++ b/damocles-manager/modules/sealer/sealer_cli.go @@ -354,9 +354,12 @@ func (s *Sealer) StoreList(ctx context.Context) ([]core.StoreDetailedInfo, error func storeConfig2StoreBasic(ocfg *objstore.Config) core.StoreBasicInfo { return core.StoreBasicInfo{ - Name: ocfg.Name, - Path: ocfg.Path, - Meta: ocfg.Meta, + Name: ocfg.Name, + Path: ocfg.Path, + Meta: ocfg.Meta, + Strict: ocfg.GetStrict(), + ReadOnly: ocfg.GetReadOnly(), + Weight: ocfg.GetWeight(), } } diff --git a/damocles-manager/pkg/objstore/filestore/store.go b/damocles-manager/pkg/objstore/filestore/store.go index 9ef5369bb..7c2a3e734 100644 --- a/damocles-manager/pkg/objstore/filestore/store.go +++ b/damocles-manager/pkg/objstore/filestore/store.go @@ -137,7 +137,7 @@ func (s *Store) open(p string, r *readRange) (io.ReadCloser, error) { } }() - if s.cfg.Strict { + if s.cfg.GetStrict() { stat, err := file.Stat() if err != nil { return nil, fmt.Errorf("obj %s: get stat: %w", p, err) @@ -211,7 +211,7 @@ func (s *Store) Get(ctx context.Context, p string) (io.ReadCloser, error) { } func (s *Store) Del(ctx context.Context, p string) error { - if s.cfg.ReadOnly { + if s.cfg.GetReadOnly() { return objstore.ErrReadOnlyStore } @@ -268,7 +268,7 @@ func (s *Store) getAbsPath(p string) (string, error) { } func (s *Store) Put(ctx context.Context, p string, r io.Reader) (int64, error) { - if s.cfg.ReadOnly { + if s.cfg.GetReadOnly() { return 0, objstore.ErrReadOnlyStore } diff --git a/damocles-manager/pkg/objstore/mgr.go b/damocles-manager/pkg/objstore/mgr.go index 12d4531d3..67a597884 100644 --- a/damocles-manager/pkg/objstore/mgr.go +++ b/damocles-manager/pkg/objstore/mgr.go @@ -208,11 +208,11 @@ func (m *StoreManager) ReserveSpace(ctx context.Context, sid abi.SectorID, size } // readonly, or not enough space - if info.Config.ReadOnly || info.Free < resSize+size { + if info.Config.GetReadOnly() || info.Free < resSize+size { continue } - weight := info.Config.Weight + weight := info.Config.GetWeight() if weight == 0 { weight = 1 } diff --git a/damocles-manager/pkg/objstore/mgr_test.go b/damocles-manager/pkg/objstore/mgr_test.go index d1d18d1d3..bbb742e1b 100644 --- a/damocles-manager/pkg/objstore/mgr_test.go +++ b/damocles-manager/pkg/objstore/mgr_test.go @@ -10,6 +10,11 @@ import ( "github.com/stretchr/testify/require" ) +var ( + TRUE = true + THOUNSAND = uint(1000) +) + func (m *StoreManager) resetReserved(ctx context.Context) error { err := m.metadb.Del(ctx, storeReserveSummaryKey) if err != nil { @@ -38,7 +43,7 @@ func TestStoreManagerReserverSpace(t *testing.T) { storeRO, err := NewMockStore(Config{ Name: storeNameReadOnly, - ReadOnly: true, + ReadOnly: &TRUE, }, 1<<30) require.NoError(t, err, "construct store-RO") @@ -188,20 +193,18 @@ func TestStoreManagerReserverSpaceWeighed(t *testing.T) { storeNameReadOnly := "store-readonly" store1, err := NewMockStore(Config{ - Name: storeName1, - Weight: 1, + Name: storeName1, }, 1<<20) require.NoError(t, err, "construct store-1") store1K, err := NewMockStore(Config{ Name: storeName1K, - Weight: 1000, + Weight: &THOUNSAND, }, 1<<20) require.NoError(t, err, "construct store-1K") storeRO, err := NewMockStore(Config{ - Name: storeNameReadOnly, - ReadOnly: true, + Name: storeNameReadOnly, }, 1<<30) require.NoError(t, err, "construct store-RO") diff --git a/damocles-manager/pkg/objstore/mock_store.go b/damocles-manager/pkg/objstore/mock_store.go index 5cbca18f8..a521498e9 100644 --- a/damocles-manager/pkg/objstore/mock_store.go +++ b/damocles-manager/pkg/objstore/mock_store.go @@ -81,7 +81,7 @@ func (ms *MockStore) Get(ctx context.Context, p string) (io.ReadCloser, error) { } func (ms *MockStore) Del(ctx context.Context, p string) error { - if ms.cfg.ReadOnly { + if ms.cfg.GetReadOnly() { return ErrReadOnlyStore } @@ -114,7 +114,7 @@ func (ms *MockStore) Stat(ctx context.Context, p string) (Stat, error) { } func (ms *MockStore) Put(ctx context.Context, p string, r io.Reader) (int64, error) { - if ms.cfg.ReadOnly { + if ms.cfg.GetReadOnly() { return 0, ErrReadOnlyStore } diff --git a/damocles-manager/pkg/piecestore/proxy.go b/damocles-manager/pkg/piecestore/proxy.go index e916f54cb..830fdaf48 100644 --- a/damocles-manager/pkg/piecestore/proxy.go +++ b/damocles-manager/pkg/piecestore/proxy.go @@ -79,7 +79,7 @@ func (p *Proxy) handlePut(rw http.ResponseWriter, req *http.Request) { continue } - if storeInfo.Config.ReadOnly { + if storeInfo.Config.GetReadOnly() { continue } @@ -119,7 +119,7 @@ func (p *Proxy) Put(ctx context.Context, pieceCid cid.Cid, data io.Reader) (int6 continue } - if storeInfo.Config.ReadOnly { + if storeInfo.Config.GetReadOnly() { continue } diff --git a/damocles-manager/pkg/piecestore/proxy_test.go b/damocles-manager/pkg/piecestore/proxy_test.go index e4ff248b5..bb1b9c017 100644 --- a/damocles-manager/pkg/piecestore/proxy_test.go +++ b/damocles-manager/pkg/piecestore/proxy_test.go @@ -20,6 +20,11 @@ import ( "github.com/filecoin-project/venus/venus-shared/api/market/v1/mock" ) +var ( + FALSE = false + ONE = uint(1) +) + func setupStoreProxy(t *testing.T, resourceEndPoint string) *Proxy { st, err := filestore.Open(objstore.Config{ Name: "mock test", diff --git a/manager-plugin/objstore/objstore.go b/manager-plugin/objstore/objstore.go index 030e525ad..b0c98ff02 100644 --- a/manager-plugin/objstore/objstore.go +++ b/manager-plugin/objstore/objstore.go @@ -17,12 +17,16 @@ var ( ) type Config struct { - Name string - Path string - Meta map[string]string - Strict *bool + Name string + Path string + Meta map[string]string + + // Strict should never be used directly, use GetStrict() instead + Strict *bool + // ReadOnly should never be used directly, use GetReadOnly() instead ReadOnly *bool - Weight *uint + // Weight should never be used directly, use GetWeight() instead + Weight *uint } func (c Config) GetStrict() bool { From f796d3735d75eb3a72f9cdb7823e16793433c859 Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 27 Jul 2023 14:32:24 +0800 Subject: [PATCH 3/6] feat: compatible for storage.json --- damocles-manager/modules/config.go | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/damocles-manager/modules/config.go b/damocles-manager/modules/config.go index d4c943d31..19d1ba38e 100644 --- a/damocles-manager/modules/config.go +++ b/damocles-manager/modules/config.go @@ -2,7 +2,9 @@ package modules import ( "bytes" + "encoding/json" "fmt" + "os" "sync" "time" @@ -13,9 +15,12 @@ import ( "github.com/ipfs-force-community/damocles/damocles-manager/pkg/messager" "github.com/ipfs-force-community/damocles/damocles-manager/pkg/confmgr" + "github.com/ipfs-force-community/damocles/damocles-manager/pkg/logging" "github.com/ipfs-force-community/damocles/damocles-manager/pkg/objstore" ) +var log = logging.New("config") + func init() { fake, err := address.NewFromString("f1abjxfbp274xpdqcpuaykwkfb43omjotacm2p3za") if err != nil { @@ -162,6 +167,17 @@ type PieceStorePreset struct { AllowMiners []abi.ActorID DenyMiners []abi.ActorID + + // compatibility for storage.json with lotus + StorageConfigPath *string +} + +type StoragePathConfig struct { + StoragePaths []LocalPath +} + +type LocalPath struct { + Path string } type ProvingConfig struct { @@ -257,6 +273,40 @@ func (c CommonConfig) GetPersistStores() []PersistStoreConfig { ret = append(ret, ps) } + + if preset.StorageConfigPath != nil { + p := *preset.StorageConfigPath + if p != "" { + cfg := StoragePathConfig{} + file, err := os.Open(p) + if err != nil { + log.Errorf("open storage config file %s failed: %s", p, err) + } else { + defer file.Close() + err := json.NewDecoder(file).Decode(&cfg) + if err != nil { + log.Errorf("decode storage config file %s failed: %s", p, err) + } else { + for _, lp := range cfg.StoragePaths { + ret = append(ret, PersistStoreConfig{ + Config: objstore.Config{ + Path: lp.Path, + Meta: preset.Meta, + Strict: preset.Strict, + ReadOnly: preset.ReadOnly, + Weight: preset.Weight, + }, + StoreSelectPolicy: objstore.StoreSelectPolicy{ + AllowMiners: preset.AllowMiners, + DenyMiners: preset.DenyMiners, + }, + }) + } + log.Infof("load storage config file %s success", p) + } + } + } + } return ret } From 3b8cafe24c923278ab2268653e039ca3bd9d06b5 Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 27 Jul 2023 15:54:53 +0800 Subject: [PATCH 4/6] feat: update documents about PieceStorePreset --- damocles-manager/modules/config.go | 14 ++++ ...15\347\275\256\350\247\243\346\236\220.md" | 74 +++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/damocles-manager/modules/config.go b/damocles-manager/modules/config.go index 19d1ba38e..e78ca1c21 100644 --- a/damocles-manager/modules/config.go +++ b/damocles-manager/modules/config.go @@ -367,6 +367,20 @@ func defaultCommonConfig(example bool) CommonConfig { PluginName: "s3store", }) + cfg.PieceStorePreset = PieceStorePreset{ + Meta: map[string]string{"SomeKey": "SomeValue"}, + Strict: new(bool), + ReadOnly: new(bool), + Weight: new(uint), + + AllowMiners: []abi.ActorID{1, 2}, + DenyMiners: []abi.ActorID{3, 4}, + + StorageConfigPath: new(string), + } + *cfg.PieceStorePreset.Weight = 1 + *cfg.PieceStorePreset.StorageConfigPath = "/optional/path/to/your/storage.json" + cfg.PersistStores = append(cfg.PersistStores, PersistStoreConfig{ Config: objstore.Config{ Name: exampleCfg.Name, diff --git "a/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" "b/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" index 7c0d1f6ea..d6150934f 100644 --- "a/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" +++ "b/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" @@ -18,11 +18,22 @@ [[Common.PieceStores]] #Name = "{store_name}" #Path = "{store_path}" +#ReadOnly = false #Plugin = "" #PluginName = "s3store" [Common.PieceStores.Meta] #SomeKey = "SomeValue" # +[Common.PieceStorePreset] +#Strict = false +#ReadOnly = false +#Weight = 1 +#AllowMiners = [1, 2] +#DenyMiners = [3, 4] +#StorageConfigPath = "/optional/path/to/your/storage.json" +[Common.PieceStorePreset.Meta] +#SomeKey = "SomeValue" +# [[Common.PersistStores]] #Name = "{store_name}" #Path = "{store_path}" @@ -228,6 +239,69 @@ Path = "/mnt/mass/piece1" # ``` +### [Common.PieceStorePreset] + + +`PersistStore` 指的是扇区持久化数据存储。与之对应的是 `damocles-worker` 中的 `attached` 概念。 + +而 `PieceStorePreset` 为全局的所有 `PersistStore` 配置，提供了一套可自定义的预设值，与后一章节中的 `[[Common.PersistStores]]` 配置项相呼应。合理利用 `PieceStorePreset` 可以大大简化 `[[Common.PersistStores]]` 的配置。 + +同时，`Damocles-Manager` 还支持 `lotus` 风格的存储配置文件（例如 `lotus-miner` repo 下默认生成的 `storage.json` 文件）。`Damocles-Manager` 会根据 `PieceStorePreset` 为该配置文件下的每一个路径在内存中生成一个对应的 `PersistStore` 配置项。 + + +#### 基础配置范例 + +```toml +[[Common.PersistStores]] + +# 只读，选填项，布尔类型 +# 默认值为 false +# 自 v0.4.0 起，持久化存储分配逻辑转到 damocles-manager 上 +# 可通过此配置设置存储是否可以继续写入 +ReadOnly = false + +# 可选项，布尔类型 +# 默认 false +# 是否验证`Path`路径是否为通常文件，true时，`Path`为软连接等非通常文件时会报错 +Strict = false + +# 权重，选填项，数字类型 +# 默认值为 1 +# 当填写值为 0 时，等效于 1 +# 自 v0.4.0 起，持久化存储分配逻辑转到 damocles-manager 上 +# 可通过此配置设置多个持久化存储之间的权重配比 +# 每个持久化存储被选中的概率为 `weight / sum`, `sum` 是所有可用的持久化存储权重的和 +# 例: 配置 3 个 持久化存储, weight 分别为 2, 1, 1。 则被选中的概率分别为 50%, 25%, 25% +Weight = 1 + + +# 允许进行分配的矿工号列表，选填项，数字数组类型 +# 默认为 null +# 当不设置时，视为允许全部矿工号；当设置时，则相当于白名单，仅允许分配给列出的矿工号 +# 如果一个矿工号同时出现在 AllowMiners 和 DenyMiners 中时，DenyMiners 优先生效，即视为拒绝 +AllowMiners = [1, 2] + +# 拒绝进行分配的矿工号列表，选填项，数字数组类型 +# 默认为 null +# 当不设置时，视为不拒绝任何矿工号；当设置时，则相当于黑名单，将拒绝为列出的矿工号分配 +# 如果一个矿工号同时出现在 AllowMiners 和 DenyMiners 中时，DenyMiners 优先生效，即视为拒绝 +DenyMiners = [3, 4] + +# lotus 风格的存储配置文件的路径，选填项，字符串类型 +# 默认为空字符串 +# 如果只是希望简单地沿用 lotus 的存储路径配置，可以选择填写此项 +# 但是如果希望更加深入地对每个存储路径进行细致的设置，建议使用 `[[Common.PersistStores]]` 配置项 +StorageConfigPath = "/optional/path/to/your/storage.json" + +# 元信息，选填项，字典类型 +# 内部值为 Key = "Value" 的格式 +# 默认值为 null +# 用于支持不同类型存储方案的预备，目前没有任何作用 +[Common.PersistStores.Meta] +#SomeKey = "SomeValue" +# +``` + ### [[Common.PersistStores]] `Common.PersistStores` 用于配置扇区持久化数据存储。与之对应的是 `damocles-worker` 中的 `attached` 概念。 From 9f643b714dc19488a51cd22a414b14858df0b42b Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 27 Jul 2023 16:25:05 +0800 Subject: [PATCH 5/6] fix: fix unit test --- damocles-manager/pkg/objstore/mgr_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/damocles-manager/pkg/objstore/mgr_test.go b/damocles-manager/pkg/objstore/mgr_test.go index bbb742e1b..897d400a1 100644 --- a/damocles-manager/pkg/objstore/mgr_test.go +++ b/damocles-manager/pkg/objstore/mgr_test.go @@ -204,7 +204,8 @@ func TestStoreManagerReserverSpaceWeighed(t *testing.T) { require.NoError(t, err, "construct store-1K") storeRO, err := NewMockStore(Config{ - Name: storeNameReadOnly, + Name: storeNameReadOnly, + ReadOnly: &TRUE, }, 1<<30) require.NoError(t, err, "construct store-RO") From abd91a9080be0b68b049bcd4bdf25ba6ae16f95d Mon Sep 17 00:00:00 2001 From: tanlang Date: Fri, 28 Jul 2023 14:49:38 +0800 Subject: [PATCH 6/6] feat: add name to storage.json --- damocles-manager/modules/config.go | 9 +++++++-- ...\215\347\275\256\350\247\243\346\236\220.md" | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/damocles-manager/modules/config.go b/damocles-manager/modules/config.go index e78ca1c21..cf1afb5dd 100644 --- a/damocles-manager/modules/config.go +++ b/damocles-manager/modules/config.go @@ -177,6 +177,7 @@ type StoragePathConfig struct { } type LocalPath struct { + Name string Path string } @@ -288,7 +289,7 @@ func (c CommonConfig) GetPersistStores() []PersistStoreConfig { log.Errorf("decode storage config file %s failed: %s", p, err) } else { for _, lp := range cfg.StoragePaths { - ret = append(ret, PersistStoreConfig{ + psc := PersistStoreConfig{ Config: objstore.Config{ Path: lp.Path, Meta: preset.Meta, @@ -300,7 +301,11 @@ func (c CommonConfig) GetPersistStores() []PersistStoreConfig { AllowMiners: preset.AllowMiners, DenyMiners: preset.DenyMiners, }, - }) + } + if lp.Name != "" { + psc.Name = lp.Name + } + ret = append(ret, psc) } log.Infof("load storage config file %s success", p) } diff --git "a/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" "b/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" index d6150934f..82f554c23 100644 --- "a/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" +++ "b/docs/zh/04.damocles-manager\347\232\204\351\205\215\347\275\256\350\247\243\346\236\220.md" @@ -252,7 +252,7 @@ Path = "/mnt/mass/piece1" #### 基础配置范例 ```toml -[[Common.PersistStores]] +[Common.PieceStorePreset] # 只读，选填项，布尔类型 # 默认值为 false @@ -302,6 +302,21 @@ StorageConfigPath = "/optional/path/to/your/storage.json" # ``` +- storage.json + +```json +{ + "StoragePaths": [ + { + "Name": "persist", + "Path": "/root/persist" + } + ] +} +``` + +其中 `Name` 属性可以省略，省略时默认使用 `Path` 属性的值作为 `Name`。 + ### [[Common.PersistStores]] `Common.PersistStores` 用于配置扇区持久化数据存储。与之对应的是 `damocles-worker` 中的 `attached` 概念。