diff --git a/collections/CHANGELOG.md b/collections/CHANGELOG.md index 101b5146f396..42c9e506fb39 100644 --- a/collections/CHANGELOG.md +++ b/collections/CHANGELOG.md @@ -31,6 +31,29 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [v0.4.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.4.0) + +### Features + +* [#17024](https://github.com/cosmos/cosmos-sdk/pull/17024) - Introduces `Triple`, a composite key with three keys. + +### API Breaking + +* [#17290](https://github.com/cosmos/cosmos-sdk/pull/17290) - Collections iteration methods (Iterate, Walk) will not error when the collection is empty. + +### Improvements + +* [#17021](https://github.com/cosmos/cosmos-sdk/pull/17021) Make collections implement the `appmodule.HasGenesis` interface. + +## [v0.3.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.3.0) + +### Features + +* [#16074](https://github.com/cosmos/cosmos-sdk/pull/16607) - Introduces `Clear` method for `Map` and `KeySet` +* [#16773](https://github.com/cosmos/cosmos-sdk/pull/16773) + * Adds `AltValueCodec` which provides a way to decode a value in two ways. + * Adds the possibility to specify an alternative way to decode the values of `KeySet`, `indexes.Multi`, `indexes.ReversePair`. + ## [v0.2.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.2.0) ### Features diff --git a/collections/README.md b/collections/README.md index 7f8278233c64..72d5f7172a58 100644 --- a/collections/README.md +++ b/collections/README.md @@ -1117,3 +1117,82 @@ func (k Keeper) GetAccount(ctx sdk.context, addr sdk.AccAddress) (sdk.AccountI, return k.Accounts.Get(ctx, addr) } ``` + +## Triple key + +The `collections.Triple` is a special type of key composed of three keys, it's identical to `collections.Pair`. + +Let's see an example. + +```go +package example + +import ( + "context" + + "cosmossdk.io/collections" + storetypes "cosmossdk.io/store/types" + "github.com/cosmos/cosmos-sdk/codec" +) + +type AccAddress = string +type ValAddress = string + +type Keeper struct { + // let's simulate we have redelegations which are stored as a triple key composed of + // the delegator, the source validator and the destination validator. + Redelegations collections.KeySet[collections.Triple[AccAddress, ValAddress, ValAddress]] +} + +func NewKeeper(storeKey *storetypes.KVStoreKey) Keeper { + sb := collections.NewSchemaBuilder(sdk.OpenKVStore(storeKey)) + return Keeper{ + Redelegations: collections.NewKeySet(sb, collections.NewPrefix(0), "redelegations", collections.TripleKeyCodec(collections.StringKey, collections.StringKey, collections.StringKey) + } +} + +// RedelegationsByDelegator iterates over all the redelegations of a given delegator and calls onResult providing +// each redelegation from source validator towards the destination validator. +func (k Keeper) RedelegationsByDelegator(ctx context.Context, delegator AccAddress, onResult func(src, dst ValAddress) (stop bool, err error)) error { + rng := collections.NewPrefixedTripleRange[AccAddress, ValAddress, ValAddress](delegator) + return k.Redelegations.Walk(ctx, rng, func(key collections.Triple[AccAddress, ValAddress, ValAddress]) (stop bool, err error) { + return onResult(key.K2(), key.K3()) + }) +} + +// RedelegationsByDelegatorAndValidator iterates over all the redelegations of a given delegator and its source validator and calls onResult for each +// destination validator. +func (k Keeper) RedelegationsByDelegatorAndValidator(ctx context.Context, delegator AccAddress, validator ValAddress, onResult func(dst ValAddress) (stop bool, err error)) error { + rng := collections.NewSuperPrefixedTripleRange[AccAddress, ValAddress, ValAddress](delegator, validator) + return k.Redelegations.Walk(ctx, rng, func(key collections.Triple[AccAddress, ValAddress, ValAddress]) (stop bool, err error) { + return onResult(key.K3()) + }) +} +``` + +## Advanced Usages + +### Alternative Value Codec + +The `codec.AltValueCodec` allows a collection to decode values using a different codec than the one used to encode them. +Basically it enables to decode two different byte representations of the same concrete value. +It can be used to lazily migrate values from one bytes representation to another, as long as the new representation is +not able to decode the old one. + +A concrete example can be found in `x/bank` where the balance was initially stored as `Coin` and then migrated to `Int`. + +```go + +var BankBalanceValueCodec = codec.NewAltValueCodec(sdk.IntValue, func(b []byte) (sdk.Int, error) { + coin := sdk.Coin{} + err := coin.Unmarshal(b) + if err != nil { + return sdk.Int{}, err + } + return coin.Amount, nil +}) +``` + +The above example shows how to create an `AltValueCodec` that can decode both `sdk.Int` and `sdk.Coin` values. The provided +decoder function will be used as a fallback in case the default decoder fails. When the value will be encoded back into state +it will use the default encoder. This allows to lazily migrate values to a new bytes representation. \ No newline at end of file diff --git a/collections/codec/alternative_value.go b/collections/codec/alternative_value.go new file mode 100644 index 000000000000..4d7902a4be33 --- /dev/null +++ b/collections/codec/alternative_value.go @@ -0,0 +1,47 @@ +package codec + +// NewAltValueCodec returns a new AltValueCodec. canonicalValueCodec is the codec that you want the value +// to be encoded and decoded as, alternativeDecoder is a function that will attempt to decode the value +// in case the canonicalValueCodec fails to decode it. +func NewAltValueCodec[V any](canonicalValueCodec ValueCodec[V], alternativeDecoder func([]byte) (V, error)) ValueCodec[V] { + return AltValueCodec[V]{ + canonicalValueCodec: canonicalValueCodec, + alternativeDecoder: alternativeDecoder, + } +} + +// AltValueCodec is a codec that can decode a value from state in an alternative format. +// This is useful for migrating data from one format to another. For example, in x/bank +// balances were initially encoded as sdk.Coin, now they are encoded as math.Int. +// The AltValueCodec will be trying to decode the value as math.Int, and if that fails, +// it will attempt to decode it as sdk.Coin. +// NOTE: if the canonical format can also decode the alternative format, then this codec +// will produce undefined and undesirable behavior. +type AltValueCodec[V any] struct { + canonicalValueCodec ValueCodec[V] + alternativeDecoder func([]byte) (V, error) +} + +// Decode will attempt to decode the value from state using the canonical value codec. +// If it fails to decode, it will attempt to decode the value using the alternative decoder. +func (a AltValueCodec[V]) Decode(b []byte) (V, error) { + v, err := a.canonicalValueCodec.Decode(b) + if err != nil { + return a.alternativeDecoder(b) + } + return v, nil +} + +// Below there is the implementation of ValueCodec relying on the canonical value codec. + +func (a AltValueCodec[V]) Encode(value V) ([]byte, error) { return a.canonicalValueCodec.Encode(value) } + +func (a AltValueCodec[V]) EncodeJSON(value V) ([]byte, error) { + return a.canonicalValueCodec.EncodeJSON(value) +} + +func (a AltValueCodec[V]) DecodeJSON(b []byte) (V, error) { return a.canonicalValueCodec.DecodeJSON(b) } + +func (a AltValueCodec[V]) Stringify(value V) string { return a.canonicalValueCodec.Stringify(value) } + +func (a AltValueCodec[V]) ValueType() string { return a.canonicalValueCodec.ValueType() } diff --git a/collections/codec/alternative_value_test.go b/collections/codec/alternative_value_test.go new file mode 100644 index 000000000000..358395427b90 --- /dev/null +++ b/collections/codec/alternative_value_test.go @@ -0,0 +1,53 @@ +package codec_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "cosmossdk.io/collections/codec" + "cosmossdk.io/collections/colltest" +) + +type altValue struct { + Value uint64 `json:"value"` +} + +func TestAltValueCodec(t *testing.T) { + // we assume we want to migrate the value from json(altValue) to just be + // the raw value uint64. + canonical := codec.KeyToValueCodec(codec.NewUint64Key[uint64]()) + alternative := func(v []byte) (uint64, error) { + var alt altValue + err := json.Unmarshal(v, &alt) + if err != nil { + return 0, err + } + return alt.Value, nil + } + + cdc := codec.NewAltValueCodec(canonical, alternative) + + t.Run("decodes alternative value", func(t *testing.T) { + expected := uint64(100) + alternativeEncodedBytes, err := json.Marshal(altValue{Value: expected}) + require.NoError(t, err) + got, err := cdc.Decode(alternativeEncodedBytes) + require.NoError(t, err) + require.Equal(t, expected, got) + }) + + t.Run("decodes canonical value", func(t *testing.T) { + expected := uint64(100) + canonicalEncodedBytes, err := cdc.Encode(expected) + require.NoError(t, err) + got, err := cdc.Decode(canonicalEncodedBytes) + require.NoError(t, err) + require.Equal(t, expected, got) + }) + + t.Run("conformance", func(t *testing.T) { + colltest.TestValueCodec(t, cdc, uint64(100)) + }) +} diff --git a/collections/codec/bool.go b/collections/codec/bool.go index 4016c8d68dcd..827af36c0715 100644 --- a/collections/codec/bool.go +++ b/collections/codec/bool.go @@ -33,7 +33,7 @@ func (b boolKey[T]) Decode(buffer []byte) (int, T, error) { } } -func (b boolKey[T]) Size(key T) int { return 1 } +func (b boolKey[T]) Size(_ T) int { return 1 } func (b boolKey[T]) EncodeJSON(value T) ([]byte, error) { return json.Marshal(value) diff --git a/collections/colltest/codec.go b/collections/colltest/codec.go index c0bb5038199c..58eb6f4ce446 100644 --- a/collections/colltest/codec.go +++ b/collections/colltest/codec.go @@ -40,6 +40,11 @@ func TestKeyCodec[T any](t *testing.T, keyCodec codec.KeyCodec[T], key T) { decoded, err := keyCodec.DecodeJSON(keyJSON) require.NoError(t, err) require.Equal(t, key, decoded, "json encoding and decoding did not produce the same results") + + // check type + require.NotEmpty(t, keyCodec.KeyType()) + // check string + _ = keyCodec.Stringify(key) } // TestValueCodec asserts the correct behavior of a ValueCodec over the type T. diff --git a/collections/genesis.go b/collections/genesis.go index 454ba496e180..0c593e4adfd8 100644 --- a/collections/genesis.go +++ b/collections/genesis.go @@ -32,7 +32,7 @@ func (m Map[K, V]) importGenesis(ctx context.Context, reader io.Reader) error { } func (m Map[K, V]) exportGenesis(ctx context.Context, writer io.Writer) error { - _, err := writer.Write([]byte("[")) + _, err := io.WriteString(writer, "[") if err != nil { return err } @@ -48,7 +48,7 @@ func (m Map[K, V]) exportGenesis(ctx context.Context, writer io.Writer) error { // add a comma before encoding the object // for all objects besides the first one. if !first { - _, err = writer.Write([]byte(",")) + _, err = io.WriteString(writer, ",") if err != nil { return err } @@ -91,7 +91,7 @@ func (m Map[K, V]) exportGenesis(ctx context.Context, writer io.Writer) error { } } - _, err = writer.Write([]byte("]")) + _, err = io.WriteString(writer, "]") return err } @@ -148,6 +148,6 @@ func (m Map[K, V]) doDecodeJSON(reader io.Reader, onEntry func(key K, value V) e } func (m Map[K, V]) defaultGenesis(writer io.Writer) error { - _, err := writer.Write([]byte(`[]`)) + _, err := io.WriteString(writer, `[]`) return err } diff --git a/collections/genesis_test.go b/collections/genesis_test.go index 3d7fd3c54955..34fb60e1414c 100644 --- a/collections/genesis_test.go +++ b/collections/genesis_test.go @@ -91,6 +91,7 @@ type testFixture struct { } func initFixture(t *testing.T) *testFixture { + t.Helper() sk, ctx := deps() schemaBuilder := NewSchemaBuilder(sk) m := NewMap(schemaBuilder, NewPrefix(1), "map", StringKey, Uint64Value) @@ -110,6 +111,7 @@ func initFixture(t *testing.T) *testFixture { } func createTestGenesisSource(t *testing.T) appmodule.GenesisSource { + t.Helper() expectedOrder := []string{"item", "key_set", "map", "sequence"} currentIndex := 0 return func(field string) (io.ReadCloser, error) { @@ -149,6 +151,7 @@ func (b *bufCloser) Close() error { } func newBufCloser(t *testing.T, str string) *bufCloser { + t.Helper() b := &bufCloser{ Buffer: bytes.NewBufferString(str), closed: false, diff --git a/collections/go.mod b/collections/go.mod index 18118e1500f4..cccc9069a77b 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -1,53 +1,53 @@ module cosmossdk.io/collections -go 1.21 +go 1.20 require ( - cosmossdk.io/core v0.11.0 - github.com/cosmos/cosmos-db v1.0.2 - github.com/stretchr/testify v1.9.0 + cosmossdk.io/core v0.10.0 + github.com/cosmos/cosmos-db v1.0.0 + github.com/stretchr/testify v1.8.4 pgregory.net/rapid v1.1.0 ) require ( - cosmossdk.io/api v0.7.5 // indirect + cosmossdk.io/api v0.7.0 // indirect cosmossdk.io/depinject v1.0.0-alpha.4 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/errors v1.10.0 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v1.1.0 // indirect + github.com/cockroachdb/pebble v0.0.0-20230525220056-bb4fc9527b3b // indirect github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect + github.com/cosmos/cosmos-proto v1.0.0-beta.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/getsentry/sentry-go v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/linxGnu/grocksdb v1.8.12 // indirect + github.com/linxGnu/grocksdb v1.7.16 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/onsi/gomega v1.20.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.47.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/spf13/cast v1.6.0 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect - google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/collections/go.sum b/collections/go.sum index 8932e2415bc5..9da2e1723ead 100644 --- a/collections/go.sum +++ b/collections/go.sum @@ -1,7 +1,7 @@ -cosmossdk.io/api v0.7.5 h1:eMPTReoNmGUm8DeiQL9DyM8sYDjEhWzL1+nLbI9DqtQ= -cosmossdk.io/api v0.7.5/go.mod h1:IcxpYS5fMemZGqyYtErK7OqvdM0C8kdW3dq8Q/XIG38= -cosmossdk.io/core v0.11.0 h1:vtIafqUi+1ZNAE/oxLOQQ7Oek2n4S48SWLG8h/+wdbo= -cosmossdk.io/core v0.11.0/go.mod h1:LaTtayWBSoacF5xNzoF8tmLhehqlA9z1SWiPuNC6X1w= +cosmossdk.io/api v0.7.0 h1:QsEMIWuv9xWDbF2HZnW4Lpu1/SejCztPu0LQx7t6MN4= +cosmossdk.io/api v0.7.0/go.mod h1:kJFAEMLN57y0viszHDPLMmieF0471o5QAwwApa+270M= +cosmossdk.io/core v0.10.0 h1:NP28Ol9YyRODmZLJg2ko/mUl40hMegeMzhJnG+XPkcY= +cosmossdk.io/core v0.10.0/go.mod h1:MygXNld9DvMgYY4yE76DM/mdZpgfeyRjy6FPjEEehlY= cosmossdk.io/depinject v1.0.0-alpha.4 h1:PLNp8ZYAMPTUKyG9IK2hsbciDWqna2z1Wsl98okJopc= cosmossdk.io/depinject v1.0.0-alpha.4/go.mod h1:HeDk7IkR5ckZ3lMGs/o91AVUc7E596vMaOmslGFM3yU= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= @@ -14,36 +14,31 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= -github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/errors v1.10.0 h1:lfxS8zZz1+OjtV4MtNWgboi/W5tyLEB6VQZBXN+0VUU= +github.com/cockroachdb/errors v1.10.0/go.mod h1:lknhIsEVQ9Ss/qKDBQS/UqFSvPQjOwNq2qyKAxtHRqE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= -github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/pebble v0.0.0-20230525220056-bb4fc9527b3b h1:LCs8gDhg6vt8A3dN7AEJxmCoETZ4qkySoVJVm3rcSJk= +github.com/cockroachdb/pebble v0.0.0-20230525220056-bb4fc9527b3b/go.mod h1:TkdVsGYRqtULUppt2RbC+YaKtTHnHoWa2apfFrSKABw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/cosmos/cosmos-db v1.0.2 h1:hwMjozuY1OlJs/uh6vddqnk9j7VamLv+0DBlbEXbAKs= -github.com/cosmos/cosmos-db v1.0.2/go.mod h1:Z8IXcFJ9PqKK6BIsVOB3QXtkKoqUOp1vRvPT39kOXEA= -github.com/cosmos/cosmos-proto v1.0.0-beta.5 h1:eNcayDLpip+zVLRLYafhzLvQlSmyab+RC5W7ZfmxJLA= -github.com/cosmos/cosmos-proto v1.0.0-beta.5/go.mod h1:hQGLpiIUloJBMdQMMWb/4wRApmI9hjHH05nefC0Ojec= +github.com/cosmos/cosmos-db v1.0.0 h1:EVcQZ+qYag7W6uorBKFPvX6gRjw6Uq2hIh4hCWjuQ0E= +github.com/cosmos/cosmos-db v1.0.0/go.mod h1:iBvi1TtqaedwLdcrZVYRSSCb6eSy61NLj4UNmdIgs0U= +github.com/cosmos/cosmos-proto v1.0.0-beta.3 h1:VitvZ1lPORTVxkmF2fAp3IiA61xVwArQYKXTdEcpW6o= +github.com/cosmos/cosmos-proto v1.0.0-beta.3/go.mod h1:t8IASdLaAq+bbHbjq4p960BvcTqtwuAxid3b/2rOD6I= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.23.0 h1:dn+QRCeJv4pPt9OjVXiMcGIBIefaTJPw/h0bZWO05nE= +github.com/getsentry/sentry-go v0.23.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -56,8 +51,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -66,21 +61,22 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linxGnu/grocksdb v1.8.12 h1:1/pCztQUOa3BX/1gR3jSZDoaKFpeHFvQ1XrqZpSvZVo= -github.com/linxGnu/grocksdb v1.8.12/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.7.16 h1:Q2co1xrpdkr5Hx3Fp+f+f7fRGhQFQhvi/+226dtLmA8= +github.com/linxGnu/grocksdb v1.7.16/go.mod h1:JkS7pl5qWpGpuVb3bPqTz8nC12X3YtPZT+Xq7+QfQo4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -97,30 +93,29 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9 github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= -github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= -github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -128,8 +123,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= +golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -141,14 +136,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -166,16 +161,16 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -186,10 +181,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e h1:S83+ibolgyZ0bqz7KEsUOPErxcv4VzlszxY+31OfB/E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -198,11 +193,10 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -212,7 +206,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/collections/indexes/multi.go b/collections/indexes/multi.go index 9e567aaab425..49ad2fb6f378 100644 --- a/collections/indexes/multi.go +++ b/collections/indexes/multi.go @@ -8,6 +8,21 @@ import ( "cosmossdk.io/collections/codec" ) +type multiOptions struct { + uncheckedValue bool +} + +// WithMultiUncheckedValue is an option that can be passed to NewMulti to +// ignore index values different from '[]byte{}' and continue with the operation. +// This should be used only to behave nicely in case you have used values different +// from '[]byte{}' in your storage before migrating to collections. Refer to +// WithKeySetUncheckedValue for more information. +func WithMultiUncheckedValue() func(*multiOptions) { + return func(o *multiOptions) { + o.uncheckedValue = true + } +} + // Multi defines the most common index. It can be used to create a reference between // a field of value and its primary key. Multiple primary keys can be mapped to the same // reference key as the index does not enforce uniqueness constraints. @@ -27,7 +42,19 @@ func NewMulti[ReferenceKey, PrimaryKey, Value any]( refCodec codec.KeyCodec[ReferenceKey], pkCodec codec.KeyCodec[PrimaryKey], getRefKeyFunc func(pk PrimaryKey, value Value) (ReferenceKey, error), + options ...func(*multiOptions), ) *Multi[ReferenceKey, PrimaryKey, Value] { + o := new(multiOptions) + for _, opt := range options { + opt(o) + } + if o.uncheckedValue { + return &Multi[ReferenceKey, PrimaryKey, Value]{ + getRefKey: getRefKeyFunc, + refKeys: collections.NewKeySet(schema, prefix, name, collections.PairKeyCodec(refCodec, pkCodec), collections.WithKeySetUncheckedValue()), + } + } + return &Multi[ReferenceKey, PrimaryKey, Value]{ getRefKey: getRefKeyFunc, refKeys: collections.NewKeySet(schema, prefix, name, collections.PairKeyCodec(refCodec, pkCodec)), diff --git a/collections/indexes/multi_test.go b/collections/indexes/multi_test.go index 428b880bfb71..73521ee03e38 100644 --- a/collections/indexes/multi_test.go +++ b/collections/indexes/multi_test.go @@ -61,3 +61,52 @@ func TestMultiIndex(t *testing.T) { require.False(t, iter.Valid()) require.NoError(t, iter.Close()) } + +func TestMultiUnchecked(t *testing.T) { + sk, ctx := deps() + schema := collections.NewSchemaBuilder(sk) + + uncheckedMi := NewMulti(schema, collections.NewPrefix("prefix"), "multi_index", collections.StringKey, collections.Uint64Key, func(_ uint64, value company) (string, error) { + return value.City, nil + }, WithMultiUncheckedValue()) + + mi := NewMulti(schema, collections.NewPrefix("prefix"), "multi_index", collections.StringKey, collections.Uint64Key, func(_ uint64, value company) (string, error) { + return value.City, nil + }) + + rawKey, err := collections.EncodeKeyWithPrefix( + collections.NewPrefix("prefix"), + uncheckedMi.KeyCodec(), + collections.Join("milan", uint64(2))) + require.NoError(t, err) + + // set value to be something different from []byte{} + require.NoError(t, sk.OpenKVStore(ctx).Set(rawKey, []byte("something"))) + + // normal multi index will fail. + err = mi.Walk(ctx, nil, func(indexingKey string, indexedKey uint64) (stop bool, err error) { + return true, err + }) + require.ErrorIs(t, err, collections.ErrEncoding) + + // unchecked multi index will not fail. + err = uncheckedMi.Walk(ctx, nil, func(indexingKey string, indexedKey uint64) (stop bool, err error) { + require.Equal(t, "milan", indexingKey) + require.Equal(t, uint64(2), indexedKey) + return true, err + }) + require.NoError(t, err) + + // unchecked multi will also reset the value + err = mi.Reference(ctx, 2, company{City: "milan"}, func() (company, error) { + return company{ + City: "milan", + }, nil + }) + require.NoError(t, err) + + // value reset to []byte{} + rawValue, err := sk.OpenKVStore(ctx).Get(rawKey) + require.NoError(t, err) + require.Equal(t, []byte{}, rawValue) +} diff --git a/collections/indexes/reverse_pair.go b/collections/indexes/reverse_pair.go index 243b2d289ad8..0e2cc301374e 100644 --- a/collections/indexes/reverse_pair.go +++ b/collections/indexes/reverse_pair.go @@ -7,6 +7,21 @@ import ( "cosmossdk.io/collections/codec" ) +type reversePairOptions struct { + uncheckedValue bool +} + +// WithReversePairUncheckedValue is an option that can be passed to NewReversePair to +// ignore index values different from '[]byte{}' and continue with the operation. +// This should be used only if you are migrating to collections and have used a different +// placeholder value in your storage index keys. +// Refer to WithKeySetUncheckedValue for more information. +func WithReversePairUncheckedValue() func(*reversePairOptions) { + return func(o *reversePairOptions) { + o.uncheckedValue = true + } +} + // ReversePair is an index that is used with collections.Pair keys. It indexes objects by their second part of the key. // When the value is being indexed by collections.IndexedMap then ReversePair will create a relationship between // the second part of the primary key and the first part. @@ -31,8 +46,19 @@ func NewReversePair[Value, K1, K2 any]( prefix collections.Prefix, name string, pairCodec codec.KeyCodec[collections.Pair[K1, K2]], + options ...func(*reversePairOptions), ) *ReversePair[K1, K2, Value] { pkc := pairCodec.(pairKeyCodec[K1, K2]) + o := new(reversePairOptions) + for _, option := range options { + option(o) + } + if o.uncheckedValue { + return &ReversePair[K1, K2, Value]{ + refKeys: collections.NewKeySet(sb, prefix, name, collections.PairKeyCodec(pkc.KeyCodec2(), pkc.KeyCodec1()), collections.WithKeySetUncheckedValue()), + } + } + mi := &ReversePair[K1, K2, Value]{ refKeys: collections.NewKeySet(sb, prefix, name, collections.PairKeyCodec(pkc.KeyCodec2(), pkc.KeyCodec1())), } diff --git a/collections/indexes/reverse_pair_test.go b/collections/indexes/reverse_pair_test.go index e4036aaaafdd..da1c1c961166 100644 --- a/collections/indexes/reverse_pair_test.go +++ b/collections/indexes/reverse_pair_test.go @@ -64,6 +64,45 @@ func TestReversePair(t *testing.T) { // assert if we remove address1 atom balance, we can no longer find it in the index err = indexedMap.Remove(ctx, collections.Join("address1", "atom")) require.NoError(t, err) - _, err = indexedMap.Indexes.Denom.MatchExact(ctx, "atom") - require.ErrorIs(t, collections.ErrInvalidIterator, err) + iter, err = indexedMap.Indexes.Denom.MatchExact(ctx, "atom") + require.NoError(t, err) + defer iter.Close() + pks, err = iter.PrimaryKeys() + require.NoError(t, err) + require.Empty(t, pks) +} + +func TestUncheckedReversePair(t *testing.T) { + sk, ctx := deps() + sb := collections.NewSchemaBuilder(sk) + prefix := collections.NewPrefix("prefix") + keyCodec := collections.PairKeyCodec(collections.StringKey, collections.StringKey) + + uncheckedRp := NewReversePair[Amount](sb, prefix, "denom_index", keyCodec, WithReversePairUncheckedValue()) + rp := NewReversePair[Amount](sb, prefix, "denom_index", keyCodec) + + rawKey, err := collections.EncodeKeyWithPrefix(prefix, uncheckedRp.KeyCodec(), collections.Join("atom", "address1")) + require.NoError(t, err) + + require.NoError(t, sk.OpenKVStore(ctx).Set(rawKey, []byte("i should not be here"))) + + // normal reverse pair fails + err = rp.Walk(ctx, nil, func(denom, address string) (bool, error) { + return false, nil + }) + require.ErrorIs(t, err, collections.ErrEncoding) + + // unchecked reverse pair succeeds + err = uncheckedRp.Walk(ctx, nil, func(indexingKey, indexedKey string) (stop bool, err error) { + require.Equal(t, "atom", indexingKey) + return true, nil + }) + require.NoError(t, err) + + // unchecked reverse pair lazily updates + err = uncheckedRp.Reference(ctx, collections.Join("address1", "atom"), 0, nil) + require.NoError(t, err) + rawValue, err := sk.OpenKVStore(ctx).Get(rawKey) + require.NoError(t, err) + require.Equal(t, []byte{}, rawValue) } diff --git a/collections/item.go b/collections/item.go index 1b36bf46dad6..155cb729aa24 100644 --- a/collections/item.go +++ b/collections/item.go @@ -37,7 +37,7 @@ func (i Item[V]) Set(ctx context.Context, value V) error { } // Has reports whether the item exists in the store or not. -// Returns an error in case encoding fails. +// Returns an error in case func (i Item[V]) Has(ctx context.Context) (bool, error) { return (Map[noKey, V])(i).Has(ctx, noKey{}) } diff --git a/collections/iter.go b/collections/iter.go index c728f1994187..06f2a9f96029 100644 --- a/collections/iter.go +++ b/collections/iter.go @@ -1,6 +1,7 @@ package collections import ( + "bytes" "context" "errors" "fmt" @@ -129,39 +130,51 @@ func (r *Range[K]) RangeValues() (start, end *RangeKey[K], order Order, err erro return r.start, r.end, r.order, nil } -// iteratorFromRanger generates an Iterator instance, with the proper prefixing and ranging. -// a nil Ranger can be seen as an ascending iteration over all the possible keys. -func iteratorFromRanger[K, V any](ctx context.Context, m Map[K, V], r Ranger[K]) (iter Iterator[K, V], err error) { +// parseRangeInstruction converts a Ranger into start bytes, end bytes and order of a store iteration. +func parseRangeInstruction[K any](prefix []byte, keyCodec codec.KeyCodec[K], r Ranger[K]) ([]byte, []byte, Order, error) { var ( start *RangeKey[K] end *RangeKey[K] order = OrderAscending + err error ) if r != nil { start, end, order, err = r.RangeValues() if err != nil { - return iter, err + return nil, nil, 0, err } } - startBytes := m.prefix + startBytes := prefix if start != nil { - startBytes, err = encodeRangeBound(m.prefix, m.kc, start) + startBytes, err = encodeRangeBound(prefix, keyCodec, start) if err != nil { - return iter, err + return nil, nil, 0, err } } var endBytes []byte if end != nil { - endBytes, err = encodeRangeBound(m.prefix, m.kc, end) + endBytes, err = encodeRangeBound(prefix, keyCodec, end) if err != nil { - return iter, err + return nil, nil, 0, err } } else { - endBytes = nextBytesPrefixKey(m.prefix) + endBytes = nextBytesPrefixKey(prefix) } + if bytes.Compare(startBytes, endBytes) == 1 { + return nil, nil, 0, ErrInvalidIterator + } + return startBytes, endBytes, order, nil +} +// iteratorFromRanger generates an Iterator instance, with the proper prefixing and ranging. +// a nil Ranger can be seen as an ascending iteration over all the possible keys. +func iteratorFromRanger[K, V any](ctx context.Context, m Map[K, V], r Ranger[K]) (iter Iterator[K, V], err error) { + startBytes, endBytes, order, err := parseRangeInstruction(m.prefix, m.kc, r) + if err != nil { + return Iterator[K, V]{}, err + } return newIterator(ctx, startBytes, endBytes, order, m) } @@ -182,9 +195,6 @@ func newIterator[K, V any](ctx context.Context, start, end []byte, order Order, if err != nil { return Iterator[K, V]{}, err } - if !iter.Valid() { - return Iterator[K, V]{}, ErrInvalidIterator - } return Iterator[K, V]{ kc: m.kc, diff --git a/collections/iter_test.go b/collections/iter_test.go index dfe690bb97ba..e18bad20a864 100644 --- a/collections/iter_test.go +++ b/collections/iter_test.go @@ -10,11 +10,13 @@ import ( func TestIteratorBasic(t *testing.T) { sk, ctx := deps() // safety check to ensure that iteration does not cross prefix boundaries - sk.OpenKVStore(ctx).Set([]byte{0, 0}, []byte("before prefix")) - sk.OpenKVStore(ctx).Set([]byte{2, 1}, []byte("after prefix")) + err := sk.OpenKVStore(ctx).Set([]byte{0, 0}, []byte("before prefix")) + require.NoError(t, err) + err = sk.OpenKVStore(ctx).Set([]byte{2, 1}, []byte("after prefix")) + require.NoError(t, err) schemaBuilder := NewSchemaBuilder(sk) m := NewMap(schemaBuilder, NewPrefix(1), "m", StringKey, Uint64Value) - _, err := schemaBuilder.Build() + _, err = schemaBuilder.Build() require.NoError(t, err) for i := uint64(1); i <= 2; i++ { diff --git a/collections/keyset.go b/collections/keyset.go index 408086b67262..37c3bd268e37 100644 --- a/collections/keyset.go +++ b/collections/keyset.go @@ -8,14 +8,43 @@ import ( "cosmossdk.io/collections/codec" ) +// WithKeySetUncheckedValue changes the behavior of the KeySet when it encounters +// a value different from '[]byte{}', by default the KeySet errors when this happens. +// This option allows to ignore the value and continue with the operation, in turn +// the value will be cleared out and set to '[]byte{}'. +// You should never use this option if you're creating a new state object from scratch. +// This should be used only to behave nicely in case you have used values different +// from '[]byte{}' in your storage before migrating to collections. +func WithKeySetUncheckedValue() func(opt *keySetOptions) { + return func(opt *keySetOptions) { + opt.uncheckedValue = true + } +} + +type keySetOptions struct{ uncheckedValue bool } + // KeySet builds on top of a Map and represents a collection retaining only a set // of keys and no value. It can be used, for example, in an allow list. type KeySet[K any] Map[K, NoValue] // NewKeySet returns a KeySet given a Schema, Prefix a human name for the collection // and a KeyCodec for the key K. -func NewKeySet[K any](schema *SchemaBuilder, prefix Prefix, name string, keyCodec codec.KeyCodec[K]) KeySet[K] { - return (KeySet[K])(NewMap(schema, prefix, name, keyCodec, noValueCodec)) +func NewKeySet[K any]( + schema *SchemaBuilder, + prefix Prefix, + name string, + keyCodec codec.KeyCodec[K], + options ...func(opt *keySetOptions), +) KeySet[K] { + o := new(keySetOptions) + for _, opt := range options { + opt(o) + } + vc := noValueCodec + if o.uncheckedValue { + vc = codec.NewAltValueCodec(vc, func(_ []byte) (NoValue, error) { return NoValue{}, nil }) + } + return (KeySet[K])(NewMap(schema, prefix, name, keyCodec, vc)) } // Set adds the key to the KeySet. Errors on encoding problems. @@ -57,6 +86,12 @@ func (k KeySet[K]) Walk(ctx context.Context, ranger Ranger[K], walkFunc func(key return (Map[K, NoValue])(k).Walk(ctx, ranger, func(key K, value NoValue) (bool, error) { return walkFunc(key) }) } +// Clear clears the KeySet using the provided Ranger. Refer to Map.Clear for +// behavioral documentation. +func (k KeySet[K]) Clear(ctx context.Context, ranger Ranger[K]) error { + return (Map[K, NoValue])(k).Clear(ctx, ranger) +} + func (k KeySet[K]) KeyCodec() codec.KeyCodec[K] { return (Map[K, NoValue])(k).KeyCodec() } func (k KeySet[K]) ValueCodec() codec.ValueCodec[NoValue] { return (Map[K, NoValue])(k).ValueCodec() } @@ -73,6 +108,7 @@ var noValueCodec codec.ValueCodec[NoValue] = NoValue{} const noValueValueType = "no_value" +// NoValue is a type that can be used to represent a non-existing value. type NoValue struct{} func (n NoValue) EncodeJSON(_ NoValue) ([]byte, error) { @@ -92,7 +128,7 @@ func (NoValue) Encode(_ NoValue) ([]byte, error) { func (NoValue) Decode(b []byte) (NoValue, error) { if !bytes.Equal(b, []byte{}) { - return NoValue{}, fmt.Errorf("%w: invalid value, wanted an empty non-nil byte slice", ErrEncoding) + return NoValue{}, fmt.Errorf("%w: invalid value, wanted an empty non-nil byte slice, got: %x", ErrEncoding, b) } return NoValue{}, nil } diff --git a/collections/keyset_test.go b/collections/keyset_test.go index 6fe3e7eedfb0..cabc2ead865b 100644 --- a/collections/keyset_test.go +++ b/collections/keyset_test.go @@ -67,3 +67,34 @@ func Test_noValue(t *testing.T) { _, err = noValueCodec.Decode([]byte("bad")) require.ErrorIs(t, err, ErrEncoding) } + +func TestUncheckedKeySet(t *testing.T) { + sk, ctx := deps() + schema := NewSchemaBuilder(sk) + uncheckedKs := NewKeySet(schema, NewPrefix("keyset"), "keyset", StringKey, WithKeySetUncheckedValue()) + ks := NewKeySet(schema, NewPrefix("keyset"), "keyset", StringKey) + // we set a NoValue unfriendly value. + require.NoError(t, sk.OpenKVStore(ctx).Set([]byte("keyset1"), []byte("A"))) + require.NoError(t, sk.OpenKVStore(ctx).Set([]byte("keyset2"), []byte("B"))) + + // the standard KeySet errors here, because it doesn't like the fact that the value is []byte("A") + // and not []byte{}. + err := ks.Walk(ctx, nil, func(key string) (stop bool, err error) { + return true, nil + }) + require.ErrorIs(t, err, ErrEncoding) + + // the unchecked KeySet doesn't care about the value, so it works. + err = uncheckedKs.Walk(ctx, nil, func(key string) (stop bool, err error) { + require.Equal(t, "1", key) + return true, nil + }) + require.NoError(t, err) + + // now we set it again + require.NoError(t, uncheckedKs.Set(ctx, "1")) + // and we will see that the value which was []byte("A") has been cleared to be []byte{} + raw, err := sk.OpenKVStore(ctx).Get([]byte("keyset1")) + require.NoError(t, err) + require.Equal(t, []byte{}, raw) +} diff --git a/collections/map.go b/collections/map.go index 525110cdf9c5..810b5a06f809 100644 --- a/collections/map.go +++ b/collections/map.go @@ -1,6 +1,7 @@ package collections import ( + "bytes" "context" "fmt" @@ -64,8 +65,7 @@ func (m Map[K, V]) Set(ctx context.Context, key K, value V) error { } kvStore := m.sa(ctx) - kvStore.Set(bytesKey, valueBytes) - return nil + return kvStore.Set(bytesKey, valueBytes) } // Get returns the value associated with the provided key, @@ -149,6 +149,58 @@ func (m Map[K, V]) Walk(ctx context.Context, ranger Ranger[K], walkFunc func(key return nil } +// Clear clears the collection contained within the provided key range. +// A nil ranger equals to clearing the whole collection. +// NOTE: this API needs to be used with care, considering that as of today +// cosmos-sdk stores the deletion records to be committed in a memory cache, +// clearing a lot of data might make the node go OOM. +func (m Map[K, V]) Clear(ctx context.Context, ranger Ranger[K]) error { + startBytes, endBytes, _, err := parseRangeInstruction(m.prefix, m.kc, ranger) + if err != nil { + return err + } + return deleteDomain(m.sa(ctx), startBytes, endBytes) +} + +const clearBatchSize = 10000 + +// deleteDomain deletes the domain of an iterator, the key difference +// is that it uses batches to clear the store meaning that it will read +// the keys within the domain close the iterator and then delete them. +func deleteDomain(s store.KVStore, start, end []byte) error { + for { + iter, err := s.Iterator(start, end) + if err != nil { + return err + } + + keys := make([][]byte, 0, clearBatchSize) + for ; iter.Valid() && len(keys) < clearBatchSize; iter.Next() { + keys = append(keys, iter.Key()) + } + + // we close the iterator here instead of deferring + err = iter.Close() + if err != nil { + return err + } + + for _, key := range keys { + err = s.Delete(key) + if err != nil { + return err + } + } + + // If we've retrieved less than the batchSize, we're done. + if len(keys) < clearBatchSize { + break + } + } + + return nil +} + // IterateRaw iterates over the collection. The iteration range is untyped, it uses raw // bytes. The resulting Iterator is typed. // A nil start iterates from the first key contained in the collection. @@ -164,6 +216,10 @@ func (m Map[K, V]) IterateRaw(ctx context.Context, start, end []byte, order Orde prefixedEnd = append(m.prefix, end...) } + if bytes.Compare(prefixedStart, prefixedEnd) == 1 { + return Iterator[K, V]{}, ErrInvalidIterator + } + s := m.sa(ctx) var ( storeIter store.Iterator @@ -181,9 +237,6 @@ func (m Map[K, V]) IterateRaw(ctx context.Context, start, end []byte, order Orde return Iterator[K, V]{}, err } - if !storeIter.Valid() { - return Iterator[K, V]{}, ErrInvalidIterator - } return Iterator[K, V]{ kc: m.kc, vc: m.vc, diff --git a/collections/map_test.go b/collections/map_test.go index 2887693d53a2..f95935c5720e 100644 --- a/collections/map_test.go +++ b/collections/map_test.go @@ -1,6 +1,8 @@ package collections import ( + "context" + "fmt" "testing" "github.com/stretchr/testify/require" @@ -36,6 +38,42 @@ func TestMap(t *testing.T) { require.False(t, has) } +func TestMap_Clear(t *testing.T) { + makeTest := func() (context.Context, Map[uint64, uint64]) { + sk, ctx := deps() + m := NewMap(NewSchemaBuilder(sk), NewPrefix(0), "test", Uint64Key, Uint64Value) + for i := uint64(0); i < clearBatchSize*2; i++ { + require.NoError(t, m.Set(ctx, i, i)) + } + return ctx, m + } + + t.Run("nil ranger", func(t *testing.T) { + ctx, m := makeTest() + err := m.Clear(ctx, nil) + require.NoError(t, err) + err = m.Walk(ctx, nil, func(key, value uint64) (bool, error) { + return false, fmt.Errorf("should never be called") + }) + require.NoError(t, err) + }) + + t.Run("custom ranger", func(t *testing.T) { + ctx, m := makeTest() + // delete from 0 to 100 + err := m.Clear(ctx, new(Range[uint64]).StartInclusive(0).EndInclusive(100)) + require.NoError(t, err) + + iter, err := m.Iterate(ctx, nil) + require.NoError(t, err) + keys, err := iter.Keys() + require.NoError(t, err) + require.Len(t, keys, clearBatchSize*2-101) + require.Equal(t, keys[0], uint64(101)) + require.Equal(t, keys[len(keys)-1], uint64(clearBatchSize*2-1)) + }) +} + func TestMap_IterateRaw(t *testing.T) { sk, ctx := deps() // safety check to ensure prefix boundaries are not crossed @@ -69,6 +107,15 @@ func TestMap_IterateRaw(t *testing.T) { keys, err = iter.Keys() require.NoError(t, err) require.Equal(t, []uint64{2, 1, 0}, keys) + + // test invalid iter + _, err = m.IterateRaw(ctx, []byte{0x2, 0x0}, []byte{0x0, 0x0}, OrderAscending) + require.ErrorIs(t, err, ErrInvalidIterator) + + // test on empty collection iterating does not error + require.NoError(t, m.Clear(ctx, nil)) + _, err = m.IterateRaw(ctx, nil, nil, OrderAscending) + require.NoError(t, err) } func Test_encodeKey(t *testing.T) { diff --git a/collections/schema.go b/collections/schema.go index e05058f144c1..8cc851166f61 100644 --- a/collections/schema.go +++ b/collections/schema.go @@ -161,6 +161,12 @@ func NewSchemaFromAccessor(accessor func(context.Context) store.KVStore) Schema } } +// IsOnePerModuleType implements the depinject.OnePerModuleType interface. +func (s Schema) IsOnePerModuleType() {} + +// IsAppModule implements the appmodule.AppModule interface. +func (s Schema) IsAppModule() {} + // DefaultGenesis implements the appmodule.HasGenesis.DefaultGenesis method. func (s Schema) DefaultGenesis(target appmodule.GenesisTarget) error { for _, name := range s.collectionsOrdered { diff --git a/collections/triple.go b/collections/triple.go new file mode 100644 index 000000000000..9733d9984099 --- /dev/null +++ b/collections/triple.go @@ -0,0 +1,303 @@ +package collections + +import ( + "encoding/json" + "fmt" + "strings" + + "cosmossdk.io/collections/codec" +) + +// Triple defines a multipart key composed of three keys. +type Triple[K1, K2, K3 any] struct { + k1 *K1 + k2 *K2 + k3 *K3 +} + +// Join3 instantiates a new Triple instance composed of the three provided keys, in order. +func Join3[K1, K2, K3 any](k1 K1, k2 K2, k3 K3) Triple[K1, K2, K3] { + return Triple[K1, K2, K3]{&k1, &k2, &k3} +} + +// K1 returns the first part of the key. If nil, the zero value is returned. +func (t Triple[K1, K2, K3]) K1() (x K1) { + if t.k1 != nil { + return *t.k1 + } + return x +} + +// K2 returns the second part of the key. If nil, the zero value is returned. +func (t Triple[K1, K2, K3]) K2() (x K2) { + if t.k2 != nil { + return *t.k2 + } + return x +} + +// K3 returns the third part of the key. If nil, the zero value is returned. +func (t Triple[K1, K2, K3]) K3() (x K3) { + if t.k3 != nil { + return *t.k3 + } + return x +} + +// TriplePrefix creates a new Triple instance composed only of the first part of the key. +func TriplePrefix[K1, K2, K3 any](k1 K1) Triple[K1, K2, K3] { + return Triple[K1, K2, K3]{k1: &k1} +} + +// TripleSuperPrefix creates a new Triple instance composed only of the first two parts of the key. +func TripleSuperPrefix[K1, K2, K3 any](k1 K1, k2 K2) Triple[K1, K2, K3] { + return Triple[K1, K2, K3]{k1: &k1, k2: &k2} +} + +// TripleKeyCodec instantiates a new KeyCodec instance that can encode the Triple, given +// the KeyCodecs of the three parts of the key, in order. +func TripleKeyCodec[K1, K2, K3 any](keyCodec1 codec.KeyCodec[K1], keyCodec2 codec.KeyCodec[K2], keyCodec3 codec.KeyCodec[K3]) codec.KeyCodec[Triple[K1, K2, K3]] { + return tripleKeyCodec[K1, K2, K3]{ + keyCodec1: keyCodec1, + keyCodec2: keyCodec2, + keyCodec3: keyCodec3, + } +} + +type tripleKeyCodec[K1, K2, K3 any] struct { + keyCodec1 codec.KeyCodec[K1] + keyCodec2 codec.KeyCodec[K2] + keyCodec3 codec.KeyCodec[K3] +} + +type jsonTripleKey [3]json.RawMessage + +func (t tripleKeyCodec[K1, K2, K3]) EncodeJSON(value Triple[K1, K2, K3]) ([]byte, error) { + json1, err := t.keyCodec1.EncodeJSON(*value.k1) + if err != nil { + return nil, err + } + + json2, err := t.keyCodec2.EncodeJSON(*value.k2) + if err != nil { + return nil, err + } + + json3, err := t.keyCodec3.EncodeJSON(*value.k3) + if err != nil { + return nil, err + } + + return json.Marshal(jsonTripleKey{json1, json2, json3}) +} + +func (t tripleKeyCodec[K1, K2, K3]) DecodeJSON(b []byte) (Triple[K1, K2, K3], error) { + var jsonKey jsonTripleKey + err := json.Unmarshal(b, &jsonKey) + if err != nil { + return Triple[K1, K2, K3]{}, err + } + + key1, err := t.keyCodec1.DecodeJSON(jsonKey[0]) + if err != nil { + return Triple[K1, K2, K3]{}, err + } + + key2, err := t.keyCodec2.DecodeJSON(jsonKey[1]) + if err != nil { + return Triple[K1, K2, K3]{}, err + } + + key3, err := t.keyCodec3.DecodeJSON(jsonKey[2]) + if err != nil { + return Triple[K1, K2, K3]{}, err + } + + return Join3(key1, key2, key3), nil +} + +func (t tripleKeyCodec[K1, K2, K3]) Stringify(key Triple[K1, K2, K3]) string { + b := new(strings.Builder) + b.WriteByte('(') + if key.k1 != nil { + b.WriteByte('"') + b.WriteString(t.keyCodec1.Stringify(*key.k1)) + b.WriteByte('"') + } else { + b.WriteString("") + } + + b.WriteString(", ") + if key.k2 != nil { + b.WriteByte('"') + b.WriteString(t.keyCodec2.Stringify(*key.k2)) + b.WriteByte('"') + } else { + b.WriteString("") + } + + b.WriteString(", ") + if key.k3 != nil { + b.WriteByte('"') + b.WriteString(t.keyCodec3.Stringify(*key.k3)) + b.WriteByte('"') + } else { + b.WriteString("") + } + + b.WriteByte(')') + return b.String() +} + +func (t tripleKeyCodec[K1, K2, K3]) KeyType() string { + return fmt.Sprintf("Triple[%s,%s,%s]", t.keyCodec1.KeyType(), t.keyCodec2.KeyType(), t.keyCodec3.KeyType()) +} + +func (t tripleKeyCodec[K1, K2, K3]) Encode(buffer []byte, key Triple[K1, K2, K3]) (int, error) { + writtenTotal := 0 + if key.k1 != nil { + written, err := t.keyCodec1.EncodeNonTerminal(buffer, *key.k1) + if err != nil { + return 0, err + } + writtenTotal += written + } + if key.k2 != nil { + written, err := t.keyCodec2.EncodeNonTerminal(buffer[writtenTotal:], *key.k2) + if err != nil { + return 0, err + } + writtenTotal += written + } + if key.k3 != nil { + written, err := t.keyCodec3.Encode(buffer[writtenTotal:], *key.k3) + if err != nil { + return 0, err + } + writtenTotal += written + } + return writtenTotal, nil +} + +func (t tripleKeyCodec[K1, K2, K3]) Decode(buffer []byte) (int, Triple[K1, K2, K3], error) { + readTotal := 0 + read, key1, err := t.keyCodec1.DecodeNonTerminal(buffer) + if err != nil { + return 0, Triple[K1, K2, K3]{}, err + } + readTotal += read + read, key2, err := t.keyCodec2.DecodeNonTerminal(buffer[readTotal:]) + if err != nil { + return 0, Triple[K1, K2, K3]{}, err + } + readTotal += read + read, key3, err := t.keyCodec3.Decode(buffer[readTotal:]) + if err != nil { + return 0, Triple[K1, K2, K3]{}, err + } + readTotal += read + return readTotal, Join3(key1, key2, key3), nil +} + +func (t tripleKeyCodec[K1, K2, K3]) Size(key Triple[K1, K2, K3]) int { + size := 0 + if key.k1 != nil { + size += t.keyCodec1.SizeNonTerminal(*key.k1) + } + if key.k2 != nil { + size += t.keyCodec2.SizeNonTerminal(*key.k2) + } + if key.k3 != nil { + size += t.keyCodec3.Size(*key.k3) + } + return size +} + +func (t tripleKeyCodec[K1, K2, K3]) EncodeNonTerminal(buffer []byte, key Triple[K1, K2, K3]) (int, error) { + writtenTotal := 0 + if key.k1 != nil { + written, err := t.keyCodec1.EncodeNonTerminal(buffer, *key.k1) + if err != nil { + return 0, err + } + writtenTotal += written + } + if key.k2 != nil { + written, err := t.keyCodec2.EncodeNonTerminal(buffer[writtenTotal:], *key.k2) + if err != nil { + return 0, err + } + writtenTotal += written + } + if key.k3 != nil { + written, err := t.keyCodec3.EncodeNonTerminal(buffer[writtenTotal:], *key.k3) + if err != nil { + return 0, err + } + writtenTotal += written + } + return writtenTotal, nil +} + +func (t tripleKeyCodec[K1, K2, K3]) DecodeNonTerminal(buffer []byte) (int, Triple[K1, K2, K3], error) { + readTotal := 0 + read, key1, err := t.keyCodec1.DecodeNonTerminal(buffer) + if err != nil { + return 0, Triple[K1, K2, K3]{}, err + } + readTotal += read + read, key2, err := t.keyCodec2.DecodeNonTerminal(buffer[readTotal:]) + if err != nil { + return 0, Triple[K1, K2, K3]{}, err + } + readTotal += read + read, key3, err := t.keyCodec3.DecodeNonTerminal(buffer[readTotal:]) + if err != nil { + return 0, Triple[K1, K2, K3]{}, err + } + readTotal += read + return readTotal, Join3(key1, key2, key3), nil +} + +func (t tripleKeyCodec[K1, K2, K3]) SizeNonTerminal(key Triple[K1, K2, K3]) int { + size := 0 + if key.k1 != nil { + size += t.keyCodec1.SizeNonTerminal(*key.k1) + } + if key.k2 != nil { + size += t.keyCodec2.SizeNonTerminal(*key.k2) + } + if key.k3 != nil { + size += t.keyCodec3.SizeNonTerminal(*key.k3) + } + return size +} + +// NewPrefixUntilTripleRange defines a collection query which ranges until the provided Pair prefix. +// Unstable: this API might change in the future. +func NewPrefixUntilTripleRange[K1, K2, K3 any](k1 K1) Ranger[Triple[K1, K2, K3]] { + key := TriplePrefix[K1, K2, K3](k1) + return &Range[Triple[K1, K2, K3]]{ + end: RangeKeyPrefixEnd(key), + } +} + +// NewPrefixedTripleRange provides a Range for all keys prefixed with the given +// first part of the Triple key. +func NewPrefixedTripleRange[K1, K2, K3 any](k1 K1) Ranger[Triple[K1, K2, K3]] { + key := TriplePrefix[K1, K2, K3](k1) + return &Range[Triple[K1, K2, K3]]{ + start: RangeKeyExact(key), + end: RangeKeyPrefixEnd(key), + } +} + +// NewSuperPrefixedTripleRange provides a Range for all keys prefixed with the given +// first and second parts of the Triple key. +func NewSuperPrefixedTripleRange[K1, K2, K3 any](k1 K1, k2 K2) Ranger[Triple[K1, K2, K3]] { + key := TripleSuperPrefix[K1, K2, K3](k1, k2) + return &Range[Triple[K1, K2, K3]]{ + start: RangeKeyExact(key), + end: RangeKeyPrefixEnd(key), + } +} diff --git a/collections/triple_test.go b/collections/triple_test.go new file mode 100644 index 000000000000..fea1b662c15c --- /dev/null +++ b/collections/triple_test.go @@ -0,0 +1,52 @@ +package collections_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "cosmossdk.io/collections" + "cosmossdk.io/collections/colltest" +) + +func TestTriple(t *testing.T) { + kc := collections.TripleKeyCodec(collections.Uint64Key, collections.StringKey, collections.BytesKey) + + t.Run("conformance", func(t *testing.T) { + colltest.TestKeyCodec(t, kc, collections.Join3(uint64(1), "2", []byte("3"))) + }) +} + +func TestTripleRange(t *testing.T) { + sk, ctx := colltest.MockStore() + schema := collections.NewSchemaBuilder(sk) + // this is a key composed of 3 parts: uint64, string, []byte + kc := collections.TripleKeyCodec(collections.Uint64Key, collections.StringKey, collections.BytesKey) + + keySet := collections.NewKeySet(schema, collections.NewPrefix(0), "triple", kc) + + keys := []collections.Triple[uint64, string, []byte]{ + collections.Join3(uint64(1), "A", []byte("1")), + collections.Join3(uint64(1), "A", []byte("2")), + collections.Join3(uint64(1), "B", []byte("3")), + collections.Join3(uint64(2), "B", []byte("4")), + } + + for _, k := range keys { + require.NoError(t, keySet.Set(ctx, k)) + } + + // we prefix over (1) we expect 3 results + iter, err := keySet.Iterate(ctx, collections.NewPrefixedTripleRange[uint64, string, []byte](uint64(1))) + require.NoError(t, err) + gotKeys, err := iter.Keys() + require.NoError(t, err) + require.Equal(t, keys[:3], gotKeys) + + // we super prefix over Join(1, "A") we expect 2 results + iter, err = keySet.Iterate(ctx, collections.NewSuperPrefixedTripleRange[uint64, string, []byte](1, "A")) + require.NoError(t, err) + gotKeys, err = iter.Keys() + require.NoError(t, err) + require.Equal(t, keys[:2], gotKeys) +}