From f63ba6aeea506748c0447caa1ed6bf4e5fc01b9a Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 22 Aug 2024 15:03:27 +0200 Subject: [PATCH 1/9] OAK commands WiP --- cmd/aem/oak.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ pkg/instance.go | 4 ++++ 2 files changed, 51 insertions(+) create mode 100644 cmd/aem/oak.go diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go new file mode 100644 index 0000000..a192316 --- /dev/null +++ b/cmd/aem/oak.go @@ -0,0 +1,47 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +func (c *CLI) oakCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "oak", + Short: "Manages OAK repository", + } + cmd.AddCommand(c.oakIndexCmd()) + return cmd +} + +func (c *CLI) oakIndexCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "index", + Aliases: []string{"idx"}, + Short: "Manage OAK indexes", + } + cmd.AddCommand(c.oakIndexListCmd()) + return cmd +} + +func (c *CLI) oakIndexListCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List OAK indexes", + Aliases: []string{"ls"}, + Run: func(cmd *cobra.Command, args []string) { + instance, err := c.aem.InstanceManager().One() + if err != nil { + c.Error(err) + return + } + bundles, err := instance.OAK().IndexManager().List() + if err != nil { + c.Error(err) + return + } + c.SetOutput("indexes", bundles) + c.Ok("indexes listed") + }, + } + return cmd +} diff --git a/pkg/instance.go b/pkg/instance.go index 40a8189..b4cda62 100644 --- a/pkg/instance.go +++ b/pkg/instance.go @@ -345,3 +345,7 @@ func (i Instance) LockDir() string { } return i.CacheDir() } + +func (i Instance) OAK() { + // TODO +} From 5c28f923be95cf0c209fca21c4041ff163a6b4b8 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 22 Aug 2024 16:21:04 +0200 Subject: [PATCH 2/9] OAK index list --- cmd/aem/oak.go | 4 +- cmd/aem/root.go | 1 + pkg/instance.go | 9 +++-- pkg/instance_manager.go | 1 + pkg/oak.go | 20 ++++++++++ pkg/oak/constants.go | 5 +++ pkg/oak/index.go | 62 ++++++++++++++++++++++++++++++ pkg/oak_index.go | 81 ++++++++++++++++++++++++++++++++++++++++ pkg/oak_index_manager.go | 52 ++++++++++++++++++++++++++ 9 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 pkg/oak.go create mode 100644 pkg/oak/constants.go create mode 100644 pkg/oak/index.go create mode 100644 pkg/oak_index.go create mode 100644 pkg/oak_index_manager.go diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index a192316..85c0da9 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -34,12 +34,12 @@ func (c *CLI) oakIndexListCmd() *cobra.Command { c.Error(err) return } - bundles, err := instance.OAK().IndexManager().List() + indexes, err := instance.OAK().IndexManager().List() if err != nil { c.Error(err) return } - c.SetOutput("indexes", bundles) + c.SetOutput("indexes", indexes) c.Ok("indexes listed") }, } diff --git a/cmd/aem/root.go b/cmd/aem/root.go index d823594..af95fb5 100644 --- a/cmd/aem/root.go +++ b/cmd/aem/root.go @@ -23,6 +23,7 @@ func (c *CLI) rootCmd() *cobra.Command { cmd.AddCommand(c.configCmd()) cmd.AddCommand(c.instanceCmd()) cmd.AddCommand(c.osgiCmd()) + cmd.AddCommand(c.oakCmd()) cmd.AddCommand(c.pkgCmd()) cmd.AddCommand(c.repoCmd()) cmd.AddCommand(c.replCmd()) diff --git a/pkg/instance.go b/pkg/instance.go index b4cda62..910972c 100644 --- a/pkg/instance.go +++ b/pkg/instance.go @@ -25,6 +25,7 @@ type Instance struct { http *HTTP status *Status repo *Repo + oak *OAK osgi *OSGi sling *Sling crypto *Crypto @@ -93,6 +94,10 @@ func (i Instance) OSGI() *OSGi { return i.osgi } +func (i Instance) OAK() *OAK { + return i.oak +} + func (i Instance) Sling() *Sling { return i.sling } @@ -345,7 +350,3 @@ func (i Instance) LockDir() string { } return i.CacheDir() } - -func (i Instance) OAK() { - // TODO -} diff --git a/pkg/instance_manager.go b/pkg/instance_manager.go index 87dc18b..a895ca9 100644 --- a/pkg/instance_manager.go +++ b/pkg/instance_manager.go @@ -266,6 +266,7 @@ func (im *InstanceManager) New(id, url, user, password string) *Instance { res.workflowManager = NewWorkflowManager(res) res.contentManager = NewContentManager(res) res.osgi = NewOSGi(res) + res.oak = NewOAK(res) res.sling = NewSling(res) res.crypto = NewCrypto(res) res.ssl = NewSSL(res) diff --git a/pkg/oak.go b/pkg/oak.go new file mode 100644 index 0000000..77aec21 --- /dev/null +++ b/pkg/oak.go @@ -0,0 +1,20 @@ +package pkg + +// OAK Facade for managing OAK repository. +type OAK struct { + instance *Instance + + indexManager *OAKIndexManager +} + +func NewOAK(instance *Instance) *OAK { + return &OAK{ + instance: instance, + + indexManager: NewOAKIndexManager(instance), + } +} + +func (o *OAK) IndexManager() *OAKIndexManager { + return o.indexManager +} diff --git a/pkg/oak/constants.go b/pkg/oak/constants.go new file mode 100644 index 0000000..97a34c6 --- /dev/null +++ b/pkg/oak/constants.go @@ -0,0 +1,5 @@ +package oak + +const ( + IndexListJson = "/oak:index.harray.1.json" +) diff --git a/pkg/oak/index.go b/pkg/oak/index.go new file mode 100644 index 0000000..632e7b5 --- /dev/null +++ b/pkg/oak/index.go @@ -0,0 +1,62 @@ +package oak + +import ( + "bytes" + "fmt" + "github.com/samber/lo" + "github.com/wttech/aemc/pkg/common/fmtx" + "sort" + "strings" +) + +type IndexList struct { + List []IndexListItem `json:"__children__"` +} + +type IndexListItem struct { + Name string `json:"__name__"` + Type string `json:"type"` + Async any `json:"async"` /* string or []string */ + IncludedPaths any `json:"includedPaths"` /* string or []string */ + QueryPaths any `json:"queryPaths"` /* string or []string */ + Reindex bool `json:"reindex"` + ReindexCount int `json:"reindexCount"` + EvaluatePathRestrictions bool `json:"evaluatePathRestrictions"` + DeclaringNodeTypes []string `json:"declaringNodeTypes"` + PropertyNames []string `json:"propertyNames"` + Tags []string `json:"tags"` +} + +func (il *IndexList) Total() int { + return len(il.List) +} + +func (il IndexList) MarshalText() string { + bs := bytes.NewBufferString("") + bs.WriteString(fmtx.TblMap("stats", "stat", "value", map[string]any{ + "total": il.Total(), + })) + bs.WriteString("\n") + + var indexesSorted []IndexListItem + indexesSorted = append(indexesSorted, il.List...) + sort.SliceStable(indexesSorted, func(i, j int) bool { + return strings.Compare(indexesSorted[i].Name, indexesSorted[j].Name) < 0 + }) + + bs.WriteString(fmtx.TblRows("list", false, []string{"name", "type", "async", "reindex", "reindex count", "tags"}, lo.Map(indexesSorted, func(i IndexListItem, _ int) map[string]any { + return map[string]any{ + "name": i.Name, + "type": i.Type, + "async": i.Async, + "reindex": i.Reindex, + "reindex count": i.ReindexCount, + "tags": i.Tags, + } + }))) + return bs.String() +} + +func (i IndexListItem) String() string { + return fmt.Sprintf("index '%s' (reindex: %v)", i.Name, i.Reindex) +} diff --git a/pkg/oak_index.go b/pkg/oak_index.go new file mode 100644 index 0000000..e95941f --- /dev/null +++ b/pkg/oak_index.go @@ -0,0 +1,81 @@ +package pkg + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/wttech/aemc/pkg/common/fmtx" + "github.com/wttech/aemc/pkg/oak" +) + +type OAKIndex struct { + manager *OAKIndexManager + + name string +} + +func (b OAKIndex) Name() string { + return b.name +} + +type OAKIndexState struct { + data *oak.IndexListItem + + Name string `yaml:"name" json:"name"` + Exists bool `yaml:"exists" json:"exists"` + Details map[string]any `yaml:"details" json:"details"` +} + +func (b OAKIndex) State() (*OAKIndexState, error) { + data, err := b.manager.Find(b.name) + if err != nil { + return nil, err + } + if data == nil { + return &OAKIndexState{ + Name: b.name, + Exists: false, + }, nil + } + + return &OAKIndexState{ + data: data, + + Name: b.name, + Exists: true, + Details: map[string]any{ + "reindex": data.Reindex, + }, + }, nil +} + +func (b OAKIndex) String() string { + return fmt.Sprintf("index '%s'", b.name) +} + +func (b OAKIndex) MarshalJSON() ([]byte, error) { + state, err := b.State() + if err != nil { + return nil, err + } + return json.Marshal(state) +} + +func (b OAKIndex) MarshalYAML() (interface{}, error) { + return b.State() +} + +func (b OAKIndex) MarshalText() string { + state, err := b.State() + if err != nil { + return fmt.Sprintf("name '%s' state cannot be read\n", b.name) + } + sb := bytes.NewBufferString("") + if state.Exists { + sb.WriteString(fmt.Sprintf("name '%s'\n", b.name)) + sb.WriteString(fmtx.TblProps(state.Details)) + } else { + sb.WriteString(fmt.Sprintf("name '%s' cannot be found\n", b.name)) + } + return sb.String() +} diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go new file mode 100644 index 0000000..878d082 --- /dev/null +++ b/pkg/oak_index_manager.go @@ -0,0 +1,52 @@ +package pkg + +import ( + "fmt" + "github.com/samber/lo" + "github.com/wttech/aemc/pkg/common/fmtx" + "github.com/wttech/aemc/pkg/oak" +) + +type OAKIndexManager struct { + instance *Instance +} + +func NewOAKIndexManager(instance *Instance) *OAKIndexManager { + return &OAKIndexManager{ + instance: instance, + } +} + +func (im *OAKIndexManager) New(name string) OAKIndex { + return OAKIndex{ + manager: im, + name: name, + } +} + +func (im *OAKIndexManager) Find(name string) (*oak.IndexListItem, error) { + indexes, err := im.List() + if err != nil { + return nil, fmt.Errorf("%s > cannot find index '%s'", im.instance.IDColor(), name) + } + item, found := lo.Find(indexes.List, func(i oak.IndexListItem) bool { return name == i.Name }) + if found { + return &item, nil + } + return nil, nil +} + +func (im *OAKIndexManager) List() (*oak.IndexList, error) { + resp, err := im.instance.http.Request().Get(oak.IndexListJson) + if err != nil { + return nil, fmt.Errorf("%s > cannot request index list: %w", im.instance.IDColor(), err) + } + if resp.IsError() { + return nil, fmt.Errorf("%s > cannot request index list: %s", im.instance.IDColor(), resp.Status()) + } + var res oak.IndexList + if err = fmtx.UnmarshalJSON(resp.RawBody(), &res); err != nil { + return nil, fmt.Errorf("%s > cannot parse index list: %w", im.instance.IDColor(), err) + } + return &res, nil +} From 138fc2fb5376719645af49eb85170e2051f64f20 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 22 Aug 2024 16:32:12 +0200 Subject: [PATCH 3/9] OAK index read --- cmd/aem/oak.go | 41 +++++++++++++++++++++++++++++++++++++++++ pkg/oak_index.go | 11 ++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index 85c0da9..7126390 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -1,7 +1,9 @@ package main import ( + "fmt" "github.com/spf13/cobra" + "github.com/wttech/aemc/pkg" ) func (c *CLI) oakCmd() *cobra.Command { @@ -20,6 +22,7 @@ func (c *CLI) oakIndexCmd() *cobra.Command { Short: "Manage OAK indexes", } cmd.AddCommand(c.oakIndexListCmd()) + cmd.AddCommand(c.oakIndexReadCmd()) return cmd } @@ -45,3 +48,41 @@ func (c *CLI) oakIndexListCmd() *cobra.Command { } return cmd } + +func (c *CLI) oakIndexReadCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "read", + Short: "Read OAK index details", + Aliases: []string{"get", "find"}, + Run: func(cmd *cobra.Command, args []string) { + instance, err := c.aem.InstanceManager().One() + if err != nil { + c.Error(err) + return + } + bundle, err := oakIndexByFlags(cmd, *instance) + if err != nil { + c.Error(err) + return + } + c.SetOutput("index", bundle) + c.Ok("index read") + }, + } + oakIndexDefineFlags(cmd) + return cmd +} + +func oakIndexDefineFlags(cmd *cobra.Command) { + cmd.Flags().String("name", "", "Name") + _ = cmd.MarkFlagRequired("name") +} + +func oakIndexByFlags(cmd *cobra.Command, i pkg.Instance) (*pkg.OAKIndex, error) { + name, _ := cmd.Flags().GetString("name") + if len(name) > 0 { + index := i.OAK().IndexManager().New(name) + return &index, nil + } + return nil, fmt.Errorf("flag 'name' is required") +} diff --git a/pkg/oak_index.go b/pkg/oak_index.go index e95941f..d6ee51b 100644 --- a/pkg/oak_index.go +++ b/pkg/oak_index.go @@ -44,7 +44,16 @@ func (b OAKIndex) State() (*OAKIndexState, error) { Name: b.name, Exists: true, Details: map[string]any{ - "reindex": data.Reindex, + "type": data.Type, + "async": data.Async, + "includePaths": data.IncludedPaths, + "queryPaths": data.QueryPaths, + "reindex": data.Reindex, + "reindexCount": data.ReindexCount, + "evaluatePathRestrictions": data.EvaluatePathRestrictions, + "declaringNodeTypes": data.DeclaringNodeTypes, + "propertyNames": data.PropertyNames, + "tags": data.Tags, }, }, nil } From 35cc4e417c612c883928ee6f8723984d2f854cc4 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Thu, 22 Aug 2024 16:50:26 +0200 Subject: [PATCH 4/9] Reindex works --- cmd/aem/oak.go | 48 ++++++++++++++++++++ pkg/oak_index.go | 96 +++++++++++++++++++++++++++++++++------- pkg/oak_index_manager.go | 8 ++++ 3 files changed, 135 insertions(+), 17 deletions(-) diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index 7126390..e4c89bf 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/spf13/cobra" "github.com/wttech/aemc/pkg" + "github.com/wttech/aemc/pkg/common/mapsx" ) func (c *CLI) oakCmd() *cobra.Command { @@ -23,6 +24,7 @@ func (c *CLI) oakIndexCmd() *cobra.Command { } cmd.AddCommand(c.oakIndexListCmd()) cmd.AddCommand(c.oakIndexReadCmd()) + cmd.AddCommand(c.oakIndexReindexCmd()) return cmd } @@ -73,6 +75,52 @@ func (c *CLI) oakIndexReadCmd() *cobra.Command { return cmd } +func (c *CLI) oakIndexReindexCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "reindex", + Short: "Reindex OAK index", + Run: func(cmd *cobra.Command, args []string) { + instances, err := c.aem.InstanceManager().Some() + if err != nil { + c.Error(err) + return + } + started, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { + index, err := oakIndexByFlags(cmd, instance) + if err != nil { + return nil, err + } + changed, err := index.ReindexWithChanged() // TODO assess if idempotent does make sense + if err != nil { + return nil, err + } + if changed { + if err = index.AwaitNotReindexed(); err != nil { + return nil, err + } + } + return map[string]any{ + OutputChanged: changed, + "instance": instance, + "index": index, + }, nil + }) + if err != nil { + c.Error(err) + return + } + c.SetOutput("started", started) + if mapsx.SomeHas(started, OutputChanged, true) { + c.Changed("index reindexed") + } else { + c.Ok("index already re-indexed (in-progress)") + } + }, + } + oakIndexDefineFlags(cmd) + return cmd +} + func oakIndexDefineFlags(cmd *cobra.Command) { cmd.Flags().String("name", "", "Name") _ = cmd.MarkFlagRequired("name") diff --git a/pkg/oak_index.go b/pkg/oak_index.go index d6ee51b..c56adbb 100644 --- a/pkg/oak_index.go +++ b/pkg/oak_index.go @@ -4,8 +4,10 @@ import ( "bytes" "encoding/json" "fmt" + log "github.com/sirupsen/logrus" "github.com/wttech/aemc/pkg/common/fmtx" "github.com/wttech/aemc/pkg/oak" + "time" ) type OAKIndex struct { @@ -14,8 +16,8 @@ type OAKIndex struct { name string } -func (b OAKIndex) Name() string { - return b.name +func (i OAKIndex) Name() string { + return i.name } type OAKIndexState struct { @@ -26,14 +28,48 @@ type OAKIndexState struct { Details map[string]any `yaml:"details" json:"details"` } -func (b OAKIndex) State() (*OAKIndexState, error) { - data, err := b.manager.Find(b.name) +func (i OAKIndex) assumeExists() (*OAKIndexState, error) { + state, err := i.State() + if err != nil { + return state, err + } + if !state.Exists { + return state, fmt.Errorf("%s > index '%s' does not exist", i.manager.instance.IDColor(), i.name) + } + return state, nil +} + +func (i OAKIndex) ReindexWithChanged() (bool, error) { + state, err := i.assumeExists() + if err != nil { + return false, err + } + if state.data.Reindex { + return false, nil + } + return true, i.manager.Reindex(state.data.Name) +} + +func (i OAKIndex) Reindex() error { + state, err := i.assumeExists() + if err != nil { + return err + } + err = i.manager.Reindex(state.data.Name) + if err != nil { + return fmt.Errorf("%s > cannot reindex index '%s': %w", i.manager.instance.IDColor(), i.name, err) + } + return nil +} + +func (i OAKIndex) State() (*OAKIndexState, error) { + data, err := i.manager.Find(i.name) if err != nil { return nil, err } if data == nil { return &OAKIndexState{ - Name: b.name, + Name: i.name, Exists: false, }, nil } @@ -41,7 +77,7 @@ func (b OAKIndex) State() (*OAKIndexState, error) { return &OAKIndexState{ data: data, - Name: b.name, + Name: i.name, Exists: true, Details: map[string]any{ "type": data.Type, @@ -58,33 +94,59 @@ func (b OAKIndex) State() (*OAKIndexState, error) { }, nil } -func (b OAKIndex) String() string { - return fmt.Sprintf("index '%s'", b.name) +func (i OAKIndex) String() string { + return fmt.Sprintf("index '%s'", i.name) } -func (b OAKIndex) MarshalJSON() ([]byte, error) { - state, err := b.State() +func (i OAKIndex) MarshalJSON() ([]byte, error) { + state, err := i.State() if err != nil { return nil, err } return json.Marshal(state) } -func (b OAKIndex) MarshalYAML() (interface{}, error) { - return b.State() +func (i OAKIndex) MarshalYAML() (interface{}, error) { + return i.State() } -func (b OAKIndex) MarshalText() string { - state, err := b.State() +func (i OAKIndex) MarshalText() string { + state, err := i.State() if err != nil { - return fmt.Sprintf("name '%s' state cannot be read\n", b.name) + return fmt.Sprintf("name '%s' state cannot be read\n", i.name) } sb := bytes.NewBufferString("") if state.Exists { - sb.WriteString(fmt.Sprintf("name '%s'\n", b.name)) + sb.WriteString(fmt.Sprintf("name '%s'\n", i.name)) sb.WriteString(fmtx.TblProps(state.Details)) } else { - sb.WriteString(fmt.Sprintf("name '%s' cannot be found\n", b.name)) + sb.WriteString(fmt.Sprintf("name '%s' cannot be found\n", i.name)) } return sb.String() } + +func (i OAKIndex) AwaitNotReindexed() error { + return i.Await("not reindexed", func() bool { + state, err := i.State() + if err != nil { + log.Warn(err) + return false + } + return state.Exists && !state.data.Reindex + }, time.Minute*1) +} + +func (i OAKIndex) Await(state string, condition func() bool, timeout time.Duration) error { + started := time.Now() + for { + if condition() { + break + } + if time.Now().After(started.Add(timeout)) { + return fmt.Errorf("%s > awaiting index '%s' state '%s' reached timeout after %s", i.manager.instance.IDColor(), i.name, state, timeout) + } + log.Infof("%s > awaiting index '%s' state '%s'", i.manager.instance.IDColor(), i.name, state) + time.Sleep(time.Second * 5) + } + return nil +} diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go index 878d082..4bd9934 100644 --- a/pkg/oak_index_manager.go +++ b/pkg/oak_index_manager.go @@ -50,3 +50,11 @@ func (im *OAKIndexManager) List() (*oak.IndexList, error) { } return &res, nil } + +func (im *OAKIndexManager) Reindex(name string) error { + node := im.instance.Repo().Node(fmt.Sprintf("/oak:index/%s", name)) + if err := node.SaveProp("reindex", true); err != nil { + return fmt.Errorf("%s > cannot reindex '%s': %w", im.instance.IDColor(), name, err) + } + return nil +} From 59d3e0cc32202c4f71c9a6a72f937b63bb423cb7 Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Fri, 23 Aug 2024 08:17:11 +0200 Subject: [PATCH 5/9] Reindex all MVP --- cmd/aem/oak.go | 48 +++++++++++++++++++++++++++++++++++----- pkg/oak/constants.go | 3 ++- pkg/oak/index.go | 5 ++++- pkg/oak_index.go | 4 +++- pkg/oak_index_manager.go | 25 +++++++++++++++++++++ 5 files changed, 77 insertions(+), 8 deletions(-) diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index e4c89bf..3db900d 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -25,6 +25,7 @@ func (c *CLI) oakIndexCmd() *cobra.Command { cmd.AddCommand(c.oakIndexListCmd()) cmd.AddCommand(c.oakIndexReadCmd()) cmd.AddCommand(c.oakIndexReindexCmd()) + cmd.AddCommand(c.oakIndexReindexAllCmd()) return cmd } @@ -85,12 +86,12 @@ func (c *CLI) oakIndexReindexCmd() *cobra.Command { c.Error(err) return } - started, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { + reindexed, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { index, err := oakIndexByFlags(cmd, instance) if err != nil { return nil, err } - changed, err := index.ReindexWithChanged() // TODO assess if idempotent does make sense + changed, err := index.ReindexWithChanged() if err != nil { return nil, err } @@ -109,11 +110,11 @@ func (c *CLI) oakIndexReindexCmd() *cobra.Command { c.Error(err) return } - c.SetOutput("started", started) - if mapsx.SomeHas(started, OutputChanged, true) { + c.SetOutput("reindexed", reindexed) + if mapsx.SomeHas(reindexed, OutputChanged, true) { c.Changed("index reindexed") } else { - c.Ok("index already re-indexed (in-progress)") + c.Ok("index already reindexed (in progress)") } }, } @@ -121,6 +122,43 @@ func (c *CLI) oakIndexReindexCmd() *cobra.Command { return cmd } +func (c *CLI) oakIndexReindexAllCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "reindex-all", + Short: "Reindex all OAK indexes", + Run: func(cmd *cobra.Command, args []string) { + instances, err := c.aem.InstanceManager().Some() + if err != nil { + c.Error(err) + return + } + reindexed, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { + indexes, err := instance.OAK().IndexManager().ReindexAll() + if err != nil { + return nil, err + } + + return map[string]any{ + OutputChanged: true, + "instance": instance, + "indexes": indexes, + }, nil + }) + if err != nil { + c.Error(err) + return + } + c.SetOutput("reindexed", reindexed) + if mapsx.SomeHas(reindexed, OutputChanged, true) { + c.Changed("indexes reindexed") + } else { + c.Ok("indexes already reindexed (up-to-date)") + } + }, + } + return cmd +} + func oakIndexDefineFlags(cmd *cobra.Command) { cmd.Flags().String("name", "", "Name") _ = cmd.MarkFlagRequired("name") diff --git a/pkg/oak/constants.go b/pkg/oak/constants.go index 97a34c6..19ecf27 100644 --- a/pkg/oak/constants.go +++ b/pkg/oak/constants.go @@ -1,5 +1,6 @@ package oak const ( - IndexListJson = "/oak:index.harray.1.json" + IndexListJson = "/oak:index.harray.1.json" + IndexPrimaryType = "oak:QueryIndexDefinition" ) diff --git a/pkg/oak/index.go b/pkg/oak/index.go index 632e7b5..0026c7b 100644 --- a/pkg/oak/index.go +++ b/pkg/oak/index.go @@ -14,10 +14,13 @@ type IndexList struct { } type IndexListItem struct { + PrimaryType string `json:"jcr:primaryType,omitempty"` Name string `json:"__name__"` Type string `json:"type"` - Async any `json:"async"` /* string or []string */ + Async any `json:"async"` /* string or []string */ + Unique bool `json:"unique"` IncludedPaths any `json:"includedPaths"` /* string or []string */ + ExcludedPaths any `json:"excludedPaths"` /* string or []string */ QueryPaths any `json:"queryPaths"` /* string or []string */ Reindex bool `json:"reindex"` ReindexCount int `json:"reindexCount"` diff --git a/pkg/oak_index.go b/pkg/oak_index.go index c56adbb..07454bb 100644 --- a/pkg/oak_index.go +++ b/pkg/oak_index.go @@ -82,7 +82,9 @@ func (i OAKIndex) State() (*OAKIndexState, error) { Details: map[string]any{ "type": data.Type, "async": data.Async, - "includePaths": data.IncludedPaths, + "unique": data.Unique, + "includedPaths": data.IncludedPaths, + "excludedPaths": data.ExcludedPaths, "queryPaths": data.QueryPaths, "reindex": data.Reindex, "reindexCount": data.ReindexCount, diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go index 4bd9934..dd9de82 100644 --- a/pkg/oak_index_manager.go +++ b/pkg/oak_index_manager.go @@ -3,6 +3,7 @@ package pkg import ( "fmt" "github.com/samber/lo" + log "github.com/sirupsen/logrus" "github.com/wttech/aemc/pkg/common/fmtx" "github.com/wttech/aemc/pkg/oak" ) @@ -48,6 +49,7 @@ func (im *OAKIndexManager) List() (*oak.IndexList, error) { if err = fmtx.UnmarshalJSON(resp.RawBody(), &res); err != nil { return nil, fmt.Errorf("%s > cannot parse index list: %w", im.instance.IDColor(), err) } + res.List = lo.Filter(res.List, func(i oak.IndexListItem, _ int) bool { return i.PrimaryType == oak.IndexPrimaryType }) return &res, nil } @@ -58,3 +60,26 @@ func (im *OAKIndexManager) Reindex(name string) error { } return nil } + +func (im *OAKIndexManager) ReindexAll() (*oak.IndexList, error) { + indexes, err := im.List() + if err != nil { + return nil, err + } + + for _, i := range indexes.List { + if i.Reindex { + log.Warnf("%s > index '%s' is currently being reindexed, skipping", im.instance.IDColor(), i.Name) + continue + } + index := im.New(i.Name) + if err = im.Reindex(i.Name); err != nil { + return nil, err + } + if err = index.AwaitNotReindexed(); err != nil { + return nil, err + } + } + + return indexes, nil +} From dd81d366809fc61da2319db28debbcd373bc34be Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Fri, 23 Aug 2024 08:29:57 +0200 Subject: [PATCH 6/9] Batch reindex nice --- pkg/oak_index_manager.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go index dd9de82..1c2c910 100644 --- a/pkg/oak_index_manager.go +++ b/pkg/oak_index_manager.go @@ -5,6 +5,7 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/wttech/aemc/pkg/common/fmtx" + "github.com/wttech/aemc/pkg/common/stringsx" "github.com/wttech/aemc/pkg/oak" ) @@ -67,11 +68,21 @@ func (im *OAKIndexManager) ReindexAll() (*oak.IndexList, error) { return nil, err } + count := 0 + total := indexes.Total() + + log.Infof("%s > reindexing all indexes (%d)", im.instance.IDColor(), total) + for _, i := range indexes.List { + count++ + percent := stringsx.PercentExplained(count, total, 0) + if i.Reindex { - log.Warnf("%s > index '%s' is currently being reindexed, skipping", im.instance.IDColor(), i.Name) + log.Warnf("%s > reindexing '%s' skipped as already in progress (%s)", im.instance.IDColor(), i.Name, percent) continue } + log.Infof("%s > reindexing '%s' (%s)", im.instance.IDColor(), i.Name, percent) + index := im.New(i.Name) if err = im.Reindex(i.Name); err != nil { return nil, err @@ -80,6 +91,7 @@ func (im *OAKIndexManager) ReindexAll() (*oak.IndexList, error) { return nil, err } } + log.Infof("%s > reindexed all indexes (%d)", im.instance.IDColor(), total) return indexes, nil } From 933ed18e0ccc1d2ff55b792671eaee2108dfdd6c Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Fri, 23 Aug 2024 09:30:03 +0200 Subject: [PATCH 7/9] Reindex batch --- cmd/aem/oak.go | 19 +-- examples/docker/src/aem/default/etc/aem.yml | 5 + pkg/cfg/defaults.go | 2 + pkg/oak_index.go | 8 +- pkg/oak_index_manager.go | 110 +++++++++++++++--- .../app_classic/aem/default/etc/aem.yml | 5 + pkg/project/app_cloud/aem/default/etc/aem.yml | 5 + pkg/project/instance/aem/default/etc/aem.yml | 5 + 8 files changed, 135 insertions(+), 24 deletions(-) diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index 3db900d..71c7d89 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -25,7 +25,7 @@ func (c *CLI) oakIndexCmd() *cobra.Command { cmd.AddCommand(c.oakIndexListCmd()) cmd.AddCommand(c.oakIndexReadCmd()) cmd.AddCommand(c.oakIndexReindexCmd()) - cmd.AddCommand(c.oakIndexReindexAllCmd()) + cmd.AddCommand(c.oakIndexReindexBatchCmd()) return cmd } @@ -122,24 +122,25 @@ func (c *CLI) oakIndexReindexCmd() *cobra.Command { return cmd } -func (c *CLI) oakIndexReindexAllCmd() *cobra.Command { +func (c *CLI) oakIndexReindexBatchCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "reindex-all", - Short: "Reindex all OAK indexes", + Use: "reindex-batch", + Short: "Reindex OAK indexes in batch", Run: func(cmd *cobra.Command, args []string) { instances, err := c.aem.InstanceManager().Some() if err != nil { c.Error(err) return } + reason, _ := cmd.Flags().GetString("reason") reindexed, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { - indexes, err := instance.OAK().IndexManager().ReindexAll() + indexes, changed, err := instance.OAK().IndexManager().ReindexAllWithChanged(reason) if err != nil { return nil, err } return map[string]any{ - OutputChanged: true, + OutputChanged: changed, "instance": instance, "indexes": indexes, }, nil @@ -150,12 +151,14 @@ func (c *CLI) oakIndexReindexAllCmd() *cobra.Command { } c.SetOutput("reindexed", reindexed) if mapsx.SomeHas(reindexed, OutputChanged, true) { - c.Changed("indexes reindexed") + c.Changed("indexes batch reindexed") } else { - c.Ok("indexes already reindexed (up-to-date)") + c.Ok("indexes batch already reindexed (up-to-date)") } }, } + cmd.Flags().StringP("reason", "r", "", "Reason for reindexing") + _ = cmd.MarkFlagRequired("reason") return cmd } diff --git a/examples/docker/src/aem/default/etc/aem.yml b/examples/docker/src/aem/default/etc/aem.yml index 200a6da..2837a49 100755 --- a/examples/docker/src/aem/default/etc/aem.yml +++ b/examples/docker/src/aem/default/etc/aem.yml @@ -197,6 +197,11 @@ instance: # Use checksums to avoid re-installations when snapshot OSGi bundles are unchanged snapshot_install_skipping: true + # OAK Repository + oak: + index: + await_not_reindexed_timeout: 60m + # Crypto Support crypto: key_bundle_symbolic_name: com.adobe.granite.crypto.file diff --git a/pkg/cfg/defaults.go b/pkg/cfg/defaults.go index 8eee378..ebc0597 100644 --- a/pkg/cfg/defaults.go +++ b/pkg/cfg/defaults.go @@ -124,6 +124,8 @@ func (c *Config) setDefaults() { v.SetDefault("instance.osgi.bundle.snapshot_ignored", false) v.SetDefault("instance.osgi.bundle.snapshot_patterns", []string{"**/*-SNAPSHOT.jar"}) + v.SetDefault("instance.oak.index.await_not_reindexed_timeout", time.Minute*60) + v.SetDefault("instance.ssl.setup_timeout", time.Second*30) v.SetDefault("instance.crypto.key_bundle_symbolic_name", "com.adobe.granite.crypto.file") diff --git a/pkg/oak_index.go b/pkg/oak_index.go index 07454bb..92040b6 100644 --- a/pkg/oak_index.go +++ b/pkg/oak_index.go @@ -28,6 +28,10 @@ type OAKIndexState struct { Details map[string]any `yaml:"details" json:"details"` } +func (i OAKIndexState) Reindexed() bool { + return i.data.Reindex +} + func (i OAKIndex) assumeExists() (*OAKIndexState, error) { state, err := i.State() if err != nil { @@ -134,8 +138,8 @@ func (i OAKIndex) AwaitNotReindexed() error { log.Warn(err) return false } - return state.Exists && !state.data.Reindex - }, time.Minute*1) + return state.Exists && !state.Reindexed() + }, i.manager.awaitNotReindexedTimeout) } func (i OAKIndex) Await(state string, condition func() bool, timeout time.Duration) error { diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go index 1c2c910..e4d6ddc 100644 --- a/pkg/oak_index_manager.go +++ b/pkg/oak_index_manager.go @@ -5,17 +5,27 @@ import ( "github.com/samber/lo" log "github.com/sirupsen/logrus" "github.com/wttech/aemc/pkg/common/fmtx" + "github.com/wttech/aemc/pkg/common/osx" "github.com/wttech/aemc/pkg/common/stringsx" "github.com/wttech/aemc/pkg/oak" + "sort" + "strings" + "time" ) type OAKIndexManager struct { instance *Instance + + awaitNotReindexedTimeout time.Duration } func NewOAKIndexManager(instance *Instance) *OAKIndexManager { + cv := instance.manager.aem.config.Values() + return &OAKIndexManager{ instance: instance, + + awaitNotReindexedTimeout: cv.GetDuration("instance.oak.index.await_not_reindexed_timeout"), } } @@ -62,36 +72,108 @@ func (im *OAKIndexManager) Reindex(name string) error { return nil } -func (im *OAKIndexManager) ReindexAll() (*oak.IndexList, error) { - indexes, err := im.List() +func (im *OAKIndexManager) ReindexBatchWithChanged(names []string, reason string) ([]OAKIndex, bool, error) { + lock := osx.NewLock(fmt.Sprintf("%s/oak/reindex-batch/%s.yml", im.instance.LockDir(), reason), func() (oakReindexAllLock, error) { + namesSorted := append([]string(nil), names...) + sort.Strings(namesSorted) + return oakReindexAllLock{Names: strings.Join(namesSorted, ",")}, nil + }) + lockState, err := lock.State() + if err != nil { + return nil, false, err + } + if lockState.UpToDate { + log.Debugf("%s > reindexing '%s' already done (up-to-date)", im.instance.IDColor(), reason) + return nil, false, nil + } + indexes, err := im.ReindexBatch(names) + if err != nil { + return nil, false, err + } + if err = lock.Lock(); err != nil { + return nil, false, err + } + return indexes, true, nil +} + +type oakReindexAllLock struct { + Names string `yaml:"names"` +} + +func (im *OAKIndexManager) ReindexBatch(names []string) ([]OAKIndex, error) { + indexes, err := im.FindByName(names) if err != nil { return nil, err } - count := 0 - total := indexes.Total() + total := len(names) + log.Infof("%s > reindexing batch of indexes (%d)", im.instance.IDColor(), total) - log.Infof("%s > reindexing all indexes (%d)", im.instance.IDColor(), total) + for i, index := range indexes { + percent := stringsx.PercentExplained(i+1, total, 0) - for _, i := range indexes.List { - count++ - percent := stringsx.PercentExplained(count, total, 0) + state, err := index.State() + if err != nil { + return nil, err + } - if i.Reindex { - log.Warnf("%s > reindexing '%s' skipped as already in progress (%s)", im.instance.IDColor(), i.Name, percent) + if state.Reindexed() { + log.Warnf("%s > reindexing '%s' skipped as already in progress (%s)", im.instance.IDColor(), index.Name(), percent) continue } - log.Infof("%s > reindexing '%s' (%s)", im.instance.IDColor(), i.Name, percent) + log.Infof("%s > reindexing '%s' (%s)", im.instance.IDColor(), index.Name(), percent) - index := im.New(i.Name) - if err = im.Reindex(i.Name); err != nil { + if err = index.Reindex(); err != nil { return nil, err } if err = index.AwaitNotReindexed(); err != nil { return nil, err } } - log.Infof("%s > reindexed all indexes (%d)", im.instance.IDColor(), total) + log.Infof("%s > reindexed batch of indexes (%d)", im.instance.IDColor(), total) return indexes, nil } + +func (im *OAKIndexManager) FindByName(names []string) ([]OAKIndex, error) { + indexes, err := im.List() + if err != nil { + return nil, err + } + var res []OAKIndex + for _, name := range names { + item, found := lo.Find(indexes.List, func(i oak.IndexListItem) bool { return name == i.Name }) + if !found { + return nil, fmt.Errorf("%s > index '%s' cannot be found", im.instance.IDColor(), name) + } + res = append(res, OAKIndex{ + manager: im, + name: item.Name, + }) + } + return res, nil +} + +func (im *OAKIndexManager) Names() ([]string, error) { + list, err := im.List() + if err != nil { + return nil, err + } + return lo.Map(list.List, func(i oak.IndexListItem, _ int) string { return i.Name }), nil +} + +func (im *OAKIndexManager) ReindexAll() ([]OAKIndex, error) { + names, err := im.Names() + if err != nil { + return nil, err + } + return im.ReindexBatch(names) +} + +func (im *OAKIndexManager) ReindexAllWithChanged(reason string) ([]OAKIndex, bool, error) { + names, err := im.Names() + if err != nil { + return nil, false, err + } + return im.ReindexBatchWithChanged(names, reason) +} diff --git a/pkg/project/app_classic/aem/default/etc/aem.yml b/pkg/project/app_classic/aem/default/etc/aem.yml index 7e3a269..701caee 100755 --- a/pkg/project/app_classic/aem/default/etc/aem.yml +++ b/pkg/project/app_classic/aem/default/etc/aem.yml @@ -198,6 +198,11 @@ instance: # Use checksums to avoid re-installations when snapshot OSGi bundles are unchanged snapshot_install_skipping: true + # OAK Repository + oak: + index: + await_not_reindexed_timeout: 60m + # Crypto Support crypto: key_bundle_symbolic_name: com.adobe.granite.crypto.file diff --git a/pkg/project/app_cloud/aem/default/etc/aem.yml b/pkg/project/app_cloud/aem/default/etc/aem.yml index e4a78fe..e216b86 100755 --- a/pkg/project/app_cloud/aem/default/etc/aem.yml +++ b/pkg/project/app_cloud/aem/default/etc/aem.yml @@ -196,6 +196,11 @@ instance: # Use checksums to avoid re-installations when snapshot OSGi bundles are unchanged snapshot_install_skipping: true + # OAK Repository + oak: + index: + await_not_reindexed_timeout: 60m + # Crypto Support crypto: key_bundle_symbolic_name: com.adobe.granite.crypto.file diff --git a/pkg/project/instance/aem/default/etc/aem.yml b/pkg/project/instance/aem/default/etc/aem.yml index e92f5c0..67ba7ed 100755 --- a/pkg/project/instance/aem/default/etc/aem.yml +++ b/pkg/project/instance/aem/default/etc/aem.yml @@ -199,6 +199,11 @@ instance: # Use checksums to avoid re-installations when snapshot OSGi bundles are unchanged snapshot_install_skipping: true + # OAK Repository + oak: + index: + await_not_reindexed_timeout: 60m + # Crypto Support crypto: key_bundle_symbolic_name: com.adobe.granite.crypto.file From 85b027e6e3e5501937a4d782b4b972a23ba6c7da Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Fri, 23 Aug 2024 11:22:04 +0200 Subject: [PATCH 8/9] OAK batch reindex idempotent --- cmd/aem/oak.go | 30 ++++++++++++--- pkg/oak_index_manager.go | 80 ++++++++++++++-------------------------- 2 files changed, 51 insertions(+), 59 deletions(-) diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index 71c7d89..5a45aa1 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/samber/lo" "github.com/spf13/cobra" "github.com/wttech/aemc/pkg" "github.com/wttech/aemc/pkg/common/mapsx" @@ -132,17 +133,33 @@ func (c *CLI) oakIndexReindexBatchCmd() *cobra.Command { c.Error(err) return } - reason, _ := cmd.Flags().GetString("reason") + + batchId, _ := cmd.Flags().GetString("batch-id") + namePatterns, _ := cmd.Flags().GetStringSlice("name-pattern") + force, _ := cmd.Flags().GetBool("force") + reindexed, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { - indexes, changed, err := instance.OAK().IndexManager().ReindexAllWithChanged(reason) + if force { + indexes, err := instance.OAK().IndexManager().ReindexBatch(namePatterns) + if err != nil { + return nil, err + } + return map[string]any{ + OutputChanged: true, + "instance": instance, + "indexNames": lo.Map(indexes, func(i pkg.OAKIndex, _ int) any { return i.Name() }), + }, nil + } + + indexes, changed, err := instance.OAK().IndexManager().ReindexBatchWithChanged(batchId, namePatterns) if err != nil { return nil, err } - return map[string]any{ OutputChanged: changed, "instance": instance, - "indexes": indexes, + "batchId": batchId, + "indexNames": lo.Map(indexes, func(i pkg.OAKIndex, _ int) any { return i.Name() }), }, nil }) if err != nil { @@ -157,8 +174,9 @@ func (c *CLI) oakIndexReindexBatchCmd() *cobra.Command { } }, } - cmd.Flags().StringP("reason", "r", "", "Reason for reindexing") - _ = cmd.MarkFlagRequired("reason") + cmd.Flags().StringP("batch-id", "b", "all", "Batch ID") + cmd.Flags().StringSliceP("name-pattern", "n", []string{"*"}, "Index name pattern(s)") + cmd.Flags().BoolP("force", "f", false, "Reindex even if already indexed") return cmd } diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go index e4d6ddc..e614332 100644 --- a/pkg/oak_index_manager.go +++ b/pkg/oak_index_manager.go @@ -72,9 +72,13 @@ func (im *OAKIndexManager) Reindex(name string) error { return nil } -func (im *OAKIndexManager) ReindexBatchWithChanged(names []string, reason string) ([]OAKIndex, bool, error) { - lock := osx.NewLock(fmt.Sprintf("%s/oak/reindex-batch/%s.yml", im.instance.LockDir(), reason), func() (oakReindexAllLock, error) { - namesSorted := append([]string(nil), names...) +func (im *OAKIndexManager) ReindexBatchWithChanged(batchId string, namePatterns []string) ([]OAKIndex, bool, error) { + indexes, err := im.FindByName(namePatterns) + if err != nil { + return nil, false, err + } + lock := osx.NewLock(fmt.Sprintf("%s/oak/reindex-batch/%s.yml", im.instance.LockDir(), batchId), func() (oakReindexAllLock, error) { + namesSorted := lo.Map(indexes, func(i OAKIndex, _ int) string { return i.Name() }) sort.Strings(namesSorted) return oakReindexAllLock{Names: strings.Join(namesSorted, ",")}, nil }) @@ -83,11 +87,10 @@ func (im *OAKIndexManager) ReindexBatchWithChanged(names []string, reason string return nil, false, err } if lockState.UpToDate { - log.Debugf("%s > reindexing '%s' already done (up-to-date)", im.instance.IDColor(), reason) + log.Debugf("%s > reindexing '%s' already done (up-to-date)", im.instance.IDColor(), batchId) return nil, false, nil } - indexes, err := im.ReindexBatch(names) - if err != nil { + if err := im.reindexBatch(indexes); err != nil { return nil, false, err } if err = lock.Lock(); err != nil { @@ -100,21 +103,23 @@ type oakReindexAllLock struct { Names string `yaml:"names"` } -func (im *OAKIndexManager) ReindexBatch(names []string) ([]OAKIndex, error) { - indexes, err := im.FindByName(names) +func (im *OAKIndexManager) ReindexBatch(namePatterns []string) ([]OAKIndex, error) { + indexes, err := im.FindByName(namePatterns) if err != nil { return nil, err } + return indexes, im.reindexBatch(indexes) +} - total := len(names) +func (im *OAKIndexManager) reindexBatch(indexes []OAKIndex) error { + total := len(indexes) log.Infof("%s > reindexing batch of indexes (%d)", im.instance.IDColor(), total) - for i, index := range indexes { percent := stringsx.PercentExplained(i+1, total, 0) state, err := index.State() if err != nil { - return nil, err + return err } if state.Reindexed() { @@ -124,56 +129,25 @@ func (im *OAKIndexManager) ReindexBatch(names []string) ([]OAKIndex, error) { log.Infof("%s > reindexing '%s' (%s)", im.instance.IDColor(), index.Name(), percent) if err = index.Reindex(); err != nil { - return nil, err + return err } if err = index.AwaitNotReindexed(); err != nil { - return nil, err + return err } } log.Infof("%s > reindexed batch of indexes (%d)", im.instance.IDColor(), total) - - return indexes, nil -} - -func (im *OAKIndexManager) FindByName(names []string) ([]OAKIndex, error) { - indexes, err := im.List() - if err != nil { - return nil, err - } - var res []OAKIndex - for _, name := range names { - item, found := lo.Find(indexes.List, func(i oak.IndexListItem) bool { return name == i.Name }) - if !found { - return nil, fmt.Errorf("%s > index '%s' cannot be found", im.instance.IDColor(), name) - } - res = append(res, OAKIndex{ - manager: im, - name: item.Name, - }) - } - return res, nil -} - -func (im *OAKIndexManager) Names() ([]string, error) { - list, err := im.List() - if err != nil { - return nil, err - } - return lo.Map(list.List, func(i oak.IndexListItem, _ int) string { return i.Name }), nil + return nil } -func (im *OAKIndexManager) ReindexAll() ([]OAKIndex, error) { - names, err := im.Names() +func (im *OAKIndexManager) FindByName(namePatterns []string) ([]OAKIndex, error) { + items, err := im.List() if err != nil { return nil, err } - return im.ReindexBatch(names) -} - -func (im *OAKIndexManager) ReindexAllWithChanged(reason string) ([]OAKIndex, bool, error) { - names, err := im.Names() - if err != nil { - return nil, false, err - } - return im.ReindexBatchWithChanged(names, reason) + indexes := lo.Map(lo.Filter(items.List, func(i oak.IndexListItem, _ int) bool { + return stringsx.MatchSome(i.Name, namePatterns) + }), func(i oak.IndexListItem, _ int) OAKIndex { + return im.New(i.Name) + }) + return indexes, nil } From 5432289b853be1f18ca25a478809fba7e1f4fb3d Mon Sep 17 00:00:00 2001 From: Krystian Panek Date: Fri, 23 Aug 2024 11:26:44 +0200 Subject: [PATCH 9/9] Minor --- cmd/aem/oak.go | 12 +++++++----- pkg/oak_index_manager.go | 28 ++++++++-------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/cmd/aem/oak.go b/cmd/aem/oak.go index 5a45aa1..04d0bd5 100644 --- a/cmd/aem/oak.go +++ b/cmd/aem/oak.go @@ -139,19 +139,21 @@ func (c *CLI) oakIndexReindexBatchCmd() *cobra.Command { force, _ := cmd.Flags().GetBool("force") reindexed, err := pkg.InstanceProcess(c.aem, instances, func(instance pkg.Instance) (map[string]any, error) { + indexes, err := instance.OAK().IndexManager().FindByName(namePatterns) + indexNames := lo.Map(indexes, func(i pkg.OAKIndex, _ int) any { return i.Name() }) + if force { - indexes, err := instance.OAK().IndexManager().ReindexBatch(namePatterns) - if err != nil { + if err := instance.OAK().IndexManager().ReindexBatch(indexes); err != nil { return nil, err } return map[string]any{ OutputChanged: true, "instance": instance, - "indexNames": lo.Map(indexes, func(i pkg.OAKIndex, _ int) any { return i.Name() }), + "indexNames": indexNames, }, nil } - indexes, changed, err := instance.OAK().IndexManager().ReindexBatchWithChanged(batchId, namePatterns) + changed, err := instance.OAK().IndexManager().ReindexBatchWithChanged(batchId, indexes) if err != nil { return nil, err } @@ -159,7 +161,7 @@ func (c *CLI) oakIndexReindexBatchCmd() *cobra.Command { OutputChanged: changed, "instance": instance, "batchId": batchId, - "indexNames": lo.Map(indexes, func(i pkg.OAKIndex, _ int) any { return i.Name() }), + "indexNames": indexNames, }, nil }) if err != nil { diff --git a/pkg/oak_index_manager.go b/pkg/oak_index_manager.go index e614332..7e7737f 100644 --- a/pkg/oak_index_manager.go +++ b/pkg/oak_index_manager.go @@ -72,11 +72,7 @@ func (im *OAKIndexManager) Reindex(name string) error { return nil } -func (im *OAKIndexManager) ReindexBatchWithChanged(batchId string, namePatterns []string) ([]OAKIndex, bool, error) { - indexes, err := im.FindByName(namePatterns) - if err != nil { - return nil, false, err - } +func (im *OAKIndexManager) ReindexBatchWithChanged(batchId string, indexes []OAKIndex) (bool, error) { lock := osx.NewLock(fmt.Sprintf("%s/oak/reindex-batch/%s.yml", im.instance.LockDir(), batchId), func() (oakReindexAllLock, error) { namesSorted := lo.Map(indexes, func(i OAKIndex, _ int) string { return i.Name() }) sort.Strings(namesSorted) @@ -84,34 +80,26 @@ func (im *OAKIndexManager) ReindexBatchWithChanged(batchId string, namePatterns }) lockState, err := lock.State() if err != nil { - return nil, false, err + return false, err } if lockState.UpToDate { log.Debugf("%s > reindexing '%s' already done (up-to-date)", im.instance.IDColor(), batchId) - return nil, false, nil + return false, nil } - if err := im.reindexBatch(indexes); err != nil { - return nil, false, err + if err := im.ReindexBatch(indexes); err != nil { + return false, err } if err = lock.Lock(); err != nil { - return nil, false, err + return false, err } - return indexes, true, nil + return true, nil } type oakReindexAllLock struct { Names string `yaml:"names"` } -func (im *OAKIndexManager) ReindexBatch(namePatterns []string) ([]OAKIndex, error) { - indexes, err := im.FindByName(namePatterns) - if err != nil { - return nil, err - } - return indexes, im.reindexBatch(indexes) -} - -func (im *OAKIndexManager) reindexBatch(indexes []OAKIndex) error { +func (im *OAKIndexManager) ReindexBatch(indexes []OAKIndex) error { total := len(indexes) log.Infof("%s > reindexing batch of indexes (%d)", im.instance.IDColor(), total) for i, index := range indexes {