From 7b2d93e9918165dfcb6e5839e8a03a1154c9f57a Mon Sep 17 00:00:00 2001 From: kmetin Date: Fri, 22 Sep 2023 11:55:47 +0300 Subject: [PATCH] remove member logs in test --- base/commands/migration/migration_stages.go | 50 +++++++------------ base/commands/migration/migration_start.go | 14 +++++- base/commands/migration/migration_status.go | 14 +++++- .../migration/start_stages_it_test.go | 11 ++++ .../migration/status_stages_it_test.go | 31 +++++++++++- 5 files changed, 83 insertions(+), 37 deletions(-) diff --git a/base/commands/migration/migration_stages.go b/base/commands/migration/migration_stages.go index a1dcec26..51f102b3 100644 --- a/base/commands/migration/migration_stages.go +++ b/base/commands/migration/migration_stages.go @@ -16,21 +16,16 @@ import ( "github.com/hazelcast/hazelcast-commandline-client/internal/plug" "github.com/hazelcast/hazelcast-go-client" "github.com/hazelcast/hazelcast-go-client/serialization" - "golang.org/x/exp/slices" ) var timeoutErr = fmt.Errorf("migration could not be completed: reached timeout while reading status: "+ "please ensure that you are using Hazelcast's migration cluster distribution and your DMT config points to that cluster: %w", context.DeadlineExceeded) -func migrationStages(ctx context.Context, ec plug.ExecContext, migrationID, reportOutputDir string, statusMap *hazelcast.Map) ([]stage.Stage[any], error) { - ci, err := ec.ClientInternal(ctx) - if err != nil { - return nil, err - } +func createMigrationStages(ctx context.Context, ec plug.ExecContext, ci *hazelcast.ClientInternal, migrationID string) ([]stage.Stage[any], error) { childCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if err = waitForMigrationToBeCreated(childCtx, ci, migrationID); err != nil { + if err := waitForMigrationToBeCreated(childCtx, ci, migrationID); err != nil { return nil, fmt.Errorf("waiting migration to be created: %w", err) } var stages []stage.Stage[any] @@ -56,20 +51,6 @@ func migrationStages(ctx context.Context, ec plug.ExecContext, migrationID, repo if err != nil { return nil, fmt.Errorf("reading migration status: %w", err) } - if slices.Contains([]Status{StatusComplete, StatusFailed, StatusCanceled}, Status(generalStatus)) { - err = saveMemberLogs(ctx, ec, ci, migrationID) - if err != nil { - return nil, err - } - var name string - if reportOutputDir == "" { - name = fmt.Sprintf("migration_report_%s.txt", migrationID) - } - err = saveReportToFile(ctx, ci, migrationID, name) - if err != nil { - return nil, fmt.Errorf("saving report to file: %w", err) - } - } switch Status(generalStatus) { case StatusComplete: return nil, nil @@ -167,7 +148,7 @@ func saveMemberLogs(ctx context.Context, ec plug.ExecContext, ci *hazelcast.Clie return err } for _, line := range logs { - ec.Logger().Debugf(fmt.Sprintf("[%s_%s] %s", migrationID, m.UUID.String(), line.(string))) + ec.Logger().Info(fmt.Sprintf("[%s_%s] %s", migrationID, m.UUID.String(), line.(string))) } } return nil @@ -230,9 +211,6 @@ func fetchMigrationStatus(ctx context.Context, ci *hazelcast.ClientInternal, mig if err != nil { return "", err } - if err != nil { - return "", err - } it, err := res.Iterator() if err != nil { return "", err @@ -258,9 +236,6 @@ func fetchMigrationReport(ctx context.Context, ci *hazelcast.ClientInternal, mig if err != nil { return "", err } - if err != nil { - return "", err - } it, err := res.Iterator() if err != nil { return "", err @@ -286,9 +261,6 @@ func fetchMigrationErrors(ctx context.Context, ci *hazelcast.ClientInternal, mig if err != nil { return "", err } - if err != nil { - return "", err - } it, err := res.Iterator() if err != nil { return "", err @@ -308,3 +280,19 @@ func fetchMigrationErrors(ctx context.Context, ci *hazelcast.ClientInternal, mig } return strings.Join(errs, "\n"), nil } + +func finalizeMigration(ctx context.Context, ec plug.ExecContext, ci *hazelcast.ClientInternal, migrationID, reportOutputDir string) error { + err := saveMemberLogs(ctx, ec, ci, migrationID) + if err != nil { + return err + } + var name string + if reportOutputDir == "" { + name = fmt.Sprintf("migration_report_%s.txt", migrationID) + } + err = saveReportToFile(ctx, ci, migrationID, name) + if err != nil { + return fmt.Errorf("saving report to file: %w", err) + } + return nil +} diff --git a/base/commands/migration/migration_start.go b/base/commands/migration/migration_start.go index 9bc0cb33..d509fd08 100644 --- a/base/commands/migration/migration_start.go +++ b/base/commands/migration/migration_start.go @@ -4,6 +4,7 @@ package migration import ( "context" + "fmt" "github.com/hazelcast/hazelcast-commandline-client/clc" "github.com/hazelcast/hazelcast-commandline-client/clc/ux/stage" @@ -29,6 +30,10 @@ func (StartCmd) Init(cc plug.InitContext) error { } func (StartCmd) Exec(ctx context.Context, ec plug.ExecContext) error { + ci, err := ec.ClientInternal(ctx) + if err != nil { + return err + } ec.PrintlnUnnecessary("") ec.PrintlnUnnecessary(`Hazelcast Data Migration Tool v5.3.0 (c) 2023 Hazelcast, Inc. @@ -52,12 +57,17 @@ Selected data structures in the source cluster will be migrated to the target cl if _, err := stage.Execute(ctx, ec, any(nil), sp); err != nil { return err } - mStages, err := migrationStages(ctx, ec, mID, ec.Props().GetString(flagOutputDir), sts.statusMap) + mStages, err := createMigrationStages(ctx, ec, ci, mID) if err != nil { return err } mp := stage.NewFixedProvider(mStages...) - if _, err := stage.Execute(ctx, ec, any(nil), mp); err != nil { + _, err = stage.Execute(ctx, ec, any(nil), mp) + err2 := finalizeMigration(ctx, ec, ci, mID, ec.Props().GetString(flagOutputDir)) + if err2 != nil { + return fmt.Errorf("finalizing migration: %w", err2) + } + if err != nil { return err } ec.PrintlnUnnecessary("") diff --git a/base/commands/migration/migration_status.go b/base/commands/migration/migration_status.go index 8b8627b0..84f6782e 100644 --- a/base/commands/migration/migration_status.go +++ b/base/commands/migration/migration_status.go @@ -4,6 +4,7 @@ package migration import ( "context" + "fmt" "github.com/hazelcast/hazelcast-commandline-client/clc/ux/stage" "github.com/hazelcast/hazelcast-commandline-client/internal/check" @@ -24,6 +25,10 @@ func (s StatusCmd) Init(cc plug.InitContext) error { } func (s StatusCmd) Exec(ctx context.Context, ec plug.ExecContext) error { + ci, err := ec.ClientInternal(ctx) + if err != nil { + return err + } ec.PrintlnUnnecessary("") ec.PrintlnUnnecessary(banner) sts := NewStatusStages() @@ -32,12 +37,17 @@ func (s StatusCmd) Exec(ctx context.Context, ec plug.ExecContext) error { if err != nil { return err } - mStages, err := migrationStages(ctx, ec, mID.(string), ec.Props().GetString(flagOutputDir), sts.statusMap) + mStages, err := createMigrationStages(ctx, ec, ci, mID.(string)) if err != nil { return err } mp := stage.NewFixedProvider(mStages...) - if _, err := stage.Execute(ctx, ec, any(nil), mp); err != nil { + _, err = stage.Execute(ctx, ec, any(nil), mp) + err2 := finalizeMigration(ctx, ec, ci, mID.(string), ec.Props().GetString(flagOutputDir)) + if err2 != nil { + return fmt.Errorf("finalizing migration: %w", err2) + } + if err != nil { return err } ec.PrintlnUnnecessary("") diff --git a/base/commands/migration/start_stages_it_test.go b/base/commands/migration/start_stages_it_test.go index 5e02c2fb..dd4d5b03 100644 --- a/base/commands/migration/start_stages_it_test.go +++ b/base/commands/migration/start_stages_it_test.go @@ -14,8 +14,10 @@ import ( _ "github.com/hazelcast/hazelcast-commandline-client/base" _ "github.com/hazelcast/hazelcast-commandline-client/base/commands" "github.com/hazelcast/hazelcast-commandline-client/base/commands/migration" + "github.com/hazelcast/hazelcast-commandline-client/clc/paths" . "github.com/hazelcast/hazelcast-commandline-client/internal/check" "github.com/hazelcast/hazelcast-commandline-client/internal/it" + hz "github.com/hazelcast/hazelcast-go-client" "github.com/hazelcast/hazelcast-go-client/serialization" "github.com/stretchr/testify/require" ) @@ -54,6 +56,9 @@ func startMigrationTest(t *testing.T, expectedOutput string, statusMapStateFiles tcx := it.TestContext{T: t} ctx := context.Background() tcx.Tester(func(tcx it.TestContext) { + ci := hz.NewClientInternal(tcx.Client) + createMemberLogs(t, ctx, ci) + defer removeMembersLogs(ctx, ci) var wg sync.WaitGroup wg.Add(1) go tcx.WithReset(func() { @@ -73,6 +78,12 @@ func startMigrationTest(t *testing.T, expectedOutput string, statusMapStateFiles f := fmt.Sprintf("migration_report_%s.txt", mID) require.Equal(t, true, fileExists(f)) Must(os.Remove(f)) + b := MustValue(os.ReadFile(paths.DefaultLogPath(time.Now()))) + for _, m := range ci.OrderedMembers() { + require.Contains(t, string(b), fmt.Sprintf("[%s_%s] log1", mID, m.UUID.String())) + require.Contains(t, string(b), fmt.Sprintf("[%s_%s] log2", mID, m.UUID.String())) + require.Contains(t, string(b), fmt.Sprintf("[%s_%s] log3", mID, m.UUID.String())) + } }) }) } diff --git a/base/commands/migration/status_stages_it_test.go b/base/commands/migration/status_stages_it_test.go index 43880689..98a98b90 100644 --- a/base/commands/migration/status_stages_it_test.go +++ b/base/commands/migration/status_stages_it_test.go @@ -14,8 +14,10 @@ import ( _ "github.com/hazelcast/hazelcast-commandline-client/base" _ "github.com/hazelcast/hazelcast-commandline-client/base/commands" "github.com/hazelcast/hazelcast-commandline-client/base/commands/migration" + "github.com/hazelcast/hazelcast-commandline-client/clc/paths" . "github.com/hazelcast/hazelcast-commandline-client/internal/check" "github.com/hazelcast/hazelcast-commandline-client/internal/it" + hz "github.com/hazelcast/hazelcast-go-client" "github.com/hazelcast/hazelcast-go-client/serialization" "github.com/stretchr/testify/require" ) @@ -52,7 +54,9 @@ func statusTest(t *testing.T) { tcx := it.TestContext{T: t} ctx := context.Background() tcx.Tester(func(tcx it.TestContext) { - mID := preStatusRunner(t, tcx, ctx) + ci := hz.NewClientInternal(tcx.Client) + mID := preStatusRunner(t, tcx, ctx, ci) + defer removeMembersLogs(ctx, ci) var wg sync.WaitGroup wg.Add(1) go tcx.WithReset(func() { @@ -68,12 +72,19 @@ func statusTest(t *testing.T) { f := fmt.Sprintf("migration_report_%s.txt", mID) require.Equal(t, true, fileExists(f)) Must(os.Remove(f)) + b := MustValue(os.ReadFile(paths.DefaultLogPath(time.Now()))) + for _, m := range ci.OrderedMembers() { + require.Contains(t, string(b), fmt.Sprintf("[%s_%s] log1", mID, m.UUID.String())) + require.Contains(t, string(b), fmt.Sprintf("[%s_%s] log2", mID, m.UUID.String())) + require.Contains(t, string(b), fmt.Sprintf("[%s_%s] log3", mID, m.UUID.String())) + } }) }) } -func preStatusRunner(t *testing.T, tcx it.TestContext, ctx context.Context) string { +func preStatusRunner(t *testing.T, tcx it.TestContext, ctx context.Context, ci *hz.ClientInternal) string { createMapping(ctx, tcx) + createMemberLogs(t, ctx, ci) mID := migration.MakeMigrationID() l := MustValue(tcx.Client.GetList(ctx, migration.MigrationsInProgressList)) m := MustValue(json.Marshal(migration.MigrationInProgress{ @@ -86,6 +97,22 @@ func preStatusRunner(t *testing.T, tcx it.TestContext, ctx context.Context) stri return mID } +func createMemberLogs(t *testing.T, ctx context.Context, ci *hz.ClientInternal) { + for _, m := range ci.OrderedMembers() { + l := MustValue(ci.Client().GetList(ctx, migration.DebugLogsListPrefix+m.UUID.String())) + require.Equal(t, true, MustValue(l.Add(ctx, "log1"))) + require.Equal(t, true, MustValue(l.Add(ctx, "log2"))) + require.Equal(t, true, MustValue(l.Add(ctx, "log3"))) + } +} + +func removeMembersLogs(ctx context.Context, ci *hz.ClientInternal) { + for _, m := range ci.OrderedMembers() { + l := MustValue(ci.Client().GetList(ctx, migration.DebugLogsListPrefix+m.UUID.String())) + Must(l.Destroy(ctx)) + } +} + func statusRunner(t *testing.T, migrationID string, tcx it.TestContext, ctx context.Context) { statusMap := MustValue(tcx.Client.GetMap(ctx, migration.StatusMapName)) b := MustValue(os.ReadFile("testdata/start/migration_success_completed.json"))