greenplum: separate restore points fetch & find operations - so we can (

wal-g#1793) reuse fetched metadata. As a result, faster garbage deletion expected.
kubedb · Sep 30, 2024 · 8204361 · 8204361
1 parent 041e31e
commit 8204361
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 94 deletions.
diff --git a/cmd/gp/backup_fetch.go b/cmd/gp/backup_fetch.go
@@ -59,7 +59,9 @@ var backupFetchCmd = &cobra.Command{
 		}
 
 		if restorePointTS != "" {
-			restorePoint, err = greenplum.FindRestorePointBeforeTS(restorePointTS, storage.RootFolder())
+			restorePoints, err := greenplum.FetchAllRestorePoints(storage.RootFolder())
+			tracelog.ErrorLogger.FatalOnError(err)
+			restorePoint, err = greenplum.FindRestorePointBeforeTS(restorePointTS, restorePoints)
 			tracelog.ErrorLogger.FatalOnError(err)
 		}
 

diff --git a/internal/databases/greenplum/restore_point.go b/internal/databases/greenplum/restore_point.go
@@ -204,29 +204,34 @@ func NewNoRestorePointsFoundError() NoRestorePointsFoundError {
 	return NoRestorePointsFoundError{fmt.Errorf("no restore points found")}
 }
 
-// FindRestorePointBeforeTS finds restore point that was created before the provided timestamp
-// and finish time closest to the provided timestamp
-func FindRestorePointBeforeTS(timestampStr string, folder storage.Folder) (string, error) {
-	ts, err := time.Parse(time.RFC3339, timestampStr)
-	if err != nil {
-		return "", fmt.Errorf("timestamp parse error: %v", err)
-	}
+func FetchAllRestorePoints(folder storage.Folder) ([]RestorePointMetadata, error) {
+	restorePointMetas := make([]RestorePointMetadata, 0)
 
 	restorePointTimes, err := GetRestorePoints(folder.GetSubFolder(utility.BaseBackupPath))
 	if err != nil {
-		return "", err
+		return restorePointMetas, err
 	}
 
-	restorePointMetas := make([]RestorePointMetadata, 0)
 	for _, rp := range restorePointTimes {
 		meta, err := FetchRestorePointMetadata(folder, rp.Name)
 		if err != nil {
-			return "", fmt.Errorf("fetch restore point %s metadata: %v", rp.Name, err)
+			return restorePointMetas, fmt.Errorf("fetch restore point %s metadata: %v", rp.Name, err)
 		}
 
 		restorePointMetas = append(restorePointMetas, meta)
 	}
 
+	return restorePointMetas, nil
+}
+
+// FindRestorePointBeforeTS finds restore point that was created before the provided timestamp
+// and finish time closest to the provided timestamp
+func FindRestorePointBeforeTS(timestampStr string, restorePointMetas []RestorePointMetadata) (string, error) {
+	ts, err := time.Parse(time.RFC3339, timestampStr)
+	if err != nil {
+		return "", fmt.Errorf("timestamp parse error: %v", err)
+	}
+
 	var targetPoint *RestorePointMetadata
 	for i := range restorePointMetas {
 		meta := restorePointMetas[i]
@@ -251,29 +256,14 @@ func FindRestorePointBeforeTS(timestampStr string, folder storage.Folder) (strin
 }
 
 // Finds restore point that contains timestamp
-func FindRestorePointWithTS(timestampStr string, folder storage.Folder) (string, error) {
+func FindRestorePointWithTS(timestampStr string, restorePointMetas []RestorePointMetadata) (string, error) {
 	ts, err := time.Parse(time.RFC3339, timestampStr)
 	if err != nil {
 		return "", fmt.Errorf("timestamp parse error: %v", err)
 	}
 	// add second because we round down when formatting
 	ts = ts.Add(time.Second)
 
-	restorePointTimes, err := GetRestorePoints(folder.GetSubFolder(utility.BaseBackupPath))
-	if err != nil {
-		return "", err
-	}
-
-	restorePointMetas := make([]RestorePointMetadata, 0)
-	for _, rp := range restorePointTimes {
-		meta, err := FetchRestorePointMetadata(folder, rp.Name)
-		if err != nil {
-			return "", fmt.Errorf("fetch restore point %s metadata: %v", rp.Name, err)
-		}
-
-		restorePointMetas = append(restorePointMetas, meta)
-	}
-
 	var targetPoint *RestorePointMetadata
 	for i := range restorePointMetas {
 		meta := restorePointMetas[i]

diff --git a/internal/databases/greenplum/restore_point_test.go b/internal/databases/greenplum/restore_point_test.go
@@ -2,12 +2,9 @@ package greenplum_test
 
 import (
 	"bytes"
-	"encoding/json"
 	"testing"
 	"time"
 
-	"github.com/wal-g/wal-g/utility"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/wal-g/wal-g/internal"
 	"github.com/wal-g/wal-g/internal/databases/greenplum"
@@ -59,9 +56,6 @@ func TestGetRestorePointTimeSlices_OrderCheck(t *testing.T) {
 }
 
 func TestFindRestorePointBeforeTS_Correct(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
-	baseBackupsFolder := folder.GetSubFolder(utility.BaseBackupPath)
-
 	targetTs := time.Now()
 	restorePoints := []greenplum.RestorePointMetadata{
 		{
@@ -78,21 +72,12 @@ func TestFindRestorePointBeforeTS_Correct(t *testing.T) {
 			FinishTime: targetTs.Add(-10 * time.Second),
 		}}
 
-	for _, rp := range restorePoints {
-		rpBytes, _ := json.Marshal(rp)
-		_ = baseBackupsFolder.PutObject(rp.Name+greenplum.RestorePointSuffix, bytes.NewBuffer(rpBytes))
-		time.Sleep(time.Microsecond)
-	}
-
-	found, err := greenplum.FindRestorePointBeforeTS(targetTs.Format(time.RFC3339), folder)
+	found, err := greenplum.FindRestorePointBeforeTS(targetTs.Format(time.RFC3339), restorePoints)
 	assert.NoError(t, err)
 	assert.Equal(t, "expected_restore_point", found)
 }
 
 func TestFindRestorePointWithTS_StartInPast(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
-	baseBackupsFolder := folder.GetSubFolder(utility.BaseBackupPath)
-
 	targetTs := time.Now()
 	restorePoints := []greenplum.RestorePointMetadata{
 		{
@@ -115,21 +100,12 @@ func TestFindRestorePointWithTS_StartInPast(t *testing.T) {
 			FinishTime: targetTs.Add(time.Second),
 		}}
 
-	for _, rp := range restorePoints {
-		rpBytes, _ := json.Marshal(rp)
-		_ = baseBackupsFolder.PutObject(rp.Name+greenplum.RestorePointSuffix, bytes.NewBuffer(rpBytes))
-		time.Sleep(time.Microsecond)
-	}
-
-	found, err := greenplum.FindRestorePointWithTS(targetTs.Format(time.RFC3339), folder)
+	found, err := greenplum.FindRestorePointWithTS(targetTs.Format(time.RFC3339), restorePoints)
 	assert.NoError(t, err)
 	assert.Equal(t, "target_restore_point", found)
 }
 
 func TestFindRestorePointWithTS_StartInTS(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
-	baseBackupsFolder := folder.GetSubFolder(utility.BaseBackupPath)
-
 	targetTs := time.Now()
 	restorePoints := []greenplum.RestorePointMetadata{
 		{
@@ -152,21 +128,12 @@ func TestFindRestorePointWithTS_StartInTS(t *testing.T) {
 			FinishTime: targetTs.Add(1 * time.Second),
 		}}
 
-	for _, rp := range restorePoints {
-		rpBytes, _ := json.Marshal(rp)
-		_ = baseBackupsFolder.PutObject(rp.Name+greenplum.RestorePointSuffix, bytes.NewBuffer(rpBytes))
-		time.Sleep(time.Microsecond)
-	}
-
-	found, err := greenplum.FindRestorePointWithTS(targetTs.Format(time.RFC3339), folder)
+	found, err := greenplum.FindRestorePointWithTS(targetTs.Format(time.RFC3339), restorePoints)
 	assert.NoError(t, err)
 	assert.Equal(t, "target_restore_point", found)
 }
 
 func TestFindRestorePointWithTS_StartInFuture(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
-	baseBackupsFolder := folder.GetSubFolder(utility.BaseBackupPath)
-
 	targetTs := time.Now()
 	restorePoints := []greenplum.RestorePointMetadata{
 		{
@@ -184,31 +151,21 @@ func TestFindRestorePointWithTS_StartInFuture(t *testing.T) {
 			FinishTime: targetTs.Add(12 * time.Second),
 		}}
 
-	for _, rp := range restorePoints {
-		rpBytes, _ := json.Marshal(rp)
-		_ = baseBackupsFolder.PutObject(rp.Name+greenplum.RestorePointSuffix, bytes.NewBuffer(rpBytes))
-		time.Sleep(time.Microsecond)
-	}
-
-	found, err := greenplum.FindRestorePointWithTS(targetTs.Format(time.RFC3339), folder)
+	found, err := greenplum.FindRestorePointWithTS(targetTs.Format(time.RFC3339), restorePoints)
 	assert.NoError(t, err)
 	assert.Equal(t, "target_restore_point", found)
 }
 
 func TestFindRestorePointBeforeTS_NoRestorePoints(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
 	targetTs := time.Now()
 
-	found, err := greenplum.FindRestorePointBeforeTS(targetTs.Format(time.RFC3339), folder)
+	found, err := greenplum.FindRestorePointBeforeTS(targetTs.Format(time.RFC3339), []greenplum.RestorePointMetadata{})
 	assert.Error(t, err)
 	assert.IsType(t, greenplum.NoRestorePointsFoundError{}, err)
 	assert.Equal(t, "", found)
 }
 
 func TestFindRestorePointBeforeTS_NoMatches(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
-	baseBackupsFolder := folder.GetSubFolder(utility.BaseBackupPath)
-
 	targetTs := time.Now()
 	restorePoints := []greenplum.RestorePointMetadata{
 		{
@@ -225,22 +182,13 @@ func TestFindRestorePointBeforeTS_NoMatches(t *testing.T) {
 			FinishTime: targetTs.Add(1 * time.Second),
 		}}
 
-	for _, rp := range restorePoints {
-		rpBytes, _ := json.Marshal(rp)
-		_ = baseBackupsFolder.PutObject(rp.Name+greenplum.RestorePointSuffix, bytes.NewBuffer(rpBytes))
-		time.Sleep(time.Microsecond)
-	}
-
-	found, err := greenplum.FindRestorePointBeforeTS(targetTs.Format(time.RFC3339), folder)
+	found, err := greenplum.FindRestorePointBeforeTS(targetTs.Format(time.RFC3339), restorePoints)
 	assert.Error(t, err)
 	assert.IsType(t, greenplum.NoRestorePointsFoundError{}, err)
 	assert.Equal(t, "", found)
 }
 
 func TestFindRestorePointBeforeTS_ExactTime(t *testing.T) {
-	folder := testtools.MakeDefaultInMemoryStorageFolder()
-	baseBackupsFolder := folder.GetSubFolder(utility.BaseBackupPath)
-
 	targetStr := "2022-12-22T14:00:02.37584Z"
 	targetTs, _ := time.Parse(time.RFC3339, targetStr)
 	restorePoints := []greenplum.RestorePointMetadata{
@@ -258,13 +206,7 @@ func TestFindRestorePointBeforeTS_ExactTime(t *testing.T) {
 			FinishTime: targetTs.Add(1 * time.Nanosecond),
 		}}
 
-	for _, rp := range restorePoints {
-		rpBytes, _ := json.Marshal(rp)
-		_ = baseBackupsFolder.PutObject(rp.Name+greenplum.RestorePointSuffix, bytes.NewBuffer(rpBytes))
-		time.Sleep(time.Microsecond)
-	}
-
-	found, err := greenplum.FindRestorePointBeforeTS(targetStr, folder)
+	found, err := greenplum.FindRestorePointBeforeTS(targetStr, restorePoints)
 	assert.NoError(t, err)
 	assert.Equal(t, "expected_restore_point", found)
 }
diff --git a/internal/databases/greenplum/segment_delete_handler.go b/internal/databases/greenplum/segment_delete_handler.go
@@ -140,7 +140,13 @@ func GetPermanentBackupsAndWals(rootFolder storage.Folder, contentID int) (map[p
 	folder := rootFolder.GetSubFolder(FormatSegmentStoragePrefix(contentID))
 	backupTimes, err := internal.GetBackups(folder.GetSubFolder(utility.BaseBackupPath))
 	if err != nil {
-		tracelog.WarningLogger.Println("Error while time")
+		tracelog.WarningLogger.Println("Error while fetching backups")
+		return map[postgres.PermanentObject]bool{}, map[postgres.PermanentObject]bool{}
+	}
+
+	restorePointMetas, err := FetchAllRestorePoints(rootFolder)
+	if err != nil {
+		tracelog.WarningLogger.Println("Error while fetching restore points")
 		return map[postgres.PermanentObject]bool{}, map[postgres.PermanentObject]bool{}
 	}
 
@@ -161,7 +167,7 @@ func GetPermanentBackupsAndWals(rootFolder storage.Folder, contentID int) (map[p
 			continue
 		}
 
-		restorePoint, err := FindRestorePointWithTS(meta.StartTime.Format(time.RFC3339), rootFolder)
+		restorePoint, err := FindRestorePointWithTS(meta.StartTime.Format(time.RFC3339), restorePointMetas)
 		if err != nil {
 			internal.FatalOnUnrecoverableMetadataError(backupTime, err)
 			continue