Add support for timeout for restic backup and restore

Signed-off-by: Md. Ishtiaq Islam <[email protected]>

go.mod

@@ -23,7 +23,7 @@ require (
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	kmodules.xyz/client-go v0.30.13
 	kmodules.xyz/offshoot-api v0.29.4
-	kubestash.dev/apimachinery v0.12.0
+	kubestash.dev/apimachinery v0.12.1-0.20240913101021-d788e7a8bd3c
 	sigs.k8s.io/controller-runtime v0.18.4
 )
 

go.sum

@@ -638,8 +638,8 @@ kmodules.xyz/offshoot-api v0.29.4 h1:WQV2BIUIoVKKiqZNmZ4gAy367jEdwBhEl3dFCLZM1qA
 kmodules.xyz/offshoot-api v0.29.4/go.mod h1:e+NQ0s4gW/YTPWBWEfdISZcmk+tlTq8IjvP5SLdqvko=
 kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg=
 kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo=
-kubestash.dev/apimachinery v0.12.0 h1:33eDk/W2n/K45iRpAj8JTFGq9JzSvJBoWAjyoKTklgQ=
-kubestash.dev/apimachinery v0.12.0/go.mod h1:gtVSpHtK8LvS26+rKyLTnZvijvSSCdfG83n6GL6+Kwc=
+kubestash.dev/apimachinery v0.12.1-0.20240913101021-d788e7a8bd3c h1:BEVDrMJ0ugpVDAcE78PbXGCB9qCKHnGhV322wSPQ3/c=
+kubestash.dev/apimachinery v0.12.1-0.20240913101021-d788e7a8bd3c/go.mod h1:gtVSpHtK8LvS26+rKyLTnZvijvSSCdfG83n6GL6+Kwc=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=

pkg/trigger.go

@@ -102,6 +102,7 @@ func triggerBackup(backupConfig *coreapi.BackupConfiguration, session coreapi.Se
 			},
 			Session:   session.Name,
 			RetryLeft: 0,
+			Timeout:   session.Timeout,
 		},
 	}
 

vendor/kubestash.dev/apimachinery/apis/constant.go

@@ -77,6 +77,10 @@ const (
 	KubeStashAppRefName      = "kubestash.com/app-ref-name"
 )
 
+const (
+	AnnKubeDBAppVersion = "kubedb.com/db-version"
+)
+
 // Keys for structure logging
 const (
 	KeyTargetKind      = "target_kind"
@@ -152,3 +156,7 @@ const (
 	SnapshotVersionV1 = "v1"
 	DirRepository     = "repository"
 )
+
+const (
+	KeyRestoreSessionBeneficiary = "restoresession.kubestash.com/beneficiary"
+)

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupconfiguration_types.go

@@ -132,8 +132,8 @@ type SessionConfig struct {
 	// +optional
 	RetryConfig *RetryConfig `json:"retryConfig,omitempty"`
 
-	// Timeout specifies the maximum duration of backup. BackupSession will be considered Failed
-	// if backup does not complete within this time limit. By default, KubeStash don't set any timeout for backup.
+	// Timeout specifies the maximum duration of backup. Backup will be considered Failed
+	// if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup.
 	// +optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_helpers.go

@@ -243,3 +243,14 @@ func (b *BackupSession) checkFailureInRetentionPolicy() (bool, string) {
 	}
 	return false, ""
 }
+
+func (b *BackupSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) {
+	if b.Spec.Timeout == nil || b.Status.Deadline == nil {
+		return nil, nil
+	}
+	currentTime := metav1.Now()
+	if b.Status.Deadline.Before(&currentTime) {
+		return nil, fmt.Errorf("deadline exceeded")
+	}
+	return &metav1.Duration{Duration: b.Status.Deadline.Sub(currentTime.Time)}, nil
+}

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_types.go

@@ -63,6 +63,11 @@ type BackupSessionSpec struct {
 	// If this set to non-zero, KubeStash will create a new BackupSession if the current one fails.
 	// +optional
 	RetryLeft int32 `json:"retryLeft,omitempty"`
+
+	// Timeout specifies the maximum duration of backup. Backup will be considered Failed
+	// if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup.
+	// +optional
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
 }
 
 // BackupSessionStatus defines the observed state of BackupSession
@@ -75,10 +80,10 @@ type BackupSessionStatus struct {
 	// +optional
 	Duration string `json:"duration,omitempty"`
 
-	// Deadline specifies the deadline of backup. BackupSession will be
-	// considered Failed if backup does not complete within this deadline
+	// Deadline specifies the deadline of backup. Backup will be
+	// considered Failed if it does not complete within this deadline
 	// +optional
-	Deadline *metav1.Time `json:"sessionDeadline,omitempty"`
+	Deadline *metav1.Time `json:"backupDeadline,omitempty"`
 
 	// TotalSnapshots specifies the total number of snapshots created for this backupSession.
 	// +optional

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_helpers.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kmapi "kmodules.xyz/client-go/api/v1"
 	"kubestash.dev/apimachinery/apis"
@@ -41,8 +42,7 @@ func (rs *RestoreSession) CalculatePhase() RestorePhase {
 	}
 
 	if cutil.IsConditionTrue(rs.Status.Conditions, TypeMetricsPushed) &&
-		(cutil.IsConditionTrue(rs.Status.Conditions, TypeDeadlineExceeded) ||
-			cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) ||
+		(cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) ||
 			cutil.IsConditionFalse(rs.Status.Conditions, TypePostRestoreHooksExecutionSucceeded) ||
 			cutil.IsConditionFalse(rs.Status.Conditions, TypeRestoreExecutorEnsured)) {
 		return RestoreFailed
@@ -181,3 +181,14 @@ func (rs *RestoreSession) GetDataSourceNamespace() string {
 	}
 	return rs.Spec.DataSource.Namespace
 }
+
+func (rs *RestoreSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) {
+	if rs.Spec.Timeout == nil || rs.Status.Deadline == nil {
+		return nil, nil
+	}
+	currentTime := metav1.Now()
+	if rs.Status.Deadline.Before(&currentTime) {
+		return nil, fmt.Errorf("deadline exceeded")
+	}
+	return &metav1.Duration{Duration: rs.Status.Deadline.Sub(currentTime.Time)}, nil
+}

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_types.go

@@ -64,8 +64,8 @@ type RestoreSessionSpec struct {
 	// +optional
 	Hooks *RestoreHooks `json:"hooks,omitempty"`
 
-	// Timeout specifies a duration that KubeStash should wait for the session execution to be completed.
-	// If the session execution does not finish within this time period, KubeStash will consider this session as a failure.
+	// Timeout specifies a duration that KubeStash should wait for the restore to be completed.
+	// If the restore tasks do not finish within this time period, KubeStash will consider this restore as a failure.
 	// +optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
@@ -222,10 +222,10 @@ type RestoreSessionStatus struct {
 	// +optional
 	Duration string `json:"duration,omitempty"`
 
-	// Deadline specifies a timestamp till this session is valid. If the session does not complete within this deadline,
-	// it will be considered as failed.
+	// Deadline specifies the deadline of restore. Restore will be
+	// considered Failed if it does not complete within this deadline
 	// +optional
-	Deadline *metav1.Time `json:"deadline,omitempty"`
+	Deadline *metav1.Time `json:"restoreDeadline,omitempty"`
 
 	// TotalComponents represents the number of total components for this RestoreSession
 	// +optional

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/types.go

@@ -217,9 +217,6 @@ type RetryConfig struct {
 }
 
 const (
-	TypeDeadlineExceeded                 = "DeadlineExceeded"
-	ReasonFailedToCompleteWithinDeadline = "FailedToCompleteWithinDeadline"
-
 	// TypeMetricsPushed indicates whether Metrics are pushed or not
 	TypeMetricsPushed               = "MetricsPushed"
 	ReasonSuccessfullyPushedMetrics = "SuccessfullyPushedMetrics"

vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupbatches.yaml

@@ -36196,9 +36196,9 @@ spec:
                       type: array
                     timeout:
                       description: Timeout specifies the maximum duration of backup.
-                        BackupSession will be considered Failed if backup does not
-                        complete within this time limit. By default, KubeStash don't
-                        set any timeout for backup.
+                        Backup will be considered Failed if backup tasks do not complete
+                        within this time limit. By default, KubeStash don't set any
+                        timeout for backup.
                       type: string
                   type: object
                 type: array

vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupblueprints.yaml

@@ -37241,8 +37241,8 @@ spec:
                           type: integer
                         timeout:
                           description: Timeout specifies the maximum duration of backup.
-                            BackupSession will be considered Failed if backup does
-                            not complete within this time limit. By default, KubeStash
+                            Backup will be considered Failed if backup tasks do not
+                            complete within this time limit. By default, KubeStash
                             don't set any timeout for backup.
                           type: string
                       type: object

vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupconfigurations.yaml

@@ -34375,9 +34375,9 @@ spec:
                       type: integer
                     timeout:
                       description: Timeout specifies the maximum duration of backup.
-                        BackupSession will be considered Failed if backup does not
-                        complete within this time limit. By default, KubeStash don't
-                        set any timeout for backup.
+                        Backup will be considered Failed if backup tasks do not complete
+                        within this time limit. By default, KubeStash don't set any
+                        timeout for backup.
                       type: string
                   type: object
                 type: array

vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupsessions.yaml

@@ -87,10 +87,21 @@ spec:
                 description: Session specifies the name of the session that triggered
                   this backup
                 type: string
+              timeout:
+                description: Timeout specifies the maximum duration of backup. Backup
+                  will be considered Failed if backup tasks do not complete within
+                  this time limit. By default, KubeStash don't set any timeout for
+                  backup.
+                type: string
             type: object
           status:
             description: BackupSessionStatus defines the observed state of BackupSession
             properties:
+              backupDeadline:
+                description: Deadline specifies the deadline of backup. Backup will
+                  be considered Failed if it does not complete within this deadline
+                format: date-time
+                type: string
               conditions:
                 description: Conditions represents list of conditions regarding this
                   BackupSession
@@ -249,12 +260,6 @@ spec:
                   not. This field will exist only if the `retryConfig` has been set
                   in the respective backup invoker.
                 type: boolean
-              sessionDeadline:
-                description: Deadline specifies the deadline of backup. BackupSession
-                  will be considered Failed if backup does not complete within this
-                  deadline
-                format: date-time
-                type: string
               snapshots:
                 description: Snapshots specifies the Snapshots status
                 items:

vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_restoresessions.yaml

@@ -24332,9 +24332,9 @@ spec:
                 type: object
               timeout:
                 description: Timeout specifies a duration that KubeStash should wait
-                  for the session execution to be completed. If the session execution
-                  does not finish within this time period, KubeStash will consider
-                  this session as a failure.
+                  for the restore to be completed. If the restore tasks do not finish
+                  within this time period, KubeStash will consider this restore as
+                  a failure.
                 type: string
             type: object
           status:
@@ -24420,12 +24420,6 @@ spec:
                   - type
                   type: object
                 type: array
-              deadline:
-                description: Deadline specifies a timestamp till this session is valid.
-                  If the session does not complete within this deadline, it will be
-                  considered as failed.
-                format: date-time
-                type: string
               dependencies:
                 description: Dependencies specifies whether the objects required by
                   this RestoreSession exist or not
@@ -24530,6 +24524,11 @@ spec:
                 - Invalid
                 - Unknown
                 type: string
+              restoreDeadline:
+                description: Deadline specifies the deadline of restore. Restore will
+                  be considered Failed if it does not complete within this deadline
+                format: date-time
+                type: string
               targetFound:
                 description: TargetFound specifies whether the restore target exist
                   or not

vendor/kubestash.dev/apimachinery/pkg/restic/commands.go

@@ -35,7 +35,8 @@ import (
 )
 
 const (
-	ResticCMD = "restic"
+	ResticCMD  = "restic"
+	TimeoutCMD = "timeout"
 )
 
 type Snapshot struct {
@@ -176,7 +177,15 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) {
 	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	return w.run(Command{Name: ResticCMD, Args: args})
+	command := Command{Name: ResticCMD, Args: args}
+	if w.config.Timeout != nil {
+		timeoutArgs := []interface{}{fmt.Sprintf("%f", w.config.Timeout.Seconds())}
+		timeoutArgs = append(timeoutArgs, ResticCMD)
+		timeoutArgs = append(timeoutArgs, args...)
+		command = Command{Name: TimeoutCMD, Args: timeoutArgs}
+	}
+
+	return w.run(command)
 }
 
 func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
@@ -200,7 +209,15 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
 	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	commands = append(commands, Command{Name: ResticCMD, Args: args})
+	command := Command{Name: ResticCMD, Args: args}
+	if w.config.Timeout != nil {
+		timeoutArgs := []interface{}{fmt.Sprintf("%f", w.config.Timeout.Seconds())}
+		timeoutArgs = append(timeoutArgs, ResticCMD)
+		timeoutArgs = append(timeoutArgs, args...)
+		command = Command{Name: TimeoutCMD, Args: timeoutArgs}
+	}
+
+	commands = append(commands, command)
 	return w.run(commands...)
 }
 
@@ -246,7 +263,15 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) {
 	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	return w.run(Command{Name: ResticCMD, Args: args})
+	command := Command{Name: ResticCMD, Args: args}
+	if w.config.Timeout != nil {
+		timeoutArgs := []interface{}{fmt.Sprintf("%f", w.config.Timeout.Seconds())}
+		timeoutArgs = append(timeoutArgs, ResticCMD)
+		timeoutArgs = append(timeoutArgs, args...)
+		command = Command{Name: TimeoutCMD, Args: timeoutArgs}
+	}
+
+	return w.run(command)
 }
 
 func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) {
@@ -277,10 +302,16 @@ func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) {
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendInsecureTLSFlag(args)
 
-	// first add restic command, then add StdoutPipeCommands
-	commands := []Command{
-		{Name: ResticCMD, Args: args},
+	command := Command{Name: ResticCMD, Args: args}
+	if w.config.Timeout != nil {
+		timeoutArgs := []interface{}{fmt.Sprintf("%f", w.config.Timeout.Seconds())}
+		timeoutArgs = append(timeoutArgs, ResticCMD)
+		timeoutArgs = append(timeoutArgs, args...)
+		command = Command{Name: TimeoutCMD, Args: timeoutArgs}
 	}
+
+	// first add restic command, then add StdoutPipeCommands
+	commands := []Command{command}
 	commands = append(commands, dumpOptions.StdoutPipeCommands...)
 	return w.run(commands...)
 }
@@ -368,7 +399,7 @@ func (w *ResticWrapper) run(commands ...Command) ([]byte, error) {
 	w.sh.Stderr = io.MultiWriter(os.Stderr, errBuff)
 
 	for _, cmd := range commands {
-		if cmd.Name == ResticCMD {
+		if cmd.Name == ResticCMD || cmd.Name == TimeoutCMD {
 			// first apply NiceSettings, then apply IONiceSettings
 			cmd, err = w.applyNiceSettings(cmd)
 			if err != nil {
@@ -393,6 +424,9 @@ func (w *ResticWrapper) run(commands ...Command) ([]byte, error) {
 func formatError(err error, stdErr string) error {
 	parts := strings.Split(strings.TrimSuffix(stdErr, "\n"), "\n")
 	if len(parts) > 1 {
+		if strings.Contains(parts[1], "signal terminated") {
+			return errors.New(strings.Join(append([]string{"deadline exceeded or signal terminated"}, parts[2:]...), " "))
+		}
 		return errors.New(strings.Join(parts[1:], " "))
 	}
 	return err