From d72b6306a39a47fef2e5221ef5c7073fac17c71a Mon Sep 17 00:00:00 2001 From: "Md. Ishtiaq Islam" Date: Wed, 18 Sep 2024 16:33:11 +0600 Subject: [PATCH] Add support for timeout for restic backup and restore (#37) Signed-off-by: Md. Ishtiaq Islam --- go.mod | 2 +- go.sum | 4 +- pkg/trigger.go | 5 +- .../apimachinery/apis/constant.go | 7 +- .../v1alpha1/backupconfiguration_types.go | 6 +- .../core/v1alpha1/backupsession_helpers.go | 11 + .../apis/core/v1alpha1/backupsession_types.go | 11 +- .../core/v1alpha1/restoresession_helpers.go | 15 +- .../core/v1alpha1/restoresession_types.go | 72 +++++- .../apimachinery/apis/core/v1alpha1/types.go | 3 - .../core/v1alpha1/zz_generated.deepcopy.go | 61 ++++- .../core.kubestash.com_backupbatches.yaml | 12 +- .../core.kubestash.com_backupblueprints.yaml | 12 +- ...re.kubestash.com_backupconfigurations.yaml | 12 +- .../core.kubestash.com_backupsessions.yaml | 17 +- .../core.kubestash.com_restoresessions.yaml | 214 +++++++++++++++++- .../apimachinery/pkg/restic/commands.go | 39 +++- .../apimachinery/pkg/restic/config.go | 5 +- vendor/modules.txt | 2 +- 19 files changed, 432 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index 687a3766..a736afd0 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 kmodules.xyz/client-go v0.30.14 kmodules.xyz/offshoot-api v0.29.4 - kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a + kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1 sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 2d694b40..9c08d300 100644 --- a/go.sum +++ b/go.sum @@ -809,8 +809,8 @@ kmodules.xyz/offshoot-api v0.29.4 h1:WQV2BIUIoVKKiqZNmZ4gAy367jEdwBhEl3dFCLZM1qA kmodules.xyz/offshoot-api v0.29.4/go.mod h1:e+NQ0s4gW/YTPWBWEfdISZcmk+tlTq8IjvP5SLdqvko= kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg= kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo= -kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a h1:CiTlSi3CIfTY+gbIBFm4oMmbhTknYvrRGWn4LZC82wA= -kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a/go.mod h1:gtVSpHtK8LvS26+rKyLTnZvijvSSCdfG83n6GL6+Kwc= +kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1 h1:1aiK375epNLvl2CzSieF56sA25YsQnODBGLaDN/Vx3Q= +kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1/go.mod h1:gtVSpHtK8LvS26+rKyLTnZvijvSSCdfG83n6GL6+Kwc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/pkg/trigger.go b/pkg/trigger.go index 55e16729..aff8ef7c 100644 --- a/pkg/trigger.go +++ b/pkg/trigger.go @@ -100,8 +100,9 @@ func triggerBackup(backupConfig *coreapi.BackupConfiguration, session coreapi.Se Kind: coreapi.ResourceKindBackupConfiguration, Name: backupConfig.Name, }, - Session: session.Name, - RetryLeft: 0, + Session: session.Name, + RetryLeft: 0, + BackupTimeout: session.BackupTimeout, }, } diff --git a/vendor/kubestash.dev/apimachinery/apis/constant.go b/vendor/kubestash.dev/apimachinery/apis/constant.go index 43d278da..33efe359 100644 --- a/vendor/kubestash.dev/apimachinery/apis/constant.go +++ b/vendor/kubestash.dev/apimachinery/apis/constant.go @@ -75,7 +75,6 @@ const ( KubeStashAppRefKind = "kubestash.com/app-ref-kind" KubeStashAppRefNamespace = "kubestash.com/app-ref-namespace" KubeStashAppRefName = "kubestash.com/app-ref-name" - KubeDBAppVersion = "kubedb.com/db-version" ) // Keys for structure logging @@ -153,3 +152,9 @@ const ( SnapshotVersionV1 = "v1" DirRepository = "repository" ) + +// Annotations +const ( + AnnKubeDBAppVersion = "kubedb.com/db-version" + AnnRestoreSessionBeneficiary = "restoresession.kubestash.com/beneficiary" +) diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupconfiguration_types.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupconfiguration_types.go index 105c4ce4..5a34cb25 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupconfiguration_types.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupconfiguration_types.go @@ -132,10 +132,10 @@ type SessionConfig struct { // +optional RetryConfig *RetryConfig `json:"retryConfig,omitempty"` - // Timeout specifies the maximum duration of backup. BackupSession will be considered Failed - // if backup does not complete within this time limit. By default, KubeStash don't set any timeout for backup. + // BackupTimeout specifies the maximum duration of backup. Backup will be considered Failed + // if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup. // +optional - Timeout *metav1.Duration `json:"timeout,omitempty"` + BackupTimeout *metav1.Duration `json:"backupTimeout,omitempty"` // SessionHistoryLimit specifies how many backup Jobs and associate resources KubeStash should keep for debugging purpose. // The default value is 1. diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_helpers.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_helpers.go index d89f19e8..edf3152e 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_helpers.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_helpers.go @@ -243,3 +243,14 @@ func (b *BackupSession) checkFailureInRetentionPolicy() (bool, string) { } return false, "" } + +func (b *BackupSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) { + if b.Spec.BackupTimeout == nil || b.Status.BackupDeadline == nil { + return nil, nil + } + currentTime := metav1.Now() + if b.Status.BackupDeadline.Before(¤tTime) { + return nil, fmt.Errorf("deadline exceeded") + } + return &metav1.Duration{Duration: b.Status.BackupDeadline.Sub(currentTime.Time)}, nil +} diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_types.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_types.go index 5ccf2faf..9f094839 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_types.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_types.go @@ -63,6 +63,11 @@ type BackupSessionSpec struct { // If this set to non-zero, KubeStash will create a new BackupSession if the current one fails. // +optional RetryLeft int32 `json:"retryLeft,omitempty"` + + // BackupTimeout specifies the maximum duration of backup. Backup will be considered Failed + // if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup. + // +optional + BackupTimeout *metav1.Duration `json:"backupTimeout,omitempty"` } // BackupSessionStatus defines the observed state of BackupSession @@ -75,10 +80,10 @@ type BackupSessionStatus struct { // +optional Duration string `json:"duration,omitempty"` - // Deadline specifies the deadline of backup. BackupSession will be - // considered Failed if backup does not complete within this deadline + // BackupDeadline specifies the deadline of backup. Backup will be + // considered Failed if it does not complete within this deadline // +optional - Deadline *metav1.Time `json:"sessionDeadline,omitempty"` + BackupDeadline *metav1.Time `json:"backupDeadline,omitempty"` // TotalSnapshots specifies the total number of snapshots created for this backupSession. // +optional diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_helpers.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_helpers.go index df970cf4..11121df2 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_helpers.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_helpers.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kmapi "kmodules.xyz/client-go/api/v1" "kubestash.dev/apimachinery/apis" @@ -41,8 +42,7 @@ func (rs *RestoreSession) CalculatePhase() RestorePhase { } if cutil.IsConditionTrue(rs.Status.Conditions, TypeMetricsPushed) && - (cutil.IsConditionTrue(rs.Status.Conditions, TypeDeadlineExceeded) || - cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) || + (cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) || cutil.IsConditionFalse(rs.Status.Conditions, TypePostRestoreHooksExecutionSucceeded) || cutil.IsConditionFalse(rs.Status.Conditions, TypeRestoreExecutorEnsured)) { return RestoreFailed @@ -181,3 +181,14 @@ func (rs *RestoreSession) GetDataSourceNamespace() string { } return rs.Spec.DataSource.Namespace } + +func (rs *RestoreSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) { + if rs.Spec.RestoreTimeout == nil || rs.Status.RestoreDeadline == nil { + return nil, nil + } + currentTime := metav1.Now() + if rs.Status.RestoreDeadline.Before(¤tTime) { + return nil, fmt.Errorf("deadline exceeded") + } + return &metav1.Duration{Duration: rs.Status.RestoreDeadline.Sub(currentTime.Time)}, nil +} diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_types.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_types.go index 141a2ff3..5b737d57 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_types.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_types.go @@ -64,10 +64,10 @@ type RestoreSessionSpec struct { // +optional Hooks *RestoreHooks `json:"hooks,omitempty"` - // Timeout specifies a duration that KubeStash should wait for the session execution to be completed. - // If the session execution does not finish within this time period, KubeStash will consider this session as a failure. + // RestoreTimeout specifies a duration that KubeStash should wait for the restore to be completed. + // If the restore tasks do not finish within this time period, KubeStash will consider this restore as a failure. // +optional - Timeout *metav1.Duration `json:"timeout,omitempty"` + RestoreTimeout *metav1.Duration `json:"restoreTimeout,omitempty"` // ManifestOptions provide options to select particular manifest object to restore // +optional @@ -98,6 +98,18 @@ type ManifestRestoreOptions struct { // MSSQLServer specifies the options for selecting particular MSSQLServer components to restore in manifest restore // +optional MSSQLServer *MSSQLServerManifestOptions `json:"msSQLServer,omitempty"` + + // Druid specifies the options for selecting particular Druid components to restore in manifest restore + // +optional + Druid *DruidManifestOptions `json:"druid,omitempty"` + + // ZooKeeper specifies the options for selecting particular ZooKeeper components to restore in manifest restore + // +optional + ZooKeeper *KubeDBManifestOptions `json:"zooKeeper,omitempty"` + + // Redis specifies the options for selecting particular Redis components to restore in manifest restore + // +optional + Redis *KubeDBManifestOptions `json:"redis,omitempty"` } type MSSQLServerManifestOptions struct { @@ -126,6 +138,50 @@ type MSSQLServerManifestOptions struct { TLSIssuerRef *core.TypedLocalObjectReference `json:"tlsIssuerRef,omitempty"` } +type DruidManifestOptions struct { + // DB specifies whether to restore the DB manifest or not + // +optional + DB bool `json:"db,omitempty"` + + // DBName specifies the new name of the DB yaml after restore + // +optional + DBName string `json:"dbName,omitempty"` + + // AuthSecret specifies whether to restore the AuthSecret manifest or not + // +optional + AuthSecret bool `json:"authSecret,omitempty"` + + // AuthSecretName specifies new name of the AuthSecret yaml after restore + // +optional + AuthSecretName string `json:"authSecretName,omitempty"` + + // ConfigSecret specifies whether to restore the ConfigSecret manifest or not + // +optional + ConfigSecret bool `json:"configSecret,omitempty"` + + // ConfigSecretName specifies new name of the ConfigSecret yaml after restore + // +optional + ConfigSecretName string `json:"configSecretName,omitempty"` + + // DeepStorageSecret specifies whether to restore the DeepStorageSecret manifest or not + // +optional + DeepStorageSecret bool `json:"deepStorageSecret,omitempty"` + + // MetadataStorage specifies new configuration of the Metadata Storage after restore + // +optional + MetadataStorage bool `json:"metadataStorage,omitempty"` + + // +optional + MetadataStorageRef *kmapi.ObjectReference `json:"metadataStorageRef,omitempty"` + + // ZooKeeper specifies new configuration of the Metadata Storage after restore + // +optional + Zookeeper bool `json:"zookeeper,omitempty"` + + // +optional + ZookeeperRef *kmapi.ObjectReference `json:"zookeeperRef,omitempty"` +} + type KubeDBManifestOptions struct { // DB specifies whether to restore the DB manifest or not // +optional @@ -151,6 +207,10 @@ type KubeDBManifestOptions struct { // +optional ConfigSecretName string `json:"configSecretName,omitempty"` + // InitScript specifies whether to restore the InitScript manifest or not + // +optional + InitScript bool `json:"initScript,omitempty"` + // TLSIssuerRef specifies the name of the IssuerRef used for TLS configurations for both client and server // +optional TLSIssuerRef *core.TypedLocalObjectReference `json:"tlsIssuerRef,omitempty"` @@ -222,10 +282,10 @@ type RestoreSessionStatus struct { // +optional Duration string `json:"duration,omitempty"` - // Deadline specifies a timestamp till this session is valid. If the session does not complete within this deadline, - // it will be considered as failed. + // RestoreDeadline specifies the deadline of restore. Restore will be + // considered Failed if it does not complete within this deadline // +optional - Deadline *metav1.Time `json:"deadline,omitempty"` + RestoreDeadline *metav1.Time `json:"restoreDeadline,omitempty"` // TotalComponents represents the number of total components for this RestoreSession // +optional diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/types.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/types.go index 96dbb652..445f867b 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/types.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/types.go @@ -217,9 +217,6 @@ type RetryConfig struct { } const ( - TypeDeadlineExceeded = "DeadlineExceeded" - ReasonFailedToCompleteWithinDeadline = "FailedToCompleteWithinDeadline" - // TypeMetricsPushed indicates whether Metrics are pushed or not TypeMetricsPushed = "MetricsPushed" ReasonSuccessfullyPushedMetrics = "SuccessfullyPushedMetrics" diff --git a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/zz_generated.deepcopy.go b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/zz_generated.deepcopy.go index 4c8ecb60..a1ec3193 100644 --- a/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/zz_generated.deepcopy.go @@ -598,6 +598,11 @@ func (in *BackupSessionSpec) DeepCopyInto(out *BackupSessionSpec) { *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.BackupTimeout != nil { + in, out := &in.BackupTimeout, &out.BackupTimeout + *out = new(metav1.Duration) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSessionSpec. @@ -613,8 +618,8 @@ func (in *BackupSessionSpec) DeepCopy() *BackupSessionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupSessionStatus) DeepCopyInto(out *BackupSessionStatus) { *out = *in - if in.Deadline != nil { - in, out := &in.Deadline, &out.Deadline + if in.BackupDeadline != nil { + in, out := &in.BackupDeadline, &out.BackupDeadline *out = (*in).DeepCopy() } if in.TotalSnapshots != nil { @@ -710,6 +715,31 @@ func (in *ComponentRestoreStatus) DeepCopy() *ComponentRestoreStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DruidManifestOptions) DeepCopyInto(out *DruidManifestOptions) { + *out = *in + if in.MetadataStorageRef != nil { + in, out := &in.MetadataStorageRef, &out.MetadataStorageRef + *out = new(v1.ObjectReference) + **out = **in + } + if in.ZookeeperRef != nil { + in, out := &in.ZookeeperRef, &out.ZookeeperRef + *out = new(v1.ObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DruidManifestOptions. +func (in *DruidManifestOptions) DeepCopy() *DruidManifestOptions { + if in == nil { + return nil + } + out := new(DruidManifestOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FunctionHookExecutorSpec) DeepCopyInto(out *FunctionHookExecutorSpec) { *out = *in @@ -1084,6 +1114,21 @@ func (in *ManifestRestoreOptions) DeepCopyInto(out *ManifestRestoreOptions) { *out = new(MSSQLServerManifestOptions) (*in).DeepCopyInto(*out) } + if in.Druid != nil { + in, out := &in.Druid, &out.Druid + *out = new(DruidManifestOptions) + (*in).DeepCopyInto(*out) + } + if in.ZooKeeper != nil { + in, out := &in.ZooKeeper, &out.ZooKeeper + *out = new(KubeDBManifestOptions) + (*in).DeepCopyInto(*out) + } + if in.Redis != nil { + in, out := &in.Redis, &out.Redis + *out = new(KubeDBManifestOptions) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManifestRestoreOptions. @@ -1373,8 +1418,8 @@ func (in *RestoreSessionSpec) DeepCopyInto(out *RestoreSessionSpec) { *out = new(RestoreHooks) (*in).DeepCopyInto(*out) } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout + if in.RestoreTimeout != nil { + in, out := &in.RestoreTimeout, &out.RestoreTimeout *out = new(metav1.Duration) **out = **in } @@ -1403,8 +1448,8 @@ func (in *RestoreSessionStatus) DeepCopyInto(out *RestoreSessionStatus) { *out = new(bool) **out = **in } - if in.Deadline != nil { - in, out := &in.Deadline, &out.Deadline + if in.RestoreDeadline != nil { + in, out := &in.RestoreDeadline, &out.RestoreDeadline *out = (*in).DeepCopy() } if in.Components != nil { @@ -1585,8 +1630,8 @@ func (in *SessionConfig) DeepCopyInto(out *SessionConfig) { *out = new(RetryConfig) **out = **in } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout + if in.BackupTimeout != nil { + in, out := &in.BackupTimeout, &out.BackupTimeout *out = new(metav1.Duration) **out = **in } diff --git a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupbatches.yaml b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupbatches.yaml index 4918fbb4..790559ec 100644 --- a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupbatches.yaml +++ b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupbatches.yaml @@ -110,6 +110,12 @@ spec: description: BatchSession specifies the session configuration for the targets. properties: + backupTimeout: + description: BackupTimeout specifies the maximum duration of + backup. Backup will be considered Failed if backup tasks do + not complete within this time limit. By default, KubeStash + don't set any timeout for backup. + type: string hooks: description: Hooks specifies the backup hooks that should be executed before and/or after the backup. @@ -36194,12 +36200,6 @@ spec: type: array type: object type: array - timeout: - description: Timeout specifies the maximum duration of backup. - BackupSession will be considered Failed if backup does not - complete within this time limit. By default, KubeStash don't - set any timeout for backup. - type: string type: object type: array targets: diff --git a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupblueprints.yaml b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupblueprints.yaml index 62b405d7..f745accc 100644 --- a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupblueprints.yaml +++ b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupblueprints.yaml @@ -15716,6 +15716,12 @@ spec: type: object type: array type: object + backupTimeout: + description: BackupTimeout specifies the maximum duration + of backup. Backup will be considered Failed if backup + tasks do not complete within this time limit. By default, + KubeStash don't set any timeout for backup. + type: string hooks: description: Hooks specifies the backup hooks that should be executed before and/or after the backup. @@ -37239,12 +37245,6 @@ spec: debugging purpose. The default value is 1. format: int32 type: integer - timeout: - description: Timeout specifies the maximum duration of backup. - BackupSession will be considered Failed if backup does - not complete within this time limit. By default, KubeStash - don't set any timeout for backup. - type: string type: object type: array type: object diff --git a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupconfigurations.yaml b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupconfigurations.yaml index 30807c31..bd2a3165 100644 --- a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupconfigurations.yaml +++ b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupconfigurations.yaml @@ -14554,6 +14554,12 @@ spec: type: object type: array type: object + backupTimeout: + description: BackupTimeout specifies the maximum duration of + backup. Backup will be considered Failed if backup tasks do + not complete within this time limit. By default, KubeStash + don't set any timeout for backup. + type: string hooks: description: Hooks specifies the backup hooks that should be executed before and/or after the backup. @@ -34373,12 +34379,6 @@ spec: purpose. The default value is 1. format: int32 type: integer - timeout: - description: Timeout specifies the maximum duration of backup. - BackupSession will be considered Failed if backup does not - complete within this time limit. By default, KubeStash don't - set any timeout for backup. - type: string type: object type: array target: diff --git a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupsessions.yaml b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupsessions.yaml index 51f413b2..fe23c658 100644 --- a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupsessions.yaml +++ b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupsessions.yaml @@ -56,6 +56,12 @@ spec: description: BackupSessionSpec specifies the information related to the respective backup invoker and session. properties: + backupTimeout: + description: BackupTimeout specifies the maximum duration of backup. + Backup will be considered Failed if backup tasks do not complete + within this time limit. By default, KubeStash don't set any timeout + for backup. + type: string invoker: description: Invoker points to the respective BackupConfiguration or BackupBatch which is responsible for triggering this backup. @@ -91,6 +97,11 @@ spec: status: description: BackupSessionStatus defines the observed state of BackupSession properties: + backupDeadline: + description: BackupDeadline specifies the deadline of backup. Backup + will be considered Failed if it does not complete within this deadline + format: date-time + type: string conditions: description: Conditions represents list of conditions regarding this BackupSession @@ -249,12 +260,6 @@ spec: not. This field will exist only if the `retryConfig` has been set in the respective backup invoker. type: boolean - sessionDeadline: - description: Deadline specifies the deadline of backup. BackupSession - will be considered Failed if backup does not complete within this - deadline - format: date-time - type: string snapshots: description: Snapshots specifies the Snapshots status items: diff --git a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_restoresessions.yaml b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_restoresessions.yaml index 12af7a0d..8f15ba2a 100644 --- a/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_restoresessions.yaml +++ b/vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_restoresessions.yaml @@ -24044,6 +24044,73 @@ spec: description: ManifestOptions provide options to select particular manifest object to restore properties: + druid: + description: Druid specifies the options for selecting particular + Druid components to restore in manifest restore + properties: + authSecret: + description: AuthSecret specifies whether to restore the AuthSecret + manifest or not + type: boolean + authSecretName: + description: AuthSecretName specifies new name of the AuthSecret + yaml after restore + type: string + configSecret: + description: ConfigSecret specifies whether to restore the + ConfigSecret manifest or not + type: boolean + configSecretName: + description: ConfigSecretName specifies new name of the ConfigSecret + yaml after restore + type: string + db: + description: DB specifies whether to restore the DB manifest + or not + type: boolean + dbName: + description: DBName specifies the new name of the DB yaml + after restore + type: string + deepStorageSecret: + description: DeepStorageSecret specifies whether to restore + the DeepStorageSecret manifest or not + type: boolean + metadataStorage: + description: MetadataStorage specifies new configuration of + the Metadata Storage after restore + type: boolean + metadataStorageRef: + description: ObjectReference contains enough information to + let you inspect or modify the referred object. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + required: + - name + type: object + zookeeper: + description: ZooKeeper specifies new configuration of the + Metadata Storage after restore + type: boolean + zookeeperRef: + description: ObjectReference contains enough information to + let you inspect or modify the referred object. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + required: + - name + type: object + type: object mariaDB: description: MariaDB specifies the options for selecting particular MariaDB components to restore in manifest restore @@ -24072,6 +24139,10 @@ spec: description: DBName specifies the new name of the DB yaml after restore type: string + initScript: + description: InitScript specifies whether to restore the InitScript + manifest or not + type: boolean tlsIssuerRef: description: TLSIssuerRef specifies the name of the IssuerRef used for TLS configurations for both client and server @@ -24122,6 +24193,10 @@ spec: description: DBName specifies the new name of the DB yaml after restore type: string + initScript: + description: InitScript specifies whether to restore the InitScript + manifest or not + type: boolean tlsIssuerRef: description: TLSIssuerRef specifies the name of the IssuerRef used for TLS configurations for both client and server @@ -24235,6 +24310,10 @@ spec: description: DBName specifies the new name of the DB yaml after restore type: string + initScript: + description: InitScript specifies whether to restore the InitScript + manifest or not + type: boolean tlsIssuerRef: description: TLSIssuerRef specifies the name of the IssuerRef used for TLS configurations for both client and server @@ -24285,6 +24364,64 @@ spec: description: DBName specifies the new name of the DB yaml after restore type: string + initScript: + description: InitScript specifies whether to restore the InitScript + manifest or not + type: boolean + tlsIssuerRef: + description: TLSIssuerRef specifies the name of the IssuerRef + used for TLS configurations for both client and server + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + type: object + redis: + description: Redis specifies the options for selecting particular + Redis components to restore in manifest restore + properties: + authSecret: + description: AuthSecret specifies whether to restore the AuthSecret + manifest or not + type: boolean + authSecretName: + description: AuthSecretName specifies new name of the AuthSecret + yaml after restore + type: string + configSecret: + description: ConfigSecret specifies whether to restore the + ConfigSecret manifest or not + type: boolean + configSecretName: + description: ConfigSecretName specifies new name of the ConfigSecret + yaml after restore + type: string + db: + description: DB specifies whether to restore the DB manifest + or not + type: boolean + dbName: + description: DBName specifies the new name of the DB yaml + after restore + type: string + initScript: + description: InitScript specifies whether to restore the InitScript + manifest or not + type: boolean tlsIssuerRef: description: TLSIssuerRef specifies the name of the IssuerRef used for TLS configurations for both client and server @@ -24311,7 +24448,67 @@ spec: description: RestoreNamespace specifies the Namespace where the restored files will be applied type: string + zooKeeper: + description: ZooKeeper specifies the options for selecting particular + ZooKeeper components to restore in manifest restore + properties: + authSecret: + description: AuthSecret specifies whether to restore the AuthSecret + manifest or not + type: boolean + authSecretName: + description: AuthSecretName specifies new name of the AuthSecret + yaml after restore + type: string + configSecret: + description: ConfigSecret specifies whether to restore the + ConfigSecret manifest or not + type: boolean + configSecretName: + description: ConfigSecretName specifies new name of the ConfigSecret + yaml after restore + type: string + db: + description: DB specifies whether to restore the DB manifest + or not + type: boolean + dbName: + description: DBName specifies the new name of the DB yaml + after restore + type: string + initScript: + description: InitScript specifies whether to restore the InitScript + manifest or not + type: boolean + tlsIssuerRef: + description: TLSIssuerRef specifies the name of the IssuerRef + used for TLS configurations for both client and server + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + type: object type: object + restoreTimeout: + description: RestoreTimeout specifies a duration that KubeStash should + wait for the restore to be completed. If the restore tasks do not + finish within this time period, KubeStash will consider this restore + as a failure. + type: string target: description: Target indicates the target application where the data will be restored. The target must be in the same namespace as the @@ -24330,12 +24527,6 @@ spec: required: - name type: object - timeout: - description: Timeout specifies a duration that KubeStash should wait - for the session execution to be completed. If the session execution - does not finish within this time period, KubeStash will consider - this session as a failure. - type: string type: object status: description: RestoreSessionStatus defines the observed state of RestoreSession @@ -24420,12 +24611,6 @@ spec: - type type: object type: array - deadline: - description: Deadline specifies a timestamp till this session is valid. - If the session does not complete within this deadline, it will be - considered as failed. - format: date-time - type: string dependencies: description: Dependencies specifies whether the objects required by this RestoreSession exist or not @@ -24530,6 +24715,11 @@ spec: - Invalid - Unknown type: string + restoreDeadline: + description: RestoreDeadline specifies the deadline of restore. Restore + will be considered Failed if it does not complete within this deadline + format: date-time + type: string targetFound: description: TargetFound specifies whether the restore target exist or not diff --git a/vendor/kubestash.dev/apimachinery/pkg/restic/commands.go b/vendor/kubestash.dev/apimachinery/pkg/restic/commands.go index a9c1a0a2..9d37aeb8 100644 --- a/vendor/kubestash.dev/apimachinery/pkg/restic/commands.go +++ b/vendor/kubestash.dev/apimachinery/pkg/restic/commands.go @@ -35,7 +35,8 @@ import ( ) const ( - ResticCMD = "restic" + ResticCMD = "restic" + TimeoutCMD = "timeout" ) type Snapshot struct { @@ -176,7 +177,10 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) { args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) - return w.run(Command{Name: ResticCMD, Args: args}) + command := Command{Name: ResticCMD, Args: args} + command = w.wrapWithTimeoutIfConfigured(command) + + return w.run(command) } func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) { @@ -200,7 +204,10 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) { args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) - commands = append(commands, Command{Name: ResticCMD, Args: args}) + command := Command{Name: ResticCMD, Args: args} + command = w.wrapWithTimeoutIfConfigured(command) + + commands = append(commands, command) return w.run(commands...) } @@ -246,7 +253,10 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) { args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) - return w.run(Command{Name: ResticCMD, Args: args}) + command := Command{Name: ResticCMD, Args: args} + command = w.wrapWithTimeoutIfConfigured(command) + + return w.run(command) } func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) { @@ -277,10 +287,11 @@ func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) { args = w.appendMaxConnectionsFlag(args) args = w.appendInsecureTLSFlag(args) + command := Command{Name: ResticCMD, Args: args} + command = w.wrapWithTimeoutIfConfigured(command) + // first add restic command, then add StdoutPipeCommands - commands := []Command{ - {Name: ResticCMD, Args: args}, - } + commands := []Command{command} commands = append(commands, dumpOptions.StdoutPipeCommands...) return w.run(commands...) } @@ -368,7 +379,7 @@ func (w *ResticWrapper) run(commands ...Command) ([]byte, error) { w.sh.Stderr = io.MultiWriter(os.Stderr, errBuff) for _, cmd := range commands { - if cmd.Name == ResticCMD { + if cmd.Name == ResticCMD || cmd.Name == TimeoutCMD { // first apply NiceSettings, then apply IONiceSettings cmd, err = w.applyNiceSettings(cmd) if err != nil { @@ -393,6 +404,9 @@ func (w *ResticWrapper) run(commands ...Command) ([]byte, error) { func formatError(err error, stdErr string) error { parts := strings.Split(strings.TrimSuffix(stdErr, "\n"), "\n") if len(parts) > 1 { + if strings.Contains(parts[1], "signal terminated") { + return errors.New(strings.Join(append([]string{"deadline exceeded or signal terminated"}, parts[2:]...), " ")) + } return errors.New(strings.Join(parts[1:], " ")) } return err @@ -515,3 +529,12 @@ func (w *ResticWrapper) removeKey(params keyParams) ([]byte, error) { return w.run(Command{Name: ResticCMD, Args: args}) } + +func (w *ResticWrapper) wrapWithTimeoutIfConfigured(cmd Command) Command { + if w.config.Timeout != nil { + timeoutArgs := []interface{}{fmt.Sprintf("%f", w.config.Timeout.Seconds()), cmd.Name} + timeoutArgs = append(timeoutArgs, cmd.Args...) + return Command{Name: TimeoutCMD, Args: timeoutArgs} + } + return cmd +} diff --git a/vendor/kubestash.dev/apimachinery/pkg/restic/config.go b/vendor/kubestash.dev/apimachinery/pkg/restic/config.go index 6bf9617b..2a910c3b 100644 --- a/vendor/kubestash.dev/apimachinery/pkg/restic/config.go +++ b/vendor/kubestash.dev/apimachinery/pkg/restic/config.go @@ -18,12 +18,12 @@ package restic import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "kubestash.dev/apimachinery/apis/storage/v1alpha1" "os" "path/filepath" "sort" - "kubestash.dev/apimachinery/apis/storage/v1alpha1" - shell "gomodules.xyz/go-sh" core "k8s.io/api/core/v1" kmapi "kmodules.xyz/client-go/api/v1" @@ -103,6 +103,7 @@ type SetupOptions struct { EnableCache bool Nice *ofst.NiceSettings IONice *ofst.IONiceSettings + Timeout *metav1.Duration } type KeyOptions struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index bd887354..86ccb2e4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1377,7 +1377,7 @@ kmodules.xyz/offshoot-api/api/v1 # kmodules.xyz/prober v0.29.0 ## explicit; go 1.21.5 kmodules.xyz/prober/api/v1 -# kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a +# kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1 ## explicit; go 1.22.0 kubestash.dev/apimachinery/apis kubestash.dev/apimachinery/apis/addons/v1alpha1