archive-async by default with spool-path (#3962)

* archive-async by default with spool-path

Issue: PGO-1371 PGO-1142

internal/controller/postgrescluster/pgbackrest_test.go

@@ -2072,7 +2072,7 @@ func TestReconcileCloudBasedDataSource(t *testing.T) {
 			result: testResult{
 				configCount: 1, jobCount: 1, pvcCount: 1,
 				expectedClusterCondition: nil,
-				conf:                     "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = /pgbackrest/repo1\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
+				conf:                     "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  archive-async = y\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = /pgbackrest/repo1\n  spool-path = /pgdata/pgbackrest-spool\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
 			},
 		}, {
 			desc: "global/configuration set",
@@ -2089,7 +2089,7 @@ func TestReconcileCloudBasedDataSource(t *testing.T) {
 			result: testResult{
 				configCount: 1, jobCount: 1, pvcCount: 1,
 				expectedClusterCondition: nil,
-				conf:                     "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = elephant\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
+				conf:                     "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  archive-async = y\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = elephant\n  spool-path = /pgdata/pgbackrest-spool\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
 			},
 		}, {
 			desc: "invalid option: stanza",
@@ -2104,7 +2104,7 @@ func TestReconcileCloudBasedDataSource(t *testing.T) {
 			result: testResult{
 				configCount: 1, jobCount: 0, pvcCount: 1,
 				expectedClusterCondition: nil,
-				conf:                     "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = /pgbackrest/repo1\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
+				conf:                     "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  archive-async = y\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = /pgbackrest/repo1\n  spool-path = /pgdata/pgbackrest-spool\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
 			},
 		}, {
 			desc: "cluster bootstrapped init condition missing",
@@ -2123,7 +2123,7 @@ func TestReconcileCloudBasedDataSource(t *testing.T) {
 					Reason:  "ClusterAlreadyBootstrapped",
 					Message: "The cluster is already bootstrapped",
 				},
-				conf: "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  log-path = /pgdata/pgbackrest/log\n  repo1-path = /pgbackrest/repo1\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
+				conf: "|\n  # Generated by postgres-operator. DO NOT EDIT.\n  # Your changes will not be saved.\n\n  [global]\n  archive-async = y\n log-path = /pgdata/pgbackrest/log\n  repo1-path = /pgbackrest/repo1\n  spool-path = /pgdata/pgbackrest-spool\n\n  [db]\n  pg1-path = /pgdata/pg13\n  pg1-port = 5432\n  pg1-socket-path = /tmp/postgres\n",
 			},
 		}}
 

internal/pgbackrest/config.go

@@ -291,6 +291,10 @@ func populatePGInstanceConfigurationMap(
 	global := iniMultiSet{}
 	stanza := iniMultiSet{}
 
+	// For faster and more robust WAL archiving, we turn on pgBackRest archive-async.
+	global.Set("archive-async", "y")
+	// pgBackRest spool-path should always be co-located with the Postgres WAL path.
+	global.Set("spool-path", "/pgdata/pgbackrest-spool")
 	// pgBackRest will log to the pgData volume for commands run on the PostgreSQL instance
 	global.Set("log-path", naming.PGBackRestPGDataLogPath)
 

internal/pgbackrest/config_test.go

@@ -131,6 +131,7 @@ pg1-socket-path = /tmp/postgres
 # Your changes will not be saved.
 
 [global]
+archive-async = y
 log-path = /pgdata/pgbackrest/log
 repo1-host = repo-hostname-0.pod-service-name.test-ns.svc.`+domain+`
 repo1-host-ca-file = /etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt
@@ -151,6 +152,7 @@ repo4-s3-bucket = s-bucket
 repo4-s3-endpoint = endpoint-s
 repo4-s3-region = earth
 repo4-type = s3
+spool-path = /pgdata/pgbackrest-spool
 
 [db]
 pg1-path = /pgdata/pg12

internal/postgres/config.go

@@ -103,12 +103,17 @@ func DataDirectory(cluster *v1beta1.PostgresCluster) string {
 func WALDirectory(
 	cluster *v1beta1.PostgresCluster, instance *v1beta1.PostgresInstanceSetSpec,
 ) string {
-	// When no WAL volume is specified, store WAL files on the main data volume.
-	walStorage := dataMountPath
+	return fmt.Sprintf("%s/pg%d_wal", WALStorage(instance), cluster.Spec.PostgresVersion)
+}
+
+// WALStorage returns the absolute path to the disk where an instance stores its
+// WAL files. Use [WALDirectory] for the exact directory that Postgres uses.
+func WALStorage(instance *v1beta1.PostgresInstanceSetSpec) string {
 	if instance.WALVolumeClaimSpec != nil {
-		walStorage = walMountPath
+		return walMountPath
 	}
-	return fmt.Sprintf("%s/pg%d_wal", walStorage, cluster.Spec.PostgresVersion)
+	// When no WAL volume is specified, store WAL files on the main data volume.
+	return dataMountPath
 }
 
 // Environment returns the environment variables required to invoke PostgreSQL
@@ -307,6 +312,11 @@ chmod +x /tmp/pg_rewind_tde.sh
 		`echo Initializing ...`,
 		`results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)"`,
 
+		// The pgbackrest spool path should be co-located with wal. If a wal volume exists, symlink the spool-path to it.
+		`if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi`,
+		// When a pgwal volume is removed, the symlink will be broken; force pgbackrest to recreate spool-path.
+		`if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi`,
+
 		// Abort when the PostgreSQL version installed in the image does not
 		// match the cluster spec.
 		`results 'postgres path' "$(command -v postgres ||:)"`,

internal/postgres/reconcile_test.go

@@ -255,6 +255,8 @@ initContainers:
     )
     echo Initializing ...
     results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)"
+    if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi
+    if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi
     results 'postgres path' "$(command -v postgres ||:)"
     results 'postgres version' "${postgres_version:=$(postgres --version ||:)}"
     [[ "${postgres_version}" =~ ") ${expected_major_version}"($|[^0-9]) ]] ||

testing/kuttl/e2e/pgbackrest-init/06--check-spool-path.yaml

@@ -0,0 +1,17 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    PRIMARY=$(
+        kubectl get pod --namespace "${NAMESPACE}" \
+          --output name --selector '
+            postgres-operator.crunchydata.com/role=master'
+      )
+
+    LIST=$(
+    kubectl exec --namespace "${NAMESPACE}" -c database "${PRIMARY}" -- \
+      ls -l /pgdata
+    )
+
+    contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
+    contains "$LIST" "pgbackrest-spool" || exit 1

testing/kuttl/e2e/wal-pvc-pgupgrade/06--check-spool-path.yaml

@@ -0,0 +1,19 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    PRIMARY=$(
+        kubectl get pod --namespace "${NAMESPACE}" \
+          --output name --selector '
+            postgres-operator.crunchydata.com/role=master'
+      )
+
+    LIST=$(
+    kubectl exec --namespace "${NAMESPACE}" -c database "${PRIMARY}" -- \
+      ls -l /pgdata
+    )
+
+    contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
+
+    # Confirm that the pgbackrest spool-path has been symlinked to the wal volume.
+    contains "$LIST" "pgbackrest-spool -> /pgwal/pgbackrest-spool" || exit 1