add support use_environment_credentials option inside `clickhouse-s…

…erver` backup object disk definition, fix #691
Altinity · Jul 13, 2023 · d7d7b6b · d7d7b6b
1 parent 45665d9
commit d7d7b6b
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 40 deletions.
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,5 +1,6 @@
 # v2.3.1
 IMPROVEMENTS
+- add support `use_environment_credentials` option inside `clickhouse-server` backup object disk definition, fix [691](https://github.com/Altinity/clickhouse-backup/issues/691)
 - add but skip tests for `azure_blob_storage` backup disk for `use_embbeded_backup_restore: true`, it works, but slow, look https://github.com/ClickHouse/ClickHouse/issues/52088 for details
 
 BUG FIXES

diff --git a/pkg/storage/object_disk/object_disk.go b/pkg/storage/object_disk/object_disk.go
@@ -300,7 +300,7 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) (
 					Type: "s3",
 				}
 				if batchDeleteNode := d.SelectElement("support_batch_delete"); batchDeleteNode != nil {
-					if strings.Trim(batchDeleteNode.InnerText(), "\r\n \t") == "true" {
+					if strings.Trim(batchDeleteNode.InnerText(), "\r\n \t") == "false" {
 						creds.Type = "gcs"
 					}
 				}
@@ -314,13 +314,18 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) (
 				}
 				accessKeyNode := d.SelectElement("access_key_id")
 				secretKeyNode := d.SelectElement("secret_access_key")
-				if accessKeyNode == nil || secretKeyNode == nil {
+				useEnvironmentCredentials := d.SelectElement("use_environment_credentials")
+				if accessKeyNode != nil && secretKeyNode != nil {
+					creds.S3AccessKey = strings.Trim(accessKeyNode.InnerText(), "\r\n \t")
+					creds.S3SecretKey = strings.Trim(secretKeyNode.InnerText(), "\r\n \t")
+				} else {
 					apexLog.Warnf("%s -> /%s/storage_configuration/disks/%s doesn't contains <access_key_id> and <secret_access_key> environment variables will use", configFile, root.Data, diskName)
 					creds.S3AssumeRole = os.Getenv("AWS_ROLE_ARN")
-					break
+					if useEnvironmentCredentials != nil {
+						creds.S3AccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
+						creds.S3SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
+					}
 				}
-				creds.S3AccessKey = strings.Trim(accessKeyNode.InnerText(), "\r\n \t")
-				creds.S3SecretKey = strings.Trim(secretKeyNode.InnerText(), "\r\n \t")
 				credentials[diskName] = creds
 				break
 			case "azure_blob_storage":

diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml
@@ -1,4 +1,4 @@
- services:
+services:
   sshd:
     image: docker.io/panubo/sshd:latest
     container_name: sshd
@@ -120,6 +120,9 @@
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
       QA_AWS_BUCKET: ${QA_AWS_BUCKET}
       QA_AWS_REGION: ${QA_AWS_REGION}
+# https://github.com/Altinity/clickhouse-backup/issues/691:
+      AWS_ACCESS_KEY_ID: access-key
+      AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key
     volumes:
       - ./backup-user.xml:/etc/clickhouse-server/users.d/backup-user.xml
       - ${CLICKHOUSE_BACKUP_BIN:-../../clickhouse-backup/clickhouse-backup-race}:/usr/bin/clickhouse-backup

diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml
@@ -167,6 +167,9 @@ services:
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
       QA_AWS_BUCKET: ${QA_AWS_BUCKET}
       QA_AWS_REGION: ${QA_AWS_REGION}
+# https://github.com/Altinity/clickhouse-backup/issues/691:
+      AWS_ACCESS_KEY_ID: access-key
+      AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key
     volumes:
       - ./backup-user.xml:/etc/clickhouse-server/users.d/backup-user.xml
       - ./enable-access_management.xml:/etc/clickhouse-server/users.d/enable-access_management.xml

diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh
@@ -90,8 +90,11 @@ cat <<EOT > /etc/clickhouse-server/config.d/storage_configuration_s3.xml
       <disk_s3>
         <type>s3</type>
         <endpoint>http://minio:9000/clickhouse/disk_s3/</endpoint>
+        <!-- https://github.com/Altinity/clickhouse-backup/issues/691
         <access_key_id>access-key</access_key_id>
         <secret_access_key>it-is-my-super-secret-key</secret_access_key>
+        -->
+        <use_environment_credentials>1</use_environment_credentials>
         <send_metadata>true</send_metadata>
       </disk_s3>
     </disks>
@@ -119,8 +122,11 @@ cat <<EOT > /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.x
       <disk_s3>
         <type>s3</type>
         <endpoint>http://minio:9000/clickhouse/disk_s3/</endpoint>
+        <!-- https://github.com/Altinity/clickhouse-backup/issues/691
         <access_key_id>access-key</access_key_id>
         <secret_access_key>it-is-my-super-secret-key</secret_access_key>
+        -->
+        <use_environment_credentials>1</use_environment_credentials>
         <send_metadata>true</send_metadata>
       </disk_s3>
       <disk_s3_encrypted>
@@ -158,25 +164,28 @@ chown -R clickhouse /var/lib/clickhouse/disks/ /var/lib/clickhouse/backups_embed
 cat <<EOT > /etc/clickhouse-server/config.d/backup_storage_configuration_s3.xml
 <?xml version="1.0"?>
 <clickhouse>
-    <storage_configuration>
-        <disks>
-            <backups_s3>
-              <send_metadata>true</send_metadata>
-              <type>s3</type>
-              <endpoint>http://minio:9000/clickhouse/backups_s3/</endpoint>
-              <access_key_id>access-key</access_key_id>
-              <secret_access_key>it-is-my-super-secret-key</secret_access_key>
-              <cache_enabled>false</cache_enabled>
-            </backups_s3>
-        </disks>
-    </storage_configuration>
-    <backups>
-        <allowed_disk>backups_s3</allowed_disk>
-        <allowed_path>/var/lib/clickhouse/backups_embedded/</allowed_path>
-    </backups>
-    <merge_tree>
-        <allow_remote_fs_zero_copy_replication>1</allow_remote_fs_zero_copy_replication>
-    </merge_tree>
+  <storage_configuration>
+    <disks>
+      <backups_s3>
+        <send_metadata>true</send_metadata>
+        <type>s3</type>
+        <endpoint>http://minio:9000/clickhouse/backups_s3/</endpoint>
+        <!-- https://github.com/Altinity/clickhouse-backup/issues/691
+        <access_key_id>access-key</access_key_id>
+        <secret_access_key>it-is-my-super-secret-key</secret_access_key>
+        -->
+        <use_environment_credentials>1</use_environment_credentials>
+        <cache_enabled>false</cache_enabled>
+      </backups_s3>
+    </disks>
+  </storage_configuration>
+  <backups>
+    <allowed_disk>backups_s3</allowed_disk>
+    <allowed_path>/var/lib/clickhouse/backups_embedded/</allowed_path>
+  </backups>
+  <merge_tree>
+    <allow_remote_fs_zero_copy_replication>1</allow_remote_fs_zero_copy_replication>
+  </merge_tree>
 </clickhouse>
 EOT
 
@@ -191,21 +200,24 @@ chown -R clickhouse /var/lib/clickhouse/disks/
 cat <<EOT > /etc/clickhouse-server/config.d/backup_storage_configuration_s3_plain.xml
 <?xml version="1.0"?>
 <clickhouse>
-    <storage_configuration>
-        <disks>
-            <backups_s3_plain>
-              <type>s3_plain</type>
-              <endpoint>http://minio:9000/clickhouse/backups_s3_plain/</endpoint>
-              <access_key_id>access-key</access_key_id>
-              <secret_access_key>it-is-my-super-secret-key</secret_access_key>
-              <cache_enabled>false</cache_enabled>
-            </backups_s3_plain>
-        </disks>
-    </storage_configuration>
-    <backups>
-        <allowed_disk>backups_s3</allowed_disk>
-        <allowed_disk>backups_s3_plain</allowed_disk>
-    </backups>
+  <storage_configuration>
+    <disks>
+      <backups_s3_plain>
+        <type>s3_plain</type>
+        <endpoint>http://minio:9000/clickhouse/backups_s3_plain/</endpoint>
+        <!-- https://github.com/Altinity/clickhouse-backup/issues/691
+        <access_key_id>access-key</access_key_id>
+        <secret_access_key>it-is-my-super-secret-key</secret_access_key>
+        -->
+        <use_environment_credentials>1</use_environment_credentials>
+        <cache_enabled>false</cache_enabled>
+      </backups_s3_plain>
+    </disks>
+  </storage_configuration>
+  <backups>
+    <allowed_disk>backups_s3</allowed_disk>
+    <allowed_disk>backups_s3_plain</allowed_disk>
+  </backups>
 </clickhouse>
 EOT