diff --git a/ChangeLog.md b/ChangeLog.md index c60d2609..4e662046 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,6 @@ # v2.3.1 IMPROVEMENTS +- add support `use_environment_credentials` option inside `clickhouse-server` backup object disk definition, fix [691](https://github.com/Altinity/clickhouse-backup/issues/691) - add but skip tests for `azure_blob_storage` backup disk for `use_embbeded_backup_restore: true`, it works, but slow, look https://github.com/ClickHouse/ClickHouse/issues/52088 for details BUG FIXES diff --git a/pkg/storage/object_disk/object_disk.go b/pkg/storage/object_disk/object_disk.go index c43dc9d8..7b3d899c 100644 --- a/pkg/storage/object_disk/object_disk.go +++ b/pkg/storage/object_disk/object_disk.go @@ -300,7 +300,7 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) ( Type: "s3", } if batchDeleteNode := d.SelectElement("support_batch_delete"); batchDeleteNode != nil { - if strings.Trim(batchDeleteNode.InnerText(), "\r\n \t") == "true" { + if strings.Trim(batchDeleteNode.InnerText(), "\r\n \t") == "false" { creds.Type = "gcs" } } @@ -314,13 +314,18 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) ( } accessKeyNode := d.SelectElement("access_key_id") secretKeyNode := d.SelectElement("secret_access_key") - if accessKeyNode == nil || secretKeyNode == nil { + useEnvironmentCredentials := d.SelectElement("use_environment_credentials") + if accessKeyNode != nil && secretKeyNode != nil { + creds.S3AccessKey = strings.Trim(accessKeyNode.InnerText(), "\r\n \t") + creds.S3SecretKey = strings.Trim(secretKeyNode.InnerText(), "\r\n \t") + } else { apexLog.Warnf("%s -> /%s/storage_configuration/disks/%s doesn't contains and environment variables will use", configFile, root.Data, diskName) creds.S3AssumeRole = os.Getenv("AWS_ROLE_ARN") - break + if useEnvironmentCredentials != nil { + creds.S3AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") + creds.S3SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + } } - creds.S3AccessKey = strings.Trim(accessKeyNode.InnerText(), "\r\n \t") - creds.S3SecretKey = strings.Trim(secretKeyNode.InnerText(), "\r\n \t") credentials[diskName] = creds break case "azure_blob_storage": diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index 0d884f0f..6d4d0545 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -1,4 +1,4 @@ - services: +services: sshd: image: docker.io/panubo/sshd:latest container_name: sshd @@ -120,6 +120,9 @@ QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} QA_AWS_BUCKET: ${QA_AWS_BUCKET} QA_AWS_REGION: ${QA_AWS_REGION} +# https://github.com/Altinity/clickhouse-backup/issues/691: + AWS_ACCESS_KEY_ID: access-key + AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key volumes: - ./backup-user.xml:/etc/clickhouse-server/users.d/backup-user.xml - ${CLICKHOUSE_BACKUP_BIN:-../../clickhouse-backup/clickhouse-backup-race}:/usr/bin/clickhouse-backup diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index 16994948..b81e1547 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -167,6 +167,9 @@ services: QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} QA_AWS_BUCKET: ${QA_AWS_BUCKET} QA_AWS_REGION: ${QA_AWS_REGION} +# https://github.com/Altinity/clickhouse-backup/issues/691: + AWS_ACCESS_KEY_ID: access-key + AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key volumes: - ./backup-user.xml:/etc/clickhouse-server/users.d/backup-user.xml - ./enable-access_management.xml:/etc/clickhouse-server/users.d/enable-access_management.xml diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh index 072b999b..62561582 100644 --- a/test/integration/dynamic_settings.sh +++ b/test/integration/dynamic_settings.sh @@ -90,8 +90,11 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_s3.xml s3 http://minio:9000/clickhouse/disk_s3/ + + 1 true @@ -119,8 +122,11 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.x s3 http://minio:9000/clickhouse/disk_s3/ + + 1 true @@ -158,25 +164,28 @@ chown -R clickhouse /var/lib/clickhouse/disks/ /var/lib/clickhouse/backups_embed cat < /etc/clickhouse-server/config.d/backup_storage_configuration_s3.xml - - - - true - s3 - http://minio:9000/clickhouse/backups_s3/ - access-key - it-is-my-super-secret-key - false - - - - - backups_s3 - /var/lib/clickhouse/backups_embedded/ - - - 1 - + + + + true + s3 + http://minio:9000/clickhouse/backups_s3/ + + 1 + false + + + + + backups_s3 + /var/lib/clickhouse/backups_embedded/ + + + 1 + EOT @@ -191,21 +200,24 @@ chown -R clickhouse /var/lib/clickhouse/disks/ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_s3_plain.xml - - - - s3_plain - http://minio:9000/clickhouse/backups_s3_plain/ - access-key - it-is-my-super-secret-key - false - - - - - backups_s3 - backups_s3_plain - + + + + s3_plain + http://minio:9000/clickhouse/backups_s3_plain/ + + 1 + false + + + + + backups_s3 + backups_s3_plain + EOT