fix create and restore command for ReplicatedMergeTree tables wit…

…h `frozen_metadata.txt` parsing
Altinity · Dec 27, 2023 · db7e7ea · db7e7ea
1 parent 88bd3a8
commit db7e7ea
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 7 deletions.
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,3 +1,7 @@
+# v2.4.15
+BUG FIXES
+- fix `create` and `restore` command for ReplicatedMergeTree tables with `frozen_metadata.txt` parsing
+
 # v2.4.14
 IMPROVEMENTS
 - refactoring `semaphore.NewWeighted()` to `errgroup.SetLimit()`

diff --git a/pkg/backup/create.go b/pkg/backup/create.go
@@ -646,6 +646,9 @@ func (b *Backuper) uploadObjectDiskParts(ctx context.Context, backupName, backup
 		if fInfo.IsDir() {
 			return nil
 		}
+		if fInfo.Name() == "frozen_metadata.txt" {
+			return nil
+		}
 		objPartFileMeta, err := object_disk.ReadMetadataFromFile(fPath)
 		if err != nil {
 			return err

diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go
@@ -854,6 +854,9 @@ func (b *Backuper) downloadObjectDiskParts(ctx context.Context, backupName strin
 					if fInfo.IsDir() {
 						return nil
 					}
+					if fInfo.Name() == "frozen_metadata.txt" {
+						return nil
+					}
 					objMeta, err := object_disk.ReadMetadataFromFile(fPath)
 					if err != nil {
 						return err

diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh
@@ -222,12 +222,16 @@ cat <<EOT > /etc/clickhouse-server/config.d/backup_storage_configuration_s3.xml
     <allowed_disk>backups_s3</allowed_disk>
     <allowed_path>/var/lib/clickhouse/backups_embedded/</allowed_path>
   </backups>
+</clickhouse>
+EOT
+
+cat <<EOT > /etc/clickhouse-server/config.d/zero_copy_replication.xml
+<yandex>
   <merge_tree>
     <allow_remote_fs_zero_copy_replication>1</allow_remote_fs_zero_copy_replication>
   </merge_tree>
-</clickhouse>
+</yandex>
 EOT
-
 fi
 
 # s3_plain and azure backup configuration
@@ -372,4 +376,4 @@ cat <<EOT > /etc/clickhouse-server/config.d/replicated_user_directories.xml
 </yandex>
 EOT
 
-fi
+fi
diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go
@@ -2208,31 +2208,34 @@ func generateTestDataWithDifferentStoragePolicy(remoteStorageType string, testDa
 		//s3 disks support after 21.8
 		if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") >= 0 && remoteStorageType == "S3" {
 			testDataWithStoragePolicy.Name = "test_s3"
-			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 's3_only'"
+			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id SETTINGS storage_policy = 's3_only'"
 			addTestDataIfNotExists()
 		}
 		//encrypted disks support after 21.10
 		if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.10") >= 0 {
 			testDataWithStoragePolicy.Name = "test_hdd3_encrypted"
-			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'hdd3_only_encrypted'"
+			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id SETTINGS storage_policy = 'hdd3_only_encrypted'"
 			addTestDataIfNotExists()
 		}
 		//encrypted s3 disks support after 21.12
 		if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 && remoteStorageType == "S3" {
 			testDataWithStoragePolicy.Name = "test_s3_encrypted"
 			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 's3_only_encrypted'"
+			if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") >= 0 {
+				testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id SETTINGS storage_policy = 's3_only_encrypted'"
+			}
 			addTestDataIfNotExists()
 		}
 		//gcs over s3 support added in 22.6
 		if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.6") >= 0 && remoteStorageType == "GCS" && os.Getenv("QA_GCS_OVER_S3_BUCKET") != "" {
 			testDataWithStoragePolicy.Name = "test_gcs"
-			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'gcs_only'"
+			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id SETTINGS storage_policy = 'gcs_only'"
 			addTestDataIfNotExists()
 		}
 		//check azure_blob_storage only in 23.3+ (added in 22.1)
 		if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") >= 0 && remoteStorageType == "AZBLOB" {
 			testDataWithStoragePolicy.Name = "test_azure"
-			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'azure_only'"
+			testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id SETTINGS storage_policy = 'azure_only'"
 			addTestDataIfNotExists()
 		}
 	}