Merge branch 'master' of github.com:Altinity/clickhouse-backup

ChangeLog.md

@@ -1,3 +1,16 @@
+# v2.4.17
+BUG FIXES
+- skip CopyObject execution for keys which have zero size, to allow properly backup S3, GCS over S3 and Azure disks
+
+# v2.4.16
+BUG FIXES
+- increase `AZBLOB_TIMEOUT` to 4h, instead 15m to allow download long size data parts 
+- change `S3_MAX_PARTS_COUNT` from `5000` to `1000` and minimal `S3_PART_SIZE` from 5Mb to 25Mb from by default to improve speedup S3 uploading / downloading 
+
+# v2.4.15
+BUG FIXES
+- fix `create` and `restore` command for ReplicatedMergeTree tables with `frozen_metadata.txt` parsing
+
 # v2.4.14
 IMPROVEMENTS
 - refactoring `semaphore.NewWeighted()` to `errgroup.SetLimit()`

ReadMe.md

@@ -68,6 +68,13 @@ Build from the sources:
 GO111MODULE=on go get github.com/Altinity/clickhouse-backup/cmd/clickhouse-backup
 ```
 
+## Brief description how clickhouse-backup works
+
+Data files is immutable in `clickhouse-server`.
+During backup operation `clickhouse-backup` create file system hard-links to exists `clickhouse-server` data parts via executing `ALTER TABLE ... FREZZE` query. 
+During restore operation `clickhouse-backup` copy hard-links to `detached` folder and execute `ALTER TABLE ... ATTACH PART` query for each data part and each table in backup.
+More detailed description available here https://www.youtube.com/watch?v=megsNh9Q-dw
+
 ## Common CLI Usage
 
 ### CLI command - tables
@@ -412,8 +419,16 @@ clickhouse:
   password: ""                     # CLICKHOUSE_PASSWORD
   host: localhost                  # CLICKHOUSE_HOST, To make backup data `clickhouse-backup` requires access to the same file system as clickhouse-server, so `host` should localhost or address of another docker container on the same machine, or IP address bound to some network interface on the same host.
   port: 9000                       # CLICKHOUSE_PORT, don't use 8123, clickhouse-backup doesn't support HTTP protocol
-  # CLICKHOUSE_DISK_MAPPING, use this mapping when your `system.disks` are different between the source and destination clusters during backup and restore process
-  # The format for this env variable is "disk_name1:disk_path1,disk_name2:disk_path2". For YAML please continue using map syntax
+  # CLICKHOUSE_DISK_MAPPING, use this mapping when your `system.disks` are different between the source and destination clusters during backup and restore process.
+  # The format for this env variable is "disk_name1:disk_path1,disk_name2:disk_path2". For YAML please continue using map syntax.
+  # If destination disk is different from source backup disk then you need to specify the destination disk in the config file:
+
+  # disk_mapping:
+  #  disk_destination: /var/lib/clickhouse/disks/destination
+
+  # `disk_destination`  needs to be referenced in backup (source config), and all names from this map (`disk:path`) shall exist in `system.disks` on destination server.
+  # During download of the backup from remote location (s3), if `name` is not present in `disk_mapping` (on the destination server config too) then `default` disk path will used for download.
+  # `disk_mapping` is used to understand during download where downloaded parts shall be unpacked (which disk) on destination server and where to search for data parts directories during restore.
   disk_mapping: {}
   # CLICKHOUSE_SKIP_TABLES, the list of tables (pattern are allowed) which are ignored during backup and restore process
   # The format for this env variable is "pattern1,pattern2,pattern3". For YAML please continue using list syntax

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/urfave/cli v1.22.14
 	github.com/xyproto/gionice v1.3.0
 	github.com/yargevad/filepathx v1.0.0
-	golang.org/x/crypto v0.16.0
+	golang.org/x/crypto v0.17.0
 	golang.org/x/mod v0.13.0
 	golang.org/x/sync v0.5.0
 	google.golang.org/api v0.152.0
@@ -88,7 +88,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
@@ -100,7 +99,6 @@ require (
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mozillazg/go-httpheader v0.4.0 // indirect