--partitions=(v1,'v2') could calculate wrong partition expression i…

…f `system.columns` will return fields in different order than they described in PARTITION BY clause, fix #791 for old clickhouse-server versions

pkg/partition/partition.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Altinity/clickhouse-backup/pkg/metadata"
 	apexLog "github.com/apex/log"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 	"regexp"
 	"sort"
 	"strconv"
@@ -49,7 +50,7 @@ var SettingsRE = regexp.MustCompile(`(?mi)\s*SETTINGS.*`)
 var OrderByRE = regexp.MustCompile(`(?mi)\s*ORDER BY.*`)
 var FunctionsRE = regexp.MustCompile(`(?i)\w+\(`)
 var StringsRE = regexp.MustCompile(`(?i)'[^']+'`)
-var SpecialCharsRE = regexp.MustCompile(`(?i)[)*+\-/\\]+`)
+var SpecialCharsRE = regexp.MustCompile(`(?i)[)(*+\-/\\,]+`)
 var FieldsNamesRE = regexp.MustCompile("(?i)\\w+|`[^`]+`\\.`[^`]+`|\"[^\"]+\"")
 
 func extractPartitionByFieldNames(s string) []struct {
@@ -60,7 +61,7 @@ func extractPartitionByFieldNames(s string) []struct {
 	s = FunctionsRE.ReplaceAllString(s, "")
 	s = StringsRE.ReplaceAllString(s, "")
 	s = SpecialCharsRE.ReplaceAllString(s, "")
-	matches := FieldsNamesRE.FindStringSubmatch(s)
+	matches := FieldsNamesRE.FindAllString(s, -1)
 	columns := make([]struct {
 		Name string `ch:"name"`
 	}, len(matches))
@@ -107,7 +108,7 @@ func GetPartitionIdAndName(ctx context.Context, ch *clickhouse.ClickHouse, datab
 		columns = extractPartitionByFieldNames(partitionByMatches[1])
 		oldVersion = true
 	}
-	// to the same order of fields as described in PARTITION BY clause, fix
+	// to the same order of fields as described in PARTITION BY clause, https://github.com/Altinity/clickhouse-backup/issues/791
 	if len(partitionByMatches) == 2 && partitionByMatches[1] != "" {
 		sort.Slice(columns, func(i int, j int) bool {
 			return strings.Index(partitionByMatches[1], columns[i].Name) < strings.Index(partitionByMatches[1], columns[j].Name)
@@ -142,13 +143,13 @@ func GetPartitionIdAndName(ctx context.Context, ch *clickhouse.ClickHouse, datab
 		)
 		batch, err := ch.GetConn().PrepareBatch(ctx, sql)
 		if err != nil {
-			return "", "", err
+			return "", "", errors.Wrapf(err, "PrepareBatch sql=%s partitionInsert=%#v", sql, partitionInsert)
 		}
 		if err = batch.Append(partitionInsert...); err != nil {
-			return "", "", err
+			return "", "", errors.Wrapf(err, "batch.Append sql=%s partitionInsert=%#v", sql, partitionInsert)
 		}
 		if err = batch.Send(); err != nil {
-			return "", "", err
+			return "", "", errors.Wrapf(err, "batch.Send sql=%s partitionInsert=%#v", sql, partitionInsert)
 		}
 	}
 	if err != nil {

test/integration/integration_test.go

@@ -1985,8 +1985,8 @@ func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, ch *Test
 	ch.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName)
 	ch.queryWithNoError(r, "DROP TABLE IF EXISTS "+dbName+".t1")
 	ch.queryWithNoError(r, "DROP TABLE IF EXISTS "+dbName+".t2")
-	ch.queryWithNoError(r, "CREATE TABLE "+dbName+".t1 (dt Date, category Int8, v UInt64) ENGINE=MergeTree() PARTITION BY (category, toYYYYMMDD(dt)) ORDER BY dt")
-	ch.queryWithNoError(r, "CREATE TABLE "+dbName+".t2 (dt String, category Int8, v UInt64) ENGINE=MergeTree() PARTITION BY (category, dt) ORDER BY dt")
+	ch.queryWithNoError(r, "CREATE TABLE "+dbName+".t1 (dt Date, category Int64, v UInt64) ENGINE=MergeTree() PARTITION BY (category, toYYYYMMDD(dt)) ORDER BY dt")
+	ch.queryWithNoError(r, "CREATE TABLE "+dbName+".t2 (dt String, category Int64, v UInt64) ENGINE=MergeTree() PARTITION BY (category, dt) ORDER BY dt")
 	for _, dt := range []string{"2022-01-01", "2022-01-02", "2022-01-03", "2022-01-04"} {
 		ch.queryWithNoError(r, fmt.Sprintf("INSERT INTO "+dbName+".t1(dt, v) SELECT '%s', number FROM numbers(10)", dt))
 		ch.queryWithNoError(r, fmt.Sprintf("INSERT INTO "+dbName+".t2(dt, v) SELECT '%s', number FROM numbers(10)", dt))