diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go index 690c01655..a99c65e9d 100644 --- a/pkg/controller/chi/worker.go +++ b/pkg/controller/chi/worker.go @@ -40,6 +40,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/model/chi/schemer" "github.com/altinity/clickhouse-operator/pkg/model/chi/tags/labeler" "github.com/altinity/clickhouse-operator/pkg/model/common/action_plan" + commonConfig "github.com/altinity/clickhouse-operator/pkg/model/common/config" commonCreator "github.com/altinity/clickhouse-operator/pkg/model/common/creator" commonMacro "github.com/altinity/clickhouse-operator/pkg/model/common/macro" commonNormalizer "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer" @@ -113,7 +114,6 @@ func configGeneratorOptions(cr *api.ClickHouseInstallation) *config.GeneratorOpt } } -// newContext creates new reconcile task func (w *worker) newTask(cr *api.ClickHouseInstallation) { w.task = common.NewTask( commonCreator.NewCreator( @@ -663,12 +663,11 @@ func (w *worker) walkHosts(ctx context.Context, chi *api.ClickHouseInstallation, } // getRemoteServersGeneratorOptions build base set of RemoteServersOptions -// which are applied on each of `remote_servers` reconfiguration during reconcile cycle -func (w *worker) getRemoteServersGeneratorOptions() *config.RemoteServersOptions { - // Base chiModel.RemoteServersOptions specifies to exclude: +func (w *worker) getRemoteServersGeneratorOptions() *commonConfig.HostSelector { + // Base model specifies to exclude: // 1. all newly added hosts // 2. all explicitly excluded hosts - return config.NewRemoteServersOptions().ExcludeReconcileAttributes( + return commonConfig.NewHostSelector().ExcludeReconcileAttributes( api.NewHostReconcileAttributes(). SetAdd(). SetExclude(), @@ -682,7 +681,7 @@ func (w *worker) options() *config.FilesGeneratorOptions { return config.NewFilesGeneratorOptions().SetRemoteServersOptions(opts) } -// createCHIFromObjectMeta +// createCRFromObjectMeta func (w *worker) createCRFromObjectMeta(meta meta.Object, isCHI bool, options *commonNormalizer.Options) (*api.ClickHouseInstallation, error) { w.a.V(3).M(meta).S().P() defer w.a.V(3).M(meta).E().P() diff --git a/pkg/controller/chk/worker.go b/pkg/controller/chk/worker.go index 7823e83a1..43d15805f 100644 --- a/pkg/controller/chk/worker.go +++ b/pkg/controller/chk/worker.go @@ -37,6 +37,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/model/chk/normalizer" "github.com/altinity/clickhouse-operator/pkg/model/chk/tags/labeler" "github.com/altinity/clickhouse-operator/pkg/model/common/action_plan" + commonConfig "github.com/altinity/clickhouse-operator/pkg/model/common/config" commonCreator "github.com/altinity/clickhouse-operator/pkg/model/common/creator" commonMacro "github.com/altinity/clickhouse-operator/pkg/model/common/macro" commonNormalizer "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer" @@ -395,12 +396,26 @@ func (w *worker) walkHosts(ctx context.Context, chk *apiChk.ClickHouseKeeperInst }) } +// getRaftGeneratorOptions build base set of RaftOptions +func (w *worker) getRaftGeneratorOptions() *commonConfig.HostSelector { + // Raft specifies to exclude: + // 1. all newly added hosts + // 2. all explicitly excluded hosts + return commonConfig.NewHostSelector().ExcludeReconcileAttributes( + api.NewHostReconcileAttributes(). + SetAdd(). + SetExclude(), + ) +} + // options build FilesGeneratorOptionsClickHouse func (w *worker) options() *config.FilesGeneratorOptions { - return config.NewFilesGeneratorOptions() + opts := w.getRaftGeneratorOptions() + w.a.Info("RaftOptions: %s", opts) + return config.NewFilesGeneratorOptions().SetRaftOptions(opts) } -// createCHIFromObjectMeta +// createCRFromObjectMeta func (w *worker) createCRFromObjectMeta(meta meta.Object, isCHI bool, options *commonNormalizer.Options) (*apiChk.ClickHouseKeeperInstallation, error) { w.a.V(3).M(meta).S().P() defer w.a.V(3).M(meta).E().P() diff --git a/pkg/model/chi/config/files_generator.go b/pkg/model/chi/config/files_generator.go index 389c1fda6..36fc796fb 100644 --- a/pkg/model/chi/config/files_generator.go +++ b/pkg/model/chi/config/files_generator.go @@ -71,7 +71,6 @@ func (c *FilesGenerator) createConfigFilesGroupCommon(options *FilesGeneratorOpt } func (c *FilesGenerator) createConfigFilesGroupCommonDomain(configSections map[string]string, options *FilesGeneratorOptions) { - // remote servers util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRemoteServers), c.configGenerator.getRemoteServers(options.GetRemoteServersOptions())) } diff --git a/pkg/model/chi/config/files_generator_options.go b/pkg/model/chi/config/files_generator_options.go index 9a70cec03..a57235a06 100644 --- a/pkg/model/chi/config/files_generator_options.go +++ b/pkg/model/chi/config/files_generator_options.go @@ -14,12 +14,15 @@ package config -import api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" +import ( + api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/model/common/config" +) // FilesGeneratorOptions specifies options for configuration files generator type FilesGeneratorOptions struct { - RemoteServersOptions *RemoteServersOptions host *api.Host + RemoteServersOptions *config.HostSelector } // defaultFilesGeneratorOptions creates new default options for files generator @@ -49,7 +52,7 @@ func (o *FilesGeneratorOptions) SetHost(host *api.Host) *FilesGeneratorOptions { } // GetRemoteServersOptions gets remote-servers generator options -func (o *FilesGeneratorOptions) GetRemoteServersOptions() *RemoteServersOptions { +func (o *FilesGeneratorOptions) GetRemoteServersOptions() *config.HostSelector { if o == nil { return nil } @@ -57,7 +60,7 @@ func (o *FilesGeneratorOptions) GetRemoteServersOptions() *RemoteServersOptions } // SetRemoteServersOptions sets remote-servers generator options -func (o *FilesGeneratorOptions) SetRemoteServersOptions(opts *RemoteServersOptions) *FilesGeneratorOptions { +func (o *FilesGeneratorOptions) SetRemoteServersOptions(opts *config.HostSelector) *FilesGeneratorOptions { if o == nil { return nil } diff --git a/pkg/model/chi/config/generator.go b/pkg/model/chi/config/generator.go index 8c650f68f..9b3f2f695 100644 --- a/pkg/model/chi/config/generator.go +++ b/pkg/model/chi/config/generator.go @@ -17,6 +17,7 @@ package config import ( "bytes" "fmt" + "github.com/altinity/clickhouse-operator/pkg/model/common/config" log "github.com/altinity/clickhouse-operator/pkg/announcer" chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" @@ -180,10 +181,10 @@ func (c *Generator) getHostZookeeper(host *chi.Host) string { } // chiHostsNum count hosts according to the options -func (c *Generator) chiHostsNum(options *RemoteServersOptions) int { +func (c *Generator) chiHostsNum(selector *config.HostSelector) int { num := 0 c.cr.WalkHosts(func(host *chi.Host) error { - if options.Include(host) { + if selector.Include(host) { num++ } return nil @@ -192,21 +193,21 @@ func (c *Generator) chiHostsNum(options *RemoteServersOptions) int { } // clusterHostsNum count hosts according to the options -func (c *Generator) clusterHostsNum(cluster chi.ICluster, options *RemoteServersOptions) int { +func (c *Generator) clusterHostsNum(cluster chi.ICluster, selector *config.HostSelector) int { num := 0 // Build each shard XML cluster.WalkShards(func(index int, shard chi.IShard) error { - num += c.shardHostsNum(shard, options) + num += c.shardHostsNum(shard, selector) return nil }) return num } // shardHostsNum count hosts according to the options -func (c *Generator) shardHostsNum(shard chi.IShard, options *RemoteServersOptions) int { +func (c *Generator) shardHostsNum(shard chi.IShard, selector *config.HostSelector) int { num := 0 shard.WalkHosts(func(host *chi.Host) error { - if options.Include(host) { + if selector.Include(host) { num++ } return nil @@ -234,9 +235,9 @@ func (c *Generator) getRemoteServersReplica(host *chi.Host, b *bytes.Buffer) { } // getRemoteServers creates "remote_servers.xml" content and calculates data generation parameters for other sections -func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { - if options == nil { - options = defaultRemoteServersOptions() +func (c *Generator) getRemoteServers(selector *config.HostSelector) string { + if selector == nil { + selector = defaultRemoteServersOptions() } b := &bytes.Buffer{} @@ -250,7 +251,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { // Build each cluster XML c.cr.WalkClusters(func(cluster chi.ICluster) error { - if c.clusterHostsNum(cluster, options) < 1 { + if c.clusterHostsNum(cluster, selector) < 1 { // Skip empty cluster return nil } @@ -269,7 +270,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { // Build each shard XML cluster.WalkShards(func(index int, shard chi.IShard) error { - if c.shardHostsNum(shard, options) < 1 { + if c.shardHostsNum(shard, selector) < 1 { // Skip empty shard return nil } @@ -285,7 +286,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { } shard.WalkHosts(func(host *chi.Host) error { - if options.Include(host) { + if selector.Include(host) { c.getRemoteServersReplica(host, b) log.V(2).M(host).Info("Adding host to remote servers: %s", host.GetName()) } else { @@ -307,7 +308,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { // Auto-generated clusters - if c.chiHostsNum(options) < 1 { + if c.chiHostsNum(selector) < 1 { util.Iline(b, 8, "") } else { util.Iline(b, 8, "") @@ -321,7 +322,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { util.Iline(b, 8, " ") util.Iline(b, 8, " true") c.cr.WalkHosts(func(host *chi.Host) error { - if options.Include(host) { + if selector.Include(host) { c.getRemoteServersReplica(host, b) } return nil @@ -338,7 +339,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { clusterName = AllShardsOneReplicaClusterName util.Iline(b, 8, "<%s>", clusterName) c.cr.WalkHosts(func(host *chi.Host) error { - if options.Include(host) { + if selector.Include(host) { // // util.Iline(b, 12, "") @@ -361,7 +362,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { util.Iline(b, 8, "<%s>", clusterName) c.cr.WalkClusters(func(cluster chi.ICluster) error { cluster.WalkShards(func(index int, shard chi.IShard) error { - if c.shardHostsNum(shard, options) < 1 { + if c.shardHostsNum(shard, selector) < 1 { // Skip empty shard return nil } @@ -369,7 +370,7 @@ func (c *Generator) getRemoteServers(options *RemoteServersOptions) string { util.Iline(b, 12, " %s", shard.GetInternalReplication()) shard.WalkHosts(func(host *chi.Host) error { - if options.Include(host) { + if selector.Include(host) { c.getRemoteServersReplica(host, b) } return nil diff --git a/pkg/model/chi/config/generator_options.go b/pkg/model/chi/config/generator_options.go index 0bd03c626..419a021b1 100644 --- a/pkg/model/chi/config/generator_options.go +++ b/pkg/model/chi/config/generator_options.go @@ -15,10 +15,8 @@ package config import ( - "fmt" - "strings" - api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/model/common/config" ) type GeneratorOptions struct { @@ -31,105 +29,6 @@ type GeneratorOptions struct { Files *api.Settings } -// RemoteServersOptions specifies options for remote-servers generator -type RemoteServersOptions struct { - exclude struct { - attributes *api.HostReconcileAttributes - hosts []*api.Host - } -} - -// NewRemoteServersOptions creates new remote-servers generator options -func NewRemoteServersOptions() *RemoteServersOptions { - return &RemoteServersOptions{} -} - -// ExcludeHost specifies to exclude a host -func (o *RemoteServersOptions) ExcludeHost(host *api.Host) *RemoteServersOptions { - if (o == nil) || (host == nil) { - return o - } - - o.exclude.hosts = append(o.exclude.hosts, host) - return o -} - -// ExcludeHosts specifies to exclude list of hosts -func (o *RemoteServersOptions) ExcludeHosts(hosts ...*api.Host) *RemoteServersOptions { - if (o == nil) || (len(hosts) == 0) { - return o - } - - o.exclude.hosts = append(o.exclude.hosts, hosts...) - return o -} - -// ExcludeReconcileAttributes specifies to exclude reconcile attributes -func (o *RemoteServersOptions) ExcludeReconcileAttributes(attrs *api.HostReconcileAttributes) *RemoteServersOptions { - if (o == nil) || (attrs == nil) { - return o - } - - o.exclude.attributes = attrs - return o -} - -// Exclude tells whether to exclude the host -func (o *RemoteServersOptions) Exclude(host *api.Host) bool { - if o == nil { - return false - } - - if o.exclude.attributes.Any(host.GetReconcileAttributes()) { - // Reconcile attributes specify to exclude this host - return true - } - - for _, val := range o.exclude.hosts { - // Host is in the list to be excluded - if val == host { - return true - } - } - - return false -} - -// Include tells whether to include the host -func (o *RemoteServersOptions) Include(host *api.Host) bool { - if o == nil { - return false - } - - if o.exclude.attributes.Any(host.GetReconcileAttributes()) { - // Reconcile attributes specify to exclude this host - return false - } - - for _, val := range o.exclude.hosts { - // Host is in the list to be excluded - if val == host { - return false - } - } - - return true -} - -// String returns string representation -func (o *RemoteServersOptions) String() string { - if o == nil { - return "(nil)" - } - - var hostnames []string - for _, host := range o.exclude.hosts { - hostnames = append(hostnames, host.Name) - } - return fmt.Sprintf("exclude hosts: %s, attributes: %s", "["+strings.Join(hostnames, ",")+"]", o.exclude.attributes) -} - -// defaultRemoteServersOptions -func defaultRemoteServersOptions() *RemoteServersOptions { - return NewRemoteServersOptions() +func defaultRemoteServersOptions() *config.HostSelector { + return config.NewHostSelector() } diff --git a/pkg/model/chk/config/files_generator.go b/pkg/model/chk/config/files_generator.go index 55794dc32..98a280d49 100644 --- a/pkg/model/chk/config/files_generator.go +++ b/pkg/model/chk/config/files_generator.go @@ -71,7 +71,7 @@ func (c *FilesGenerator) createConfigFilesGroupCommon(options *FilesGeneratorOpt } func (c *FilesGenerator) createConfigFilesGroupCommonDomain(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRaft), c.configGenerator.getRaftConfig()) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRaft), c.configGenerator.getRaftConfig(options.GetRaftOptions())) } func (c *FilesGenerator) createConfigFilesGroupCommonGeneric(configSections map[string]string, options *FilesGeneratorOptions) { diff --git a/pkg/model/chk/config/files_generator_options.go b/pkg/model/chk/config/files_generator_options.go index d92657c34..ef879aef0 100644 --- a/pkg/model/chk/config/files_generator_options.go +++ b/pkg/model/chk/config/files_generator_options.go @@ -14,11 +14,15 @@ package config -import api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" +import ( + api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/model/common/config" +) // FilesGeneratorOptions specifies options for configuration files generator type FilesGeneratorOptions struct { - host *api.Host + host *api.Host + RaftOptions *config.HostSelector } // defaultFilesGeneratorOptions creates new default options for files generator @@ -46,3 +50,21 @@ func (o *FilesGeneratorOptions) SetHost(host *api.Host) *FilesGeneratorOptions { return o } + +// GetRaftOptions gets RAFT generator options +func (o *FilesGeneratorOptions) GetRaftOptions() *config.HostSelector { + if o == nil { + return nil + } + return o.RaftOptions +} + +// SetRaftOptions sets RAFT generator options +func (o *FilesGeneratorOptions) SetRaftOptions(opts *config.HostSelector) *FilesGeneratorOptions { + if o == nil { + return nil + } + o.RaftOptions = opts + + return o +} diff --git a/pkg/model/chk/config/generator.go b/pkg/model/chk/config/generator.go index 0b4ac1541..a72e76818 100644 --- a/pkg/model/chk/config/generator.go +++ b/pkg/model/chk/config/generator.go @@ -16,10 +16,14 @@ package config import ( "bytes" + "fmt" + "github.com/altinity/clickhouse-operator/pkg/model/common/config" + "strings" + + log "github.com/altinity/clickhouse-operator/pkg/announcer" chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/interfaces" "github.com/altinity/clickhouse-operator/pkg/util" - "strings" ) // Generator generates configuration files content for specified CR @@ -69,23 +73,29 @@ func (c *Generator) getSectionFromFiles(section chi.SettingsSection, includeUnsp } // getRaftConfig builds raft config for the chk -func (c *Generator) getRaftConfig() string { - // Prepare empty placeholder for RAFT config and will be replaced later in this function: +func (c *Generator) getRaftConfig(selector *config.HostSelector) string { + // Prepare empty placeholder for RAFT config which will be replaced later with real raft config // // // config := chi.NewSettings().Set("keeper_server/raft_configuration/server", chi.NewSettingScalar("")).ClickHouseConfig() + // Prepare RAFT config // 3-rd level (keeper_server/raft_configuration/server) by 4 spaces raftIndent := 12 // Prepare RAFT config for injection into main config raft := &bytes.Buffer{} c.cr.WalkHosts(func(host *chi.Host) error { - util.Iline(raft, raftIndent, "") - util.Iline(raft, raftIndent, " %d", getServerId(host)) - util.Iline(raft, raftIndent, " %s", c.namer.Name(interfaces.NameInstanceHostname, host)) - util.Iline(raft, raftIndent, " %d", host.RaftPort.Value()) - util.Iline(raft, raftIndent, "") + msg := fmt.Sprintf("SKIP host from RAFT servers: %s", host.GetName()) + if selector.Include(host) { + util.Iline(raft, raftIndent, "") + util.Iline(raft, raftIndent, " %d", getServerId(host)) + util.Iline(raft, raftIndent, " %s", c.namer.Name(interfaces.NameInstanceHostname, host)) + util.Iline(raft, raftIndent, " %d", host.RaftPort.Value()) + util.Iline(raft, raftIndent, "") + msg = fmt.Sprintf("Add host to RAFT servers: %s", host.GetName()) + } + log.V(1).M(host).Info(msg) return nil }) diff --git a/pkg/model/common/config/generator_options.go b/pkg/model/common/config/generator_options.go new file mode 100644 index 000000000..243dbf59b --- /dev/null +++ b/pkg/model/common/config/generator_options.go @@ -0,0 +1,120 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "strings" + + api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" +) + +// HostSelector specifies options for excluding host +type HostSelector struct { + exclude struct { + attributes *api.HostReconcileAttributes + hosts []*api.Host + } +} + +// NewHostSelector creates new host selector +func NewHostSelector() *HostSelector { + return &HostSelector{} +} + +// ExcludeHost specifies to exclude a host +func (o *HostSelector) ExcludeHost(host *api.Host) *HostSelector { + if (o == nil) || (host == nil) { + return o + } + + o.exclude.hosts = append(o.exclude.hosts, host) + return o +} + +// ExcludeHosts specifies to exclude list of hosts +func (o *HostSelector) ExcludeHosts(hosts ...*api.Host) *HostSelector { + if (o == nil) || (len(hosts) == 0) { + return o + } + + o.exclude.hosts = append(o.exclude.hosts, hosts...) + return o +} + +// ExcludeReconcileAttributes specifies to exclude reconcile attributes +func (o *HostSelector) ExcludeReconcileAttributes(attrs *api.HostReconcileAttributes) *HostSelector { + if (o == nil) || (attrs == nil) { + return o + } + + o.exclude.attributes = attrs + return o +} + +// Exclude tells whether to exclude the host +func (o *HostSelector) Exclude(host *api.Host) bool { + if o == nil { + return false + } + + if o.exclude.attributes.Any(host.GetReconcileAttributes()) { + // Reconcile attributes specify to exclude this host + return true + } + + for _, val := range o.exclude.hosts { + // Host is in the list to be excluded + if val == host { + return true + } + } + + return false +} + +// Include tells whether to include the host +func (o *HostSelector) Include(host *api.Host) bool { + if o == nil { + return false + } + + if o.exclude.attributes.Any(host.GetReconcileAttributes()) { + // Reconcile attributes specify to exclude this host + return false + } + + for _, val := range o.exclude.hosts { + // Host is in the list to be excluded + if val == host { + return false + } + } + + return true +} + +// String returns string representation +func (o *HostSelector) String() string { + if o == nil { + return "(nil)" + } + + var hostnames []string + for _, host := range o.exclude.hosts { + hostnames = append(hostnames, host.Name) + } + return fmt.Sprintf("exclude hosts: %s, attributes: %s", "["+strings.Join(hostnames, ",")+"]", o.exclude.attributes) +}