diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c7b422fa..085b963d7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -484,7 +484,7 @@ jobs: e2e: name: E2E - runs-on: ${{ matrix.runner || 'ubuntu-latest' }} + runs-on: ${{ matrix.runner || 'ubuntu-22.04' }} needs: - build-current - build-previous-k0s @@ -503,8 +503,6 @@ jobs: - TestHostPreflightCustomSpec - TestHostPreflightInBuiltSpec - TestUnsupportedOverrides - - TestMultiNodeInstallation - - TestMultiNodeReset - TestCommandsRequireSudo - TestInstallWithoutEmbed - TestInstallFromReplicatedApp @@ -520,19 +518,29 @@ jobs: - TestSingleNodeAirgapUpgradeFromEC18 - TestSingleNodeAirgapUpgradeCustomCIDR - TestInstallSnapshotFromReplicatedApp - - TestMultiNodeAirgapUpgrade - TestSingleNodeDisasterRecovery - TestSingleNodeDisasterRecoveryWithProxy - TestSingleNodeResumeDisasterRecovery - - TestProxiedEnvironment - - TestMultiNodeHAInstallation - - TestMultiNodeHADisasterRecovery - - TestCustomCIDR - - TestProxiedCustomCIDR - TestSingleNodeInstallationNoopUpgrade - TestInstallWithPrivateCAs - TestInstallWithMITMProxy include: + - test: TestCustomCIDR + runner: embedded-cluster + - test: TestProxiedEnvironment + runner: embedded-cluster + - test: TestProxiedCustomCIDR + runner: embedded-cluster + - test: TestMultiNodeInstallation + runner: embedded-cluster + - test: TestMultiNodeReset + runner: embedded-cluster + - test: TestMultiNodeAirgapUpgrade + runner: embedded-cluster + - test: TestMultiNodeHAInstallation + runner: embedded-cluster + - test: TestMultiNodeHADisasterRecovery + runner: embedded-cluster - test: TestMultiNodeAirgapUpgrade runner: embedded-cluster - test: TestMultiNodeAirgapUpgradeSameK0s diff --git a/Makefile b/Makefile index 7c10cbb88..189a15dab 100644 --- a/Makefile +++ b/Makefile @@ -199,22 +199,22 @@ static: pkg/goods/bins/k0s \ pkg/goods/internal/bins/kubectl-kots .PHONY: embedded-cluster-linux-amd64 -embedded-cluster-linux-amd64: OS = linux -embedded-cluster-linux-amd64: ARCH = amd64 +embedded-cluster-linux-amd64: export OS = linux +embedded-cluster-linux-amd64: export ARCH = amd64 embedded-cluster-linux-amd64: static go.mod embedded-cluster mkdir -p ./output/bin cp ./build/embedded-cluster-$(OS)-$(ARCH) ./output/bin/$(APP_NAME) .PHONY: embedded-cluster-linux-arm64 -embedded-cluster-linux-arm64: OS = linux -embedded-cluster-linux-arm64: ARCH = arm64 +embedded-cluster-linux-arm64: export OS = linux +embedded-cluster-linux-arm64: export ARCH = arm64 embedded-cluster-linux-arm64: static go.mod embedded-cluster mkdir -p ./output/bin cp ./build/embedded-cluster-$(OS)-$(ARCH) ./output/bin/$(APP_NAME) .PHONY: embedded-cluster-darwin-arm64 -embedded-cluster-darwin-arm64: OS = darwin -embedded-cluster-darwin-arm64: ARCH = arm64 +embedded-cluster-darwin-arm64: export OS = darwin +embedded-cluster-darwin-arm64: export ARCH = arm64 embedded-cluster-darwin-arm64: go.mod embedded-cluster mkdir -p ./output/bin cp ./build/embedded-cluster-$(OS)-$(ARCH) ./output/bin/$(APP_NAME) @@ -291,7 +291,7 @@ list-distros: .PHONY: create-node% create-node%: DISTRO = debian-bookworm create-node%: NODE_PORT = 30000 -create-node%: K0S_DATA_DIR = /var/lib/k0s +create-node%: K0S_DATA_DIR = /var/lib/embedded-cluster/k0s create-node%: @docker run -d \ --name node$* \ diff --git a/README.md b/README.md index 4634fba11..60801a8c4 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ $ output/bin/embedded-cluster shell ((____!___/) Type 'exit' (or CTRL+d) to exit. \0\0\0\0\/ Happy hacking. ~~~~~~~~~~~ -$ export KUBECONFIG="/var/lib/k0s/pki/admin.conf" +$ export KUBECONFIG="/var/lib/embedded-cluster/k0s/pki/admin.conf" $ export PATH="$PATH:/var/lib/embedded-cluster/bin" $ source <(kubectl completion bash) $ source /etc/bash_completion diff --git a/cmd/buildtools/openebs.go b/cmd/buildtools/openebs.go index bd429911b..b1cedca8b 100644 --- a/cmd/buildtools/openebs.go +++ b/cmd/buildtools/openebs.go @@ -64,11 +64,12 @@ var updateOpenEBSAddonCommand = &cli.Command{ current := openebs.Metadata if current.Version == nextChartVersion && !c.Bool("force") { logrus.Infof("openebs chart version is already up-to-date") - } else { - logrus.Infof("mirroring openebs chart version %s", nextChartVersion) - if err := MirrorChart(openebsRepo, "openebs", nextChartVersion); err != nil { - return fmt.Errorf("failed to mirror openebs chart: %v", err) - } + return nil + } + + logrus.Infof("mirroring openebs chart version %s", nextChartVersion) + if err := MirrorChart(openebsRepo, "openebs", nextChartVersion); err != nil { + return fmt.Errorf("failed to mirror openebs chart: %v", err) } upstream := fmt.Sprintf("%s/openebs", os.Getenv("CHARTS_DESTINATION")) diff --git a/cmd/buildtools/seaweedfs.go b/cmd/buildtools/seaweedfs.go index 71aa7c6ca..e6e386c3a 100644 --- a/cmd/buildtools/seaweedfs.go +++ b/cmd/buildtools/seaweedfs.go @@ -52,11 +52,12 @@ var updateSeaweedFSAddonCommand = &cli.Command{ current := seaweedfs.Metadata if current.Version == nextChartVersion && !c.Bool("force") { logrus.Infof("seaweedfs chart version is already up-to-date") - } else { - logrus.Infof("mirroring seaweedfs chart version %s", nextChartVersion) - if err := MirrorChart(seaweedfsRepo, "seaweedfs", nextChartVersion); err != nil { - return fmt.Errorf("failed to mirror seaweedfs chart: %v", err) - } + return nil + } + + logrus.Infof("mirroring seaweedfs chart version %s", nextChartVersion) + if err := MirrorChart(seaweedfsRepo, "seaweedfs", nextChartVersion); err != nil { + return fmt.Errorf("failed to mirror seaweedfs chart: %v", err) } upstream := fmt.Sprintf("%s/seaweedfs", os.Getenv("CHARTS_DESTINATION")) diff --git a/cmd/embedded-cluster/flags.go b/cmd/embedded-cluster/flags.go index cdd2ed7db..7044628dd 100644 --- a/cmd/embedded-cluster/flags.go +++ b/cmd/embedded-cluster/flags.go @@ -4,55 +4,73 @@ import ( "fmt" "strconv" - "github.com/replicatedhq/embedded-cluster/pkg/defaults" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" k8snet "k8s.io/utils/net" ) -func getAdminColsolePortFlag() cli.Flag { +func getAdminConsolePortFlag(runtimeConfig *ecv1beta1.RuntimeConfigSpec) cli.Flag { return &cli.StringFlag{ Name: "admin-console-port", Usage: "Port on which the Admin Console will be served", - Value: strconv.Itoa(defaults.AdminConsolePort), + Value: strconv.Itoa(ecv1beta1.DefaultAdminConsolePort), Hidden: false, + Action: func(c *cli.Context, s string) error { + if s == "" { + return nil + } + // TODO: add first class support for service node port range and validate the port + port, err := k8snet.ParsePort(s, false) + if err != nil { + return fmt.Errorf("invalid port: %w", err) + } + logrus.Debugf("Setting admin console port to %d from flag", port) + runtimeConfig.AdminConsole.Port = port + return nil + }, } } -func getAdminConsolePortFromFlag(c *cli.Context) (int, error) { - portStr := c.String("admin-console-port") - if portStr == "" { - return defaults.AdminConsolePort, nil - } - // TODO: add first class support for service node port range and validate the port - port, err := k8snet.ParsePort(portStr, false) - if err != nil { - return 0, fmt.Errorf("invalid admin console port: %w", err) - } - return port, nil -} - -func getLocalArtifactMirrorPortFlag() cli.Flag { +func getLocalArtifactMirrorPortFlag(runtimeConfig *ecv1beta1.RuntimeConfigSpec) cli.Flag { return &cli.StringFlag{ Name: "local-artifact-mirror-port", Usage: "Port on which the Local Artifact Mirror will be served", - Value: strconv.Itoa(defaults.LocalArtifactMirrorPort), + Value: strconv.Itoa(ecv1beta1.DefaultLocalArtifactMirrorPort), Hidden: false, + Action: func(c *cli.Context, s string) error { + if s == "" { + return nil + } + // TODO: add first class support for service node port range and validate the port does not + // conflict with this range + port, err := k8snet.ParsePort(s, false) + if err != nil { + return fmt.Errorf("invalid local artifact mirror port: %w", err) + } + if s == c.String("admin-console-port") { + return fmt.Errorf("local artifact mirror port cannot be the same as admin console port") + } + logrus.Debugf("Setting local artifact mirror port to %d from flag", port) + runtimeConfig.LocalArtifactMirror.Port = port + return nil + }, } } -func getLocalArtifactMirrorPortFromFlag(c *cli.Context) (int, error) { - portStr := c.String("local-artifact-mirror-port") - if portStr == "" { - return defaults.LocalArtifactMirrorPort, nil - } - // TODO: add first class support for service node port range and validate the port does not - // conflict with this range - port, err := k8snet.ParsePort(portStr, false) - if err != nil { - return 0, fmt.Errorf("invalid local artifact mirror port: %w", err) - } - if portStr == c.String("admin-console-port") { - return 0, fmt.Errorf("local artifact mirror port cannot be the same as admin console port") +func getDataDirFlag(runtimeConfig *ecv1beta1.RuntimeConfigSpec) cli.Flag { + return &cli.StringFlag{ + Name: "data-dir", + Usage: "Path to the data directory", + Value: ecv1beta1.DefaultDataDir, + Hidden: false, + Action: func(c *cli.Context, s string) error { + if s == "" { + return nil + } + logrus.Debugf("Setting data dir to %s from flag", s) + runtimeConfig.DataDir = s + return nil + }, } - return port, nil } diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index ba6684182..ca60edb34 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -27,6 +27,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/prompts" "github.com/replicatedhq/embedded-cluster/pkg/release" "github.com/replicatedhq/embedded-cluster/pkg/spinner" + "github.com/replicatedhq/embedded-cluster/pkg/support" "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" ) @@ -47,11 +48,12 @@ var ErrPreflightsHaveFail = fmt.Errorf("host preflight failures detected") // installAndEnableLocalArtifactMirror installs and enables the local artifact mirror. This // service is responsible for serving on localhost, through http, all files that are used // during a cluster upgrade. -func installAndEnableLocalArtifactMirror(port int) error { - if err := goods.MaterializeLocalArtifactMirrorUnitFile(); err != nil { +func installAndEnableLocalArtifactMirror(provider *defaults.Provider) error { + materializer := goods.NewMaterializer(provider) + if err := materializer.LocalArtifactMirrorUnitFile(); err != nil { return fmt.Errorf("failed to materialize artifact mirror unit: %w", err) } - if err := writeLocalArtifactMirrorEnvironmentFile(port); err != nil { + if err := writeLocalArtifactMirrorDropInFile(provider); err != nil { return fmt.Errorf("failed to write local artifact mirror environment file: %w", err) } if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil { @@ -67,8 +69,8 @@ func installAndEnableLocalArtifactMirror(port int) error { } // updateLocalArtifactMirrorService updates the port on which the local artifact mirror is served. -func updateLocalArtifactMirrorService(port int) error { - if err := writeLocalArtifactMirrorEnvironmentFile(port); err != nil { +func updateLocalArtifactMirrorService(provider *defaults.Provider) error { + if err := writeLocalArtifactMirrorDropInFile(provider); err != nil { return fmt.Errorf("failed to write local artifact mirror environment file: %w", err) } if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil { @@ -81,18 +83,28 @@ func updateLocalArtifactMirrorService(port int) error { } const ( - localArtifactMirrorSystemdConfFile = "/etc/systemd/system/local-artifact-mirror.service.d/embedded-cluster.conf" - localArtifactMirrorEnvironmentFileContents = `[Service] -Environment="LOCAL_ARTIFACT_MIRROR_PORT=%d"` + localArtifactMirrorSystemdConfFile = "/etc/systemd/system/local-artifact-mirror.service.d/embedded-cluster.conf" + localArtifactMirrorDropInFileContents = `[Service] +Environment="LOCAL_ARTIFACT_MIRROR_PORT=%d" +Environment="LOCAL_ARTIFACT_MIRROR_DATA_DIR=%s" +# Empty ExecStart= will clear out the previous ExecStart value +ExecStart= +ExecStart=%s serve +` ) -func writeLocalArtifactMirrorEnvironmentFile(port int) error { +func writeLocalArtifactMirrorDropInFile(provider *defaults.Provider) error { dir := filepath.Dir(localArtifactMirrorSystemdConfFile) err := os.MkdirAll(dir, 0755) if err != nil { return fmt.Errorf("create directory: %w", err) } - contents := fmt.Sprintf(localArtifactMirrorEnvironmentFileContents, port) + contents := fmt.Sprintf( + localArtifactMirrorDropInFileContents, + provider.LocalArtifactMirrorPort(), + provider.EmbeddedClusterHomeDirectory(), + provider.PathToEmbeddedClusterBinary("local-artifact-mirror"), + ) err = os.WriteFile(localArtifactMirrorSystemdConfFile, []byte(contents), 0644) if err != nil { return fmt.Errorf("write file: %w", err) @@ -103,7 +115,7 @@ func writeLocalArtifactMirrorEnvironmentFile(port int) error { // configureNetworkManager configures the network manager (if the host is using it) to ignore // the calico interfaces. This function restarts the NetworkManager service if the configuration // was changed. -func configureNetworkManager(c *cli.Context) error { +func configureNetworkManager(c *cli.Context, provider *defaults.Provider) error { if active, err := helpers.IsSystemdServiceActive(c.Context, "NetworkManager"); err != nil { return fmt.Errorf("unable to check if NetworkManager is active: %w", err) } else if !active { @@ -118,7 +130,8 @@ func configureNetworkManager(c *cli.Context) error { } logrus.Debugf("creating NetworkManager config file") - if err := goods.MaterializeCalicoNetworkManagerConfig(); err != nil { + materializer := goods.NewMaterializer(provider) + if err := materializer.CalicoNetworkManagerConfig(); err != nil { return fmt.Errorf("unable to materialize configuration: %w", err) } @@ -132,7 +145,7 @@ func configureNetworkManager(c *cli.Context) error { // RunHostPreflights runs the host preflights we found embedded in the binary // on all configured hosts. We attempt to read HostPreflights from all the // embedded Helm Charts and from the Kots Application Release files. -func RunHostPreflights(c *cli.Context, applier *addons.Applier, replicatedAPIURL, proxyRegistryURL string, isAirgap bool, proxy *ecv1beta1.ProxySpec, adminConsolePort int, localArtifactMirrorPort int) error { +func RunHostPreflights(c *cli.Context, provider *defaults.Provider, applier *addons.Applier, replicatedAPIURL, proxyRegistryURL string, isAirgap bool, proxy *ecv1beta1.ProxySpec) error { hpf, err := applier.HostPreflights() if err != nil { return fmt.Errorf("unable to read host preflights: %w", err) @@ -142,8 +155,11 @@ func RunHostPreflights(c *cli.Context, applier *addons.Applier, replicatedAPIURL ReplicatedAPIURL: replicatedAPIURL, ProxyRegistryURL: proxyRegistryURL, IsAirgap: isAirgap, - AdminConsolePort: adminConsolePort, - LocalArtifactMirrorPort: localArtifactMirrorPort, + AdminConsolePort: provider.AdminConsolePort(), + LocalArtifactMirrorPort: provider.LocalArtifactMirrorPort(), + DataDir: provider.EmbeddedClusterHomeDirectory(), + K0sDataDir: provider.EmbeddedClusterK0sSubDir(), + OpenEBSDataDir: provider.EmbeddedClusterOpenEBSLocalSubDir(), SystemArchitecture: runtime.GOARCH, } chpfs, err := preflights.GetClusterHostPreflights(c.Context, data) @@ -156,10 +172,10 @@ func RunHostPreflights(c *cli.Context, applier *addons.Applier, replicatedAPIURL hpf.Analyzers = append(hpf.Analyzers, h.Spec.Analyzers...) } - return runHostPreflights(c, hpf, proxy) + return runHostPreflights(c, provider, hpf, proxy) } -func runHostPreflights(c *cli.Context, hpf *v1beta2.HostPreflightSpec, proxy *ecv1beta1.ProxySpec) error { +func runHostPreflights(c *cli.Context, provider *defaults.Provider, hpf *v1beta2.HostPreflightSpec, proxy *ecv1beta1.ProxySpec) error { if len(hpf.Collectors) == 0 && len(hpf.Analyzers) == 0 { return nil } @@ -170,7 +186,7 @@ func runHostPreflights(c *cli.Context, hpf *v1beta2.HostPreflightSpec, proxy *ec return nil } pb.Infof("Running host preflights") - output, stderr, err := preflights.Run(c.Context, hpf, proxy) + output, stderr, err := preflights.Run(c.Context, provider, hpf, proxy) if err != nil { pb.CloseWithError() return fmt.Errorf("host preflights failed to run: %w", err) @@ -179,12 +195,12 @@ func runHostPreflights(c *cli.Context, hpf *v1beta2.HostPreflightSpec, proxy *ec logrus.Debugf("preflight stderr: %s", stderr) } - err = output.SaveToDisk() + err = output.SaveToDisk(provider.PathToEmbeddedClusterSupportFile("host-preflight-results.json")) if err != nil { logrus.Warnf("unable to save preflights output: %v", err) } - err = preflights.CopyBundleToECSupportDir() + err = preflights.CopyBundleToECSupportDir(provider) if err != nil { logrus.Warnf("unable to copy preflight bundle to embedded-cluster support dir: %v", err) } @@ -341,14 +357,18 @@ func checkAirgapMatches(c *cli.Context) error { return nil } -func materializeFiles(c *cli.Context) error { +func materializeFiles(c *cli.Context, provider *defaults.Provider) error { mat := spinner.Start() defer mat.Close() mat.Infof("Materializing files") - if err := goods.Materialize(); err != nil { + materializer := goods.NewMaterializer(provider) + if err := materializer.Materialize(); err != nil { return fmt.Errorf("unable to materialize binaries: %w", err) } + if err := support.MaterializeSupportBundleSpec(provider); err != nil { + return fmt.Errorf("unable to materialize support bundle spec: %w", err) + } if c.String("airgap-bundle") != "" { mat.Infof("Materializing airgap installation files") @@ -359,7 +379,7 @@ func materializeFiles(c *cli.Context) error { } defer rawfile.Close() - if err := airgap.MaterializeAirgap(rawfile); err != nil { + if err := airgap.MaterializeAirgap(provider, rawfile); err != nil { err = fmt.Errorf("unable to materialize airgap files: %w", err) return err } @@ -373,7 +393,7 @@ func materializeFiles(c *cli.Context) error { // createK0sConfig creates a new k0s.yaml configuration file. The file is saved in the // global location (as returned by defaults.PathToK0sConfig()). If a file already sits // there, this function returns an error. -func ensureK0sConfig(c *cli.Context, applier *addons.Applier) (*k0sconfig.ClusterConfig, error) { +func ensureK0sConfig(c *cli.Context, provider *defaults.Provider, applier *addons.Applier) (*k0sconfig.ClusterConfig, error) { cfgpath := defaults.PathToK0sConfig() if _, err := os.Stat(cfgpath); err == nil { return nil, fmt.Errorf("configuration file already exists") @@ -399,7 +419,7 @@ func ensureK0sConfig(c *cli.Context, applier *addons.Applier) (*k0sconfig.Cluste } if c.String("airgap-bundle") != "" { // update the k0s config to install with airgap - airgap.RemapHelm(cfg) + airgap.RemapHelm(provider, cfg) airgap.SetAirgapConfig(cfg) } data, err := k8syaml.Marshal(cfg) @@ -459,8 +479,8 @@ func applyUnsupportedOverrides(c *cli.Context, cfg *k0sconfig.ClusterConfig) (*k // installK0s runs the k0s install command and waits for it to finish. If no configuration // is found one is generated. -func installK0s(c *cli.Context) error { - ourbin := defaults.PathToEmbeddedClusterBinary("k0s") +func installK0s(c *cli.Context, provider *defaults.Provider) error { + ourbin := provider.PathToEmbeddedClusterBinary("k0s") hstbin := defaults.K0sBinaryPath() if err := helpers.MoveFile(ourbin, hstbin); err != nil { return fmt.Errorf("unable to move k0s binary: %w", err) @@ -469,7 +489,7 @@ func installK0s(c *cli.Context) error { if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - if _, err := helpers.RunCommand(hstbin, config.InstallFlags(nodeIP)...); err != nil { + if _, err := helpers.RunCommand(hstbin, config.InstallFlags(provider, nodeIP)...); err != nil { return fmt.Errorf("unable to install: %w", err) } if _, err := helpers.RunCommand(hstbin, "start"); err != nil { @@ -501,26 +521,26 @@ func waitForK0s() error { } // installAndWaitForK0s installs the k0s binary and waits for it to be ready -func installAndWaitForK0s(c *cli.Context, applier *addons.Applier, proxy *ecv1beta1.ProxySpec) (*k0sconfig.ClusterConfig, error) { +func installAndWaitForK0s(c *cli.Context, provider *defaults.Provider, applier *addons.Applier, proxy *ecv1beta1.ProxySpec) (*k0sconfig.ClusterConfig, error) { loading := spinner.Start() defer loading.Close() loading.Infof("Installing %s node", defaults.BinaryName()) logrus.Debugf("creating k0s configuration file") - cfg, err := ensureK0sConfig(c, applier) + cfg, err := ensureK0sConfig(c, provider, applier) if err != nil { err := fmt.Errorf("unable to create config file: %w", err) metrics.ReportApplyFinished(c, err) return nil, err } logrus.Debugf("creating systemd unit files") - if err := createSystemdUnitFiles(false, proxy, applier.GetLocalArtifactMirrorPort()); err != nil { + if err := createSystemdUnitFiles(provider, false, proxy); err != nil { err := fmt.Errorf("unable to create systemd unit files: %w", err) metrics.ReportApplyFinished(c, err) return nil, err } logrus.Debugf("installing k0s") - if err := installK0s(c); err != nil { + if err := installK0s(c, provider); err != nil { err := fmt.Errorf("unable update cluster: %w", err) metrics.ReportApplyFinished(c, err) return nil, err @@ -537,8 +557,8 @@ func installAndWaitForK0s(c *cli.Context, applier *addons.Applier, proxy *ecv1be } // runOutro calls Outro() in all enabled addons by means of Applier. -func runOutro(c *cli.Context, applier *addons.Applier, cfg *k0sconfig.ClusterConfig) error { - os.Setenv("KUBECONFIG", defaults.PathToKubeConfig()) +func runOutro(c *cli.Context, provider *defaults.Provider, applier *addons.Applier, cfg *k0sconfig.ClusterConfig) error { + os.Setenv("KUBECONFIG", provider.PathToKubeConfig()) metadata, err := gatherVersionMetadata(cfg) if err != nil { @@ -596,167 +616,169 @@ func validateAdminConsolePassword(password, passwordCheck string) bool { // installCommands executes the "install" command. This will ensure that a k0s.yaml file exists // and then run `k0s install` to apply the cluster. Once this is finished then a "kubeconfig" // file is created. Resulting kubeconfig is stored in the configuration dir. -var installCommand = &cli.Command{ - Name: "install", - Usage: fmt.Sprintf("Install %s", binName), - Subcommands: []*cli.Command{ - installRunPreflightsCommand, - }, - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("install command must be run as root") - } - if c.String("airgap-bundle") != "" { - metrics.DisableMetrics() - } - return nil - }, - Flags: withProxyFlags(withSubnetCIDRFlags( - []cli.Flag{ - &cli.StringFlag{ - Name: "admin-console-password", - Usage: fmt.Sprintf("Password for the Admin Console (minimum %d characters)", minAdminPasswordLength), - Hidden: false, - }, - &cli.StringFlag{ - Name: "airgap-bundle", - Usage: "Path to the air gap bundle. If set, the installation will complete without internet access.", - Hidden: true, - }, - &cli.StringFlag{ - Name: "license", - Aliases: []string{"l"}, - Usage: "Path to the license file", - Hidden: false, - }, - &cli.StringFlag{ - Name: "network-interface", - Usage: "The network interface to use for the cluster", - Value: "", - }, - &cli.BoolFlag{ - Name: "no-prompt", - Usage: "Disable interactive prompts. The Admin Console password will be set to password.", - Value: false, - }, - &cli.StringFlag{ - Name: "overrides", - Usage: "File with an EmbeddedClusterConfig object to override the default configuration", - Hidden: true, - }, - &cli.BoolFlag{ - Name: "skip-host-preflights", - Usage: "Skip host preflight checks. This is not recommended.", - Value: false, - }, - &cli.StringSliceFlag{ - Name: "private-ca", - Usage: "Path to a trusted private CA certificate file", - }, - getAdminColsolePortFlag(), - getLocalArtifactMirrorPortFlag(), +func installCommand() *cli.Command { + runtimeConfig := ecv1beta1.GetDefaultRuntimeConfig() + + return &cli.Command{ + Name: "install", + Usage: fmt.Sprintf("Install %s", binName), + Subcommands: []*cli.Command{ + installRunPreflightsCommand(), }, - )), - Action: func(c *cli.Context) error { - var err error - proxy := getProxySpecFromFlags(c) - proxy, err = includeLocalIPInNoProxy(c, proxy) - if err != nil { - metrics.ReportApplyFinished(c, err) - return err - } - setProxyEnv(proxy) - - logrus.Debugf("checking if %s is already installed", binName) - if installed, err := isAlreadyInstalled(); err != nil { - return err - } else if installed { - logrus.Errorf("An installation has been detected on this machine.") - logrus.Infof("If you want to reinstall, you need to remove the existing installation first.") - logrus.Infof("You can do this by running the following command:") - logrus.Infof("\n sudo ./%s reset\n", binName) - return ErrNothingElseToAdd - } - metrics.ReportApplyStarted(c) - logrus.Debugf("configuring network manager") - if err := configureNetworkManager(c); err != nil { - return fmt.Errorf("unable to configure network manager: %w", err) - } - logrus.Debugf("checking license matches") - license, err := getLicenseFromFilepath(c.String("license")) - if err != nil { - metricErr := fmt.Errorf("unable to get license: %w", err) - metrics.ReportApplyFinished(c, metricErr) - return err // do not return the metricErr, as we want the user to see the error message without a prefix - } - isAirgap := c.String("airgap-bundle") != "" - if isAirgap { - logrus.Debugf("checking airgap bundle matches binary") - if err := checkAirgapMatches(c); err != nil { - return err // we want the user to see the error message without a prefix + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("install command must be run as root") } - } - if err := preflights.ValidateApp(); err != nil { - metrics.ReportApplyFinished(c, err) - return err - } - adminConsolePwd, err := maybeAskAdminConsolePassword(c) - if err != nil { - metrics.ReportApplyFinished(c, err) - return err - } + if c.String("airgap-bundle") != "" { + metrics.DisableMetrics() + } + return nil + }, + Flags: withProxyFlags(withSubnetCIDRFlags( + []cli.Flag{ + &cli.StringFlag{ + Name: "admin-console-password", + Usage: fmt.Sprintf("Password for the Admin Console (minimum %d characters)", minAdminPasswordLength), + Hidden: false, + }, + &cli.StringFlag{ + Name: "airgap-bundle", + Usage: "Path to the air gap bundle. If set, the installation will complete without internet access.", + Hidden: true, + }, + &cli.StringFlag{ + Name: "license", + Aliases: []string{"l"}, + Usage: "Path to the license file", + Hidden: false, + }, + &cli.StringFlag{ + Name: "network-interface", + Usage: "The network interface to use for the cluster", + Value: "", + }, + &cli.BoolFlag{ + Name: "no-prompt", + Usage: "Disable interactive prompts. The Admin Console password will be set to password.", + Value: false, + }, + &cli.StringFlag{ + Name: "overrides", + Usage: "File with an EmbeddedClusterConfig object to override the default configuration", + Hidden: true, + }, + &cli.BoolFlag{ + Name: "skip-host-preflights", + Usage: "Skip host preflight checks. This is not recommended.", + Value: false, + }, + &cli.StringSliceFlag{ + Name: "private-ca", + Usage: "Path to a trusted private CA certificate file", + }, + getDataDirFlag(runtimeConfig), + getAdminConsolePortFlag(runtimeConfig), + getLocalArtifactMirrorPortFlag(runtimeConfig), + }, + )), + Action: func(c *cli.Context) error { + provider := defaults.NewProviderFromRuntimeConfig(runtimeConfig) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) - logrus.Debugf("materializing binaries") - if err := materializeFiles(c); err != nil { - metrics.ReportApplyFinished(c, err) - return err - } - applier, err := getAddonsApplier(c, adminConsolePwd, proxy) - if err != nil { - metrics.ReportApplyFinished(c, err) - return err - } - logrus.Debugf("running host preflights") - var replicatedAPIURL, proxyRegistryURL string - if license != nil { - replicatedAPIURL = license.Spec.Endpoint - proxyRegistryURL = fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) - } + defer tryRemoveTmpDirContents(provider) - adminConsolePort, err := getAdminConsolePortFromFlag(c) - if err != nil { - return fmt.Errorf("unable to parse admin console port: %w", err) - } + var err error + proxy := getProxySpecFromFlags(c) + proxy, err = includeLocalIPInNoProxy(c, proxy) + if err != nil { + metrics.ReportApplyFinished(c, err) + return err + } + setProxyEnv(proxy) + + logrus.Debugf("checking if %s is already installed", binName) + if installed, err := isAlreadyInstalled(); err != nil { + return err + } else if installed { + logrus.Errorf("An installation has been detected on this machine.") + logrus.Infof("If you want to reinstall, you need to remove the existing installation first.") + logrus.Infof("You can do this by running the following command:") + logrus.Infof("\n sudo ./%s reset\n", binName) + return ErrNothingElseToAdd + } + metrics.ReportApplyStarted(c) + logrus.Debugf("configuring network manager") + if err := configureNetworkManager(c, provider); err != nil { + return fmt.Errorf("unable to configure network manager: %w", err) + } + logrus.Debugf("checking license matches") + license, err := getLicenseFromFilepath(c.String("license")) + if err != nil { + metricErr := fmt.Errorf("unable to get license: %w", err) + metrics.ReportApplyFinished(c, metricErr) + return err // do not return the metricErr, as we want the user to see the error message without a prefix + } + isAirgap := c.String("airgap-bundle") != "" + if isAirgap { + logrus.Debugf("checking airgap bundle matches binary") + if err := checkAirgapMatches(c); err != nil { + return err // we want the user to see the error message without a prefix + } + } + if err := preflights.ValidateApp(); err != nil { + metrics.ReportApplyFinished(c, err) + return err + } + adminConsolePwd, err := maybeAskAdminConsolePassword(c) + if err != nil { + metrics.ReportApplyFinished(c, err) + return err + } - localArtifactMirrorPort, err := getLocalArtifactMirrorPortFromFlag(c) - if err != nil { - return fmt.Errorf("unable to parse local artifact mirror port: %w", err) - } + logrus.Debugf("materializing binaries") + if err := materializeFiles(c, provider); err != nil { + metrics.ReportApplyFinished(c, err) + return err + } + applier, err := getAddonsApplier(c, runtimeConfig, adminConsolePwd, proxy) + if err != nil { + metrics.ReportApplyFinished(c, err) + return err + } + logrus.Debugf("running host preflights") + var replicatedAPIURL, proxyRegistryURL string + if license != nil { + replicatedAPIURL = license.Spec.Endpoint + proxyRegistryURL = fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) + } - if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy, adminConsolePort, localArtifactMirrorPort); err != nil { - metrics.ReportApplyFinished(c, err) - if err == ErrPreflightsHaveFail { - return ErrNothingElseToAdd + if err := RunHostPreflights(c, provider, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy); err != nil { + metrics.ReportApplyFinished(c, err) + if err == ErrPreflightsHaveFail { + return ErrNothingElseToAdd + } + return err } - return err - } - cfg, err := installAndWaitForK0s(c, applier, proxy) - if err != nil { - return err - } - logrus.Debugf("running outro") - if err := runOutro(c, applier, cfg); err != nil { - metrics.ReportApplyFinished(c, err) - return err - } - metrics.ReportApplyFinished(c, nil) - return nil - }, + cfg, err := installAndWaitForK0s(c, provider, applier, proxy) + if err != nil { + return err + } + logrus.Debugf("running outro") + if err := runOutro(c, provider, applier, cfg); err != nil { + metrics.ReportApplyFinished(c, err) + return err + } + metrics.ReportApplyFinished(c, nil) + return nil + }, + } } -func getAddonsApplier(c *cli.Context, adminConsolePwd string, proxy *ecv1beta1.ProxySpec) (*addons.Applier, error) { +func getAddonsApplier(c *cli.Context, runtimeConfig *ecv1beta1.RuntimeConfigSpec, adminConsolePwd string, proxy *ecv1beta1.ProxySpec) (*addons.Applier, error) { opts := []addons.Option{} + opts = append(opts, addons.WithRuntimeConfig(runtimeConfig)) + if c.Bool("no-prompt") { opts = append(opts, addons.WithoutPrompt()) } @@ -789,18 +811,6 @@ func getAddonsApplier(c *cli.Context, adminConsolePwd string, proxy *ecv1beta1.P opts = append(opts, addons.WithPrivateCAs(privateCAs)) } - adminConsolePort, err := getAdminConsolePortFromFlag(c) - if err != nil { - return nil, err - } - opts = append(opts, addons.WithAdminConsolePort(adminConsolePort)) - - localArtifactMirrorPort, err := getLocalArtifactMirrorPortFromFlag(c) - if err != nil { - return nil, err - } - opts = append(opts, addons.WithLocalArtifactMirrorPort(localArtifactMirrorPort)) - if adminConsolePwd != "" { opts = append(opts, addons.WithAdminConsolePassword(adminConsolePwd)) } diff --git a/cmd/embedded-cluster/install_test.go b/cmd/embedded-cluster/install_test.go index 029e85a26..a58bfd7a7 100644 --- a/cmd/embedded-cluster/install_test.go +++ b/cmd/embedded-cluster/install_test.go @@ -295,7 +295,7 @@ func Test_maybeAskAdminConsolePassword(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req := require.New(t) - flags := installCommand.Flags + flags := installCommand().Flags flagSet := flag.NewFlagSet("test", 0) for _, flag := range flags { flag.Apply(flagSet) diff --git a/cmd/embedded-cluster/join.go b/cmd/embedded-cluster/join.go index 03c9800a7..5d4b9ba31 100644 --- a/cmd/embedded-cluster/join.go +++ b/cmd/embedded-cluster/join.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "os" - "strconv" "strings" "time" @@ -168,7 +167,6 @@ var joinCommand = &cli.Command{ if c.String("airgap-bundle") != "" { metrics.DisableMetrics() } - os.Setenv("KUBECONFIG", defaults.PathToKubeConfig()) return nil }, Action: func(c *cli.Context) error { @@ -193,6 +191,12 @@ var joinCommand = &cli.Command{ return fmt.Errorf("unable to get join token: %w", err) } + provider := defaults.NewProviderFromRuntimeConfig(jcmd.InstallationSpec.RuntimeConfig) + os.Setenv("KUBECONFIG", provider.PathToKubeConfig()) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + + defer tryRemoveTmpDirContents(provider) + // check to make sure the version returned by the join token is the same as the one we are running if strings.TrimPrefix(jcmd.EmbeddedClusterVersion, "v") != strings.TrimPrefix(versions.Version, "v") { return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion) @@ -218,12 +222,12 @@ var joinCommand = &cli.Command{ metrics.ReportJoinStarted(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID) logrus.Debugf("materializing %s binaries", binName) - if err := materializeFiles(c); err != nil { + if err := materializeFiles(c, provider); err != nil { metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } - applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy) + applier, err := getAddonsApplier(c, jcmd.InstallationSpec.RuntimeConfig, "", jcmd.InstallationSpec.Proxy) if err != nil { metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err @@ -232,22 +236,7 @@ var joinCommand = &cli.Command{ // jcmd.InstallationSpec.MetricsBaseURL is the replicated.app endpoint url replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) - - urlSlices := strings.Split(c.Args().Get(0), ":") - if len(urlSlices) != 2 { - return fmt.Errorf("unable to get port from url %s", c.Args().Get(0)) - } - adminConsolePort, err := strconv.Atoi(urlSlices[1]) - if err != nil { - return fmt.Errorf("unable to convert port to int: %w", err) - } - - localArtifactMirrorPort := defaults.LocalArtifactMirrorPort - if jcmd.InstallationSpec.LocalArtifactMirror != nil { - localArtifactMirrorPort = jcmd.InstallationSpec.LocalArtifactMirror.Port - } - - if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy, adminConsolePort, localArtifactMirrorPort); err != nil { + if err := RunHostPreflights(c, provider, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil { metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) if err == ErrPreflightsHaveFail { return ErrNothingElseToAdd @@ -256,7 +245,7 @@ var joinCommand = &cli.Command{ } logrus.Debugf("configuring network manager") - if err := configureNetworkManager(c); err != nil { + if err := configureNetworkManager(c, provider); err != nil { return fmt.Errorf("unable to configure network manager: %w", err) } @@ -268,7 +257,7 @@ var joinCommand = &cli.Command{ } logrus.Debugf("installing %s binaries", binName) - if err := installK0sBinary(); err != nil { + if err := installK0sBinary(provider); err != nil { err := fmt.Errorf("unable to install k0s binary: %w", err) metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err @@ -283,9 +272,8 @@ var joinCommand = &cli.Command{ } logrus.Debugf("creating systemd unit files") - // both controller and worker nodes will have 'worker' in the join command - if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.InstallationSpec.Proxy, localArtifactMirrorPort); err != nil { + if err := createSystemdUnitFiles(provider, !strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.InstallationSpec.Proxy); err != nil { err := fmt.Errorf("unable to create systemd unit files: %w", err) metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err @@ -305,7 +293,7 @@ var joinCommand = &cli.Command{ } logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(c, jcmd.K0sJoinCommand); err != nil { + if err := runK0sInstallCommand(c, provider, jcmd.K0sJoinCommand); err != nil { err := fmt.Errorf("unable to join node to cluster: %w", err) metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err @@ -478,8 +466,8 @@ func saveTokenToDisk(token string) error { } // installK0sBinary moves the embedded k0s binary to its destination. -func installK0sBinary() error { - ourbin := defaults.PathToEmbeddedClusterBinary("k0s") +func installK0sBinary(provider *defaults.Provider) error { + ourbin := provider.PathToEmbeddedClusterBinary("k0s") hstbin := defaults.K0sBinaryPath() if err := helpers.MoveFile(ourbin, hstbin); err != nil { return fmt.Errorf("unable to move k0s binary: %w", err) @@ -501,18 +489,20 @@ func systemdUnitFileName() string { // runK0sInstallCommand runs the k0s install command as provided by the kots // adm api. -func runK0sInstallCommand(c *cli.Context, fullcmd string) error { +func runK0sInstallCommand(c *cli.Context, provider *defaults.Provider, fullcmd string) error { args := strings.Split(fullcmd, " ") args = append(args, "--token-file", "/etc/k0s/join-token") - if strings.Contains(fullcmd, "controller") { - args = append(args, "--disable-components", "konnectivity-server", "--enable-dynamic-config") - } nodeIP, err := netutils.FirstValidAddress(c.String("network-interface")) if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - args = append(args, "--kubelet-extra-args", fmt.Sprintf(`"--node-ip=%s"`, nodeIP)) + + args = append(args, config.AdditionalInstallFlags(provider, nodeIP)...) + + if strings.Contains(fullcmd, "controller") { + args = append(args, config.AdditionalInstallFlagsController()...) + } if _, err := helpers.RunCommand(args[0], args[1:]...); err != nil { return err diff --git a/cmd/embedded-cluster/k0s.go b/cmd/embedded-cluster/k0s.go new file mode 100644 index 000000000..d8b729573 --- /dev/null +++ b/cmd/embedded-cluster/k0s.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "encoding/json" + "os/exec" + + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" +) + +var ( + k0s = "/usr/local/bin/k0s" +) + +type k0sStatus struct { + Role string `json:"Role"` + Vars k0sVars `json:"K0sVars"` + ClusterConfig k0sv1beta1.ClusterConfig `json:"ClusterConfig"` +} + +type k0sVars struct { + AdminKubeConfigPath string `json:"AdminKubeConfigPath"` + KubeletAuthConfigPath string `json:"KubeletAuthConfigPath"` + CertRootDir string `json:"CertRootDir"` + EtcdCertDir string `json:"EtcdCertDir"` +} + +// getK0sStatus returns the status of the k0s service. +func getK0sStatus(ctx context.Context) (*k0sStatus, error) { + // get k0s status json + out, err := exec.CommandContext(ctx, k0s, "status", "-o", "json").Output() + if err != nil { + return nil, err + } + var status k0sStatus + err = json.Unmarshal(out, &status) + if err != nil { + return nil, err + } + return &status, nil +} diff --git a/cmd/embedded-cluster/list_images.go b/cmd/embedded-cluster/list_images.go index 9966a94db..38f021e41 100644 --- a/cmd/embedded-cluster/list_images.go +++ b/cmd/embedded-cluster/list_images.go @@ -13,7 +13,6 @@ var listImagesCommand = &cli.Command{ Hidden: true, Action: func(c *cli.Context) error { k0sCfg := config.RenderK0sConfig() - metadata, err := gatherVersionMetadata(k0sCfg) if err != nil { return fmt.Errorf("failed to gather version metadata: %w", err) diff --git a/cmd/embedded-cluster/main.go b/cmd/embedded-cluster/main.go index db8f88c07..8474b668b 100644 --- a/cmd/embedded-cluster/main.go +++ b/cmd/embedded-cluster/main.go @@ -28,15 +28,15 @@ func main() { Usage: fmt.Sprintf("Install and manage %s", name), Suggest: true, Commands: []*cli.Command{ - installCommand, - shellCommand, + installCommand(), + shellCommand(), nodeCommands, versionCommand, joinCommand, - resetCommand, - materializeCommand, - updateCommand, - restoreCommand, + resetCommand(), + materializeCommand(), + updateCommand(), + restoreCommand(), }, } if err := app.RunContext(ctx, os.Args); err != nil { diff --git a/cmd/embedded-cluster/materialize.go b/cmd/embedded-cluster/materialize.go index 3cc087466..2bca0fd2f 100644 --- a/cmd/embedded-cluster/materialize.go +++ b/cmd/embedded-cluster/materialize.go @@ -6,31 +6,39 @@ import ( "github.com/urfave/cli/v2" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/goods" ) -var materializeCommand = &cli.Command{ - Name: "materialize", - Usage: "Materialize embedded assets on /var/lib/embedded-cluster", - Hidden: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "basedir", - Usage: "Base directory to materialize assets", - Value: "", +func materializeCommand() *cli.Command { + runtimeConfig := ecv1beta1.GetDefaultRuntimeConfig() + + return &cli.Command{ + Name: "materialize", + Usage: "Materialize embedded assets into the data directory", + Hidden: true, + Flags: []cli.Flag{ + getDataDirFlag(runtimeConfig), + }, + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("materialize command must be run as root") + } + return nil + }, + Action: func(c *cli.Context) error { + provider := defaults.NewProviderFromRuntimeConfig(runtimeConfig) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + + defer tryRemoveTmpDirContents(provider) + + materializer := goods.NewMaterializer(provider) + if err := materializer.Materialize(); err != nil { + return fmt.Errorf("unable to materialize: %v", err) + } + + return nil }, - }, - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("materialize command must be run as root") - } - return nil - }, - Action: func(c *cli.Context) error { - materializer := goods.NewMaterializer(c.String("basedir")) - if err := materializer.Materialize(); err != nil { - return fmt.Errorf("unable to materialize: %v", err) - } - return nil - }, + } } diff --git a/cmd/embedded-cluster/metadata.go b/cmd/embedded-cluster/metadata.go index 7a23d6a31..d5e987913 100644 --- a/cmd/embedded-cluster/metadata.go +++ b/cmd/embedded-cluster/metadata.go @@ -9,8 +9,6 @@ import ( k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" eckinds "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/kinds/types" - "github.com/urfave/cli/v2" - "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole" "github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator" @@ -20,6 +18,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/release" "github.com/replicatedhq/embedded-cluster/pkg/versions" + "github.com/urfave/cli/v2" ) var metadataCommand = &cli.Command{ diff --git a/cmd/embedded-cluster/node.go b/cmd/embedded-cluster/node.go index 28142cc20..69d97f026 100644 --- a/cmd/embedded-cluster/node.go +++ b/cmd/embedded-cluster/node.go @@ -10,6 +10,6 @@ var nodeCommands = &cli.Command{ Hidden: true, // this has been replaced by top-level commands Subcommands: []*cli.Command{ joinCommand, - resetCommand, + resetCommand(), }, } diff --git a/cmd/embedded-cluster/preflights.go b/cmd/embedded-cluster/preflights.go index 2e2e3e137..40ec32693 100644 --- a/cmd/embedded-cluster/preflights.go +++ b/cmd/embedded-cluster/preflights.go @@ -3,9 +3,9 @@ package main import ( "fmt" "os" - "strconv" "strings" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/versions" "github.com/sirupsen/logrus" @@ -13,92 +13,91 @@ import ( ) // installRunPreflightsCommand runs install host preflights. -var installRunPreflightsCommand = &cli.Command{ - Name: "run-preflights", - Hidden: true, - Usage: "Run install host preflights", - Flags: withProxyFlags(withSubnetCIDRFlags( - []cli.Flag{ - &cli.StringFlag{ - Name: "airgap-bundle", - Usage: "Path to the air gap bundle. If set, the installation will complete without internet access.", - Hidden: true, +func installRunPreflightsCommand() *cli.Command { + runtimeConfig := ecv1beta1.GetDefaultRuntimeConfig() + + return &cli.Command{ + Name: "run-preflights", + Hidden: true, + Usage: "Run install host preflights", + Flags: withProxyFlags(withSubnetCIDRFlags( + []cli.Flag{ + &cli.StringFlag{ + Name: "airgap-bundle", + Usage: "Path to the air gap bundle. If set, the installation will complete without internet access.", + Hidden: true, + }, + &cli.StringFlag{ + Name: "license", + Aliases: []string{"l"}, + Usage: "Path to the license file.", + Hidden: false, + }, + &cli.BoolFlag{ + Name: "no-prompt", + Usage: "Disable interactive prompts.", + Value: false, + }, + getDataDirFlag(runtimeConfig), + getAdminConsolePortFlag(runtimeConfig), + getLocalArtifactMirrorPortFlag(runtimeConfig), }, - &cli.StringFlag{ - Name: "license", - Aliases: []string{"l"}, - Usage: "Path to the license file.", - Hidden: false, - }, - &cli.BoolFlag{ - Name: "no-prompt", - Usage: "Disable interactive prompts.", - Value: false, - }, - getAdminColsolePortFlag(), - getLocalArtifactMirrorPortFlag(), + )), + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("run-preflights command must be run as root") + } + return nil }, - )), - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("run-preflights command must be run as root") - } - return nil - }, - Action: func(c *cli.Context) error { - var err error - proxy := getProxySpecFromFlags(c) - proxy, err = includeLocalIPInNoProxy(c, proxy) - if err != nil { - return err - } - setProxyEnv(proxy) + Action: func(c *cli.Context) error { + provider := defaults.NewProviderFromRuntimeConfig(runtimeConfig) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) - license, err := getLicenseFromFilepath(c.String("license")) - if err != nil { - return err - } + defer tryRemoveTmpDirContents(provider) - isAirgap := c.String("airgap-bundle") != "" - - logrus.Debugf("materializing binaries") - if err := materializeFiles(c); err != nil { - return err - } + var err error + proxy := getProxySpecFromFlags(c) + proxy, err = includeLocalIPInNoProxy(c, proxy) + if err != nil { + return err + } + setProxyEnv(proxy) - applier, err := getAddonsApplier(c, "", proxy) - if err != nil { - return err - } + license, err := getLicenseFromFilepath(c.String("license")) + if err != nil { + return err + } - logrus.Debugf("running host preflights") - var replicatedAPIURL, proxyRegistryURL string - if license != nil { - replicatedAPIURL = license.Spec.Endpoint - proxyRegistryURL = fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) - } + isAirgap := c.String("airgap-bundle") != "" - adminConsolePort, err := getAdminConsolePortFromFlag(c) - if err != nil { - return fmt.Errorf("unable to parse admin console port: %w", err) - } + logrus.Debugf("materializing binaries") + if err := materializeFiles(c, provider); err != nil { + return err + } - localArtifactMirrorPort, err := getLocalArtifactMirrorPortFromFlag(c) - if err != nil { - return fmt.Errorf("unable to parse local artifact mirror port: %w", err) - } + applier, err := getAddonsApplier(c, runtimeConfig, "", proxy) + if err != nil { + return err + } - if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy, adminConsolePort, localArtifactMirrorPort); err != nil { - if err == ErrPreflightsHaveFail { - return ErrNothingElseToAdd + logrus.Debugf("running host preflights") + var replicatedAPIURL, proxyRegistryURL string + if license != nil { + replicatedAPIURL = license.Spec.Endpoint + proxyRegistryURL = fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) + } + if err := RunHostPreflights(c, provider, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy); err != nil { + if err == ErrPreflightsHaveFail { + return ErrNothingElseToAdd + } + return err } - return err - } - logrus.Info("Host preflights completed successfully") + logrus.Info("Host preflights completed successfully") - return nil - }, + return nil + }, + } } // joinRunPreflightsCommand runs install host preflights. @@ -141,6 +140,11 @@ var joinRunPreflightsCommand = &cli.Command{ return fmt.Errorf("unable to get join token: %w", err) } + provider := defaults.NewProviderFromRuntimeConfig(jcmd.InstallationSpec.RuntimeConfig) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + + defer tryRemoveTmpDirContents(provider) + // check to make sure the version returned by the join token is the same as the one we are running if strings.TrimPrefix(jcmd.EmbeddedClusterVersion, "v") != strings.TrimPrefix(versions.Version, "v") { return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion) @@ -158,11 +162,11 @@ var joinRunPreflightsCommand = &cli.Command{ isAirgap := c.String("airgap-bundle") != "" logrus.Debugf("materializing binaries") - if err := materializeFiles(c); err != nil { + if err := materializeFiles(c, provider); err != nil { return err } - applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy) + applier, err := getAddonsApplier(c, jcmd.InstallationSpec.RuntimeConfig, "", jcmd.InstallationSpec.Proxy) if err != nil { return err } @@ -170,22 +174,7 @@ var joinRunPreflightsCommand = &cli.Command{ logrus.Debugf("running host preflights") replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) - - urlSlices := strings.Split(c.Args().Get(0), ":") - if len(urlSlices) != 2 { - return fmt.Errorf("unable to get port from url %s", c.Args().Get(0)) - } - adminConsolePort, err := strconv.Atoi(urlSlices[1]) - if err != nil { - return fmt.Errorf("unable to convert port to int: %w", err) - } - - localArtifactMirrorPort := defaults.LocalArtifactMirrorPort - if jcmd.InstallationSpec.LocalArtifactMirror != nil { - localArtifactMirrorPort = jcmd.InstallationSpec.LocalArtifactMirror.Port - } - - if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy, adminConsolePort, localArtifactMirrorPort); err != nil { + if err := RunHostPreflights(c, provider, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil { if err == ErrPreflightsHaveFail { return ErrNothingElseToAdd } diff --git a/cmd/embedded-cluster/provider.go b/cmd/embedded-cluster/provider.go new file mode 100644 index 000000000..7a08068f4 --- /dev/null +++ b/cmd/embedded-cluster/provider.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "fmt" + "os" + + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" +) + +// discoverKubeconfigPath finds the kubeconfig path from the k0s status command if it's available. +// Otherwise, it will use the data dir from the cli flag or the default path. +func discoverKubeconfigPath(ctx context.Context, runtimeConfig *ecv1beta1.RuntimeConfigSpec) (string, error) { + var kubeconfigPath string + + status, err := getK0sStatus(ctx) + if err == nil { + kubeconfigPath = status.Vars.AdminKubeConfigPath + } else { + // Use the data dir from the cli flag if it's set + if runtimeConfig.DataDir != "" { + provider := defaults.NewProviderFromRuntimeConfig(runtimeConfig) + kubeconfigPath = provider.PathToKubeConfig() + } else { + provider := defaults.NewProvider(ecv1beta1.DefaultDataDir) + kubeconfigPath = provider.PathToKubeConfig() + } + } + if _, err := os.Stat(kubeconfigPath); err != nil { + return "", fmt.Errorf("kubeconfig not found at %s", kubeconfigPath) + } + + return kubeconfigPath, nil +} + +// discoverBestProvider discovers the provider from the cluster (if it's up) and will fall back to +// the flag, the filesystem, or the default. +func discoverBestProvider(ctx context.Context, runtimeConfig *ecv1beta1.RuntimeConfigSpec) *defaults.Provider { + // It's possible that the cluster is not up + provider, err := getProviderFromCluster(ctx) + if err == nil { + return provider + } + + // Use the data dir from the data-dir flag if it's set + if runtimeConfig.DataDir != "" { + return defaults.NewProviderFromRuntimeConfig(runtimeConfig) + } + + // Otherwise, fall back to the filesystem + provider, err = defaults.NewProviderFromFilesystem() + if err == nil { + return provider + } + + // If we can't find a provider, use the default + return defaults.NewProvider(ecv1beta1.DefaultDataDir) +} + +// getProviderFromCluster finds the kubeconfig and discovers the provider from the cluster. If this +// is a prior version of EC, we will have to fall back to the filesystem. +func getProviderFromCluster(ctx context.Context) (*defaults.Provider, error) { + status, err := getK0sStatus(ctx) + if err != nil { + return nil, fmt.Errorf("unable to get k0s status: %w", err) + } + kubeconfigPath := status.Vars.AdminKubeConfigPath + + os.Setenv("KUBECONFIG", kubeconfigPath) + + // Discover the provider from the cluster + kcli, err := kubeutils.KubeClient() + if err != nil { + return nil, fmt.Errorf("unable to create kube client: %w", err) + } + + provider, err := defaults.NewProviderFromCluster(ctx, kcli) + if err != nil { + return nil, fmt.Errorf("unable to get config from cluster: %w", err) + } + return provider, nil +} diff --git a/cmd/embedded-cluster/reset.go b/cmd/embedded-cluster/reset.go index 80a10c61a..f472864c9 100644 --- a/cmd/embedded-cluster/reset.go +++ b/cmd/embedded-cluster/reset.go @@ -11,7 +11,6 @@ import ( "time" autopilot "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" "github.com/k0sproject/k0s/pkg/etcd" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -19,6 +18,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" @@ -41,21 +41,8 @@ type hostInfo struct { RoleName string } -type k0sStatus struct { - Role string `json:"Role"` - Vars k0sVars `json:"K0sVars"` - ClusterConfig v1beta1.ClusterConfig `json:"ClusterConfig"` -} - -type k0sVars struct { - KubeletAuthConfigPath string `json:"KubeletAuthConfigPath"` - CertRootDir string `json:"CertRootDir"` - EtcdCertDir string `json:"EtcdCertDir"` -} - var ( binName = defaults.BinaryName() - k0s = "/usr/local/bin/k0s" ) var haWarningMessage = "WARNING: High-availability clusters must maintain at least three controller nodes, but resetting this node will leave only two. This can lead to a loss of functionality and non-recoverable failures. You should re-add a third node as soon as possible." @@ -143,7 +130,7 @@ func (h *hostInfo) configureKubernetesClient() { func (h *hostInfo) getHostName() error { hostname, err := os.Hostname() if err != nil { - return nil + return fmt.Errorf("unable to get hostname: %w", err) } h.Hostname = hostname return nil @@ -243,12 +230,12 @@ func (h *hostInfo) leaveEtcdcluster() error { } // stopK0s attempts to stop the k0s service -func stopAndResetK0s() error { +func stopAndResetK0s(dataDir string) error { out, err := exec.Command(k0s, "stop").CombinedOutput() if err != nil { return fmt.Errorf("could not stop k0s service: %w, %s", err, string(out)) } - out, err = exec.Command(k0s, "reset").CombinedOutput() + out, err = exec.Command(k0s, "reset", "--data-dir", dataDir).CombinedOutput() if err != nil { return fmt.Errorf("could not reset k0s: %w, %s", err, string(out)) } @@ -256,34 +243,32 @@ func stopAndResetK0s() error { } // newHostInfo returns a populated hostInfo struct -func newHostInfo(c *cli.Context) (hostInfo, error) { +func newHostInfo(ctx context.Context) (hostInfo, error) { currentHost := hostInfo{} // populate hostname err := currentHost.getHostName() if err != nil { - currentHost.KclientError = fmt.Errorf("client not initialized") + err = fmt.Errorf("unable to get hostname: %w", err) + currentHost.KclientError = err return currentHost, err } // get k0s status json - out, err := exec.Command(k0s, "status", "-o", "json").Output() + status, err := getK0sStatus(ctx) if err != nil { - currentHost.KclientError = fmt.Errorf("client not initialized") - return currentHost, err - } - err = json.Unmarshal(out, ¤tHost.Status) - if err != nil { - currentHost.KclientError = fmt.Errorf("client not initialized") + err := fmt.Errorf("client not initialized") + currentHost.KclientError = err return currentHost, err } + currentHost.Status = *status currentHost.RoleName = currentHost.Status.Role // set up kube client currentHost.configureKubernetesClient() // fetch node object - currentHost.getNodeObject(c.Context) + currentHost.getNodeObject(ctx) // control plane only stuff if currentHost.Status.Role == "controller" { // fetch controlNode - currentHost.getControlNodeObject(c.Context) + currentHost.getControlNodeObject(ctx) } // try and get custom role name from the node labels labels := currentHost.Node.GetLabels() @@ -311,8 +296,8 @@ func checkErrPrompt(c *cli.Context, err error) bool { // maybePrintHAWarning prints a warning message when the user is running a reset a node // in a high availability cluster and there are only 3 control nodes. -func maybePrintHAWarning(c *cli.Context) error { - kubeconfig := defaults.PathToKubeConfig() +func maybePrintHAWarning(ctx context.Context, provider *defaults.Provider) error { + kubeconfig := provider.PathToKubeConfig() if _, err := os.Stat(kubeconfig); err != nil { return nil } @@ -323,7 +308,7 @@ func maybePrintHAWarning(c *cli.Context) error { return fmt.Errorf("unable to create kube client: %w", err) } - if in, err := kubeutils.GetLatestInstallation(c.Context, kubecli); err != nil { + if in, err := kubeutils.GetLatestInstallation(ctx, kubecli); err != nil { if errors.Is(err, kubeutils.ErrNoInstallations{}) { return nil // no installations found, not an HA cluster - just an incomplete install } @@ -333,7 +318,7 @@ func maybePrintHAWarning(c *cli.Context) error { return nil } - ncps, err := kubeutils.NumOfControlPlaneNodes(c.Context, kubecli) + ncps, err := kubeutils.NumOfControlPlaneNodes(ctx, kubecli) if err != nil { return fmt.Errorf("unable to check control plane nodes: %w", err) } @@ -344,158 +329,179 @@ func maybePrintHAWarning(c *cli.Context) error { return nil } -var resetCommand = &cli.Command{ - Name: "reset", - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("reset command must be run as root") - } - return nil - }, - Args: false, - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "force", - Aliases: []string{"f"}, - Usage: "Ignore errors encountered when resetting the node (implies --no-prompt)", - Value: false, +func resetCommand() *cli.Command { + runtimeConfig := ecv1beta1.GetDefaultRuntimeConfig() + + return &cli.Command{ + Name: "reset", + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("reset command must be run as root") + } + return nil }, - &cli.BoolFlag{ - Name: "no-prompt", - Usage: "Disable interactive prompts", - Value: false, + Args: false, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "force", + Aliases: []string{"f"}, + Usage: "Ignore errors encountered when resetting the node (implies --no-prompt)", + Value: false, + }, + &cli.BoolFlag{ + Name: "no-prompt", + Usage: "Disable interactive prompts", + Value: false, + }, + getDataDirFlag(runtimeConfig), }, - }, - Usage: fmt.Sprintf("Remove %s from the current node", binName), - Action: func(c *cli.Context) error { - if err := maybePrintHAWarning(c); err != nil && !c.Bool("force") { - return err - } + Usage: fmt.Sprintf("Remove %s from the current node", binName), + Action: func(c *cli.Context) error { + provider := discoverBestProvider(c.Context, runtimeConfig) + os.Setenv("KUBECONFIG", provider.PathToKubeConfig()) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) - logrus.Info("This will remove this node from the cluster and completely reset it, removing all data stored on the node.") - logrus.Info("This node will also reboot. Do not reset another node until this is complete.") - if !c.Bool("force") && !c.Bool("no-prompt") && !prompts.New().Confirm("Do you want to continue?", false) { - return fmt.Errorf("Aborting") - } - - // populate options struct with host information - currentHost, err := newHostInfo(c) - if !checkErrPrompt(c, err) { - return err - } - - // basic check to see if it's safe to remove this node from the cluster - if currentHost.Status.Role == "controller" { - safeToRemove, reason, err := currentHost.checkResetSafety(c) - if !checkErrPrompt(c, err) { + if err := maybePrintHAWarning(c.Context, provider); err != nil && !c.Bool("force") { return err } - if !safeToRemove { - return fmt.Errorf("%s\nRun reset command with --force to ignore this.", reason) - } - } - var numControllerNodes int - if currentHost.KclientError == nil { - numControllerNodes, _ = kubeutils.NumOfControlPlaneNodes(c.Context, currentHost.Kclient) - } - // do not drain node if this is the only controller node in the cluster - // if there is an error (numControllerNodes == 0), drain anyway to be safe - if currentHost.Status.Role != "controller" || numControllerNodes != 1 { - logrus.Info("Draining node...") - err = currentHost.drainNode() - if !checkErrPrompt(c, err) { - return err + logrus.Info("This will remove this node from the cluster and completely reset it, removing all data stored on the node.") + logrus.Info("This node will also reboot. Do not reset another node until this is complete.") + if !c.Bool("force") && !c.Bool("no-prompt") && !prompts.New().Confirm("Do you want to continue?", false) { + return fmt.Errorf("Aborting") } - // remove node from cluster - logrus.Info("Removing node from cluster...") - removeCtx, removeCancel := context.WithTimeout(c.Context, time.Minute) - defer removeCancel() - err = currentHost.deleteNode(removeCtx) + // populate options struct with host information + currentHost, err := newHostInfo(c.Context) if !checkErrPrompt(c, err) { return err } - // controller pre-reset + // basic check to see if it's safe to remove this node from the cluster if currentHost.Status.Role == "controller" { + safeToRemove, reason, err := currentHost.checkResetSafety(c) + if !checkErrPrompt(c, err) { + return err + } + if !safeToRemove { + return fmt.Errorf("%s\nRun reset command with --force to ignore this.", reason) + } + } - // delete controlNode object from cluster - deleteControlCtx, deleteCancel := context.WithTimeout(c.Context, time.Minute) - defer deleteCancel() - err := currentHost.deleteControlNode(deleteControlCtx) + var numControllerNodes int + if currentHost.KclientError == nil { + numControllerNodes, _ = kubeutils.NumOfControlPlaneNodes(c.Context, currentHost.Kclient) + } + // do not drain node if this is the only controller node in the cluster + // if there is an error (numControllerNodes == 0), drain anyway to be safe + if currentHost.Status.Role != "controller" || numControllerNodes != 1 { + logrus.Info("Draining node...") + err = currentHost.drainNode() if !checkErrPrompt(c, err) { return err } - // try and leave etcd cluster - err = currentHost.leaveEtcdcluster() + // remove node from cluster + logrus.Info("Removing node from cluster...") + removeCtx, removeCancel := context.WithTimeout(c.Context, time.Minute) + defer removeCancel() + err = currentHost.deleteNode(removeCtx) if !checkErrPrompt(c, err) { return err } - } - } + // controller pre-reset + if currentHost.Status.Role == "controller" { - // reset - logrus.Infof("Resetting node...") - err = stopAndResetK0s() - if !checkErrPrompt(c, err) { - return err - } + // delete controlNode object from cluster + deleteControlCtx, deleteCancel := context.WithTimeout(c.Context, time.Minute) + defer deleteCancel() + err := currentHost.deleteControlNode(deleteControlCtx) + if !checkErrPrompt(c, err) { + return err + } - if err := helpers.RemoveAll(defaults.PathToK0sConfig()); err != nil { - return fmt.Errorf("failed to remove k0s config: %w", err) - } + // try and leave etcd cluster + err = currentHost.leaveEtcdcluster() + if !checkErrPrompt(c, err) { + return err + } + + } + } - lamPath := "/etc/systemd/system/local-artifact-mirror.service" - if _, err := os.Stat(lamPath); err == nil { - if _, err := helpers.RunCommand("systemctl", "stop", "local-artifact-mirror"); err != nil { + // reset + logrus.Infof("Resetting node...") + err = stopAndResetK0s(provider.EmbeddedClusterK0sSubDir()) + if !checkErrPrompt(c, err) { return err } - } - if err := helpers.RemoveAll(lamPath); err != nil { - return fmt.Errorf("failed to remove local-artifact-mirror path: %w", err) - } - proxyControllerPath := "/etc/systemd/system/k0scontroller.service.d" - if err := helpers.RemoveAll(proxyControllerPath); err != nil { - return fmt.Errorf("failed to remove proxy controller path: %w", err) - } + if err := helpers.RemoveAll(defaults.PathToK0sConfig()); err != nil { + return fmt.Errorf("failed to remove k0s config: %w", err) + } - proxyWorkerPath := "/etc/systemd/system/k0sworker.service.d" - if err := helpers.RemoveAll(proxyWorkerPath); err != nil { - return fmt.Errorf("failed to remove proxy worker path: %w", err) - } + lamPath := "/etc/systemd/system/local-artifact-mirror.service" + if _, err := os.Stat(lamPath); err == nil { + if _, err := helpers.RunCommand("systemctl", "stop", "local-artifact-mirror"); err != nil { + return err + } + } + if err := helpers.RemoveAll(lamPath); err != nil { + return fmt.Errorf("failed to remove local-artifact-mirror service file: %w", err) + } - if err := helpers.RemoveAll(defaults.EmbeddedClusterHomeDirectory()); err != nil { - return fmt.Errorf("failed to remove embedded cluster directory: %w", err) - } + lamPathD := "/etc/systemd/system/local-artifact-mirror.service.d" + if err := helpers.RemoveAll(lamPathD); err != nil { + return fmt.Errorf("failed to remove local-artifact-mirror config directory: %w", err) + } - if err := helpers.RemoveAll(defaults.PathToK0sContainerdConfig()); err != nil { - return fmt.Errorf("failed to remove containerd config: %w", err) - } + proxyControllerPath := "/etc/systemd/system/k0scontroller.service.d" + if err := helpers.RemoveAll(proxyControllerPath); err != nil { + return fmt.Errorf("failed to remove proxy controller config directory: %w", err) + } - if err := helpers.RemoveAll(systemdUnitFileName()); err != nil { - return fmt.Errorf("failed to remove systemd unit file: %w", err) - } + proxyWorkerPath := "/etc/systemd/system/k0sworker.service.d" + if err := helpers.RemoveAll(proxyWorkerPath); err != nil { + return fmt.Errorf("failed to remove proxy worker config directory: %w", err) + } - if err := helpers.RemoveAll("/var/openebs"); err != nil { - return fmt.Errorf("failed to remove openebs storage: %w", err) - } + // Now that k0s is nested under the data directory, we see the following error in the + // dev environment because k0s is mounted in the docker container: + // "failed to remove embedded cluster directory: remove k0s: unlinkat /var/lib/embedded-cluster/k0s: device or resource busy" + if err := helpers.RemoveAll(provider.EmbeddedClusterHomeDirectory()); err != nil { + logrus.Debugf("Failed to remove embedded cluster directory: %v", err) + } - if err := helpers.RemoveAll("/etc/NetworkManager/conf.d/embedded-cluster.conf"); err != nil { - return fmt.Errorf("failed to remove NetworkManager configuration: %w", err) - } + if err := helpers.RemoveAll(defaults.EmbeddedClusterLogsSubDir()); err != nil { + return fmt.Errorf("failed to remove logs directory: %w", err) + } - if err := helpers.RemoveAll("/usr/local/bin/k0s"); err != nil { - return fmt.Errorf("failed to remove k0s binary: %w", err) - } + if err := helpers.RemoveAll(defaults.PathToK0sContainerdConfig()); err != nil { + return fmt.Errorf("failed to remove containerd config: %w", err) + } - if _, err := exec.Command("reboot").Output(); err != nil { - return err - } + if err := helpers.RemoveAll(systemdUnitFileName()); err != nil { + return fmt.Errorf("failed to remove systemd unit file: %w", err) + } - return nil - }, + if err := helpers.RemoveAll(provider.EmbeddedClusterOpenEBSLocalSubDir()); err != nil { + return fmt.Errorf("failed to remove openebs storage: %w", err) + } + + if err := helpers.RemoveAll("/etc/NetworkManager/conf.d/embedded-cluster.conf"); err != nil { + return fmt.Errorf("failed to remove NetworkManager configuration: %w", err) + } + + if err := helpers.RemoveAll("/usr/local/bin/k0s"); err != nil { + return fmt.Errorf("failed to remove k0s binary: %w", err) + } + + if _, err := exec.Command("reboot").Output(); err != nil { + return err + } + + return nil + }, + } } diff --git a/cmd/embedded-cluster/restore.go b/cmd/embedded-cluster/restore.go index 8cc1a0e32..3794628fc 100644 --- a/cmd/embedded-cluster/restore.go +++ b/cmd/embedded-cluster/restore.go @@ -202,7 +202,7 @@ func resetECRestoreState(ctx context.Context) error { // It returns an error if a backup is defined in the restore state but: // - is not found by Velero anymore. // - is not restorable by the current binary. -func getBackupFromRestoreState(ctx context.Context, isAirgap bool) (*velerov1.Backup, error) { +func getBackupFromRestoreState(ctx context.Context, provider *defaults.Provider, isAirgap bool) (*velerov1.Backup, error) { kcli, err := kubeutils.KubeClient() if err != nil { return nil, fmt.Errorf("unable to create kube client: %w", err) @@ -244,7 +244,7 @@ func getBackupFromRestoreState(ctx context.Context, isAirgap bool) (*velerov1.Ba return nil, fmt.Errorf("unable to get k0s config from disk: %w", err) } - if restorable, reason := isBackupRestorable(backup, rel, isAirgap, k0sCfg); !restorable { + if restorable, reason := isBackupRestorable(backup, provider, rel, isAirgap, k0sCfg); !restorable { return nil, fmt.Errorf("backup %q %s", backup.Name, reason) } return backup, nil @@ -305,18 +305,18 @@ func validateS3BackupStore(s *s3BackupStore) error { // RunHostPreflightsForRestore runs the host preflights we found embedded in the binary // on all configured hosts. We attempt to read HostPreflights from all the // embedded Helm Charts for restore operations. -func RunHostPreflightsForRestore(c *cli.Context, applier *addons.Applier, proxy *ecv1beta1.ProxySpec) error { +func RunHostPreflightsForRestore(c *cli.Context, provider *defaults.Provider, applier *addons.Applier, proxy *ecv1beta1.ProxySpec) error { hpf, err := applier.HostPreflightsForRestore() if err != nil { return fmt.Errorf("unable to read host preflights: %w", err) } - return runHostPreflights(c, hpf, proxy) + return runHostPreflights(c, provider, hpf, proxy) } // ensureK0sConfigForRestore creates a new k0s.yaml configuration file for restore operations. // The file is saved in the global location (as returned by defaults.PathToK0sConfig()). // If a file already sits there, this function returns an error. -func ensureK0sConfigForRestore(c *cli.Context, applier *addons.Applier) (*k0sv1beta1.ClusterConfig, error) { +func ensureK0sConfigForRestore(c *cli.Context, provider *defaults.Provider, applier *addons.Applier) (*k0sv1beta1.ClusterConfig, error) { cfgpath := defaults.PathToK0sConfig() if _, err := os.Stat(cfgpath); err == nil { return nil, fmt.Errorf("configuration file already exists") @@ -342,7 +342,7 @@ func ensureK0sConfigForRestore(c *cli.Context, applier *addons.Applier) (*k0sv1b } if c.String("airgap-bundle") != "" { // update the k0s config to install with airgap - airgap.RemapHelm(cfg) + airgap.RemapHelm(provider, cfg) airgap.SetAirgapConfig(cfg) } data, err := k8syaml.Marshal(cfg) @@ -365,7 +365,7 @@ func runOutroForRestore(c *cli.Context, applier *addons.Applier, cfg *k0sv1beta1 return applier.OutroForRestore(c.Context, cfg) } -func isBackupRestorable(backup *velerov1.Backup, rel *release.ChannelRelease, isAirgap bool, k0sCfg *k0sv1beta1.ClusterConfig) (bool, string) { +func isBackupRestorable(backup *velerov1.Backup, provider *defaults.Provider, rel *release.ChannelRelease, isAirgap bool, k0sCfg *k0sv1beta1.ClusterConfig) (bool, string) { if backup.Annotations["kots.io/embedded-cluster"] != "true" { return false, "is not an embedded cluster backup" } @@ -421,6 +421,11 @@ func isBackupRestorable(backup *velerov1.Backup, rel *release.ChannelRelease, is } } } + + if v := backup.Annotations["kots.io/embedded-cluster-data-dir"]; v != "" && v != provider.EmbeddedClusterHomeDirectory() { + return false, fmt.Sprintf("has a different data directory than the current cluster. Please rerun with '--data-dir %s'.", v) + } + return true, "" } @@ -434,7 +439,7 @@ func isHighAvailabilityBackup(backup *velerov1.Backup) (bool, error) { // waitForBackups waits for backups to become available. // It returns a list of restorable backups, or an error if none are found. -func waitForBackups(ctx context.Context, isAirgap bool) ([]velerov1.Backup, error) { +func waitForBackups(ctx context.Context, provider *defaults.Provider, isAirgap bool) ([]velerov1.Backup, error) { loading := spinner.Start() defer loading.Close() loading.Infof("Waiting for backups to become available") @@ -479,7 +484,7 @@ func waitForBackups(ctx context.Context, isAirgap bool) ([]velerov1.Backup, erro invalidReasons := []string{} for _, backup := range backupList.Items { - restorable, reason := isBackupRestorable(&backup, rel, isAirgap, k0sCfg) + restorable, reason := isBackupRestorable(&backup, provider, rel, isAirgap, k0sCfg) if restorable { validBackups = append(validBackups, backup) } else { @@ -806,25 +811,16 @@ func restoreFromBackup(ctx context.Context, backup *velerov1.Backup, drComponent } // waitForAdditionalNodes waits for for user to add additional nodes to the cluster. -func waitForAdditionalNodes(ctx context.Context, highAvailability bool, networkInterface string) error { +func waitForAdditionalNodes(ctx context.Context, provider *defaults.Provider, highAvailability bool, networkInterface string) error { kcli, err := kubeutils.KubeClient() if err != nil { return fmt.Errorf("unable to create kube client: %w", err) } - in, err := kubeutils.GetLatestInstallation(ctx, kcli) - if err != nil { - return fmt.Errorf("unable to get latest installation: %w", err) - } - adminConsolePort := defaults.AdminConsolePort - if in.Spec.AdminConsole != nil && in.Spec.AdminConsole.Port > 0 { - adminConsolePort = in.Spec.AdminConsole.Port - } - successColor := "\033[32m" colorReset := "\033[0m" joinNodesMsg := fmt.Sprintf("\nVisit the Admin Console if you need to add nodes to the cluster: %s%s%s\n", - successColor, adminconsole.GetURL(networkInterface, adminConsolePort), colorReset, + successColor, adminconsole.GetURL(networkInterface, provider.AdminConsolePort()), colorReset, ) logrus.Info(joinNodesMsg) @@ -858,23 +854,23 @@ func waitForAdditionalNodes(ctx context.Context, highAvailability bool, networkI return nil } -func installAndWaitForRestoredK0sNode(c *cli.Context, applier *addons.Applier) (*k0sv1beta1.ClusterConfig, error) { +func installAndWaitForRestoredK0sNode(c *cli.Context, provider *defaults.Provider, applier *addons.Applier) (*k0sv1beta1.ClusterConfig, error) { loading := spinner.Start() defer loading.Close() loading.Infof("Installing %s node", binName) logrus.Debugf("creating k0s configuration file") - cfg, err := ensureK0sConfigForRestore(c, applier) + cfg, err := ensureK0sConfigForRestore(c, provider, applier) if err != nil { return nil, fmt.Errorf("unable to create config file: %w", err) } proxy := getProxySpecFromFlags(c) logrus.Debugf("creating systemd unit files") - if err := createSystemdUnitFiles(false, proxy, applier.GetLocalArtifactMirrorPort()); err != nil { + if err := createSystemdUnitFiles(provider, false, proxy); err != nil { return nil, fmt.Errorf("unable to create systemd unit files: %w", err) } logrus.Debugf("installing k0s") - if err := installK0s(c); err != nil { + if err := installK0s(c, provider); err != nil { return nil, fmt.Errorf("unable update cluster: %w", err) } loading.Infof("Waiting for %s node to be ready", binName) @@ -886,370 +882,395 @@ func installAndWaitForRestoredK0sNode(c *cli.Context, applier *addons.Applier) ( return cfg, nil } -var restoreCommand = &cli.Command{ - Name: "restore", - Usage: fmt.Sprintf("Restore a %s cluster", binName), - Flags: withProxyFlags(withSubnetCIDRFlags( - []cli.Flag{ - &cli.StringFlag{ - Name: "airgap-bundle", - Usage: "Path to the air gap bundle. If set, the restore will complete without internet access.", - Hidden: true, - }, - &cli.StringFlag{ - Name: "network-interface", - Usage: "The network interface to use for the cluster", - Value: "", - }, - &cli.BoolFlag{ - Name: "no-prompt", - Usage: "Disable interactive prompts.", - Value: false, - }, - &cli.BoolFlag{ - Name: "skip-host-preflights", - Usage: "Skip host preflight checks. This is not recommended.", - Value: false, - }, - &cli.BoolFlag{ - Name: "skip-store-validation", - Usage: "Skip validation of the backup store. This is not recommended.", - Value: false, - Hidden: true, - }, - &cli.StringFlag{ - Name: "local-artifact-mirror-port", - Usage: "Port on which the Local Artifact Mirror will be served. If left empty, the port will be retrieved from the snapshot.", - // DefaultText: strconv.Itoa(defaults.LocalArtifactMirrorPort), - Hidden: false, +func restoreCommand() *cli.Command { + runtimeConfig := ecv1beta1.GetDefaultRuntimeConfig() + + return &cli.Command{ + Name: "restore", + Usage: fmt.Sprintf("Restore a %s cluster", binName), + Flags: withProxyFlags(withSubnetCIDRFlags( + []cli.Flag{ + &cli.StringFlag{ + Name: "airgap-bundle", + Usage: "Path to the air gap bundle. If set, the restore will complete without internet access.", + Hidden: true, + }, + &cli.StringFlag{ + Name: "network-interface", + Usage: "The network interface to use for the cluster", + Value: "", + }, + &cli.BoolFlag{ + Name: "no-prompt", + Usage: "Disable interactive prompts.", + Value: false, + }, + &cli.BoolFlag{ + Name: "skip-host-preflights", + Usage: "Skip host preflight checks. This is not recommended.", + Value: false, + }, + &cli.BoolFlag{ + Name: "skip-store-validation", + Usage: "Skip validation of the backup store. This is not recommended.", + Value: false, + Hidden: true, + }, + &cli.StringFlag{ + Name: "local-artifact-mirror-port", + Usage: "Port on which the Local Artifact Mirror will be served. If left empty, the port will be retrieved from the snapshot.", + // DefaultText: strconv.Itoa(ecv1beta1.DefaultLocalArtifactMirrorPort), + Hidden: false, + }, + getDataDirFlag(runtimeConfig), }, - }, - )), - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("restore command must be run as root") - } - os.Setenv("KUBECONFIG", defaults.PathToKubeConfig()) - return nil - }, - Action: func(c *cli.Context) error { - proxy := getProxySpecFromFlags(c) - setProxyEnv(proxy) - - logrus.Debugf("getting restore state") - state := getECRestoreState(c.Context) - logrus.Debugf("restore state is: %q", state) - - if state != ecRestoreStateNew { - shouldResume := prompts.New().Confirm("A previous restore operation was detected. Would you like to resume?", true) - logrus.Info("") - if !shouldResume { - state = ecRestoreStateNew - } - } - if c.String("airgap-bundle") != "" { - logrus.Debugf("checking airgap bundle matches binary") - if err := checkAirgapMatches(c); err != nil { - return err // we want the user to see the error message without a prefix - } - } - - // if the user wants to resume, check if a backup has already been picked. - var backupToRestore *velerov1.Backup - if state != ecRestoreStateNew { - logrus.Debugf("getting backup from restore state") - var err error - backupToRestore, err = getBackupFromRestoreState(c.Context, c.String("airgap-bundle") != "") - if err != nil { - return fmt.Errorf("unable to resume: %w", err) - } - if backupToRestore != nil { - completionTimestamp := backupToRestore.Status.CompletionTimestamp.Time.Format("2006-01-02 15:04:05 UTC") - logrus.Infof("Resuming restore from backup %q (%s)\n", backupToRestore.Name, completionTimestamp) + )), + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("restore command must be run as root") } - } + return nil + }, + Action: func(c *cli.Context) error { + provider := defaults.NewProviderFromRuntimeConfig(runtimeConfig) + os.Setenv("KUBECONFIG", provider.PathToKubeConfig()) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) - applier, err := getAddonsApplier(c, "", proxy) - if err != nil { - return err - } + defer tryRemoveTmpDirContents(provider) - switch state { - case ecRestoreStateNew: - logrus.Debugf("checking if %s is already installed", binName) - if installed, err := isAlreadyInstalled(); err != nil { - return err - } else if installed { - logrus.Errorf("An installation has been detected on this machine.") - logrus.Infof("If you want to restore you need to remove the existing installation") - logrus.Infof("first. You can do this by running the following command:") - logrus.Infof("\n sudo ./%s reset\n", binName) - return ErrNothingElseToAdd - } + proxy := getProxySpecFromFlags(c) + setProxyEnv(proxy) - logrus.Infof("You'll be guided through the process of restoring %s from a backup.\n", binName) - logrus.Info("Enter information to configure access to your backup storage location.\n") - s3Store := newS3BackupStore() + logrus.Debugf("getting restore state") + state := getECRestoreState(c.Context) + logrus.Debugf("restore state is: %q", state) - if !c.Bool("skip-store-validation") { - logrus.Debugf("validating backup store configuration") - if err := validateS3BackupStore(s3Store); err != nil { - return fmt.Errorf("unable to validate backup store: %w", err) + if state != ecRestoreStateNew { + shouldResume := prompts.New().Confirm("A previous restore operation was detected. Would you like to resume?", true) + logrus.Info("") + if !shouldResume { + state = ecRestoreStateNew } } - - logrus.Debugf("configuring network manager") - if err := configureNetworkManager(c); err != nil { - return fmt.Errorf("unable to configure network manager: %w", err) - } - logrus.Debugf("materializing binaries") - if err := materializeFiles(c); err != nil { - return fmt.Errorf("unable to materialize binaries: %w", err) - } - logrus.Debugf("running host preflights") - if err := RunHostPreflightsForRestore(c, applier, proxy); err != nil { - return fmt.Errorf("unable to finish preflight checks: %w", err) + if c.String("airgap-bundle") != "" { + logrus.Debugf("checking airgap bundle matches binary") + if err := checkAirgapMatches(c); err != nil { + return err // we want the user to see the error message without a prefix + } } - cfg, err := installAndWaitForRestoredK0sNode(c, applier) - if err != nil { - return err + // if the user wants to resume, check if a backup has already been picked. + var backupToRestore *velerov1.Backup + if state != ecRestoreStateNew { + logrus.Debugf("getting backup from restore state") + var err error + backupToRestore, err = getBackupFromRestoreState(c.Context, provider, c.String("airgap-bundle") != "") + if err != nil { + return fmt.Errorf("unable to resume: %w", err) + } + if backupToRestore != nil { + completionTimestamp := backupToRestore.Status.CompletionTimestamp.Time.Format("2006-01-02 15:04:05 UTC") + logrus.Infof("Resuming restore from backup %q (%s)\n", backupToRestore.Name, completionTimestamp) + + runtimeConfig, err = overrideRuntimeConfigFromBackup(c, runtimeConfig, backupToRestore) + if err != nil { + return fmt.Errorf("unable to override runtime config from backup: %w", err) + } + } } - kcli, err := kubeutils.KubeClient() + // If the installation is available, we can further augment the runtime config from the + // installation. + rc, err := getRuntimeConfigFromInstallation(c) if err != nil { - return fmt.Errorf("unable to create kube client: %w", err) + logrus.Debugf( + "Unable to get runtime config from installation, this is expected if the installation is not yet available (restore state=%s): %v", + state, err, + ) + } else { + runtimeConfig = rc } - errCh := kubeutils.WaitForKubernetes(c.Context, kcli) - defer func() { - for len(errCh) > 0 { - err := <-errCh - logrus.Error(fmt.Errorf("infrastructure failed to become ready: %w", err)) - } - }() - logrus.Debugf("running outro") - if err := runOutroForRestore(c, applier, cfg); err != nil { - return fmt.Errorf("unable to run outro: %w", err) - } + provider = defaults.NewProviderFromRuntimeConfig(runtimeConfig) + os.Setenv("KUBECONFIG", provider.PathToKubeConfig()) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) - logrus.Debugf("configuring velero backup storage location") - if err := kotscli.VeleroConfigureOtherS3(kotscli.VeleroConfigureOtherS3Options{ - Endpoint: s3Store.endpoint, - Region: s3Store.region, - Bucket: s3Store.bucket, - Path: s3Store.prefix, - AccessKeyID: s3Store.accessKeyID, - SecretAccessKey: s3Store.secretAccessKey, - Namespace: defaults.KotsadmNamespace, - }); err != nil { - return err - } - fallthrough - - case ecRestoreStateConfirmBackup: - logrus.Debugf("setting restore state to %q", ecRestoreStateConfirmBackup) - if err := setECRestoreState(c.Context, ecRestoreStateConfirmBackup, ""); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) - } + defer tryRemoveTmpDirContents(provider) - logrus.Debugf("waiting for backups to become available") - backups, err := waitForBackups(c.Context, c.String("airgap-bundle") != "") + applier, err := getAddonsApplier(c, runtimeConfig, "", proxy) if err != nil { return err } - logrus.Debugf("picking backup to restore") - backupToRestore = pickBackupToRestore(backups) + switch state { + case ecRestoreStateNew: + logrus.Debugf("checking if %s is already installed", binName) + if installed, err := isAlreadyInstalled(); err != nil { + return err + } else if installed { + logrus.Errorf("An installation has been detected on this machine.") + logrus.Infof("If you want to restore you need to remove the existing installation") + logrus.Infof("first. You can do this by running the following command:") + logrus.Infof("\n sudo ./%s reset\n", binName) + return ErrNothingElseToAdd + } - logrus.Info("") - completionTimestamp := backupToRestore.Status.CompletionTimestamp.Time.Format("2006-01-02 15:04:05 UTC") - shouldRestore := prompts.New().Confirm(fmt.Sprintf("Restore from backup %q (%s)?", backupToRestore.Name, completionTimestamp), true) - logrus.Info("") - if !shouldRestore { - logrus.Infof("Aborting restore...") - return nil - } - fallthrough + logrus.Infof("You'll be guided through the process of restoring %s from a backup.\n", binName) + logrus.Info("Enter information to configure access to your backup storage location.\n") + s3Store := newS3BackupStore() - case ecRestoreStateRestoreECInstall: - logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreECInstall) - if err := setECRestoreState(c.Context, ecRestoreStateRestoreECInstall, backupToRestore.Name); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) - } - logrus.Debugf("restoring embedded cluster installation from backup %q", backupToRestore.Name) - if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentECInstall); err != nil { - return fmt.Errorf("unable to restore from backup: %w", err) - } - logrus.Debugf("updating local artifact mirror port %q", backupToRestore.Name) - if err := restoreReconcileLocalArtifactMirrorPort(c, backupToRestore); err != nil { - return fmt.Errorf("unable to update local artifact mirror port: %w", err) - } - fallthrough + if !c.Bool("skip-store-validation") { + logrus.Debugf("validating backup store configuration") + if err := validateS3BackupStore(s3Store); err != nil { + return fmt.Errorf("unable to validate backup store: %w", err) + } + } - case ecRestoreStateRestoreAdminConsole: - logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreAdminConsole) - if err := setECRestoreState(c.Context, ecRestoreStateRestoreAdminConsole, backupToRestore.Name); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) - } - logrus.Debugf("restoring admin console from backup %q", backupToRestore.Name) - if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentAdminConsole); err != nil { - return err - } - fallthrough + logrus.Debugf("configuring network manager") + if err := configureNetworkManager(c, provider); err != nil { + return fmt.Errorf("unable to configure network manager: %w", err) + } + logrus.Debugf("materializing binaries") + if err := materializeFiles(c, provider); err != nil { + return fmt.Errorf("unable to materialize binaries: %w", err) + } + logrus.Debugf("running host preflights") + if err := RunHostPreflightsForRestore(c, provider, applier, proxy); err != nil { + return fmt.Errorf("unable to finish preflight checks: %w", err) + } - case ecRestoreStateWaitForNodes: - logrus.Debugf("setting restore state to %q", ecRestoreStateWaitForNodes) - if err := setECRestoreState(c.Context, ecRestoreStateWaitForNodes, backupToRestore.Name); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) - } - logrus.Debugf("checking if backup is high availability") - highAvailability, err := isHighAvailabilityBackup(backupToRestore) - if err != nil { - return err - } - logrus.Debugf("waiting for additional nodes to be added") - if err := waitForAdditionalNodes(c.Context, highAvailability, c.String("network-interface")); err != nil { - return err - } - fallthrough + cfg, err := installAndWaitForRestoredK0sNode(c, provider, applier) + if err != nil { + return err + } - case ecRestoreStateRestoreSeaweedFS: - // only restore seaweedfs in case of high availability and airgap - highAvailability, err := isHighAvailabilityBackup(backupToRestore) - if err != nil { - return err - } - if highAvailability && c.String("airgap-bundle") != "" { - logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreSeaweedFS) - if err := setECRestoreState(c.Context, ecRestoreStateRestoreSeaweedFS, backupToRestore.Name); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) + kcli, err := kubeutils.KubeClient() + if err != nil { + return fmt.Errorf("unable to create kube client: %w", err) + } + errCh := kubeutils.WaitForKubernetes(c.Context, kcli) + defer func() { + for len(errCh) > 0 { + err := <-errCh + logrus.Error(fmt.Errorf("infrastructure failed to become ready: %w", err)) + } + }() + + logrus.Debugf("running outro") + if err := runOutroForRestore(c, applier, cfg); err != nil { + return fmt.Errorf("unable to run outro: %w", err) } - logrus.Debugf("restoring seaweedfs from backup %q", backupToRestore.Name) - if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentSeaweedFS); err != nil { + + logrus.Debugf("configuring velero backup storage location") + if err := kotscli.VeleroConfigureOtherS3(provider, kotscli.VeleroConfigureOtherS3Options{ + Endpoint: s3Store.endpoint, + Region: s3Store.region, + Bucket: s3Store.bucket, + Path: s3Store.prefix, + AccessKeyID: s3Store.accessKeyID, + SecretAccessKey: s3Store.secretAccessKey, + Namespace: defaults.KotsadmNamespace, + }); err != nil { return err } - } - fallthrough + fallthrough - case ecRestoreStateRestoreRegistry: - // only restore registry in case of airgap - if c.String("airgap-bundle") != "" { - logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreRegistry) - if err := setECRestoreState(c.Context, ecRestoreStateRestoreRegistry, backupToRestore.Name); err != nil { + case ecRestoreStateConfirmBackup: + logrus.Debugf("setting restore state to %q", ecRestoreStateConfirmBackup) + if err := setECRestoreState(c.Context, ecRestoreStateConfirmBackup, ""); err != nil { return fmt.Errorf("unable to set restore state: %w", err) } - logrus.Debugf("restoring embedded cluster registry from backup %q", backupToRestore.Name) - if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentRegistry); err != nil { + + logrus.Debugf("waiting for backups to become available") + backups, err := waitForBackups(c.Context, provider, c.String("airgap-bundle") != "") + if err != nil { return err } - registryAddress, ok := backupToRestore.Annotations["kots.io/embedded-registry"] - if !ok { - return fmt.Errorf("unable to read registry address from backup") + + logrus.Debugf("picking backup to restore") + backupToRestore = pickBackupToRestore(backups) + + logrus.Info("") + completionTimestamp := backupToRestore.Status.CompletionTimestamp.Time.Format("2006-01-02 15:04:05 UTC") + shouldRestore := prompts.New().Confirm(fmt.Sprintf("Restore from backup %q (%s)?", backupToRestore.Name, completionTimestamp), true) + logrus.Info("") + if !shouldRestore { + logrus.Infof("Aborting restore...") + return nil } - if err := airgap.AddInsecureRegistry(registryAddress); err != nil { - return fmt.Errorf("failed to add insecure registry: %w", err) + fallthrough + + case ecRestoreStateRestoreECInstall: + logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreECInstall) + if err := setECRestoreState(c.Context, ecRestoreStateRestoreECInstall, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) } - } - fallthrough + logrus.Debugf("restoring embedded cluster installation from backup %q", backupToRestore.Name) + if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentECInstall); err != nil { + return fmt.Errorf("unable to restore from backup: %w", err) + } + logrus.Debugf("updating installation from backup %q", backupToRestore.Name) + if err := restoreReconcileInstallationFromRuntimeConfig(c.Context, runtimeConfig); err != nil { + return fmt.Errorf("unable to update installation from backup: %w", err) + } + logrus.Debugf("updating local artifact mirror service from backup %q", backupToRestore.Name) + if err := updateLocalArtifactMirrorService(provider); err != nil { + return fmt.Errorf("unable to update local artifact mirror service from backup: %w", err) + } + fallthrough - case ecRestoreStateRestoreECO: - logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreECO) - if err := setECRestoreState(c.Context, ecRestoreStateRestoreECO, backupToRestore.Name); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) - } - logrus.Debugf("restoring embedded cluster operator from backup %q", backupToRestore.Name) - if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentECO); err != nil { - return err - } - fallthrough + case ecRestoreStateRestoreAdminConsole: + logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreAdminConsole) + if err := setECRestoreState(c.Context, ecRestoreStateRestoreAdminConsole, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) + } + logrus.Debugf("restoring admin console from backup %q", backupToRestore.Name) + if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentAdminConsole); err != nil { + return err + } + fallthrough - case ecRestoreStateRestoreApp: - logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreApp) - if err := setECRestoreState(c.Context, ecRestoreStateRestoreApp, backupToRestore.Name); err != nil { - return fmt.Errorf("unable to set restore state: %w", err) - } - logrus.Debugf("restoring app from backup %q", backupToRestore.Name) - if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentApp); err != nil { - return err - } - logrus.Debugf("resetting restore state") - if err := resetECRestoreState(c.Context); err != nil { - return fmt.Errorf("unable to reset restore state: %w", err) - } + case ecRestoreStateWaitForNodes: + logrus.Debugf("setting restore state to %q", ecRestoreStateWaitForNodes) + if err := setECRestoreState(c.Context, ecRestoreStateWaitForNodes, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) + } + logrus.Debugf("checking if backup is high availability") + highAvailability, err := isHighAvailabilityBackup(backupToRestore) + if err != nil { + return err + } + logrus.Debugf("waiting for additional nodes to be added") + if err := waitForAdditionalNodes(c.Context, provider, highAvailability, c.String("network-interface")); err != nil { + return err + } + fallthrough - default: - return fmt.Errorf("unknown restore state: %q", state) - } + case ecRestoreStateRestoreSeaweedFS: + // only restore seaweedfs in case of high availability and airgap + highAvailability, err := isHighAvailabilityBackup(backupToRestore) + if err != nil { + return err + } + if highAvailability && c.String("airgap-bundle") != "" { + logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreSeaweedFS) + if err := setECRestoreState(c.Context, ecRestoreStateRestoreSeaweedFS, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) + } + logrus.Debugf("restoring seaweedfs from backup %q", backupToRestore.Name) + if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentSeaweedFS); err != nil { + return err + } + } + fallthrough + + case ecRestoreStateRestoreRegistry: + // only restore registry in case of airgap + if c.String("airgap-bundle") != "" { + logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreRegistry) + if err := setECRestoreState(c.Context, ecRestoreStateRestoreRegistry, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) + } + logrus.Debugf("restoring embedded cluster registry from backup %q", backupToRestore.Name) + if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentRegistry); err != nil { + return err + } + registryAddress, ok := backupToRestore.Annotations["kots.io/embedded-registry"] + if !ok { + return fmt.Errorf("unable to read registry address from backup") + } + if err := airgap.AddInsecureRegistry(registryAddress); err != nil { + return fmt.Errorf("failed to add insecure registry: %w", err) + } + } + fallthrough - return nil - }, -} + case ecRestoreStateRestoreECO: + logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreECO) + if err := setECRestoreState(c.Context, ecRestoreStateRestoreECO, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) + } + logrus.Debugf("restoring embedded cluster operator from backup %q", backupToRestore.Name) + if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentECO); err != nil { + return err + } + fallthrough -// restoreReconcileLocalArtifactMirrorPort will set the local artifact mirror port in the -// installation if it was explicitly set using a flag, otherwise it will update the service to use -// the port from the installation. -func restoreReconcileLocalArtifactMirrorPort(c *cli.Context, backup *velerov1.Backup) error { - if c.IsSet("local-artifact-mirror-port") { - logrus.Debugf("updating local artifact mirror port from flag %q", backup.Name) - err := restoreReconcileLocalArtifactMirrorPortFromFlag(c) - if err != nil { - return fmt.Errorf("unable to update local artifact mirror port from flag: %w", err) - } - return nil - } + case ecRestoreStateRestoreApp: + logrus.Debugf("setting restore state to %q", ecRestoreStateRestoreApp) + if err := setECRestoreState(c.Context, ecRestoreStateRestoreApp, backupToRestore.Name); err != nil { + return fmt.Errorf("unable to set restore state: %w", err) + } + logrus.Debugf("restoring app from backup %q", backupToRestore.Name) + if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentApp); err != nil { + return err + } + logrus.Debugf("resetting restore state") + if err := resetECRestoreState(c.Context); err != nil { + return fmt.Errorf("unable to reset restore state: %w", err) + } - logrus.Debugf("updating local artifact mirror port from backup %q", backup.Name) - err := restoreReconcileLocalArtifactMirrorPortFromBackup(backup) - if err != nil { - return fmt.Errorf("unable to update local artifact mirror port from backup: %w", err) + default: + return fmt.Errorf("unknown restore state: %q", state) + } + + return nil + }, } - return nil } -// restoreReconcileLocalArtifactMirrorPortFromFlag will set the local artifact mirror port in the -// installation from the flag. -func restoreReconcileLocalArtifactMirrorPortFromFlag(c *cli.Context) error { +// restoreReconcileInstallationFromRuntimeConfig will update the installation to match the runtime +// config from the original installation. +func restoreReconcileInstallationFromRuntimeConfig(ctx context.Context, runtimeConfig *ecv1beta1.RuntimeConfigSpec) error { kcli, err := kubeutils.KubeClient() if err != nil { return fmt.Errorf("create kube client: %w", err) } - in, err := kubeutils.GetLatestInstallation(c.Context, kcli) + in, err := kubeutils.GetLatestInstallation(ctx, kcli) if err != nil { return fmt.Errorf("get latest installation: %w", err) } - if in.Spec.LocalArtifactMirror == nil { - in.Spec.LocalArtifactMirror = &ecv1beta1.LocalArtifactMirrorSpec{} - } - port, err := getLocalArtifactMirrorPortFromFlag(c) - if err != nil { - return fmt.Errorf("get local artifact mirror port: %w", err) - } - if in.Spec.LocalArtifactMirror.Port == port { - return nil + + if in.Spec.RuntimeConfig == nil { + in.Spec.RuntimeConfig = &ecv1beta1.RuntimeConfigSpec{} } - logrus.Debugf("updating local artifact mirror port from flag to %d on installation %q", port, in.Name) - in.Spec.LocalArtifactMirror.Port = port - if err := kcli.Update(c.Context, in); err != nil { + + // We allow the user to override the port with a flag to the restore command. + in.Spec.RuntimeConfig.LocalArtifactMirror.Port = runtimeConfig.LocalArtifactMirror.Port + + if err := kcli.Update(ctx, in); err != nil { return fmt.Errorf("update installation: %w", err) } return nil } -// restoreReconcileLocalArtifactMirrorPortFromBackup will update the service to use the port from -// the installation. -func restoreReconcileLocalArtifactMirrorPortFromBackup(backup *velerov1.Backup) error { - portStr := backup.Annotations["kots.io/embedded-cluster-local-artifact-mirror-port"] - if portStr == "" { - return nil +// overrideRuntimeConfigFromBackup will update the runtime config from the backup. These values may +// be used during the install and set in the Installation object via the +// restoreReconcileInstallationFromRuntimeConfig function. +func overrideRuntimeConfigFromBackup(c *cli.Context, runtimeConfig *ecv1beta1.RuntimeConfigSpec, backup *velerov1.Backup) (*ecv1beta1.RuntimeConfigSpec, error) { + if !c.IsSet("local-artifact-mirror-port") { + if val := backup.Annotations["kots.io/embedded-cluster-local-artifact-mirror-port"]; val != "" { + port, err := k8snet.ParsePort(val, false) + if err != nil { + return nil, fmt.Errorf("parse local artifact mirror port: %w", err) + } + logrus.Debugf("updating local artifact mirror port to %d from backup %q", port, backup.Name) + runtimeConfig.LocalArtifactMirror.Port = port + } } - port, err := k8snet.ParsePort(portStr, false) + + return runtimeConfig, nil +} + +// getRuntimeConfigFromInstallation returns the runtime config from the latest installation. +func getRuntimeConfigFromInstallation(c *cli.Context) (*ecv1beta1.RuntimeConfigSpec, error) { + kcli, err := kubeutils.KubeClient() if err != nil { - return fmt.Errorf("unable to parse local artifact mirror port from backup: %w", err) + return nil, fmt.Errorf("unable to create kube client: %w", err) } - logrus.Debugf("updating local artifact mirror port from backup to %d", port) - if err := updateLocalArtifactMirrorService(port); err != nil { - return fmt.Errorf("unable to update local artifact mirror service: %w", err) + in, err := kubeutils.GetLatestInstallation(c.Context, kcli) + if err != nil { + return nil, fmt.Errorf("unable to get latest installation: %w", err) } - return nil + return in.Spec.RuntimeConfig, nil } diff --git a/cmd/embedded-cluster/shell.go b/cmd/embedded-cluster/shell.go index 4d02663f0..94d058028 100644 --- a/cmd/embedded-cluster/shell.go +++ b/cmd/embedded-cluster/shell.go @@ -14,6 +14,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/term" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" ) @@ -35,87 +36,96 @@ func handleResize(ch chan os.Signal, tty *os.File) { } } -var shellCommand = &cli.Command{ - Name: "shell", - Usage: "Start a shell with access to the cluster", - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("shell command must be run as root") - } - return nil - }, - Action: func(c *cli.Context) error { - cfgpath := defaults.PathToKubeConfig() - if _, err := os.Stat(cfgpath); err != nil { - return fmt.Errorf("kubeconfig not found at %s", cfgpath) - } - - shpath := os.Getenv("SHELL") - if shpath == "" { - shpath = "/bin/bash" - } - - fmt.Printf(welcome, defaults.BinaryName()) - shell := exec.Command(shpath) - shell.Env = os.Environ() - - // get the current working directory - var err error - shell.Dir, err = os.Getwd() - if err != nil { - return fmt.Errorf("unable to get current working directory: %w", err) - } - - shellpty, err := pty.Start(shell) - if err != nil { - return fmt.Errorf("unable to start shell: %w", err) - } - - sigch := make(chan os.Signal, 1) - signal.Notify(sigch, syscall.SIGWINCH) - go handleResize(sigch, shellpty) - sigch <- syscall.SIGWINCH - state, err := term.MakeRaw(int(os.Stdin.Fd())) - if err != nil { - return fmt.Errorf("unable to make raw terminal: %w", err) - } - - defer func() { - signal.Stop(sigch) - close(sigch) - fd := int(os.Stdin.Fd()) - _ = term.Restore(fd, state) - }() - - kcpath := defaults.PathToKubeConfig() - config := fmt.Sprintf("export KUBECONFIG=%q\n", kcpath) - _, _ = shellpty.WriteString(config) - _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) - - bindir := defaults.EmbeddedClusterBinsSubDir() - config = fmt.Sprintf("export PATH=\"$PATH:%s\"\n", bindir) - _, _ = shellpty.WriteString(config) - _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) - - // if /etc/bash_completion is present enable kubectl auto completion. - if _, err := os.Stat("/etc/bash_completion"); err == nil { - config = fmt.Sprintf("source <(k0s completion %s)\n", filepath.Base(shpath)) - _, _ = shellpty.WriteString(config) - _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) - - comppath := defaults.PathToEmbeddedClusterBinary("kubectl_completion_bash.sh") - config = fmt.Sprintf("source <(cat %s)\n", comppath) +func shellCommand() *cli.Command { + runtimeConfig := ecv1beta1.GetDefaultRuntimeConfig() + + return &cli.Command{ + Name: "shell", + Usage: "Start a shell with access to the cluster", + Flags: []cli.Flag{ + getDataDirFlag(runtimeConfig), + }, + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("shell command must be run as root") + } + return nil + }, + Action: func(c *cli.Context) error { + provider := discoverBestProvider(c.Context, runtimeConfig) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + + if _, err := os.Stat(provider.PathToKubeConfig()); err != nil { + return fmt.Errorf("kubeconfig not found at %s", provider.PathToKubeConfig()) + } + + shpath := os.Getenv("SHELL") + if shpath == "" { + shpath = "/bin/bash" + } + + fmt.Printf(welcome, defaults.BinaryName()) + shell := exec.Command(shpath) + shell.Env = os.Environ() + + // get the current working directory + var err error + shell.Dir, err = os.Getwd() + if err != nil { + return fmt.Errorf("unable to get current working directory: %w", err) + } + + shellpty, err := pty.Start(shell) + if err != nil { + return fmt.Errorf("unable to start shell: %w", err) + } + + sigch := make(chan os.Signal, 1) + signal.Notify(sigch, syscall.SIGWINCH) + go handleResize(sigch, shellpty) + sigch <- syscall.SIGWINCH + state, err := term.MakeRaw(int(os.Stdin.Fd())) + if err != nil { + return fmt.Errorf("unable to make raw terminal: %w", err) + } + + defer func() { + signal.Stop(sigch) + close(sigch) + fd := int(os.Stdin.Fd()) + _ = term.Restore(fd, state) + }() + + kcpath := provider.PathToKubeConfig() + config := fmt.Sprintf("export KUBECONFIG=%q\n", kcpath) _, _ = shellpty.WriteString(config) _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) - config = "source /etc/bash_completion\n" + bindir := provider.EmbeddedClusterBinsSubDir() + config = fmt.Sprintf("export PATH=\"$PATH:%s\"\n", bindir) _, _ = shellpty.WriteString(config) _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) - } - go func() { _, _ = io.Copy(shellpty, os.Stdin) }() - go func() { _, _ = io.Copy(os.Stdout, shellpty) }() - _ = shell.Wait() - return nil - }, + // if /etc/bash_completion is present enable kubectl auto completion. + if _, err := os.Stat("/etc/bash_completion"); err == nil { + config = fmt.Sprintf("source <(k0s completion %s)\n", filepath.Base(shpath)) + _, _ = shellpty.WriteString(config) + _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) + + comppath := provider.PathToEmbeddedClusterBinary("kubectl_completion_bash.sh") + config = fmt.Sprintf("source <(cat %s)\n", comppath) + _, _ = shellpty.WriteString(config) + _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) + + config = "source /etc/bash_completion\n" + _, _ = shellpty.WriteString(config) + _, _ = io.CopyN(io.Discard, shellpty, int64(len(config)+1)) + } + + go func() { _, _ = io.Copy(shellpty, os.Stdin) }() + go func() { _, _ = io.Copy(os.Stdout, shellpty) }() + _ = shell.Wait() + return nil + }, + } } diff --git a/cmd/embedded-cluster/update.go b/cmd/embedded-cluster/update.go index cfccce2a0..6a33f0272 100644 --- a/cmd/embedded-cluster/update.go +++ b/cmd/embedded-cluster/update.go @@ -12,48 +12,58 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/release" ) -var updateCommand = &cli.Command{ - Name: "update", - Usage: fmt.Sprintf("Update %s", binName), - Hidden: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "airgap-bundle", - Usage: "Path to the airgap bundle", - Required: true, +func updateCommand() *cli.Command { + return &cli.Command{ + Name: "update", + Usage: fmt.Sprintf("Update %s", binName), + Hidden: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "airgap-bundle", + Usage: "Path to the airgap bundle", + Required: true, + }, }, - }, - Before: func(c *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("update command must be run as root") - } - os.Setenv("KUBECONFIG", defaults.PathToKubeConfig()) - return nil - }, - Action: func(c *cli.Context) error { - if c.String("airgap-bundle") != "" { - logrus.Debugf("checking airgap bundle matches binary") - if err := checkAirgapMatches(c); err != nil { - return err // we want the user to see the error message without a prefix + Before: func(c *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("update command must be run as root") } - } - - rel, err := release.GetChannelRelease() - if err != nil { - return fmt.Errorf("unable to get channel release: %w", err) - } - if rel == nil { - return fmt.Errorf("no channel release found") - } - - if err := kotscli.AirgapUpdate(kotscli.AirgapUpdateOptions{ - AppSlug: rel.AppSlug, - Namespace: defaults.KotsadmNamespace, - AirgapBundle: c.String("airgap-bundle"), - }); err != nil { - return err - } - - return nil - }, + return nil + }, + Action: func(c *cli.Context) error { + provider, err := getProviderFromCluster(c.Context) + if err != nil { + return err + } + os.Setenv("KUBECONFIG", provider.PathToKubeConfig()) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + + defer tryRemoveTmpDirContents(provider) + + if c.String("airgap-bundle") != "" { + logrus.Debugf("checking airgap bundle matches binary") + if err := checkAirgapMatches(c); err != nil { + return err // we want the user to see the error message without a prefix + } + } + + rel, err := release.GetChannelRelease() + if err != nil { + return fmt.Errorf("unable to get channel release: %w", err) + } + if rel == nil { + return fmt.Errorf("no channel release found") + } + + if err := kotscli.AirgapUpdate(provider, kotscli.AirgapUpdateOptions{ + AppSlug: rel.AppSlug, + Namespace: defaults.KotsadmNamespace, + AirgapBundle: c.String("airgap-bundle"), + }); err != nil { + return err + } + + return nil + }, + } } diff --git a/cmd/embedded-cluster/util.go b/cmd/embedded-cluster/util.go index f3ef286e6..3edaa3801 100644 --- a/cmd/embedded-cluster/util.go +++ b/cmd/embedded-cluster/util.go @@ -6,13 +6,14 @@ import ( "path/filepath" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/sirupsen/logrus" ) // createSystemdUnitFiles links the k0s systemd unit file. this also creates a new // systemd unit file for the local artifact mirror service. -func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec, localArtifactMirrorPort int) error { +func createSystemdUnitFiles(provider *defaults.Provider, isWorker bool, proxy *ecv1beta1.ProxySpec) error { dst := systemdUnitFileName() if _, err := os.Lstat(dst); err == nil { if err := os.Remove(dst); err != nil { @@ -36,7 +37,7 @@ func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec, localArti if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil { return fmt.Errorf("unable to get reload systemctl daemon: %w", err) } - if err := installAndEnableLocalArtifactMirror(localArtifactMirrorPort); err != nil { + if err := installAndEnableLocalArtifactMirror(provider); err != nil { return fmt.Errorf("unable to install and enable local artifact mirror: %w", err) } return nil @@ -68,3 +69,10 @@ Environment="NO_PROXY=%s"`, return nil } + +func tryRemoveTmpDirContents(provider *defaults.Provider) { + err := helpers.RemoveAll(provider.EmbeddedClusterTmpSubDir()) + if err != nil { + logrus.Debugf("failed to remove tmp dir contents: %v", err) + } +} diff --git a/cmd/embedded-cluster/version.go b/cmd/embedded-cluster/version.go index b89835de3..4ec8421ae 100644 --- a/cmd/embedded-cluster/version.go +++ b/cmd/embedded-cluster/version.go @@ -25,8 +25,11 @@ var versionCommand = &cli.Command{ listImagesCommand, }, Action: func(c *cli.Context) error { - opts := []addons.Option{addons.Quiet(), addons.WithoutPrompt()} - applierVersions, err := addons.NewApplier(opts...).Versions(config.AdditionalCharts()) + applierVersions, err := addons.NewApplier( + addons.WithoutPrompt(), + addons.OnlyDefaults(), + addons.Quiet(), + ).Versions(config.AdditionalCharts()) if err != nil { return fmt.Errorf("unable to get versions: %w", err) } diff --git a/cmd/local-artifact-mirror/pull.go b/cmd/local-artifact-mirror/pull.go index ce57d00b7..75a546355 100644 --- a/cmd/local-artifact-mirror/pull.go +++ b/cmd/local-artifact-mirror/pull.go @@ -9,7 +9,7 @@ import ( "os/exec" "path/filepath" - "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" @@ -18,7 +18,6 @@ import ( "github.com/urfave/cli/v2" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -64,7 +63,13 @@ var imagesCommand = &cli.Command{ } return nil }, + Flags: []cli.Flag{ + getPullDataDirFlag(), + }, Action: func(c *cli.Context) error { + provider := defaults.NewProvider(c.String("data-dir")) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + in, err := fetchAndValidateInstallation(c.Context, c.Args().First()) if err != nil { return err @@ -81,7 +86,7 @@ var imagesCommand = &cli.Command{ os.RemoveAll(location) }() - dst := filepath.Join(defaults.EmbeddedClusterImagesSubDir(), ImagesArtifactName) + dst := filepath.Join(provider.EmbeddedClusterImagesSubDir(), ImagesArtifactName) src := filepath.Join(location, ImagesArtifactName) logrus.Infof("%s > %s", src, dst) if err := helpers.MoveFile(src, dst); err != nil { @@ -110,7 +115,13 @@ var helmChartsCommand = &cli.Command{ } return nil }, + Flags: []cli.Flag{ + getPullDataDirFlag(), + }, Action: func(c *cli.Context) error { + provider := defaults.NewProvider(c.String("data-dir")) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + in, err := fetchAndValidateInstallation(c.Context, c.Args().First()) if err != nil { return err @@ -127,7 +138,7 @@ var helmChartsCommand = &cli.Command{ os.RemoveAll(location) }() - dst := defaults.EmbeddedClusterChartsSubDir() + dst := provider.EmbeddedClusterChartsSubDir() src := filepath.Join(location, HelmChartsArtifactName) logrus.Infof("uncompressing %s", src) if err := tgzutils.Decompress(src, dst); err != nil { @@ -156,7 +167,13 @@ var binariesCommand = &cli.Command{ } return nil }, + Flags: []cli.Flag{ + getPullDataDirFlag(), + }, Action: func(c *cli.Context) error { + provider := defaults.NewProvider(c.String("data-dir")) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + in, err := fetchAndValidateInstallation(c.Context, c.Args().First()) if err != nil { return err @@ -183,7 +200,7 @@ var binariesCommand = &cli.Command{ } out := bytes.NewBuffer(nil) - cmd := exec.Command(namedBin, "materialize") + cmd := exec.Command(namedBin, "materialize", "--data-dir", provider.EmbeddedClusterHomeDirectory()) cmd.Stdout = out cmd.Stderr = out @@ -201,10 +218,10 @@ var binariesCommand = &cli.Command{ // fetchAndValidateInstallation fetches an Installation object from its name or directly decodes it // and checks if it is valid for an airgap cluster deployment. -func fetchAndValidateInstallation(ctx context.Context, iname string) (*v1beta1.Installation, error) { +func fetchAndValidateInstallation(ctx context.Context, iname string) (*ecv1beta1.Installation, error) { in, err := decodeInstallation(ctx, iname) if err != nil { - in, err = fetchInstallationFromCluster(ctx, iname) + in, err = kubeutils.GetInstallation(ctx, kubecli, iname) if err != nil { return nil, err } @@ -219,21 +236,8 @@ func fetchAndValidateInstallation(ctx context.Context, iname string) (*v1beta1.I return in, nil } -// fetchInstallationFromCluster fetches an Installation object from the cluster. -func fetchInstallationFromCluster(ctx context.Context, iname string) (*v1beta1.Installation, error) { - logrus.Infof("reading installation from cluster %q", iname) - - nsn := types.NamespacedName{Name: iname} - var in v1beta1.Installation - if err := kubecli.Get(ctx, nsn, &in); err != nil { - return nil, fmt.Errorf("unable to get installation: %w", err) - } - - return &in, nil -} - // decodeInstallation decodes an Installation object from a string. -func decodeInstallation(ctx context.Context, data string) (*v1beta1.Installation, error) { +func decodeInstallation(ctx context.Context, data string) (*ecv1beta1.Installation, error) { logrus.Info("decoding installation") decoded, err := base64.StdEncoding.DecodeString(data) @@ -242,7 +246,7 @@ func decodeInstallation(ctx context.Context, data string) (*v1beta1.Installation } scheme := runtime.NewScheme() - err = v1beta1.AddToScheme(scheme) + err = ecv1beta1.AddToScheme(scheme) if err != nil { return nil, fmt.Errorf("add to scheme: %w", err) } @@ -253,10 +257,20 @@ func decodeInstallation(ctx context.Context, data string) (*v1beta1.Installation return nil, fmt.Errorf("decode: %w", err) } - in, ok := obj.(*v1beta1.Installation) + in, ok := obj.(*ecv1beta1.Installation) if !ok { return nil, fmt.Errorf("unexpected object type: %T", obj) } return in, nil } + +func getPullDataDirFlag() cli.Flag { + return &cli.StringFlag{ + Name: "data-dir", + Usage: "Path to the data directory", + Value: ecv1beta1.DefaultDataDir, + EnvVars: []string{"LOCAL_ARTIFACT_MIRROR_DATA_DIR"}, + Required: true, + } +} diff --git a/cmd/local-artifact-mirror/serve.go b/cmd/local-artifact-mirror/serve.go index d48143c86..ce59d1f81 100644 --- a/cmd/local-artifact-mirror/serve.go +++ b/cmd/local-artifact-mirror/serve.go @@ -12,22 +12,32 @@ import ( "syscall" "time" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/urfave/cli/v2" k8snet "k8s.io/utils/net" ) -// serveCommand starts a http server that serves files from the /var/lib/embedded-cluster -// directory. This server listen only on localhost and is used to serve files needed by -// the autopilot during an upgrade. +var ( + whitelistServeDirs = []string{"bin", "charts", "images"} +) + +// serveCommand starts a http server that serves files from the data directory. This server listen +// only on localhost and is used to serve files needed by the autopilot during an upgrade. var serveCommand = &cli.Command{ Name: "serve", - Usage: "Serve /var/lib/embedded-cluster files over HTTP", + Usage: "Serve files from the data directory over HTTP", Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "data-dir", + Usage: "Path to the data directory", + Value: ecv1beta1.DefaultDataDir, + EnvVars: []string{"LOCAL_ARTIFACT_MIRROR_DATA_DIR"}, + }, &cli.StringFlag{ Name: "port", Usage: "Port to listen on", - Value: strconv.Itoa(defaults.LocalArtifactMirrorPort), + Value: strconv.Itoa(ecv1beta1.DefaultLocalArtifactMirrorPort), EnvVars: []string{"LOCAL_ARTIFACT_MIRROR_PORT"}, }, }, @@ -38,29 +48,35 @@ var serveCommand = &cli.Command{ return nil }, Action: func(c *cli.Context) error { - dir := defaults.EmbeddedClusterHomeDirectory() + var provider *defaults.Provider + if c.IsSet("data-dir") { + provider = defaults.NewProvider(c.String("data-dir")) + } else { + var err error + provider, err = defaults.NewProviderFromFilesystem() + if err != nil { + panic(fmt.Errorf("unable to get provider from filesystem: %w", err)) + } + } + + port, err := k8snet.ParsePort(c.String("port"), false) + if err != nil { + panic(fmt.Errorf("unable to parse port from flag: %w", err)) + } - fileServer := http.FileServer(http.Dir(dir)) + os.Setenv("TMPDIR", provider.EmbeddedClusterTmpSubDir()) + + fileServer := http.FileServer(http.Dir(provider.EmbeddedClusterHomeDirectory())) loggedFileServer := logAndFilterRequest(fileServer) http.Handle("/", loggedFileServer) stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - if err := startBinaryWatcher(stop); err != nil { + if err := startBinaryWatcher(provider, stop); err != nil { panic(err) } - port := defaults.LocalArtifactMirrorPort - portStr := c.String("port") - if portStr != "" { - var err error - port, err = k8snet.ParsePort(portStr, false) - if err != nil { - panic(fmt.Errorf("unable to parse port: %w", err)) - } - } - addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) server := &http.Server{Addr: addr} go func() { @@ -88,8 +104,8 @@ var serveCommand = &cli.Command{ // startBinaryWatcher starts a loop that observes the binary until its modification // time changes. When the modification time changes a SIGTERM is send in the provided // channel. -func startBinaryWatcher(stop chan os.Signal) error { - fpath := defaults.PathToEmbeddedClusterBinary("local-artifact-mirror") +func startBinaryWatcher(provider *defaults.Provider, stop chan os.Signal) error { + fpath := provider.PathToEmbeddedClusterBinary("local-artifact-mirror") stat, err := os.Stat(fpath) if err != nil { return fmt.Errorf("unable to stat %s: %s", fpath, err) @@ -118,10 +134,20 @@ func startBinaryWatcher(stop chan os.Signal) error { func logAndFilterRequest(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Printf("%s %s %s\n", r.RemoteAddr, r.Method, r.URL) - if strings.HasPrefix(r.URL.Path, "/logs") { - w.WriteHeader(http.StatusNotFound) - return + for _, dir := range whitelistServeDirs { + if !strings.HasPrefix(dir, "/") { + dir = "/" + dir + } + if !strings.HasSuffix(dir, "/") { + dir = dir + "/" + } + if strings.HasPrefix(r.URL.Path, dir) { + fmt.Printf("serving %s\n", r.URL.Path) + handler.ServeHTTP(w, r) + return + } } - handler.ServeHTTP(w, r) + fmt.Printf("not serving %s\n", r.URL.Path) + w.WriteHeader(http.StatusNotFound) }) } diff --git a/e2e/install_test.go b/e2e/install_test.go index fc4e06a7a..b78ead875 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -542,12 +542,14 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { RequireEnvVars(t, []string{"SHORT_SHA"}) + withEnv := WithEnv(map[string]string{"KUBECONFIG": "/var/lib/k0s/pki/admin.conf"}) + tc := cluster.NewTestCluster(&cluster.Input{ T: t, Nodes: 1, Image: "debian/12", }) - defer cleanupCluster(t, tc) + defer cleanupCluster(t, tc, withEnv) t.Logf("%s: downloading embedded-cluster 1.8.0+k8s-1.28 on node 0", time.Now().Format(time.RFC3339)) line := []string{"vandoor-prepare.sh", "1.8.0+k8s-1.28", os.Getenv("LICENSE_ID"), "false"} @@ -557,17 +559,21 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { t.Logf("%s: installing embedded-cluster 1.8.0+k8s-1.28 on node 0", time.Now().Format(time.RFC3339)) line = []string{"single-node-install.sh", "ui"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } - if _, _, err := setupPlaywrightAndRunTest(t, tc, "deploy-ec18-app-version"); err != nil { + if err := setupPlaywright(t, tc, withEnv); err != nil { + t.Fatalf("fail to setup playwright: %v", err) + } + + if _, _, err := runPlaywrightTest(t, tc, "deploy-ec18-app-version"); err != nil { t.Fatalf("fail to run playwright test deploy-ec18-app-version: %v", err) } t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) line = []string{"check-installation-state.sh", "1.8.0+k8s-1.28", "v1.28.11"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check installation state: %v", err) } @@ -581,7 +587,7 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-postupgrade-state.sh", k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -728,12 +734,14 @@ func TestOldVersionUpgrade(t *testing.T) { RequireEnvVars(t, []string{"SHORT_SHA"}) + withEnv := WithEnv(map[string]string{"KUBECONFIG": "/var/lib/k0s/pki/admin.conf"}) + tc := cluster.NewTestCluster(&cluster.Input{ T: t, Nodes: 1, Image: "debian/12", }) - defer cleanupCluster(t, tc) + defer cleanupCluster(t, tc, withEnv) t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339)) line := []string{"vandoor-prepare.sh", fmt.Sprintf("appver-%s-pre-minio-removal", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} @@ -743,25 +751,25 @@ func TestOldVersionUpgrade(t *testing.T) { t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) line = []string{"pre-minio-removal-install.sh", "cli"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) line = []string{"check-pre-minio-removal-installation-state.sh", fmt.Sprintf("%s-pre-minio-removal", os.Getenv("SHORT_SHA"))} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check installation state: %v", err) } t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339)) line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to run kots upstream upgrade: %v", err) } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-postupgrade-state.sh", k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -967,6 +975,8 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"}) + withEnv := WithEnv(map[string]string{"KUBECONFIG": "/var/lib/k0s/pki/admin.conf"}) + t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339)) airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz" airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz" @@ -986,7 +996,7 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { AirgapInstallBundlePath: airgapInstallBundlePath, AirgapUpgradeBundlePath: airgapUpgradeBundlePath, }) - defer cleanupCluster(t, tc) + defer cleanupCluster(t, tc, withEnv) // delete airgap bundles once they've been copied to the nodes if err := os.Remove(airgapInstallBundlePath); err != nil { @@ -1007,7 +1017,7 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) line = []string{"single-node-airgap-install.sh"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } // remove the airgap bundle after installation @@ -1016,7 +1026,11 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err) } - if _, _, err := setupPlaywrightAndRunTest(t, tc, "deploy-ec18-app-version"); err != nil { + if err := setupPlaywright(t, tc, withEnv); err != nil { + t.Fatalf("fail to setup playwright: %v", err) + } + + if _, _, err := runPlaywrightTest(t, tc, "deploy-ec18-app-version"); err != nil { t.Fatalf("fail to run playwright test deploy-ec18-app-version: %v", err) } @@ -1027,13 +1041,13 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { // the '+' character is problematic in the regex used to validate the version, so we use '.' instead "1.8.0.k8s-1.28", "v1.28.11"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check installation state: %v", err) } t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339)) line = []string{"airgap-update.sh"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to run airgap update: %v", err) } // remove the airgap bundle after upgrade @@ -1052,7 +1066,7 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) line = []string{"check-postupgrade-state.sh", k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -1924,10 +1938,10 @@ func setupPlaywrightAndRunTest(t *testing.T, tc *cluster.Output, testName string return runPlaywrightTest(t, tc, testName, args...) } -func setupPlaywright(t *testing.T, tc *cluster.Output) error { +func setupPlaywright(t *testing.T, tc *cluster.Output, commandOpts ...RunCommandOption) error { t.Logf("%s: bypassing kurl-proxy on node 0", time.Now().Format(time.RFC3339)) line := []string{"bypass-kurl-proxy.sh"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, commandOpts...); err != nil { return fmt.Errorf("fail to bypass kurl-proxy on node %s: %v", tc.Nodes[0], err) } @@ -1965,7 +1979,7 @@ func runPlaywrightTest(t *testing.T, tc *cluster.Output, testName string, args . return stdout, stderr, nil } -func generateAndCopySupportBundle(t *testing.T, tc *cluster.Output) { +func generateAndCopySupportBundle(t *testing.T, tc *cluster.Output, commandOpts ...RunCommandOption) { wg := sync.WaitGroup{} wg.Add(len(tc.Nodes)) @@ -1974,7 +1988,7 @@ func generateAndCopySupportBundle(t *testing.T, tc *cluster.Output) { defer wg.Done() t.Logf("%s: generating host support bundle from node %s", time.Now().Format(time.RFC3339), tc.Nodes[i]) line := []string{"collect-support-bundle-host.sh"} - if stdout, stderr, err := RunCommandOnNode(t, tc, i, line); err != nil { + if stdout, stderr, err := RunCommandOnNode(t, tc, i, line, commandOpts...); err != nil { t.Logf("stdout: %s", stdout) t.Logf("stderr: %s", stderr) t.Logf("fail to generate support from node %s bundle: %v", tc.Nodes[i], err) @@ -1991,7 +2005,7 @@ func generateAndCopySupportBundle(t *testing.T, tc *cluster.Output) { node := tc.Nodes[0] t.Logf("%s: generating cluster support bundle from node %s", time.Now().Format(time.RFC3339), node) line := []string{"collect-support-bundle-cluster.sh"} - if stdout, stderr, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if stdout, stderr, err := RunCommandOnNode(t, tc, 0, line, commandOpts...); err != nil { t.Logf("stdout: %s", stdout) t.Logf("stderr: %s", stderr) t.Logf("fail to generate cluster support from node %s bundle: %v", node, err) @@ -2030,8 +2044,8 @@ func copyPlaywrightReport(t *testing.T, tc *cluster.Output) { } } -func cleanupCluster(t *testing.T, tc *cluster.Output) { - generateAndCopySupportBundle(t, tc) +func cleanupCluster(t *testing.T, tc *cluster.Output, commandOpts ...RunCommandOption) { + generateAndCopySupportBundle(t, tc, commandOpts...) copyPlaywrightReport(t, tc) } @@ -2197,7 +2211,7 @@ func TestInstallWithPrivateCAs(t *testing.T) { t.Logf("checking if the configmap was created with the right values") line = []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} - stdout, _, err := RunCommandOnNode(t, tc, 0, line, WithECShelEnv()) + stdout, _, err := RunCommandOnNode(t, tc, 0, line, WithECShellEnv("/var/lib/embedded-cluster")) require.NoError(t, err, "unable get kotsadm-private-cas configmap") var cm corev1.ConfigMap diff --git a/e2e/local-artifact-mirror_test.go b/e2e/local-artifact-mirror_test.go index 58bf89488..3d41b32e7 100644 --- a/e2e/local-artifact-mirror_test.go +++ b/e2e/local-artifact-mirror_test.go @@ -42,12 +42,12 @@ func TestLocalArtifactMirror(t *testing.T) { t.Fatalf("fail testing local artifact mirror: %v", err) } - command := []string{"cp", "/etc/passwd", "/var/lib/embedded-cluster/logs/passwd"} + command := []string{"cp", "/etc/passwd", "/var/log/embedded-cluster/passwd"} if _, _, err := RunCommandOnNode(t, tc, 0, command); err != nil { t.Fatalf("fail to copy file: %v", err) } - command = []string{"curl", "-O", "--fail", "127.0.0.1:50001/logs/passwd"} + command = []string{"curl", "-O", "--fail", "127.0.0.1:50001/passwd"} t.Logf("running %v", command) if _, _, err := RunCommandOnNode(t, tc, 0, command); err == nil { t.Fatalf("we should not be able to fetch logs from local artifact mirror") @@ -59,6 +59,12 @@ func TestLocalArtifactMirror(t *testing.T) { t.Fatalf("we should not be able to fetch paths with ../") } + command = []string{"curl", "-I", "--fail", "127.0.0.1:50001/bin/kubectl"} + t.Logf("running %v", command) + if _, _, err := RunCommandOnNode(t, tc, 0, command); err != nil { + t.Fatalf("we should be able to fetch the kubectl binary in the bin directory: %v", err) + } + t.Logf("testing local artifact mirror restart after materialize") command = []string{"embedded-cluster", "materialize"} if _, _, err := RunCommandOnNode(t, tc, 0, command); err != nil { diff --git a/e2e/restore_test.go b/e2e/restore_test.go index 458944336..204cf397e 100644 --- a/e2e/restore_test.go +++ b/e2e/restore_test.go @@ -597,6 +597,11 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Fatal(err) } + // Use an alternate data directory + withEnv := WithEnv(map[string]string{ + "EMBEDDED_CLUSTER_BASE_DIR": "/var/lib/ec", + }) + tc := cluster.NewTestCluster(&cluster.Input{ T: t, Nodes: 3, @@ -604,7 +609,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { WithProxy: true, AirgapInstallBundlePath: airgapInstallBundlePath, }) - defer cleanupCluster(t, tc) + defer cleanupCluster(t, tc, withEnv) // install "expect" dependency on node 0 as that's where the restore process will be initiated. // install "expect" dependency on node 2 as that's where the HA join command will run. @@ -618,17 +623,21 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) line := []string{"airgap-prepare.sh"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) } t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) - line = []string{"single-node-airgap-install.sh", "--proxy"} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { + line = []string{"single-node-airgap-install.sh", "--proxy", "--data-dir", "/var/lib/ec"} + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } - if _, _, err := setupPlaywrightAndRunTest(t, tc, "deploy-app"); err != nil { + if err := setupPlaywright(t, tc, withEnv); err != nil { + t.Fatalf("fail to setup playwright: %v", err) + } + + if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } @@ -645,11 +654,12 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Log("controller join token command:", command) t.Logf("%s: preparing embedded cluster airgap files on node 1", time.Now().Format(time.RFC3339)) line = []string{"airgap-prepare.sh"} - if _, _, err := RunCommandOnNode(t, tc, 1, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 1, line, withEnv); err != nil { t.Fatalf("fail to prepare airgap files on node 1: %v", err) } t.Logf("%s: joining node 1 to the cluster (controller)", time.Now().Format(time.RFC3339)) - if _, _, err := RunCommandOnNode(t, tc, 1, strings.Split(command, " ")); err != nil { + line = strings.Split(command, " ") + if _, _, err := RunCommandOnNode(t, tc, 1, line, withEnv); err != nil { t.Fatalf("fail to join node 1 as a controller: %v", err) } @@ -666,18 +676,19 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Log("controller join token command:", command) t.Logf("%s: preparing embedded cluster airgap files on node 2", time.Now().Format(time.RFC3339)) line = []string{"airgap-prepare.sh"} - if _, _, err := RunCommandOnNode(t, tc, 2, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 2, line, withEnv); err != nil { t.Fatalf("fail to prepare airgap files on node 2: %v", err) } t.Logf("%s: joining node 2 to the cluster (controller) in ha mode", time.Now().Format(time.RFC3339)) line = append([]string{"join-ha.exp"}, []string{command}...) - if _, _, err := RunCommandOnNode(t, tc, 2, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 2, line, withEnv); err != nil { t.Fatalf("fail to join node 2 as a controller in ha mode: %v", err) } // wait for the nodes to report as ready. t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) - stdout, _, err = RunCommandOnNode(t, tc, 0, []string{"wait-for-ready-nodes.sh", "3"}) + line = []string{"wait-for-ready-nodes.sh", "3"} + stdout, _, err = RunCommandOnNode(t, tc, 0, line, withEnv) if err != nil { t.Log(stdout) t.Fatalf("fail to wait for ready nodes: %v", err) @@ -685,7 +696,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Logf("%s: checking installation state after enabling high availability", time.Now().Format(time.RFC3339)) line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check post ha state: %v", err) } @@ -694,17 +705,17 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { } // reset the cluster - line = []string{"reset-installation.sh", "--force"} + line = []string{"reset-installation.sh", "--force", "--data-dir", "/var/lib/ec"} t.Logf("%s: resetting the installation on node 2", time.Now().Format(time.RFC3339)) - if _, _, err := RunCommandOnNode(t, tc, 2, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 2, line, withEnv); err != nil { t.Fatalf("fail to reset the installation: %v", err) } t.Logf("%s: resetting the installation on node 1", time.Now().Format(time.RFC3339)) - if _, _, err := RunCommandOnNode(t, tc, 1, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 1, line, withEnv); err != nil { t.Fatalf("fail to reset the installation: %v", err) } t.Logf("%s: resetting the installation on node 0", time.Now().Format(time.RFC3339)) - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to reset the installation: %v", err) } @@ -715,7 +726,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { // begin restoring the cluster t.Logf("%s: restoring the installation: phase 1", time.Now().Format(time.RFC3339)) line = append([]string{"restore-multi-node-airgap-phase1.exp"}, testArgs...) - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to restore phase 1 of the installation: %v", err) } @@ -756,7 +767,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { // wait for the nodes to report as ready. t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) - stdout, _, err = RunCommandOnNode(t, tc, 0, []string{"wait-for-ready-nodes.sh", "3", "true"}) + stdout, _, err = RunCommandOnNode(t, tc, 0, []string{"wait-for-ready-nodes.sh", "3", "true"}, withEnv) if err != nil { t.Log(stdout) t.Fatalf("fail to wait for ready nodes: %v", err) @@ -764,13 +775,13 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Logf("%s: restoring the installation: phase 2", time.Now().Format(time.RFC3339)) line = []string{"restore-multi-node-airgap-phase2.exp"} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to restore phase 2 of the installation: %v", err) } t.Logf("%s: checking installation state after restoring the high availability backup", time.Now().Format(time.RFC3339)) line = []string{"check-airgap-post-ha-state.sh", os.Getenv("SHORT_SHA"), k8sVersion(), "true"} - if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withEnv); err != nil { t.Fatalf("fail to check post ha state: %v", err) } diff --git a/e2e/scripts/airgap-update.sh b/e2e/scripts/airgap-update.sh index 4bdbe47a1..c8b834492 100755 --- a/e2e/scripts/airgap-update.sh +++ b/e2e/scripts/airgap-update.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash set -euox pipefail +DIR=/usr/local/bin +. $DIR/common.sh + main() { echo "upgrading from airgap bundle" embedded-cluster-upgrade update --airgap-bundle /assets/upgrade/release.airgap } -export KUBECONFIG=/var/lib/k0s/pki/admin.conf -export PATH=$PATH:/var/lib/embedded-cluster/bin main "$@" diff --git a/e2e/scripts/bypass-kurl-proxy.sh b/e2e/scripts/bypass-kurl-proxy.sh index 66e66181e..bfd3e03cb 100755 --- a/e2e/scripts/bypass-kurl-proxy.sh +++ b/e2e/scripts/bypass-kurl-proxy.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euox pipefail +DIR=/usr/local/bin +. $DIR/common.sh + main() { # create a nodeport service directly to kotsadm cat < 0 { + if i.Deprecated_AdminConsole != nil && i.Deprecated_AdminConsole.Port > 0 { if i.RuntimeConfig == nil { i.RuntimeConfig = &RuntimeConfigSpec{} } if i.RuntimeConfig.AdminConsole.Port == 0 { - i.RuntimeConfig.AdminConsole.Port = i.AdminConsole.Port + i.RuntimeConfig.AdminConsole.Port = i.Deprecated_AdminConsole.Port } } - if i.LocalArtifactMirror != nil && i.LocalArtifactMirror.Port > 0 { + if i.Deprecated_LocalArtifactMirror != nil && i.Deprecated_LocalArtifactMirror.Port > 0 { if i.RuntimeConfig == nil { i.RuntimeConfig = &RuntimeConfigSpec{} } if i.RuntimeConfig.LocalArtifactMirror.Port == 0 { - i.RuntimeConfig.LocalArtifactMirror.Port = i.LocalArtifactMirror.Port + i.RuntimeConfig.LocalArtifactMirror.Port = i.Deprecated_LocalArtifactMirror.Port } } return nil diff --git a/kinds/apis/v1beta1/installation_types_test.go b/kinds/apis/v1beta1/installation_types_test.go index ddf7e8fdc..10ac01ef9 100644 --- a/kinds/apis/v1beta1/installation_types_test.go +++ b/kinds/apis/v1beta1/installation_types_test.go @@ -72,7 +72,7 @@ spec: Port: 31111, }, }, - AdminConsole: &AdminConsoleSpec{ + Deprecated_AdminConsole: &AdminConsoleSpec{ Port: 31111, }, }, @@ -101,7 +101,7 @@ spec: Port: 51111, }, }, - LocalArtifactMirror: &LocalArtifactMirrorSpec{ + Deprecated_LocalArtifactMirror: &LocalArtifactMirrorSpec{ Port: 51111, }, }, diff --git a/kinds/apis/v1beta1/zz_generated.deepcopy.go b/kinds/apis/v1beta1/zz_generated.deepcopy.go index 5b3259d6f..93af0cbdd 100644 --- a/kinds/apis/v1beta1/zz_generated.deepcopy.go +++ b/kinds/apis/v1beta1/zz_generated.deepcopy.go @@ -348,13 +348,13 @@ func (in *InstallationSpec) DeepCopyInto(out *InstallationSpec) { *out = new(NetworkSpec) **out = **in } - if in.AdminConsole != nil { - in, out := &in.AdminConsole, &out.AdminConsole + if in.Deprecated_AdminConsole != nil { + in, out := &in.Deprecated_AdminConsole, &out.Deprecated_AdminConsole *out = new(AdminConsoleSpec) **out = **in } - if in.LocalArtifactMirror != nil { - in, out := &in.LocalArtifactMirror, &out.LocalArtifactMirror + if in.Deprecated_LocalArtifactMirror != nil { + in, out := &in.Deprecated_LocalArtifactMirror, &out.Deprecated_LocalArtifactMirror *out = new(LocalArtifactMirrorSpec) **out = **in } diff --git a/operator/Makefile b/operator/Makefile index fb4fea9e4..ca1c502c6 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -62,12 +62,8 @@ manifests: kustomize controller-gen ## Generate WebhookConfiguration, ClusterRol fmt: ## Run go fmt against code. go fmt ./... -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - .PHONY: test -test: manifests fmt vet envtest ## Run tests. +test: manifests envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out ##@ Build @@ -80,7 +76,7 @@ build: ## Build manager binary. -o bin/manager main.go .PHONY: run -run: manifests fmt vet ## Run a controller from your host. +run: manifests fmt ## Run a controller from your host. go run ./main.go ##@ Build Dependencies diff --git a/operator/controllers/installation_controller.go b/operator/controllers/installation_controller.go index 32b451b8d..226b0e8bc 100644 --- a/operator/controllers/installation_controller.go +++ b/operator/controllers/installation_controller.go @@ -29,6 +29,8 @@ import ( k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" apcore "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core" "github.com/k0sproject/version" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -67,11 +69,11 @@ var requeueAfter = time.Hour const copyHostPreflightResultsJobPrefix = "copy-host-preflight-results-" const ecNamespace = "embedded-cluster" -// copyHostPreflightResultsJob is a job we create everytime we need to copy -// host preflight results from a newly added node in the cluster. Host preflight -// are run on installation, join or restore operations. The results are stored -// in /var/lib/embedded-cluster/support/host-preflight-results.json. During a -// reconcile cycle we will populate the node selector, any env variables and labels. +// copyHostPreflightResultsJob is a job we create everytime we need to copy host preflight results +// from a newly added node in the cluster. Host preflight are run on installation, join or restore +// operations. The results are stored in the data directory in +// /support/host-preflight-results.json. During a reconcile cycle we will populate the node +// selector, any env variables and labels. var copyHostPreflightResultsJob = &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Namespace: ecNamespace, @@ -87,7 +89,7 @@ var copyHostPreflightResultsJob = &batchv1.Job{ Name: "host", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/embedded-cluster", + Path: v1beta1.DefaultDataDir, Type: ptr.To[corev1.HostPathType]("Directory"), }, }, @@ -102,22 +104,22 @@ var copyHostPreflightResultsJob = &batchv1.Job{ "/bin/sh", "-e", "-c", - "if [ -f /var/lib/embedded-cluster/support/host-preflight-results.json ]; " + + "if [ -f /embedded-cluster/support/host-preflight-results.json ]; " + "then " + - "/var/lib/embedded-cluster/bin/kubectl create configmap ${HSPF_CM_NAME} " + - "--from-file=results.json=/var/lib/embedded-cluster/support/host-preflight-results.json " + + "/embedded-cluster/bin/kubectl create configmap ${HSPF_CM_NAME} " + + "--from-file=results.json=/embedded-cluster/support/host-preflight-results.json " + "-n embedded-cluster --dry-run=client -oyaml | " + - "/var/lib/embedded-cluster/bin/kubectl label -f - embedded-cluster/host-preflight-result=${EC_NODE_NAME} --local -o yaml | " + - "/var/lib/embedded-cluster/bin/kubectl apply -f - && " + - "/var/lib/embedded-cluster/bin/kubectl annotate configmap ${HSPF_CM_NAME} \"update-timestamp=$(date +'%Y-%m-%dT%H:%M:%SZ')\" --overwrite; " + + "/embedded-cluster/bin/kubectl label -f - embedded-cluster/host-preflight-result=${EC_NODE_NAME} --local -o yaml | " + + "/embedded-cluster/bin/kubectl apply -f - && " + + "/embedded-cluster/bin/kubectl annotate configmap ${HSPF_CM_NAME} \"update-timestamp=$(date +'%Y-%m-%dT%H:%M:%SZ')\" --overwrite; " + "else " + - "echo '/var/lib/embedded-cluster/support/host-preflight-results.json does not exist'; " + + "echo '/embedded-cluster/support/host-preflight-results.json does not exist'; " + "fi", }, VolumeMounts: []corev1.VolumeMount{ { Name: "host", - MountPath: "/var/lib/embedded-cluster", + MountPath: "/embedded-cluster", ReadOnly: false, }, }, @@ -293,7 +295,7 @@ func (r *InstallationReconciler) ReportInstallationChanges(ctx context.Context, // HasOnlyOneInstallation returns true if only one Installation object exists in the cluster. func (r *InstallationReconciler) HasOnlyOneInstallation(ctx context.Context) (bool, error) { - ins, err := r.listInstallations(ctx) + ins, err := kubeutils.ListInstallations(ctx, r.Client) if err != nil { return false, fmt.Errorf("failed to list installations: %w", err) } @@ -624,19 +626,6 @@ func (r *InstallationReconciler) StartAutopilotUpgrade(ctx context.Context, in * return upgrade.StartAutopilotUpgrade(ctx, r.Client, in, meta) } -// listInstallations returns a list of all the installation objects in the cluster in order. -func (r *InstallationReconciler) listInstallations(ctx context.Context) ([]v1beta1.Installation, error) { - var list v1beta1.InstallationList - if err := r.List(ctx, &list); err != nil { - return nil, fmt.Errorf("list installations: %w", err) - } - items := list.Items - sort.SliceStable(items, func(i, j int) bool { - return items[j].Name < items[i].Name - }) - return items, nil -} - // CoalesceInstallations goes through all the installation objects and make sure that the // status of the newest one is coherent with whole cluster status. Returns the newest // installation object. @@ -711,7 +700,7 @@ func (r *InstallationReconciler) CopyHostPreflightResultsFromNodes(ctx context.C for _, event := range events.NodesAdded { log.Info("Creating job to copy host preflight results from node", "node", event.NodeName, "installation", in.Name) - job := constructHostPreflightResultsJob(event.NodeName, in.Name) + job := constructHostPreflightResultsJob(in, event.NodeName) // overrides the job image if the environment says so. if img := os.Getenv("EMBEDDEDCLUSTER_UTILS_IMAGE"); img != "" { @@ -727,10 +716,12 @@ func (r *InstallationReconciler) CopyHostPreflightResultsFromNodes(ctx context.C return nil } -func constructHostPreflightResultsJob(nodeName, installationName string) *batchv1.Job { +func constructHostPreflightResultsJob(in *v1beta1.Installation, nodeName string) *batchv1.Job { + provider := defaults.NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig) + labels := map[string]string{ "embedded-cluster/node-name": nodeName, - "embedded-cluster/installation": installationName, + "embedded-cluster/installation": in.Name, } job := copyHostPreflightResultsJob.DeepCopy() @@ -738,6 +729,7 @@ func constructHostPreflightResultsJob(nodeName, installationName string) *batchv job.Spec.Template.Labels, job.Labels = labels, labels job.Spec.Template.Spec.NodeName = nodeName + job.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path = provider.EmbeddedClusterHomeDirectory() job.Spec.Template.Spec.Containers[0].Env = append( job.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "EC_NODE_NAME", Value: nodeName}, @@ -764,7 +756,7 @@ func (r *InstallationReconciler) Reconcile(ctx context.Context, req ctrl.Request // are going to operate only on the newest one (sorting by installation // name). log := ctrl.LoggerFrom(ctx) - installs, err := r.listInstallations(ctx) + installs, err := kubeutils.ListInstallations(ctx, r.Client) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to list installations: %w", err) } diff --git a/operator/controllers/installation_controller_test.go b/operator/controllers/installation_controller_test.go index 88bf4c343..fa3f409b9 100644 --- a/operator/controllers/installation_controller_test.go +++ b/operator/controllers/installation_controller_test.go @@ -3,17 +3,31 @@ package controllers import ( "testing" + "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestInstallationReconciler_constructCreateCMCommand(t *testing.T) { - job := constructHostPreflightResultsJob("my-node", "install-name") + in := &v1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "install-name", + }, + Spec: v1beta1.InstallationSpec{ + RuntimeConfig: &v1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/embedded-cluster", + }, + }, + } + job := constructHostPreflightResultsJob(in, "my-node") + require.Len(t, job.Spec.Template.Spec.Volumes, 1) + require.Equal(t, "/var/lib/embedded-cluster", job.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path) require.Len(t, job.Spec.Template.Spec.Containers, 1) require.Len(t, job.Spec.Template.Spec.Containers[0].Command, 4) kctlCmd := job.Spec.Template.Spec.Containers[0].Command[3] - expected := "if [ -f /var/lib/embedded-cluster/support/host-preflight-results.json ]; then /var/lib/embedded-cluster/bin/kubectl create configmap ${HSPF_CM_NAME} --from-file=results.json=/var/lib/embedded-cluster/support/host-preflight-results.json -n embedded-cluster --dry-run=client -oyaml | /var/lib/embedded-cluster/bin/kubectl label -f - embedded-cluster/host-preflight-result=${EC_NODE_NAME} --local -o yaml | /var/lib/embedded-cluster/bin/kubectl apply -f - && /var/lib/embedded-cluster/bin/kubectl annotate configmap ${HSPF_CM_NAME} \"update-timestamp=$(date +'%Y-%m-%dT%H:%M:%SZ')\" --overwrite; else echo '/var/lib/embedded-cluster/support/host-preflight-results.json does not exist'; fi" + expected := "if [ -f /embedded-cluster/support/host-preflight-results.json ]; then /embedded-cluster/bin/kubectl create configmap ${HSPF_CM_NAME} --from-file=results.json=/embedded-cluster/support/host-preflight-results.json -n embedded-cluster --dry-run=client -oyaml | /embedded-cluster/bin/kubectl label -f - embedded-cluster/host-preflight-result=${EC_NODE_NAME} --local -o yaml | /embedded-cluster/bin/kubectl apply -f - && /embedded-cluster/bin/kubectl annotate configmap ${HSPF_CM_NAME} \"update-timestamp=$(date +'%Y-%m-%dT%H:%M:%SZ')\" --overwrite; else echo '/embedded-cluster/support/host-preflight-results.json does not exist'; fi" assert.Equal(t, expected, kctlCmd) require.Len(t, job.Spec.Template.Spec.Containers[0].Env, 2) assert.Equal(t, v1.EnvVar{ diff --git a/operator/controllers/k0s.go b/operator/controllers/k0s.go index 5f3bac354..c80bc1ae7 100644 --- a/operator/controllers/k0s.go +++ b/operator/controllers/k0s.go @@ -8,6 +8,7 @@ import ( "github.com/k0sproject/version" clusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/operator/pkg/release" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" ctrl "sigs.k8s.io/controller-runtime" ) @@ -52,7 +53,7 @@ func (r *InstallationReconciler) shouldUpgradeK0s(ctx context.Context, in *clust // discoverPreviousK0sVersion gets the k0s version from the previous installation object. func (r *InstallationReconciler) discoverPreviousK0sVersion(ctx context.Context, in *clusterv1beta1.Installation) (string, error) { - ins, err := r.listInstallations(ctx) + ins, err := kubeutils.ListInstallations(ctx, r.Client) if err != nil { return "", fmt.Errorf("list installations: %w", err) } diff --git a/operator/pkg/artifacts/upgrade.go b/operator/pkg/artifacts/upgrade.go index d7ce229c2..bc8fe5083 100644 --- a/operator/pkg/artifacts/upgrade.go +++ b/operator/pkg/artifacts/upgrade.go @@ -34,10 +34,10 @@ const ( ArtifactsConfigHashAnnotation = "embedded-cluster.replicated.com/artifacts-config-hash" ) -// copyArtifactsJob is a job we create everytime we need to sync files into all nodes. -// This job mounts /var/lib/embedded-cluster from the node and uses binaries that are -// present there. This is not yet a complete version of the job as it misses some env -// variables and a node selector, those are populated during the reconcile cycle. +// copyArtifactsJob is a job we create everytime we need to sync files into all nodes. This job +// mounts the data directory from the node and uses binaries that are present there. This is not +// yet a complete version of the job as it misses some env variables and a node selector, those are +// populated during the reconcile cycle. var copyArtifactsJob = &batchv1.Job{ TypeMeta: metav1.TypeMeta{ APIVersion: "batch/v1", @@ -56,7 +56,7 @@ var copyArtifactsJob = &batchv1.Job{ Name: "host", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/embedded-cluster", + Path: clusterv1beta1.DefaultDataDir, Type: ptr.To[corev1.HostPathType]("Directory"), }, }, @@ -69,7 +69,7 @@ var copyArtifactsJob = &batchv1.Job{ VolumeMounts: []corev1.VolumeMount{ { Name: "host", - MountPath: "/var/lib/embedded-cluster", + MountPath: "/embedded-cluster", ReadOnly: false, }, }, @@ -77,12 +77,12 @@ var copyArtifactsJob = &batchv1.Job{ "/bin/sh", "-ex", "-c", - "/usr/local/bin/local-artifact-mirror pull binaries $INSTALLATION_DATA\n" + - "/usr/local/bin/local-artifact-mirror pull images $INSTALLATION_DATA\n" + - "/usr/local/bin/local-artifact-mirror pull helmcharts $INSTALLATION_DATA\n" + - "mv /var/lib/embedded-cluster/bin/k0s /var/lib/embedded-cluster/bin/k0s-upgrade\n" + - "rm /var/lib/embedded-cluster/images/images-amd64-* || true\n" + - "cd /var/lib/embedded-cluster/images/\n" + + "/usr/local/bin/local-artifact-mirror pull binaries --data-dir /embedded-cluster $INSTALLATION_DATA\n" + + "/usr/local/bin/local-artifact-mirror pull images --data-dir /embedded-cluster $INSTALLATION_DATA\n" + + "/usr/local/bin/local-artifact-mirror pull helmcharts --data-dir /embedded-cluster $INSTALLATION_DATA\n" + + "mv /embedded-cluster/bin/k0s /embedded-cluster/bin/k0s-upgrade\n" + + "rm /embedded-cluster/images/images-amd64-* || true\n" + + "cd /embedded-cluster/images/\n" + "mv images-amd64.tar images-amd64-${INSTALLATION}.tar\n" + "echo 'done'", }, @@ -203,6 +203,8 @@ func ensureArtifactsJobForNode(ctx context.Context, cli client.Client, in *clust } func getArtifactJobForNode(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation, node corev1.Node, localArtifactMirrorImage string) (*batchv1.Job, error) { + provider := defaults.NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig) + hash, err := HashForAirgapConfig(in) if err != nil { return nil, fmt.Errorf("failed to hash airgap config: %w", err) @@ -219,6 +221,7 @@ func getArtifactJobForNode(ctx context.Context, cli client.Client, in *clusterv1 job.ObjectMeta.Labels = applyECOperatorLabels(job.ObjectMeta.Labels, "upgrader") job.ObjectMeta.Annotations = applyArtifactsJobAnnotations(job.GetAnnotations(), in, hash) job.Spec.Template.Spec.NodeName = node.Name + job.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path = provider.EmbeddedClusterHomeDirectory() job.Spec.Template.Spec.Containers[0].Env = append( job.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "INSTALLATION", Value: in.Name}, @@ -241,6 +244,8 @@ func getArtifactJobForNode(ctx context.Context, cli client.Client, in *clusterv1 // CreateAutopilotAirgapPlanCommand creates the plan to execute an aigrap upgrade in all nodes. The // return of this function is meant to be used as part of an autopilot plan. func CreateAutopilotAirgapPlanCommand(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) (*autopilotv1beta2.PlanCommand, error) { + provider := defaults.NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig) + meta, err := release.MetadataFor(ctx, in, cli) if err != nil { return nil, fmt.Errorf("failed to get release metadata: %w", err) @@ -256,11 +261,10 @@ func CreateAutopilotAirgapPlanCommand(ctx context.Context, cli client.Client, in allNodes = append(allNodes, node.Name) } - port := defaults.LocalArtifactMirrorPort - if in.Spec.LocalArtifactMirror != nil && in.Spec.LocalArtifactMirror.Port > 0 { - port = in.Spec.LocalArtifactMirror.Port - } - imageURL := fmt.Sprintf("http://127.0.0.1:%d/images/images-amd64-%s.tar", port, in.Name) + imageURL := fmt.Sprintf( + "http://127.0.0.1:%d/images/images-amd64-%s.tar", + provider.LocalArtifactMirrorPort(), in.Name, + ) return &autopilotv1beta2.PlanCommand{ AirgapUpdate: &autopilotv1beta2.PlanCommandAirgapUpdate{ diff --git a/operator/pkg/charts/charts.go b/operator/pkg/charts/charts.go index 6e147938b..9d3021946 100644 --- a/operator/pkg/charts/charts.go +++ b/operator/pkg/charts/charts.go @@ -16,6 +16,7 @@ import ( "github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil" "github.com/replicatedhq/embedded-cluster/operator/pkg/registry" "github.com/replicatedhq/embedded-cluster/operator/pkg/util" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/helm" ) @@ -39,7 +40,7 @@ func K0sHelmExtensionsFromInstallation( // if in airgap mode then all charts are already on the node's disk. we just need to // make sure that the helm charts are pointing to the right location on disk and that // we do not have any kind of helm repository configuration. - combinedConfigs = patchExtensionsForAirGap(combinedConfigs) + combinedConfigs = patchExtensionsForAirGap(in, combinedConfigs) } combinedConfigs, err = applyUserProvidedAddonOverrides(in, combinedConfigs) @@ -91,6 +92,12 @@ func mergeHelmConfigs(ctx context.Context, meta *ectypes.ReleaseMetadata, in *cl combinedConfigs.Charts = append(combinedConfigs.Charts, registryConfig.Charts...) combinedConfigs.Repositories = append(combinedConfigs.Repositories, registryConfig.Repositories...) } + } else { + registryConfig, ok := meta.BuiltinConfigs["registry"] + if ok { + combinedConfigs.Charts = append(combinedConfigs.Charts, registryConfig.Charts...) + combinedConfigs.Repositories = append(combinedConfigs.Repositories, registryConfig.Repositories...) + } } } else { registryConfig, ok := meta.BuiltinConfigs["registry"] @@ -127,6 +134,8 @@ func mergeHelmConfigs(ctx context.Context, meta *ectypes.ReleaseMetadata, in *cl // updateInfraChartsFromInstall updates the infrastructure charts with dynamic values from the installation spec func updateInfraChartsFromInstall(in *v1beta1.Installation, clusterConfig *k0sv1beta1.ClusterConfig, charts []v1beta1.Chart) ([]v1beta1.Chart, error) { + provider := defaults.NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig) + for i, chart := range charts { ecCharts := []string{ "admin-console", @@ -171,8 +180,8 @@ func updateInfraChartsFromInstall(in *v1beta1.Installation, clusterConfig *k0sv1 } } - if in.Spec.AdminConsole != nil && in.Spec.AdminConsole.Port > 0 { - newVals, err = helm.SetValue(newVals, "kurlProxy.nodePort", in.Spec.AdminConsole.Port) + if port := provider.AdminConsolePort(); port > 0 { + newVals, err = helm.SetValue(newVals, "kurlProxy.nodePort", port) if err != nil { return nil, fmt.Errorf("set helm values admin-console.kurlProxy.nodePort: %w", err) } @@ -305,11 +314,12 @@ func applyUserProvidedAddonOverrides(in *clusterv1beta1.Installation, combinedCo // sure that all helm charts point to a chart stored on disk as a tgz file. These files are already // expected to be present on the disk and, during an upgrade, are laid down on disk by the artifact // copy job. -func patchExtensionsForAirGap(config *v1beta1.Helm) *v1beta1.Helm { +func patchExtensionsForAirGap(in *clusterv1beta1.Installation, config *v1beta1.Helm) *v1beta1.Helm { + provider := defaults.NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig) config.Repositories = nil for idx, chart := range config.Charts { chartName := fmt.Sprintf("%s-%s.tgz", chart.Name, chart.Version) - chartPath := filepath.Join("/var/lib/embedded-cluster/charts", chartName) + chartPath := filepath.Join(provider.EmbeddedClusterHomeDirectory(), "charts", chartName) config.Charts[idx].ChartName = chartPath } return config diff --git a/operator/pkg/charts/charts_test.go b/operator/pkg/charts/charts_test.go index d40a7f1f2..2bbebfd28 100644 --- a/operator/pkg/charts/charts_test.go +++ b/operator/pkg/charts/charts_test.go @@ -431,6 +431,9 @@ func Test_mergeHelmConfigs(t *testing.T) { { Name: "seaweedfsrepo", }, + { + Name: "registryrepo", + }, }, Charts: []v1beta1.Chart{ { @@ -441,6 +444,10 @@ func Test_mergeHelmConfigs(t *testing.T) { Name: "seaweedfschart", Order: 100, }, + { + Name: "registrychart", + Order: 100, + }, }, }, }, diff --git a/operator/pkg/migrations/registry.go b/operator/pkg/migrations/registry.go index 6e03e83c1..7b9f7fdbf 100644 --- a/operator/pkg/migrations/registry.go +++ b/operator/pkg/migrations/registry.go @@ -19,9 +19,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// RegistryData runs a migration that copies data from the disk (/var/lib/embedded-cluster/registry) -// to the seaweedfs s3 store. If it fails, it will scale the registry deployment back to 1. If it -// succeeds, it will create a secret used to indicate success to the operator. +// RegistryData runs a migration that copies data on disk in the registry-data PVC to the seaweedfs +// s3 store. If it fails, it will scale the registry deployment back to 1. If it succeeds, it will +// create a secret used to indicate success to the operator. func RegistryData(ctx context.Context) error { // if the migration fails, we need to scale the registry back to 1 success := false @@ -64,7 +64,7 @@ func RegistryData(ctx context.Context) error { } fmt.Printf("Running registry data migration\n") - err = filepath.Walk("/var/lib/embedded-cluster/registry", func(path string, info os.FileInfo, err error) error { + err = filepath.Walk("/registry", func(path string, info os.FileInfo, err error) error { if err != nil { return fmt.Errorf("walk: %w", err) } @@ -79,7 +79,7 @@ func RegistryData(ctx context.Context) error { } defer f.Close() - relPath, err := filepath.Rel("/var/lib/embedded-cluster", path) + relPath, err := filepath.Rel("/", path) if err != nil { return fmt.Errorf("get relative path: %w", err) } @@ -110,6 +110,9 @@ func RegistryData(ctx context.Context) error { ObjectMeta: metav1.ObjectMeta{ Name: registry.RegistryDataMigrationCompleteSecretName, Namespace: registry.RegistryNamespace(), + Labels: map[string]string{ + "replicated.com/disaster-recovery": "ec-install", + }, }, TypeMeta: metav1.TypeMeta{ Kind: "Secret", diff --git a/operator/pkg/registry/migrate.go b/operator/pkg/registry/migrate.go index a2b7325fb..e2c0318aa 100644 --- a/operator/pkg/registry/migrate.go +++ b/operator/pkg/registry/migrate.go @@ -30,22 +30,10 @@ const registryS3SecretName = "seaweedfs-s3-rw" // before finally creating a 'migration is complete' secret in the registry namespace // if this secret is present, the function will return without reattempting the migration func MigrateRegistryData(ctx context.Context, in *clusterv1beta1.Installation, cli client.Client) error { - migrationStatus := k8sutil.CheckConditionStatus(in.Status, RegistryMigrationStatusConditionType) - if migrationStatus == metav1.ConditionTrue { - return nil - } - - hasMigrated, err := HasRegistryMigrated(ctx, cli) + hasMigrated, err := EnsureRegistryMigrationCompleteCondition(ctx, in, cli) if err != nil { - return fmt.Errorf("check if registry has migrated before running migration: %w", err) - } - if hasMigrated { - in.Status.SetCondition(metav1.Condition{ - Type: RegistryMigrationStatusConditionType, - Status: metav1.ConditionTrue, - Reason: "MigrationJobCompleted", - ObservedGeneration: in.Generation, - }) + return err + } else if hasMigrated { return nil } @@ -127,6 +115,31 @@ func MigrateRegistryData(ctx context.Context, in *clusterv1beta1.Installation, c return nil } +// EnsureRegistryMigrationCompleteCondition will check if the registry migration has been +// completed. If so, it will set the condition and return true. +func EnsureRegistryMigrationCompleteCondition(ctx context.Context, in *clusterv1beta1.Installation, cli client.Client) (bool, error) { + migrationStatus := k8sutil.CheckConditionStatus(in.Status, RegistryMigrationStatusConditionType) + if migrationStatus == metav1.ConditionTrue { + return true, nil + } + + hasMigrated, err := HasRegistryMigrated(ctx, cli) + if err != nil { + return false, fmt.Errorf("check if registry has migrated before running migration: %w", err) + } + if !hasMigrated { + return false, nil + } + + in.Status.SetCondition(metav1.Condition{ + Type: RegistryMigrationStatusConditionType, + Status: metav1.ConditionTrue, + Reason: "MigrationJobCompleted", + ObservedGeneration: in.Generation, + }) + return true, nil +} + // HasRegistryMigrated checks if the registry data has been migrated by looking for the 'migration complete' secret in the registry namespace func HasRegistryMigrated(ctx context.Context, cli client.Client) (bool, error) { sec := corev1.Secret{} @@ -180,7 +193,7 @@ func newMigrationJob(in *clusterv1beta1.Installation, cli client.Client) (batchv VolumeMounts: []corev1.VolumeMount{ { Name: "registry-data", - MountPath: "/var/lib/embedded-cluster/registry", + MountPath: "/registry", }, }, EnvFrom: []corev1.EnvFromSource{ diff --git a/operator/pkg/upgrade/autopilot.go b/operator/pkg/upgrade/autopilot.go index 777206696..ddebde433 100644 --- a/operator/pkg/upgrade/autopilot.go +++ b/operator/pkg/upgrade/autopilot.go @@ -48,6 +48,8 @@ func DetermineUpgradeTargets(ctx context.Context, cli client.Client) (apv1b2.Pla // StartAutopilotUpgrade creates an autopilot plan to upgrade to version specified in spec.config.version. func StartAutopilotUpgrade(ctx context.Context, cli client.Client, in *v1beta1.Installation, meta *ectypes.ReleaseMetadata) error { + provider := defaults.NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig) + targets, err := DetermineUpgradeTargets(ctx, cli) if err != nil { return fmt.Errorf("failed to determine upgrade targets: %w", err) @@ -58,11 +60,7 @@ func StartAutopilotUpgrade(ctx context.Context, cli client.Client, in *v1beta1.I // if we are running in an airgap environment all assets are already present in the // node and are served by the local-artifact-mirror binary listening on localhost // port 50000. we just need to get autopilot to fetch the k0s binary from there. - port := defaults.LocalArtifactMirrorPort - if in.Spec.LocalArtifactMirror != nil && in.Spec.LocalArtifactMirror.Port > 0 { - port = in.Spec.LocalArtifactMirror.Port - } - k0surl = fmt.Sprintf("http://127.0.0.1:%d/bin/k0s-upgrade", port) + k0surl = fmt.Sprintf("http://127.0.0.1:%d/bin/k0s-upgrade", provider.LocalArtifactMirrorPort()) } else { artifact := meta.Artifacts["k0s"] if strings.HasPrefix(artifact, "https://") || strings.HasPrefix(artifact, "http://") { diff --git a/operator/pkg/upgrade/installation.go b/operator/pkg/upgrade/installation.go index b15eca4ec..24fa8c228 100644 --- a/operator/pkg/upgrade/installation.go +++ b/operator/pkg/upgrade/installation.go @@ -9,6 +9,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" clusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" ) func CreateInstallation(ctx context.Context, cli client.Client, original *clusterv1beta1.Installation) error { @@ -25,7 +27,12 @@ func CreateInstallation(ctx context.Context, cli client.Client, original *cluste } log.Info(fmt.Sprintf("Creating installation %s", in.Name)) - err := cli.Create(ctx, in) + in, err := maybeOverrideInstallationDataDirs(ctx, cli, in) + if err != nil { + return fmt.Errorf("override installation data dirs: %w", err) + } + + err = cli.Create(ctx, in) if err != nil { return fmt.Errorf("create installation: %w", err) } @@ -72,3 +79,32 @@ func reApplyInstallation(ctx context.Context, cli client.Client, in *clusterv1be return nil } + +// maybeOverrideInstallationDataDirs checks if the installation has an annotation indicating that +// it was created or updated by a version that stored the location of the data directories in the +// installation object. If it is not set, it will set the annotation and update the installation +// object with the old location of the data directories. +func maybeOverrideInstallationDataDirs(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) (*clusterv1beta1.Installation, error) { + previous, err := kubeutils.GetLatestInstallation(ctx, cli) + if err != nil { + return in, fmt.Errorf("get latest installation: %w", err) + } + + if ok := previous.Annotations[embeddedclusteroperator.AnnotationHasDataDirectories]; ok == "true" { + return in, nil + } + if in.ObjectMeta.Annotations == nil { + in.ObjectMeta.Annotations = map[string]string{} + } + in.ObjectMeta.Annotations[embeddedclusteroperator.AnnotationHasDataDirectories] = "true" + + if in.Spec.RuntimeConfig == nil { + in.Spec.RuntimeConfig = &clusterv1beta1.RuntimeConfigSpec{} + } + + // In prior versions, the data directories are not a subdirectory of /var/lib/embedded-cluster. + in.Spec.RuntimeConfig.K0sDataDirOverride = "/var/lib/k0s" + in.Spec.RuntimeConfig.OpenEBSDataDirOverride = "/var/openebs" + + return in, nil +} diff --git a/operator/pkg/upgrade/upgrade.go b/operator/pkg/upgrade/upgrade.go index c59014c2e..b1c34b4fc 100644 --- a/operator/pkg/upgrade/upgrade.go +++ b/operator/pkg/upgrade/upgrade.go @@ -11,6 +11,7 @@ import ( "github.com/replicatedhq/embedded-cluster/operator/pkg/autopilot" "github.com/replicatedhq/embedded-cluster/operator/pkg/charts" "github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil" + "github.com/replicatedhq/embedded-cluster/operator/pkg/registry" "github.com/replicatedhq/embedded-cluster/operator/pkg/release" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" @@ -31,6 +32,11 @@ func Upgrade(ctx context.Context, cli client.Client, in *clusterv1beta1.Installa return fmt.Errorf("k0s upgrade: %w", err) } + err = registryMigrationStatus(ctx, cli, in) + if err != nil { + return fmt.Errorf("registry migration status: %w", err) + } + err = chartUpgrade(ctx, cli, in) if err != nil { return fmt.Errorf("chart upgrade: %w", err) @@ -136,6 +142,20 @@ func k0sUpgrade(ctx context.Context, cli client.Client, in *clusterv1beta1.Insta return nil } +// registryMigrationStatus ensures that the registry migration complete condition is set orior to +// reconciling the helm charts. Otherwise, the registry chart will revert back to non-ha mode. +func registryMigrationStatus(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) error { + if in == nil || !in.Spec.AirGap || !in.Spec.HighAvailability { + return nil + } + + _, err := registry.EnsureRegistryMigrationCompleteCondition(ctx, in, cli) + if err != nil { + return fmt.Errorf("ensure registry migration complete condition: %w", err) + } + return nil +} + func chartUpgrade(ctx context.Context, cli client.Client, original *clusterv1beta1.Installation) error { input := original.DeepCopy() input.Status.SetState(v1beta1.InstallationStateKubernetesInstalled, "", nil) diff --git a/pkg/addons/adminconsole/adminconsole.go b/pkg/addons/adminconsole/adminconsole.go index ae1ff3e42..9a1b4cb35 100644 --- a/pkg/addons/adminconsole/adminconsole.go +++ b/pkg/addons/adminconsole/adminconsole.go @@ -84,13 +84,13 @@ func init() { // AdminConsole manages the admin console helm chart installation. type AdminConsole struct { + provider *defaults.Provider namespace string password string licenseFile string airgapBundle string proxyEnv map[string]string privateCAs map[string]string - port int } // Version returns the embedded admin console version. @@ -115,7 +115,7 @@ func (a *AdminConsole) HostPreflights() (*v1beta2.HostPreflightSpec, error) { // GenerateHelmConfig generates the helm config for the adminconsole and writes the charts to // the disk. -func (a *AdminConsole) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { +func (a *AdminConsole) GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { if !onlyDefaults { helmValues["embeddedClusterID"] = metrics.ClusterID().String() if a.airgapBundle != "" { @@ -133,12 +133,12 @@ func (a *AdminConsole) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, only } helmValues["extraEnv"] = extraEnv } - } - var err error - helmValues, err = helm.SetValue(helmValues, "kurlProxy.nodePort", a.port) - if err != nil { - return nil, nil, fmt.Errorf("set helm values admin-console.kurlProxy.nodePort: %w", err) + var err error + helmValues, err = helm.SetValue(helmValues, "kurlProxy.nodePort", a.provider.AdminConsolePort()) + if err != nil { + return nil, nil, fmt.Errorf("set helm values admin-console.kurlProxy.nodePort: %w", err) + } } values, err := helm.MarshalValues(helmValues) @@ -176,7 +176,7 @@ func (a *AdminConsole) GetAdditionalImages() []string { } // Outro waits for the adminconsole to be ready. -func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { +func (a *AdminConsole) Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { loading := spinner.Start() loading.Infof("Waiting for the Admin Console to deploy") defer loading.Close() @@ -211,7 +211,7 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0s Namespace: a.namespace, AirgapBundle: a.airgapBundle, } - if err := kotscli.Install(installOpts, loading); err != nil { + if err := kotscli.Install(provider, installOpts, loading); err != nil { return err } } @@ -223,22 +223,22 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0s // New creates a new AdminConsole object. func New( + provider *defaults.Provider, namespace string, password string, licenseFile string, airgapBundle string, proxyEnv map[string]string, privateCAs map[string]string, - port int, ) (*AdminConsole, error) { return &AdminConsole{ + provider: provider, namespace: namespace, password: password, licenseFile: licenseFile, airgapBundle: airgapBundle, proxyEnv: proxyEnv, privateCAs: privateCAs, - port: GetPort(port), }, nil } @@ -288,15 +288,7 @@ func GetURL(networkInterface string, port int) string { ipaddr = "NODE-IP-ADDRESS" } } - return fmt.Sprintf("http://%s:%v", ipaddr, GetPort(port)) -} - -// GetURL returns the URL to the admin console. -func GetPort(port int) int { - if port <= 0 { - return defaults.AdminConsolePort - } - return port + return fmt.Sprintf("http://%s:%v", ipaddr, port) } func createRegistrySecret(ctx context.Context, cli client.Client, namespace string) error { diff --git a/pkg/addons/adminconsole/static/metadata.yaml b/pkg/addons/adminconsole/static/metadata.yaml index 93d2bdd80..f6bf72396 100644 --- a/pkg/addons/adminconsole/static/metadata.yaml +++ b/pkg/addons/adminconsole/static/metadata.yaml @@ -5,24 +5,24 @@ # $ make buildtools # $ output/bin/buildtools update addon # -version: 1.117.4 +version: 1.117.5 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console images: kotsadm: repo: proxy.replicated.com/anonymous/kotsadm/kotsadm tag: - amd64: v1.117.4-amd64@sha256:bc11913b3c2433c3457c4054d02a033e4f1c1fa8e4a30e44ff71775742b92efa - arm64: v1.117.4-arm64@sha256:566004e5723120bda8bedd2c47e1e6e219359f81cb04eaab5a739ba8fecc12c2 + amd64: v1.117.5-amd64@sha256:2ae245823556251943cc6f40144b894f863f9b14dbb02270d777948503d253bf + arm64: v1.117.5-arm64@sha256:ace754d367f3bb8511d7d2ba65f5b329a3635eefbb2872675bef42f17968ac4d kotsadm-migrations: repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations tag: - amd64: v1.117.4-amd64@sha256:4b0ec40866f1dfb6918383ef2e8f08983e4463a28c64987c202aef5f64c597e6 - arm64: v1.117.4-arm64@sha256:77d281c1096a46dcd86249c9da2bf2ba3e303e2b5640063ce158900dfe70db50 + amd64: v1.117.5-amd64@sha256:efd3a6614d73647b9c27a04d3dd90a7e2f07b59ba405460948e778122935901f + arm64: v1.117.5-arm64@sha256:5089d7dea86497a52c22af9cee6324c10108bfb52d0e6335207ec288eaf6779a kurl-proxy: repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy tag: - amd64: v1.117.4-amd64@sha256:90ffa0855b0da6b41fe4e2204c3d35c250c32c95e188a97d2eb3ce95181e0f10 - arm64: v1.117.4-arm64@sha256:29334a1cebe104f02ae127a50fb035f73da9647d92f745ef95dec0c737aff3a8 + amd64: v1.117.5-amd64@sha256:d2b6897d4a48ac417e3ceb37646ceca0e9dc9232ada1d9d708cdb2fd3a49d1a6 + arm64: v1.117.5-arm64@sha256:c09cdf05eda0d8dfa979a3e3c2c1b5bbeab8831f1fc2322f1d20dcbe415f04c3 rqlite: repo: proxy.replicated.com/anonymous/kotsadm/rqlite tag: diff --git a/pkg/addons/applier.go b/pkg/addons/applier.go index 244360d4f..ab096155d 100644 --- a/pkg/addons/applier.go +++ b/pkg/addons/applier.go @@ -32,8 +32,8 @@ type AddOn interface { Version() (map[string]string, error) Name() string HostPreflights() (*v1beta2.HostPreflightSpec, error) - GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) - Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error + GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) + Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error GetProtectedFields() map[string][]string GetImages() []string GetAdditionalImages() []string @@ -41,17 +41,17 @@ type AddOn interface { // Applier is an entity that applies (installs and updates) addons in the cluster. type Applier struct { - prompt bool - verbose bool - adminConsolePwd string // admin console password - licenseFile string - onlyDefaults bool - endUserConfig *ecv1beta1.Config - airgapBundle string - proxyEnv map[string]string - privateCAs map[string]string - adminConsolePort int - localArtifactMirrorPort int + prompt bool + verbose bool + adminConsolePwd string // admin console password + licenseFile string + onlyDefaults bool + endUserConfig *ecv1beta1.Config + airgapBundle string + proxyEnv map[string]string + privateCAs map[string]string + provider *defaults.Provider + runtimeConfig *ecv1beta1.RuntimeConfigSpec } // Outro runs the outro in all enabled add-ons. @@ -74,14 +74,14 @@ func (a *Applier) Outro(ctx context.Context, k0sCfg *k0sv1beta1.ClusterConfig, e }() for _, addon := range addons { - if err := addon.Outro(ctx, kcli, k0sCfg, releaseMetadata); err != nil { + if err := addon.Outro(ctx, a.provider, kcli, k0sCfg, releaseMetadata); err != nil { return err } } if err := spinForInstallation(ctx, kcli); err != nil { return err } - if err := printKotsadmLinkMessage(a.licenseFile, networkInterface, a.GetAdminConsolePort()); err != nil { + if err := printKotsadmLinkMessage(a.licenseFile, networkInterface, a.provider.AdminConsolePort()); err != nil { return fmt.Errorf("unable to print success message: %w", err) } return nil @@ -98,7 +98,7 @@ func (a *Applier) OutroForRestore(ctx context.Context, k0sCfg *k0sv1beta1.Cluste return fmt.Errorf("unable to load addons: %w", err) } for _, addon := range addons { - if err := addon.Outro(ctx, kcli, k0sCfg, nil); err != nil { + if err := addon.Outro(ctx, a.provider, kcli, k0sCfg, nil); err != nil { return err } } @@ -116,7 +116,7 @@ func (a *Applier) GenerateHelmConfigs(k0sCfg *k0sv1beta1.ClusterConfig, addition // charts required by embedded-cluster for _, addon := range addons { - addonChartConfig, addonRepositoryConfig, err := addon.GenerateHelmConfig(k0sCfg, a.onlyDefaults) + addonChartConfig, addonRepositoryConfig, err := addon.GenerateHelmConfig(a.provider, k0sCfg, a.onlyDefaults) if err != nil { return nil, nil, fmt.Errorf("unable to generate helm config for %s: %w", addon, err) } @@ -142,7 +142,7 @@ func (a *Applier) GenerateHelmConfigsForRestore(k0sCfg *k0sv1beta1.ClusterConfig // charts required for restore for _, addon := range addons { - addonChartConfig, addonRepositoryConfig, err := addon.GenerateHelmConfig(k0sCfg, a.onlyDefaults) + addonChartConfig, addonRepositoryConfig, err := addon.GenerateHelmConfig(a.provider, k0sCfg, a.onlyDefaults) if err != nil { return nil, nil, fmt.Errorf("unable to generate helm config for %s: %w", addon, err) } @@ -164,7 +164,7 @@ func (a *Applier) GetBuiltinCharts(k0sCfg *k0sv1beta1.ClusterConfig) (map[string } for name, addon := range addons { - chart, repo, err := addon.GenerateHelmConfig(k0sCfg, true) + chart, repo, err := addon.GenerateHelmConfig(a.provider, k0sCfg, true) if err != nil { return nil, fmt.Errorf("unable to generate helm config for %s: %w", name, err) } @@ -252,20 +252,6 @@ func (a *Applier) HostPreflightsForRestore() (*v1beta2.HostPreflightSpec, error) return a.hostPreflights(addons) } -func (a *Applier) GetAdminConsolePort() int { - if a.adminConsolePort <= 0 { - return defaults.AdminConsolePort - } - return a.adminConsolePort -} - -func (a *Applier) GetLocalArtifactMirrorPort() int { - if a.localArtifactMirrorPort <= 0 { - return defaults.LocalArtifactMirrorPort - } - return a.localArtifactMirrorPort -} - func (a *Applier) hostPreflights(addons []AddOn) (*v1beta2.HostPreflightSpec, error) { allpf := &v1beta2.HostPreflightSpec{} for _, addon := range addons { @@ -302,8 +288,7 @@ func (a *Applier) load() ([]AddOn, error) { a.airgapBundle != "", a.proxyEnv, a.privateCAs, - a.GetAdminConsolePort(), - a.GetLocalArtifactMirrorPort(), + a.runtimeConfig, ) if err != nil { return nil, fmt.Errorf("unable to create embedded cluster operator addon: %w", err) @@ -321,13 +306,13 @@ func (a *Applier) load() ([]AddOn, error) { addons = append(addons, vel) aconsole, err := adminconsole.New( + a.provider, defaults.KotsadmNamespace, a.adminConsolePwd, a.licenseFile, a.airgapBundle, a.proxyEnv, a.privateCAs, - a.GetAdminConsolePort(), ) if err != nil { return nil, fmt.Errorf("unable to create admin console addon: %w", err) diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go index de48d6e78..034a52369 100644 --- a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go +++ b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go @@ -31,6 +31,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + // AnnotationHasDataDirectories is an annotation on the installation object that indicates that + // it was created by an operator that stored information about the location of the data + // directories. If this is not set, the operator will update the installation object. + AnnotationHasDataDirectories = "embedded-cluster.replicated.com/has-data-directories" +) + const releaseName = "embedded-cluster-operator" var ( @@ -63,15 +70,14 @@ func init() { // EmbeddedClusterOperator manages the installation of the embedded cluster operator // helm chart. type EmbeddedClusterOperator struct { - namespace string - deployName string - endUserConfig *ecv1beta1.Config - licenseFile string - airgap bool - proxyEnv map[string]string - privateCAs map[string]string - adminConsolePort int - localArtifactMirrorPort int + namespace string + deployName string + endUserConfig *ecv1beta1.Config + runtimeConfig *ecv1beta1.RuntimeConfigSpec + licenseFile string + airgap bool + proxyEnv map[string]string + privateCAs map[string]string } // Version returns the version of the embedded cluster operator chart. @@ -99,7 +105,7 @@ func (e *EmbeddedClusterOperator) GetProtectedFields() map[string][]string { } // GenerateHelmConfig generates the helm config for the embedded cluster operator chart. -func (e *EmbeddedClusterOperator) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { +func (e *EmbeddedClusterOperator) GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { chartConfig := ecv1beta1.Chart{ Name: releaseName, ChartName: Metadata.Location, @@ -202,7 +208,7 @@ func createCAConfigmap(ctx context.Context, cli client.Client, namespace string, // Outro is executed after the cluster deployment. Waits for the embedded cluster operator // to finish its deployment, creates the version metadata configmap (if in airgap) and // the installation object. -func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { +func (e *EmbeddedClusterOperator) Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { loading := spinner.Start() loading.Infof("Waiting for Embedded Cluster Operator to be ready") @@ -261,20 +267,18 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client, Labels: map[string]string{ "replicated.com/disaster-recovery": "ec-install", }, + Annotations: map[string]string{ + AnnotationHasDataDirectories: "true", + }, }, Spec: ecv1beta1.InstallationSpec{ - ClusterID: metrics.ClusterID().String(), - MetricsBaseURL: metrics.BaseURL(license), - AirGap: e.airgap, - Proxy: proxySpec, - Network: k0sConfigToNetworkSpec(k0sCfg), - AdminConsole: &ecv1beta1.AdminConsoleSpec{ - Port: e.adminConsolePort, - }, - LocalArtifactMirror: &ecv1beta1.LocalArtifactMirrorSpec{ - Port: e.localArtifactMirrorPort, - }, + ClusterID: metrics.ClusterID().String(), + MetricsBaseURL: metrics.BaseURL(license), + AirGap: e.airgap, + Proxy: proxySpec, + Network: k0sConfigToNetworkSpec(k0sCfg), Config: cfgspec, + RuntimeConfig: e.runtimeConfig, EndUserK0sConfigOverrides: euOverrides, BinaryName: defaults.BinaryName(), LicenseInfo: &ecv1beta1.LicenseInfo{ @@ -295,19 +299,17 @@ func New( airgapEnabled bool, proxyEnv map[string]string, privateCAs map[string]string, - adminConsolePort int, - localArtifactMirrorPort int, + runtimeConfig *ecv1beta1.RuntimeConfigSpec, ) (*EmbeddedClusterOperator, error) { return &EmbeddedClusterOperator{ - namespace: "embedded-cluster", - deployName: "embedded-cluster-operator", - endUserConfig: endUserConfig, - licenseFile: licenseFile, - airgap: airgapEnabled, - proxyEnv: proxyEnv, - privateCAs: privateCAs, - adminConsolePort: adminConsolePort, - localArtifactMirrorPort: localArtifactMirrorPort, + namespace: "embedded-cluster", + deployName: "embedded-cluster-operator", + endUserConfig: endUserConfig, + licenseFile: licenseFile, + airgap: airgapEnabled, + proxyEnv: proxyEnv, + privateCAs: privateCAs, + runtimeConfig: runtimeConfig, }, nil } diff --git a/pkg/addons/openebs/openebs.go b/pkg/addons/openebs/openebs.go index c79b8c965..5446403dc 100644 --- a/pkg/addons/openebs/openebs.go +++ b/pkg/addons/openebs/openebs.go @@ -15,6 +15,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" + "github.com/replicatedhq/embedded-cluster/pkg/helm" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/replicatedhq/embedded-cluster/pkg/release" "github.com/replicatedhq/embedded-cluster/pkg/spinner" @@ -74,7 +76,7 @@ func (o *OpenEBS) GetProtectedFields() map[string][]string { } // GenerateHelmConfig generates the helm config for the OpenEBS chart. -func (o *OpenEBS) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { +func (o *OpenEBS) GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { chartConfig := ecv1beta1.Chart{ Name: releaseName, ChartName: Metadata.Location, @@ -84,6 +86,14 @@ func (o *OpenEBS) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefau Order: 1, } + if !onlyDefaults { + var err error + helmValues, err = helm.SetValue(helmValues, `["localpv-provisioner"].localpv.basePath`, provider.EmbeddedClusterOpenEBSLocalSubDir()) + if err != nil { + return nil, nil, fmt.Errorf("set helm values localpv-provisioner.localpv.basePath: %w", err) + } + } + valuesStringData, err := yaml.Marshal(helmValues) if err != nil { return nil, nil, fmt.Errorf("unable to marshal helm values: %w", err) @@ -110,7 +120,7 @@ func (o *OpenEBS) GetAdditionalImages() []string { } // Outro is executed after the cluster deployment. -func (o *OpenEBS) Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { +func (o *OpenEBS) Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { loading := spinner.Start() loading.Infof("Waiting for Storage to be ready") if err := kubeutils.WaitForDeployment(ctx, cli, namespace, "openebs-localpv-provisioner"); err != nil { diff --git a/pkg/addons/openebs/static/values.tpl.yaml b/pkg/addons/openebs/static/values.tpl.yaml index f629c7e7c..1d80376be 100644 --- a/pkg/addons/openebs/static/values.tpl.yaml +++ b/pkg/addons/openebs/static/values.tpl.yaml @@ -20,13 +20,14 @@ localpv-provisioner: hostpathClass: enabled: true isDefaultClass: true -{{- if .ReplaceImages }} localpv: +{{- if .ReplaceImages }} image: registry: proxy.replicated.com/anonymous/ repository: '{{ TrimPrefix "proxy.replicated.com/anonymous/" (index .Images "openebs-provisioner-localpv").Repo }}' tag: '{{ index (index .Images "openebs-provisioner-localpv").Tag .GOARCH }}' {{- end }} + basePath: "/var/lib/embedded-cluster/openebs-local" lvm-localpv: enabled: false mayastor: diff --git a/pkg/addons/options.go b/pkg/addons/options.go index 6eeef9564..5328f991d 100644 --- a/pkg/addons/options.go +++ b/pkg/addons/options.go @@ -1,7 +1,8 @@ package addons import ( - embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" ) // Option sets and option on an Applier reference. @@ -21,20 +22,6 @@ func WithPrivateCAs(privateCAs map[string]string) Option { } } -// WithAdminConsolePort sets the port on which the admin console will be served. -func WithAdminConsolePort(port int) Option { - return func(a *Applier) { - a.adminConsolePort = port - } -} - -// WithLocalArtifactMirrorPort sets the port on which the local artifact mirror will be served. -func WithLocalArtifactMirrorPort(port int) Option { - return func(a *Applier) { - a.localArtifactMirrorPort = port - } -} - // Quiet disables logging for addons. func Quiet() Option { return func(a *Applier) { @@ -46,18 +33,29 @@ func Quiet() Option { func OnlyDefaults() Option { return func(a *Applier) { a.onlyDefaults = true + a.runtimeConfig = ecv1beta1.GetDefaultRuntimeConfig() + a.provider = defaults.NewProviderFromRuntimeConfig(a.runtimeConfig) } } // WithEndUserConfig sets the end user config passed in by the customer // at install time. This configuration is similar to the one embedded // in the cluster through a Kots Release. -func WithEndUserConfig(config *embeddedclusterv1beta1.Config) Option { +func WithEndUserConfig(config *ecv1beta1.Config) Option { return func(a *Applier) { a.endUserConfig = config } } +// WithRuntimeConfig sets the runtime config passed in by the customer +// at install time. +func WithRuntimeConfig(runtimeConfig *ecv1beta1.RuntimeConfigSpec) Option { + return func(a *Applier) { + a.runtimeConfig = runtimeConfig + a.provider = defaults.NewProviderFromRuntimeConfig(runtimeConfig) + } +} + // WithLicense sets the license for the application. func WithLicense(licenseFile string) Option { return func(a *Applier) { diff --git a/pkg/addons/registry/registry.go b/pkg/addons/registry/registry.go index 5acc33845..bcfd2d820 100644 --- a/pkg/addons/registry/registry.go +++ b/pkg/addons/registry/registry.go @@ -21,6 +21,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/airgap" "github.com/replicatedhq/embedded-cluster/pkg/certs" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/replicatedhq/embedded-cluster/pkg/release" @@ -99,7 +100,7 @@ func (o *Registry) GetProtectedFields() map[string][]string { } // GenerateHelmConfig generates the helm config for the Registry chart. -func (o *Registry) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { +func (o *Registry) GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { if !o.isAirgap { return nil, nil, nil } @@ -262,7 +263,7 @@ func (o *Registry) generateRegistryMigrationRole(ctx context.Context, cli client } // Outro is executed after the cluster deployment. -func (o *Registry) Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { +func (o *Registry) Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { if !o.isAirgap { return nil } diff --git a/pkg/addons/registry/static/values-ha.tpl.yaml b/pkg/addons/registry/static/values-ha.tpl.yaml index 45b9052ef..aa3d84f06 100644 --- a/pkg/addons/registry/static/values-ha.tpl.yaml +++ b/pkg/addons/registry/static/values-ha.tpl.yaml @@ -41,3 +41,7 @@ secrets: s3: secretRef: seaweedfs-s3-rw storage: s3 +service: + name: registry + type: ClusterIP + port: 5000 diff --git a/pkg/addons/registry/static/values.tpl.yaml b/pkg/addons/registry/static/values.tpl.yaml index ab6be39bb..aa939076c 100644 --- a/pkg/addons/registry/static/values.tpl.yaml +++ b/pkg/addons/registry/static/values.tpl.yaml @@ -25,3 +25,7 @@ podAnnotations: backup.velero.io/backup-volumes: data replicaCount: 1 storage: filesystem +service: + name: registry + type: ClusterIP + port: 5000 diff --git a/pkg/addons/seaweedfs/seaweedfs.go b/pkg/addons/seaweedfs/seaweedfs.go index 939771710..ba6b80e8b 100644 --- a/pkg/addons/seaweedfs/seaweedfs.go +++ b/pkg/addons/seaweedfs/seaweedfs.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "fmt" + "path/filepath" "time" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" @@ -15,6 +16,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" + "github.com/replicatedhq/embedded-cluster/pkg/helm" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/replicatedhq/embedded-cluster/pkg/release" "github.com/replicatedhq/embedded-cluster/pkg/spinner" @@ -73,7 +76,7 @@ func (o *SeaweedFS) GetProtectedFields() map[string][]string { } // GenerateHelmConfig generates the helm config for the SeaweedFS chart. -func (o *SeaweedFS) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { +func (o *SeaweedFS) GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { if !o.isAirgap { return nil, nil, nil } @@ -87,6 +90,20 @@ func (o *SeaweedFS) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDef Order: 2, } + if !onlyDefaults { + var err error + dataPath := filepath.Join(provider.EmbeddedClusterSeaweedfsSubDir(), "ssd") + helmValues, err = helm.SetValue(helmValues, "global.data.hostPathPrefix", dataPath) + if err != nil { + return nil, nil, fmt.Errorf("set helm values global.data.hostPathPrefix: %w", err) + } + logsPath := filepath.Join(provider.EmbeddedClusterSeaweedfsSubDir(), "storage") + helmValues, err = helm.SetValue(helmValues, "global.logs.hostPathPrefix", logsPath) + if err != nil { + return nil, nil, fmt.Errorf("set helm values global.logs.hostPathPrefix: %w", err) + } + } + valuesStringData, err := yaml.Marshal(helmValues) if err != nil { return nil, nil, fmt.Errorf("unable to marshal helm values: %w", err) @@ -109,7 +126,7 @@ func (o *SeaweedFS) GetAdditionalImages() []string { } // Outro is executed after the cluster deployment. -func (o *SeaweedFS) Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { +func (o *SeaweedFS) Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { // SeaweedFS is applied by the operator return nil } diff --git a/pkg/addons/velero/static/values.tpl.yaml b/pkg/addons/velero/static/values.tpl.yaml index 26eb40167..4b8453cdc 100644 --- a/pkg/addons/velero/static/values.tpl.yaml +++ b/pkg/addons/velero/static/values.tpl.yaml @@ -26,5 +26,5 @@ configMaps: image: '{{ ImageString (index .Images "velero-restore-helper") }}' {{- end }} nodeAgent: - podVolumePath: /var/lib/k0s/kubelet/pods + podVolumePath: /var/lib/embedded-cluster/k0s/kubelet/pods snapshotsEnabled: false diff --git a/pkg/addons/velero/velero.go b/pkg/addons/velero/velero.go index 4e235d3f4..5e445d4f7 100644 --- a/pkg/addons/velero/velero.go +++ b/pkg/addons/velero/velero.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "fmt" + "path/filepath" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" @@ -15,6 +16,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" + "github.com/replicatedhq/embedded-cluster/pkg/helm" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/replicatedhq/embedded-cluster/pkg/release" "github.com/replicatedhq/embedded-cluster/pkg/spinner" @@ -77,7 +80,7 @@ func (o *Velero) GetProtectedFields() map[string][]string { } // GenerateHelmConfig generates the helm config for the Velero chart. -func (o *Velero) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { +func (o *Velero) GenerateHelmConfig(provider *defaults.Provider, k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaults bool) ([]ecv1beta1.Chart, []ecv1beta1.Repository, error) { if !o.isEnabled { return nil, nil, nil } @@ -91,13 +94,22 @@ func (o *Velero) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, onlyDefaul Order: 3, } - if len(o.proxyEnv) > 0 { - extraEnvVars := map[string]interface{}{} - for k, v := range o.proxyEnv { - extraEnvVars[k] = v + if !onlyDefaults { + if len(o.proxyEnv) > 0 { + extraEnvVars := map[string]interface{}{} + for k, v := range o.proxyEnv { + extraEnvVars[k] = v + } + helmValues["configuration"] = map[string]interface{}{ + "extraEnvVars": extraEnvVars, + } } - helmValues["configuration"] = map[string]interface{}{ - "extraEnvVars": extraEnvVars, + + var err error + podVolumePath := filepath.Join(provider.EmbeddedClusterK0sSubDir(), "kubelet/pods") + helmValues, err = helm.SetValue(helmValues, "nodeAgent.podVolumePath", podVolumePath) + if err != nil { + return nil, nil, fmt.Errorf("set helm values nodeAgent.podVolumePath: %w", err) } } @@ -130,7 +142,7 @@ func (o *Velero) GetAdditionalImages() []string { } // Outro is executed after the cluster deployment. -func (o *Velero) Outro(ctx context.Context, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { +func (o *Velero) Outro(ctx context.Context, provider *defaults.Provider, cli client.Client, k0sCfg *k0sv1beta1.ClusterConfig, releaseMetadata *types.ReleaseMetadata) error { if !o.isEnabled { return nil } diff --git a/pkg/airgap/materialize.go b/pkg/airgap/materialize.go index ffc0fdaaa..2fbe82c3a 100644 --- a/pkg/airgap/materialize.go +++ b/pkg/airgap/materialize.go @@ -11,12 +11,12 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/defaults" ) -const K0sImagePath = "/var/lib/k0s/images/images-amd64.tar" +const K0sImagePath = "images/images-amd64.tar" // MaterializeAirgap places the airgap image bundle for k0s and the embedded cluster charts on disk. // - image bundle should be located at 'images-amd64.tar' within the embedded-cluster directory within the airgap bundle. // - charts should be located at 'charts.tar.gz' within the embedded-cluster directory within the airgap bundle. -func MaterializeAirgap(airgapReader io.Reader) error { +func MaterializeAirgap(provider *defaults.Provider, airgapReader io.Reader) error { // decompress tarball ungzip, err := gzip.NewReader(airgapReader) if err != nil { @@ -37,7 +37,7 @@ func MaterializeAirgap(airgapReader io.Reader) error { } if nextFile.Name == "embedded-cluster/images-amd64.tar" { - err = writeOneFile(tarreader, K0sImagePath, nextFile.Mode) + err = writeOneFile(tarreader, filepath.Join(provider.EmbeddedClusterK0sSubDir(), K0sImagePath), nextFile.Mode) if err != nil { return fmt.Errorf("failed to write k0s images file: %w", err) } @@ -45,7 +45,7 @@ func MaterializeAirgap(airgapReader io.Reader) error { } if nextFile.Name == "embedded-cluster/charts.tar.gz" { - err = writeChartFiles(tarreader) + err = writeChartFiles(provider, tarreader) if err != nil { return fmt.Errorf("failed to write chart files: %w", err) } @@ -82,7 +82,7 @@ func writeOneFile(reader io.Reader, path string, mode int64) error { } // take in a stream of a tarball and write the charts contained within to disk -func writeChartFiles(reader io.Reader) error { +func writeChartFiles(provider *defaults.Provider, reader io.Reader) error { // decompress tarball ungzip, err := gzip.NewReader(reader) if err != nil { @@ -105,7 +105,7 @@ func writeChartFiles(reader io.Reader) error { continue } - subdir := defaults.EmbeddedClusterChartsSubDir() + subdir := provider.EmbeddedClusterChartsSubDir() dst := filepath.Join(subdir, nextFile.Name) if err := writeOneFile(tarreader, dst, nextFile.Mode); err != nil { return fmt.Errorf("failed to write chart file: %w", err) diff --git a/pkg/airgap/remap.go b/pkg/airgap/remap.go index 81ba800c5..bf056002c 100644 --- a/pkg/airgap/remap.go +++ b/pkg/airgap/remap.go @@ -10,17 +10,17 @@ import ( // RemapHelm removes all helm repositories from the cluster config, and changes the upstreams of all helm charts // to paths on the host within the charts directory -func RemapHelm(cfg *v1beta1.ClusterConfig) { +func RemapHelm(provider *defaults.Provider, cfg *v1beta1.ClusterConfig) { // there's no upstream to reach, so we can zero out the repositories cfg.Spec.Extensions.Helm.Repositories = nil // replace each chart's name with the path on the host it should be found at // see https://docs.k0sproject.io/v1.29.2+k0s.0/helm-charts/#example for idx := range cfg.Spec.Extensions.Helm.Charts { - cfg.Spec.Extensions.Helm.Charts[idx].ChartName = helmChartHostPath(cfg.Spec.Extensions.Helm.Charts[idx]) + cfg.Spec.Extensions.Helm.Charts[idx].ChartName = helmChartHostPath(provider, cfg.Spec.Extensions.Helm.Charts[idx]) } } -func helmChartHostPath(chart v1beta1.Chart) string { - return filepath.Join(defaults.EmbeddedClusterChartsSubDir(), fmt.Sprintf("%s-%s.tgz", chart.Name, chart.Version)) +func helmChartHostPath(provider *defaults.Provider, chart v1beta1.Chart) string { + return filepath.Join(provider.EmbeddedClusterChartsSubDir(), fmt.Sprintf("%s-%s.tgz", chart.Name, chart.Version)) } diff --git a/pkg/config/config.go b/pkg/config/config.go index ed03c5e6c..0c798411c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -170,18 +170,32 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C } // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster. -func InstallFlags(nodeIP string) []string { - return []string{ +func InstallFlags(provider *defaults.Provider, nodeIP string) []string { + flags := []string{ "install", "controller", - "--disable-components", "konnectivity-server", "--labels", strings.Join(nodeLabels(), ","), "--enable-worker", "--no-taints", - "--enable-dynamic-config", - "--kubelet-extra-args", fmt.Sprintf(`"--node-ip=%s"`, nodeIP), "-c", defaults.PathToK0sConfig(), } + flags = append(flags, AdditionalInstallFlags(provider, nodeIP)...) + flags = append(flags, AdditionalInstallFlagsController()...) + return flags +} + +func AdditionalInstallFlags(provider *defaults.Provider, nodeIP string) []string { + return []string{ + "--kubelet-extra-args", fmt.Sprintf(`"--node-ip=%s"`, nodeIP), + "--data-dir", provider.EmbeddedClusterK0sSubDir(), + } +} + +func AdditionalInstallFlagsController() []string { + return []string{ + "--disable-components", "konnectivity-server", + "--enable-dynamic-config", + } } func ControllerLabels() map[string]string { diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 99a0347ef..d7aa33225 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -3,9 +3,16 @@ // these should not happen in the first place. package defaults -var ( - // DefaultProvider holds the default provider and is used by the exported functions. - DefaultProvider = NewProvider("") +import ( + "io" + "net" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/gosimple/slug" + "github.com/sirupsen/logrus" ) // Holds the default no proxy values. @@ -18,79 +25,96 @@ const SeaweedFSNamespace = "seaweedfs" const RegistryNamespace = "registry" const VeleroNamespace = "velero" -const AdminConsolePort = 30000 -const LocalArtifactMirrorPort = 50000 - -// BinaryName calls BinaryName on the default provider. +// BinaryName returns the binary name, this is useful for places where we +// need to present the name of the binary to the user (the name may vary if +// the binary is renamed). We make sure the name does not contain invalid +// characters for a filename. func BinaryName() string { - return DefaultProvider.BinaryName() -} - -// EmbeddedClusterBinsSubDir calls EmbeddedClusterBinsSubDir on the default provider. -func EmbeddedClusterBinsSubDir() string { - return DefaultProvider.EmbeddedClusterBinsSubDir() + exe, err := os.Executable() + if err != nil { + logrus.Fatalf("unable to get executable path: %s", err) + } + base := filepath.Base(exe) + return slug.Make(base) } -// EmbeddedClusterChartsSubDir calls EmbeddedClusterChartsSubDir on the default provider. -func EmbeddedClusterChartsSubDir() string { - return DefaultProvider.EmbeddedClusterChartsSubDir() -} - -// EmbeddedClusterImagesSubDir calls EmbeddedClusterImagesSubDir on the default provider. -func EmbeddedClusterImagesSubDir() string { - return DefaultProvider.EmbeddedClusterImagesSubDir() -} - -// EmbeddedClusterLogsSubDir calls EmbeddedClusterLogsSubDir on the default provider. +// EmbeddedClusterLogsSubDir returns the path to the directory where embedded-cluster logs +// are stored. func EmbeddedClusterLogsSubDir() string { - return DefaultProvider.EmbeddedClusterLogsSubDir() -} - -// K0sBinaryPath calls K0sBinaryPath on the default provider. -func K0sBinaryPath() string { - return DefaultProvider.K0sBinaryPath() -} - -// PathToEmbeddedClusterBinary calls PathToEmbeddedClusterBinary on the default provider. -func PathToEmbeddedClusterBinary(name string) string { - return DefaultProvider.PathToEmbeddedClusterBinary(name) + path := "/var/log/embedded-cluster" + if err := os.MkdirAll(path, 0755); err != nil { + logrus.Fatalf("unable to create embedded-cluster logs dir: %s", err) + } + return path } -// PathToLog calls PathToLog on the default provider. +// PathToLog returns the full path to a log file. This function does not check +// if the file exists. func PathToLog(name string) string { - return DefaultProvider.PathToLog(name) + return filepath.Join(EmbeddedClusterLogsSubDir(), name) } -// PathToKubeConfig calls PathToKubeConfig on the default provider. -func PathToKubeConfig() string { - return DefaultProvider.PathToKubeConfig() +// K0sBinaryPath returns the path to the k0s binary when it is installed on the node. This +// does not return the binary just after we materilized it but the path we want it to be +// once it is installed. +func K0sBinaryPath() string { + return "/usr/local/bin/k0s" } -// TryDiscoverPublicIP calls TryDiscoverPublicIP on the default provider. -func TryDiscoverPublicIP() string { - return DefaultProvider.TryDiscoverPublicIP() -} +// TryDiscoverPublicIP tries to discover the public IP of the node by querying +// a list of known providers. If the public IP cannot be discovered, an empty +// string is returned. -// PathToK0sConfig calls PathToK0sConfig on the default provider. -func PathToK0sConfig() string { - return DefaultProvider.PathToK0sConfig() +func TryDiscoverPublicIP() string { + // List of providers and their respective metadata URLs + providers := []struct { + name string + url string + headers map[string]string + }{ + {"gce", "http://169.254.169.254/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip", map[string]string{"Metadata-Flavor": "Google"}}, + {"ec2", "http://169.254.169.254/latest/meta-data/public-ipv4", nil}, + {"azure", "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-08-01&format=text", map[string]string{"Metadata": "true"}}, + } + + for _, provider := range providers { + client := &http.Client{ + Timeout: 5 * time.Second, + } + req, _ := http.NewRequest("GET", provider.url, nil) + for k, v := range provider.headers { + req.Header.Add(k, v) + } + resp, err := client.Do(req) + if err != nil { + return "" + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + publicIP := string(bodyBytes) + if net.ParseIP(publicIP).To4() != nil { + return publicIP + } else { + return "" + } + } + } + return "" } -// PathToK0sStatusSocket calls PathToK0sStatusSocket on the default provider. +// PathToK0sStatusSocket returns the full path to the k0s status socket. func PathToK0sStatusSocket() string { - return DefaultProvider.PathToK0sStatusSocket() + return "/run/k0s/status.sock" } -func PathToK0sContainerdConfig() string { - return DefaultProvider.PathToK0sContainerdConfig() -} - -// EmbeddedClusterHomeDirectory calls EmbeddedClusterHomeDirectory on the default provider. -func EmbeddedClusterHomeDirectory() string { - return DefaultProvider.EmbeddedClusterHomeDirectory() +// PathToK0sConfig returns the full path to the k0s configuration file. +func PathToK0sConfig() string { + return "/etc/k0s/k0s.yaml" } -// PathToEmbeddedClusterSupportFile calls PathToEmbeddedClusterSupportFile on the default provider. -func PathToEmbeddedClusterSupportFile(name string) string { - return DefaultProvider.PathToEmbeddedClusterSupportFile(name) +// PathToK0sContainerdConfig returns the full path to the k0s containerd configuration directory +func PathToK0sContainerdConfig() string { + return "/etc/k0s/containerd.d/" } diff --git a/pkg/defaults/provider.go b/pkg/defaults/provider.go index 2e79ef1e9..cd37978d0 100644 --- a/pkg/defaults/provider.go +++ b/pkg/defaults/provider.go @@ -1,65 +1,98 @@ package defaults import ( - "io" - "net" - "net/http" + "context" + "fmt" "os" "path/filepath" - "time" - "github.com/gosimple/slug" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// NewProvider returns a new Provider using the provided base dir. -// Base is the base directory inside which all the other directories are +// NewProvider returns a new Provider using the provided data dir. +// Data is the base directory inside which all the other directories are // created. -func NewProvider(base string) *Provider { - obj := &Provider{Base: base} - obj.Init() +func NewProvider(dataDir string) *Provider { + return NewProviderFromRuntimeConfig(&ecv1beta1.RuntimeConfigSpec{ + DataDir: dataDir, + }) +} + +// NewProviderFromRuntimeConfig returns a new Provider using the provided runtime config. +func NewProviderFromRuntimeConfig(runtimeConfig *ecv1beta1.RuntimeConfigSpec) *Provider { + obj := &Provider{ + runtimeConfig: runtimeConfig, + } return obj } +// NewProviderFromCluster discovers the provider from the installation object. If there is no +// runtime config, this is probably a prior version of EC so we will have to fall back to the +// filesystem. +func NewProviderFromCluster(ctx context.Context, cli client.Client) (*Provider, error) { + in, err := kubeutils.GetLatestInstallation(ctx, cli) + if err != nil { + return nil, fmt.Errorf("get latest installation: %w", err) + } + + if in.Spec.RuntimeConfig == nil { + // If there is no runtime config, this is probably a prior version of EC so we will have to + // fall back to the filesystem. + return NewProviderFromFilesystem() + } + return NewProviderFromRuntimeConfig(in.Spec.RuntimeConfig), nil +} + +// NewProviderFromFilesystem returns a new provider from the filesystem. It supports older versions +// of EC that used a different directory for k0s and openebs. +func NewProviderFromFilesystem() (*Provider, error) { + provider := NewProvider(ecv1beta1.DefaultDataDir) + _, err := os.Stat(provider.PathToKubeConfig()) + if err == nil { + return provider, nil + } + // Handle versions prior to consolidation of data dirs + provider = NewProviderFromRuntimeConfig(&ecv1beta1.RuntimeConfigSpec{ + DataDir: ecv1beta1.DefaultDataDir, + K0sDataDirOverride: "/var/lib/k0s", + OpenEBSDataDirOverride: "/var/openebs", + }) + _, err = os.Stat(provider.PathToKubeConfig()) + if err == nil { + return provider, nil + } + return nil, fmt.Errorf("unable to discover provider from filesystem") +} + // Provider is an entity that provides default values used during // EmbeddedCluster installation. type Provider struct { - Base string + runtimeConfig *ecv1beta1.RuntimeConfigSpec } -// Init makes sure all the necessary directory exists on the system. -func (d *Provider) Init() {} - -// BinaryName returns the binary name, this is useful for places where we -// need to present the name of the binary to the user (the name may vary if -// the binary is renamed). We make sure the name does not contain invalid -// characters for a filename. -func (d *Provider) BinaryName() string { - exe, err := os.Executable() - if err != nil { - logrus.Fatalf("unable to get executable path: %s", err) +// EmbeddedClusterHomeDirectory returns the parent directory. Inside this parent directory we +// store all the embedded-cluster related files. +func (d *Provider) EmbeddedClusterHomeDirectory() string { + if d.runtimeConfig != nil && d.runtimeConfig.DataDir != "" { + return d.runtimeConfig.DataDir } - base := filepath.Base(exe) - return slug.Make(base) + return ecv1beta1.DefaultDataDir } -// EmbeddedClusterLogsSubDir returns the path to the directory where embedded-cluster logs -// are stored. -func (d *Provider) EmbeddedClusterLogsSubDir() string { - path := filepath.Join(d.EmbeddedClusterHomeDirectory(), "logs") +// EmbeddedClusterTmpSubDir returns the path to the tmp directory where embedded-cluster +// stores temporary files. +func (d *Provider) EmbeddedClusterTmpSubDir() string { + path := filepath.Join(d.EmbeddedClusterHomeDirectory(), "tmp") + if err := os.MkdirAll(path, 0755); err != nil { - logrus.Fatalf("unable to create embedded-cluster logs dir: %s", err) + logrus.Fatalf("unable to create embedded-cluster tmp dir: %s", err) } - return path } -// PathToLog returns the full path to a log file. This function does not check -// if the file exists. -func (d *Provider) PathToLog(name string) string { - return filepath.Join(d.EmbeddedClusterLogsSubDir(), name) -} - // EmbeddedClusterBinsSubDir returns the path to the directory where embedded-cluster binaries // are stored. func (d *Provider) EmbeddedClusterBinsSubDir() string { @@ -91,17 +124,25 @@ func (d *Provider) EmbeddedClusterImagesSubDir() string { return path } -// EmbeddedClusterHomeDirectory returns the parent directory. Inside this parent directory we -// store all the embedded-cluster related files. -func (d *Provider) EmbeddedClusterHomeDirectory() string { - return filepath.Join(d.Base, "/var/lib/embedded-cluster") +// EmbeddedClusterK0sSubDir returns the path to the directory where k0s data is stored. +func (d *Provider) EmbeddedClusterK0sSubDir() string { + if d.runtimeConfig != nil && d.runtimeConfig.K0sDataDirOverride != "" { + return d.runtimeConfig.K0sDataDirOverride + } + return filepath.Join(d.EmbeddedClusterHomeDirectory(), "k0s") +} + +// EmbeddedClusterSeaweedfsSubDir returns the path to the directory where seaweedfs data is stored. +func (d *Provider) EmbeddedClusterSeaweedfsSubDir() string { + return filepath.Join(d.EmbeddedClusterHomeDirectory(), "seaweedfs") } -// K0sBinaryPath returns the path to the k0s binary when it is installed on the node. This -// does not return the binary just after we materilized it but the path we want it to be -// once it is installed. -func (d *Provider) K0sBinaryPath() string { - return "/usr/local/bin/k0s" +// EmbeddedClusterOpenEBSLocalSubDir returns the path to the directory where OpenEBS local data is stored. +func (d *Provider) EmbeddedClusterOpenEBSLocalSubDir() string { + if d.runtimeConfig != nil && d.runtimeConfig.OpenEBSDataDirOverride != "" { + return d.runtimeConfig.OpenEBSDataDirOverride + } + return filepath.Join(d.EmbeddedClusterHomeDirectory(), "openebs-local") } // PathToEmbeddedClusterBinary is an utility function that returns the full path to a @@ -111,65 +152,9 @@ func (d *Provider) PathToEmbeddedClusterBinary(name string) string { return filepath.Join(d.EmbeddedClusterBinsSubDir(), name) } +// PathToKubeConfig returns the path to the kubeconfig file. func (d *Provider) PathToKubeConfig() string { - return "/var/lib/k0s/pki/admin.conf" -} - -// TryDiscoverPublicIP tries to discover the public IP of the node by querying -// a list of known providers. If the public IP cannot be discovered, an empty -// string is returned. -func (d *Provider) TryDiscoverPublicIP() string { - // List of providers and their respective metadata URLs - providers := []struct { - name string - url string - headers map[string]string - }{ - {"gce", "http://169.254.169.254/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip", map[string]string{"Metadata-Flavor": "Google"}}, - {"ec2", "http://169.254.169.254/latest/meta-data/public-ipv4", nil}, - {"azure", "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-08-01&format=text", map[string]string{"Metadata": "true"}}, - } - - for _, provider := range providers { - client := &http.Client{ - Timeout: 5 * time.Second, - } - req, _ := http.NewRequest("GET", provider.url, nil) - for k, v := range provider.headers { - req.Header.Add(k, v) - } - resp, err := client.Do(req) - if err != nil { - return "" - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - bodyBytes, _ := io.ReadAll(resp.Body) - publicIP := string(bodyBytes) - if net.ParseIP(publicIP).To4() != nil { - return publicIP - } else { - return "" - } - } - } - return "" -} - -// PathToK0sStatusSocket returns the full path to the k0s status socket. -func (d *Provider) PathToK0sStatusSocket() string { - return "/run/k0s/status.sock" -} - -// PathToK0sConfig returns the full path to the k0s configuration file. -func (d *Provider) PathToK0sConfig() string { - return "/etc/k0s/k0s.yaml" -} - -// PathToK0sContainerdConfig returns the full path to the k0s containerd configuration directory -func (d *Provider) PathToK0sContainerdConfig() string { - return "/etc/k0s/containerd.d/" + return filepath.Join(d.EmbeddedClusterK0sSubDir(), "pki/admin.conf") } // EmbeddedClusterSupportSubDir returns the path to the directory where embedded-cluster @@ -188,3 +173,17 @@ func (d *Provider) EmbeddedClusterSupportSubDir() string { func (d *Provider) PathToEmbeddedClusterSupportFile(name string) string { return filepath.Join(d.EmbeddedClusterSupportSubDir(), name) } + +func (d *Provider) LocalArtifactMirrorPort() int { + if d.runtimeConfig != nil && d.runtimeConfig.LocalArtifactMirror.Port > 0 { + return d.runtimeConfig.LocalArtifactMirror.Port + } + return ecv1beta1.DefaultLocalArtifactMirrorPort +} + +func (d *Provider) AdminConsolePort() int { + if d.runtimeConfig != nil && d.runtimeConfig.AdminConsole.Port > 0 { + return d.runtimeConfig.AdminConsole.Port + } + return ecv1beta1.DefaultAdminConsolePort +} diff --git a/pkg/defaults/provider_test.go b/pkg/defaults/provider_test.go deleted file mode 100644 index 5727ac0ee..000000000 --- a/pkg/defaults/provider_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package defaults - -import ( - "os" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestInit(t *testing.T) { - tmpdir, err := os.MkdirTemp("", "embedded-cluster") - assert.NoError(t, err) - defer os.RemoveAll(tmpdir) - def := NewProvider(tmpdir) - assert.DirExists(t, def.EmbeddedClusterLogsSubDir(), "logs dir should exist") - assert.DirExists(t, def.EmbeddedClusterBinsSubDir(), "embedded-cluster binary dir should exist") -} - -func TestEnsureAllDirectoriesAreInsideBase(t *testing.T) { - tmpdir, err := os.MkdirTemp("", "embedded-cluster") - assert.NoError(t, err) - defer os.RemoveAll(tmpdir) - def := NewProvider(tmpdir) - for _, fn := range []func() string{ - def.EmbeddedClusterBinsSubDir, - def.EmbeddedClusterLogsSubDir, - } { - assert.Contains(t, fn(), tmpdir, "directory should be inside base") - count := strings.Count(fn(), tmpdir) - assert.Equal(t, 1, count, "base directory should not repeat") - } -} diff --git a/pkg/goods/goods.go b/pkg/goods/goods.go index 97c83d90e..b3490e87c 100644 --- a/pkg/goods/goods.go +++ b/pkg/goods/goods.go @@ -3,12 +3,14 @@ package goods import ( + "crypto/sha256" "embed" + "encoding/hex" + "fmt" + "io" ) var ( - // materializer is our default instace of the artifact materializer. - materializer = NewMaterializer("") //go:embed bins/* binfs embed.FS //go:embed support/* @@ -21,25 +23,14 @@ var ( // K0sBinarySHA256 returns the SHA256 checksum of the embedded k0s binary. func K0sBinarySHA256() (string, error) { - return materializer.K0sBinarySHA256() -} - -// Materialize writes to disk all embedded assets using the default materializer. -func Materialize() error { - return materializer.Materialize() -} - -// MaterializeCalicoNetworkManagerConfig is a helper function that uses the default materializer. -func MaterializeCalicoNetworkManagerConfig() error { - return materializer.CalicoNetworkManagerConfig() -} - -// MaterializeLocalArtifactMirrorUnitFile is a helper function that uses the default materializer. -func MaterializeLocalArtifactMirrorUnitFile() error { - return materializer.LocalArtifactMirrorUnitFile() -} - -// MaterializeInternalBinary is a helper for the default materializer. -func MaterializeInternalBinary(name string) (string, error) { - return materializer.InternalBinary(name) + fp, err := binfs.Open("bins/k0s") + if err != nil { + return "", fmt.Errorf("unable to open embedded k0s binary: %w", err) + } + defer fp.Close() + hasher := sha256.New() + if _, err := io.Copy(hasher, fp); err != nil { + return "", fmt.Errorf("unable to copy embedded k0s binary: %w", err) + } + return hex.EncodeToString(hasher.Sum(nil)), nil } diff --git a/pkg/goods/materializer.go b/pkg/goods/materializer.go index 430f141ad..40acb5186 100644 --- a/pkg/goods/materializer.go +++ b/pkg/goods/materializer.go @@ -1,8 +1,6 @@ package goods import ( - "crypto/sha256" - "encoding/hex" "fmt" "io" "os" @@ -22,8 +20,8 @@ type Materializer struct { // NewMaterializer returns a new entity capable of materialize (write to disk) embedded // assets. Other operations on embedded assets are also available. -func NewMaterializer(basedir string) *Materializer { - return &Materializer{def: defaults.NewProvider(basedir)} +func NewMaterializer(provider *defaults.Provider) *Materializer { + return &Materializer{def: provider} } // InternalBinary materializes an internal binary from inside internal/bins directory @@ -171,7 +169,7 @@ func (m *Materializer) Kubectl() error { // https://github.com/k0sproject/k0s/blob/5d48d20767851fe8e299aacd3d5aae6fcfbeab37/main.go#L40 dstpath := m.def.PathToEmbeddedClusterBinary("kubectl") _ = os.RemoveAll(dstpath) - k0spath := m.def.K0sBinaryPath() + k0spath := defaults.K0sBinaryPath() content := fmt.Sprintf(kubectlScript, k0spath) if err := os.WriteFile(dstpath, []byte(content), 0755); err != nil { return fmt.Errorf("write kubectl completion: %w", err) @@ -189,20 +187,6 @@ func (m *Materializer) Kubectl() error { return nil } -// K0sBinarySHA256 returns the SHA256 checksum of the embedded k0s binary. -func (m *Materializer) K0sBinarySHA256() (string, error) { - fp, err := binfs.Open("bins/k0s") - if err != nil { - return "", fmt.Errorf("unable to open embedded k0s binary: %w", err) - } - defer fp.Close() - hasher := sha256.New() - if _, err := io.Copy(hasher, fp); err != nil { - return "", fmt.Errorf("unable to copy embedded k0s binary: %w", err) - } - return hex.EncodeToString(hasher.Sum(nil)), nil -} - // Ourselves makes a copy of the embedded-cluster binary into the PathToEmbeddedClusterBinary() directory. // We are doing this copy for three reasons: 1. We make sure we have it in a standard location across all // installations. 2. We can overwrite it during cluster upgrades. 3. we can serve a copy of the binary @@ -213,7 +197,7 @@ func (m *Materializer) Ourselves() error { return fmt.Errorf("unable to get our own executable path: %w", err) } - dstpath := m.def.PathToEmbeddedClusterBinary(m.def.BinaryName()) + dstpath := m.def.PathToEmbeddedClusterBinary(defaults.BinaryName()) if srcpath == dstpath { return nil } diff --git a/pkg/goods/support/host-support-bundle.yaml b/pkg/goods/support/host-support-bundle.tmpl.yaml similarity index 87% rename from pkg/goods/support/host-support-bundle.yaml rename to pkg/goods/support/host-support-bundle.tmpl.yaml index 32ba98136..832c1ee61 100644 --- a/pkg/goods/support/host-support-bundle.yaml +++ b/pkg/goods/support/host-support-bundle.tmpl.yaml @@ -3,7 +3,6 @@ kind: SupportBundle metadata: name: embedded-cluster-host-support-bundle spec: - uri: https://raw.githubusercontent.com/replicatedhq/embedded-cluster/main/pkg/goods/support/host-support-bundle.yaml hostCollectors: - ipv4Interfaces: {} - hostServices: {} @@ -14,28 +13,25 @@ spec: - time: {} - certificate: collectorName: k8s-api-keypair - certificatePath: /var/lib/k0s/pki/k0s-api.crt - keyPath: /var/lib/k0s/pki/k0s-api.key + certificatePath: {{ .K0sDataDir }}/pki/k0s-api.crt + keyPath: {{ .K0sDataDir }}/pki/k0s-api.key - certificate: collectorName: etcd-keypair - certificatePath: /var/lib/k0s/pki/etcd/server.crt - keyPath: /var/lib/k0s/pki/etcd/server.key - # Disk usage for commonly used directories + certificatePath: {{ .K0sDataDir }}/pki/etcd/server.crt + keyPath: {{ .K0sDataDir }}/pki/etcd/server.key + # Disk usage for commonly used directories - diskUsage: collectorName: root-disk-usage path: / - diskUsage: - collectorName: openebs-disk-usage - path: /var/openebs/local + collectorName: openebs-path-usage + path: {{ .OpenEBSDataDir }} - diskUsage: collectorName: embedded-cluster-path-usage - path: /var/lib/embedded-cluster + path: {{ .DataDir }} - diskUsage: collectorName: k0s-path-usage - path: /var/lib/k0s - - diskUsage: - collectorName: openebs-path-usage - path: /opt/openebs + path: {{ .K0sDataDir }} - diskUsage: collectorName: tmp-path-usage path: /tmp @@ -46,11 +42,11 @@ spec: args: [ '--cert', - '/var/lib/k0s/pki/admin.crt', + '{{ .K0sDataDir }}/pki/admin.crt', '--key', - '/var/lib/k0s/pki/admin.key', + '{{ .K0sDataDir }}/pki/admin.key', '--cacert', - '/var/lib/k0s/pki/ca.crt', + '{{ .K0sDataDir }}/pki/ca.crt', '-i', 'https://localhost:6443/healthz?verbose', ] @@ -60,11 +56,11 @@ spec: args: [ '--cert', - '/var/lib/k0s/pki/apiserver-etcd-client.crt', + '{{ .K0sDataDir }}/pki/apiserver-etcd-client.crt', '--key', - '/var/lib/k0s/pki/apiserver-etcd-client.key', + '{{ .K0sDataDir }}/pki/apiserver-etcd-client.key', '--cacert', - '/var/lib/k0s/pki/etcd/ca.crt', + '{{ .K0sDataDir }}/pki/etcd/ca.crt', '-i', 'https://localhost:2379/health', ] @@ -110,10 +106,13 @@ spec: args: [ "sysinfo" ] - copy: collectorName: installer-logs - path: /var/lib/embedded-cluster/logs/*.log + path: /var/log/embedded-cluster/*.log + - copy: + collectorName: installer-logs-old + path: {{ .DataDir }}/logs/*.log - copy: collectorName: installer-support-files - path: /var/lib/embedded-cluster/support/* + path: {{ .DataDir }}/support/* - run: collectorName: network-manager-logs command: journalctl @@ -125,7 +124,7 @@ spec: - run: collectorName: k0s-images-dir command: ls - args: [ "-alh", "/var/lib/k0s/images" ] + args: [ "-alh", "{{ .K0sDataDir }}/images" ] # External k0s runtime dependencies # https://docs.k0sproject.io/stable/external-runtime-deps/ - kernelConfigs: {} @@ -184,36 +183,36 @@ spec: outcomes: - fail: when: 'total < 40Gi' - message: The filesystem at /var/lib/embedded-cluster has less than 40Gi of total space + message: The filesystem at {{ .DataDir }} has less than 40 Gi of total space. Ensure sufficient space is available, or use the --data-dir flag to specify an alternative data directory. - pass: - message: The filesystem at /var/lib/embedded-cluster has sufficient space + message: The filesystem at {{ .DataDir }} has sufficient space - diskUsage: checkName: k0s Disk Space collectorName: k0s-path-usage outcomes: - fail: when: 'total < 40Gi' - message: The filesystem at /var/lib/k0s has less than 40Gi of total space + message: The filesystem at {{ .K0sDataDir }} has less than 40Gi of total space - fail: when: 'used/total > 80%' - message: The filesystem at /var/lib/k0s is more than 80% full + message: The filesystem at {{ .K0sDataDir }} is more than 80% full - pass: - message: The filesystem at /var/lib/k0s has sufficient space + message: The filesystem at {{ .K0sDataDir }} has sufficient space - diskUsage: checkName: OpenEBS disk usage - collectorName: openebs-disk-usage + collectorName: openebs-path-usage outcomes: - fail: when: "total < 40Gi" - message: The disk containing OpenEBS volumes has less than 40Gi of space + message: The disk containing {{ .OpenEBSDataDir }} volumes has less than 40Gi of space - fail: when: "used/total > 80%" - message: The disk containing OpenEBS volumes is more than 80% full + message: The disk containing {{ .OpenEBSDataDir }} volumes is more than 80% full - fail: when: "available < 10Gi" - message: The disk containing OpenEBS volumes has less than 10Gi of disk space available + message: The disk containing {{ .OpenEBSDataDir }} volumes has less than 10Gi of disk space available - pass: - message: The disk containing directory OpenEBS volumes has sufficient space + message: The disk containing directory {{ .OpenEBSDataDir }} volumes has sufficient space - diskUsage: checkName: tmp Disk Space collectorName: tmp-path-usage diff --git a/pkg/helm/values.go b/pkg/helm/values.go index 4ca5f6bc9..ea40895d1 100644 --- a/pkg/helm/values.go +++ b/pkg/helm/values.go @@ -24,6 +24,9 @@ func MarshalValues(values map[string]interface{}) (string, error) { return string(newValuesYaml), nil } +// SetValue sets the value at the given path in the values map. +// NOTE: this function does not support creating new maps. It only supports setting values in +// existing ones. func SetValue(values map[string]interface{}, path string, newValue interface{}) (map[string]interface{}, error) { newValuesMap := dig.Mapping(values) diff --git a/pkg/helm/values_test.go b/pkg/helm/values_test.go new file mode 100644 index 000000000..0e22cbc64 --- /dev/null +++ b/pkg/helm/values_test.go @@ -0,0 +1,81 @@ +package helm + +import ( + "reflect" + "testing" +) + +func TestSetValue(t *testing.T) { + type args struct { + values map[string]interface{} + path string + newValue interface{} + } + tests := []struct { + name string + args args + want map[string]interface{} + wantErr bool + }{ + { + name: "set value", + args: args{ + values: map[string]interface{}{ + "foo": "bar", + }, + path: "foo", + newValue: "new value", + }, + want: map[string]interface{}{ + "foo": "new value", + }, + wantErr: false, + }, + { + name: "set value in nested map", + args: args{ + values: map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": "baz", + }, + }, + path: "foo.bar", + newValue: "new value", + }, + want: map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": "new value", + }, + }, + wantErr: false, + }, + { + name: "set value in empty map", + args: args{ + values: map[string]interface{}{ + "foo": map[string]interface{}{}, + }, + path: "foo.bar", + newValue: "new value", + }, + want: map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": "new value", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SetValue(tt.args.values, tt.args.path, tt.args.newValue) + if (err != nil) != tt.wantErr { + t.Errorf("SetValue() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("SetValue() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/helpers/command.go b/pkg/helpers/command.go index 415de788a..86c5185f3 100644 --- a/pkg/helpers/command.go +++ b/pkg/helpers/command.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "os" "os/exec" "github.com/sirupsen/logrus" @@ -30,10 +29,11 @@ func RunCommandWithOptions(opts RunCommandOptions, bin string, args ...string) e cmd.Stdout = io.MultiWriter(opts.Writer, stdout) } cmd.Stderr = stderr - cmd.Env = os.Environ() + cmdEnv := cmd.Environ() for k, v := range opts.Env { - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v)) } + cmd.Env = cmdEnv if err := cmd.Run(); err != nil { logrus.Debugf("failed to run command:") logrus.Debugf("stdout: %s", stdout.String()) diff --git a/pkg/helpers/fs.go b/pkg/helpers/fs.go index 9cded7b6e..13afe1c2f 100644 --- a/pkg/helpers/fs.go +++ b/pkg/helpers/fs.go @@ -7,6 +7,25 @@ import ( "path/filepath" ) +type MultiError struct { + Errors []error +} + +func (e *MultiError) Add(err error) { + e.Errors = append(e.Errors, err) +} + +func (e *MultiError) ErrorOrNil() error { + switch len(e.Errors) { + case 0: + return nil + case 1: + return e.Errors[0] + default: + return fmt.Errorf("errors: %q", e.Errors) + } +} + // MoveFile moves a file from one location to another, overwriting the destination if it // exists. File mode is preserved. func MoveFile(src, dst string) error { @@ -69,10 +88,11 @@ func RemoveAll(path string) error { if err != nil { return fmt.Errorf("read directory: %w", err) } + var me MultiError for _, name := range names { if err := os.RemoveAll(filepath.Join(path, name)); err != nil { - return fmt.Errorf("remove %s: %w", name, err) + me.Add(fmt.Errorf("remove %s: %w", name, err)) } } - return nil + return me.ErrorOrNil() } diff --git a/pkg/kotscli/kotscli.go b/pkg/kotscli/kotscli.go index 96962249d..d3ab8c8c1 100644 --- a/pkg/kotscli/kotscli.go +++ b/pkg/kotscli/kotscli.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/goods" "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/metrics" @@ -24,8 +25,9 @@ type InstallOptions struct { AirgapBundle string } -func Install(opts InstallOptions, msg *spinner.MessageWriter) error { - kotsBinPath, err := goods.MaterializeInternalBinary("kubectl-kots") +func Install(provider *defaults.Provider, opts InstallOptions, msg *spinner.MessageWriter) error { + materializer := goods.NewMaterializer(provider) + kotsBinPath, err := materializer.InternalBinary("kubectl-kots") if err != nil { return fmt.Errorf("unable to materialize kubectl-kots binary: %w", err) } @@ -88,8 +90,9 @@ type AirgapUpdateOptions struct { AirgapBundle string } -func AirgapUpdate(opts AirgapUpdateOptions) error { - kotsBinPath, err := goods.MaterializeInternalBinary("kubectl-kots") +func AirgapUpdate(provider *defaults.Provider, opts AirgapUpdateOptions) error { + materializer := goods.NewMaterializer(provider) + kotsBinPath, err := materializer.InternalBinary("kubectl-kots") if err != nil { return fmt.Errorf("unable to materialize kubectl-kots binary: %w", err) } @@ -133,8 +136,9 @@ type VeleroConfigureOtherS3Options struct { Namespace string } -func VeleroConfigureOtherS3(opts VeleroConfigureOtherS3Options) error { - kotsBinPath, err := goods.MaterializeInternalBinary("kubectl-kots") +func VeleroConfigureOtherS3(provider *defaults.Provider, opts VeleroConfigureOtherS3Options) error { + materializer := goods.NewMaterializer(provider) + kotsBinPath, err := materializer.InternalBinary("kubectl-kots") if err != nil { return fmt.Errorf("unable to materialize kubectl-kots binary: %w", err) } diff --git a/pkg/kubeutils/kubeutils.go b/pkg/kubeutils/kubeutils.go index 240b9c86f..cfe046226 100644 --- a/pkg/kubeutils/kubeutils.go +++ b/pkg/kubeutils/kubeutils.go @@ -173,29 +173,42 @@ func WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner return nil } +func ListInstallations(ctx context.Context, cli client.Client) ([]embeddedclusterv1beta1.Installation, error) { + var list embeddedclusterv1beta1.InstallationList + if err := cli.List(ctx, &list); err != nil { + return nil, err + } + installs := list.Items + sort.SliceStable(installs, func(i, j int) bool { + return installs[j].Name < installs[i].Name + }) + return installs, nil +} + +func GetInstallation(ctx context.Context, cli client.Client, name string) (*embeddedclusterv1beta1.Installation, error) { + nsn := types.NamespacedName{Name: name} + var install embeddedclusterv1beta1.Installation + if err := cli.Get(ctx, nsn, &install); err != nil { + return nil, fmt.Errorf("unable to get installation: %w", err) + } + return &install, nil +} + func GetLatestInstallation(ctx context.Context, cli client.Client) (*embeddedclusterv1beta1.Installation, error) { - var installList embeddedclusterv1beta1.InstallationList - if err := cli.List(ctx, &installList); meta.IsNoMatchError(err) { + installs, err := ListInstallations(ctx, cli) + if meta.IsNoMatchError(err) { // this will happen if the CRD is not yet installed return nil, ErrNoInstallations{} } else if err != nil { return nil, fmt.Errorf("unable to list installations: %v", err) } - installs := installList.Items if len(installs) == 0 { return nil, ErrNoInstallations{} } - // sort the installations - sort.SliceStable(installs, func(i, j int) bool { - return installs[j].Name < installs[i].Name - }) - // get the latest installation - lastInstall := installs[0] - - return &lastInstall, nil + return &installs[0], nil } func writeStatusMessage(writer *spinner.MessageWriter, install *embeddedclusterv1beta1.Installation) { diff --git a/pkg/preflights/host-preflight.yaml b/pkg/preflights/host-preflight.yaml index 60b246970..e6e85b868 100644 --- a/pkg/preflights/host-preflight.yaml +++ b/pkg/preflights/host-preflight.yaml @@ -9,16 +9,7 @@ spec: path: / - diskUsage: collectorName: embedded-cluster-path-usage - path: /var/lib/embedded-cluster - - diskUsage: - collectorName: k0s-path-usage - path: /var/lib/k0s - - diskUsage: - collectorName: openebs-path-usage - path: /var/openebs - - diskUsage: - collectorName: tmp-path-usage - path: /tmp + path: {{ .DataDir }} - memory: {} - cpu: {} - time: {} @@ -66,7 +57,7 @@ spec: - filesystemPerformance: collectorName: filesystem-write-latency-etcd timeout: 5m - directory: /var/lib/k0s/etcd + directory: {{ .K0sDataDir }}/etcd fileSize: 22Mi operationSize: 2300 datasync: true @@ -158,39 +149,12 @@ spec: outcomes: - fail: when: 'total < 40Gi' - message: The filesystem at /var/lib/embedded-cluster has less than 40Gi of total space - - pass: - message: The filesystem at /var/lib/embedded-cluster has sufficient space - - diskUsage: - checkName: k0s Disk Space - collectorName: k0s-path-usage - outcomes: - - fail: - when: 'total < 40Gi' - message: The filesystem at /var/lib/k0s has less than 40Gi of total space + message: The filesystem at {{ .DataDir }} has less than 40 Gi of total space. Ensure sufficient space is available, or use the --data-dir flag to specify an alternative data directory. - fail: when: 'used/total > 80%' - message: The filesystem at /var/lib/k0s is more than 80% full - - pass: - message: The filesystem at /var/lib/k0s has sufficient space - - diskUsage: - checkName: OpenEBS Disk Space - collectorName: openebs-path-usage - outcomes: - - fail: - when: 'total < 5Gi' - message: The filesystem at /var/openebs has less than 5Gi of total space - - pass: - message: The filesystem at /var/openebs has sufficient space - - diskUsage: - checkName: tmp Disk Space - collectorName: tmp-path-usage - outcomes: - - fail: - when: 'total < 5Gi' - message: The filesystem at /tmp has less than 5Gi of total space + message: The filesystem at {{ .DataDir }} is more than 80% full. Ensure sufficient space is available, or use the --data-dir flag to specify an alternative data directory. - pass: - message: The filesystem at /tmp has sufficient space + message: The filesystem at {{ .DataDir }} has sufficient space - textAnalyze: checkName: Default Route fileName: host-collectors/run-host/ip-route-table.txt @@ -442,9 +406,9 @@ spec: outcomes: - pass: when: "p99 < 10ms" - message: 'P99 write latency for the disk at /var/lib/k0s/etcd is {{ "{{" }} .P99 {{ "}}" }}, which is better than the 10 ms requirement.' + message: 'P99 write latency for the disk at {{ .K0sDataDir }}/etcd is {{ "{{" }} .P99 {{ "}}" }}, which is better than the 10 ms requirement.' - fail: - message: 'P99 write latency for the disk at /var/lib/k0s/etcd is {{ "{{" }} .P99 {{ "}}" }}, but it must be less than 10 ms. A higher-performance disk is required.' + message: 'P99 write latency for the disk at {{ .K0sDataDir }}/etcd is {{ "{{" }} .P99 {{ "}}" }}, but it must be less than 10 ms. A higher-performance disk is required.' - tcpPortStatus: checkName: ETCD Internal Port Availability collectorName: ETCD Internal Port diff --git a/pkg/preflights/output.go b/pkg/preflights/output.go index 6c7e56a5b..11baef494 100644 --- a/pkg/preflights/output.go +++ b/pkg/preflights/output.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/jedib0t/go-pretty/v6/table" - "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/sirupsen/logrus" "golang.org/x/term" ) @@ -107,7 +106,7 @@ func (o Output) printTable() { logrus.Info("Please address this issue and try again.") } -func (o Output) SaveToDisk() error { +func (o Output) SaveToDisk(path string) error { // Store results on disk of the host that ran the preflights data, err := json.MarshalIndent(o, "", " ") if err != nil { @@ -116,7 +115,6 @@ func (o Output) SaveToDisk() error { // If we ever want to store multiple preflight results // we can add a timestamp to the filename. - path := defaults.PathToEmbeddedClusterSupportFile("host-preflight-results.json") if err := os.WriteFile(path, data, 0644); err != nil { return fmt.Errorf("unable to write preflight results to %s: %w", path, err) } diff --git a/pkg/preflights/preflights.go b/pkg/preflights/preflights.go index c37df946b..8ec871c16 100644 --- a/pkg/preflights/preflights.go +++ b/pkg/preflights/preflights.go @@ -33,7 +33,7 @@ func SerializeSpec(spec *troubleshootv1beta2.HostPreflightSpec) ([]byte, error) // Run runs the provided host preflight spec locally. This function is meant to be // used when upgrading a local node. -func Run(ctx context.Context, spec *troubleshootv1beta2.HostPreflightSpec, proxy *ecv1beta1.ProxySpec) (*Output, string, error) { +func Run(ctx context.Context, provider *defaults.Provider, spec *troubleshootv1beta2.HostPreflightSpec, proxy *ecv1beta1.ProxySpec) (*Output, string, error) { // Deduplicate collectors and analyzers before running preflights spec.Collectors = dedup(spec.Collectors) spec.Analyzers = dedup(spec.Analyzers) @@ -44,13 +44,14 @@ func Run(ctx context.Context, spec *troubleshootv1beta2.HostPreflightSpec, proxy } defer os.Remove(fpath) - binpath := defaults.PathToEmbeddedClusterBinary("kubectl-preflight") + binpath := provider.PathToEmbeddedClusterBinary("kubectl-preflight") stdout := bytes.NewBuffer(nil) stderr := bytes.NewBuffer(nil) cmd := exec.Command(binpath, "--interactive=false", "--format=json", fpath) - cmd.Env = os.Environ() - cmd.Env = proxyEnv(cmd.Env, proxy) - cmd.Env = pathEnv(cmd.Env) + cmdEnv := cmd.Environ() + cmdEnv = proxyEnv(cmdEnv, proxy) + cmdEnv = pathEnv(cmdEnv, provider) + cmd.Env = cmdEnv cmd.Stdout, cmd.Stderr = stdout, stderr if err = cmd.Run(); err == nil { out, err := OutputFromReader(stdout) @@ -65,7 +66,7 @@ func Run(ctx context.Context, spec *troubleshootv1beta2.HostPreflightSpec, proxy return out, stderr.String(), err } -func CopyBundleToECSupportDir() error { +func CopyBundleToECSupportDir(provider *defaults.Provider) error { matches, err := filepath.Glob("preflightbundle-*.tar.gz") if err != nil { return fmt.Errorf("find preflight bundle: %w", err) @@ -80,7 +81,7 @@ func CopyBundleToECSupportDir() error { src = match } } - dst := defaults.PathToEmbeddedClusterSupportFile("preflight-bundle.tar.gz") + dst := provider.PathToEmbeddedClusterSupportFile("preflight-bundle.tar.gz") if err := helpers.MoveFile(src, dst); err != nil { return fmt.Errorf("move preflight bundle to %s: %w", dst, err) } @@ -145,7 +146,7 @@ func proxyEnv(env []string, proxy *ecv1beta1.ProxySpec) []string { return next } -func pathEnv(env []string) []string { +func pathEnv(env []string, provider *defaults.Provider) []string { path := "" next := []string{} for _, e := range env { @@ -158,9 +159,9 @@ func pathEnv(env []string) []string { } } if path != "" { - next = append(next, fmt.Sprintf("PATH=%s:%s", path, defaults.EmbeddedClusterBinsSubDir())) + next = append(next, fmt.Sprintf("PATH=%s:%s", path, provider.EmbeddedClusterBinsSubDir())) } else { - next = append(next, fmt.Sprintf("PATH=%s", defaults.EmbeddedClusterBinsSubDir())) + next = append(next, fmt.Sprintf("PATH=%s", provider.EmbeddedClusterBinsSubDir())) } return next } diff --git a/pkg/preflights/preflights_test.go b/pkg/preflights/preflights_test.go index a09409820..42318bdda 100644 --- a/pkg/preflights/preflights_test.go +++ b/pkg/preflights/preflights_test.go @@ -2,7 +2,6 @@ package preflights import ( - "os" "strings" "testing" @@ -128,18 +127,8 @@ func Test_proxyEnv(t *testing.T) { } func Test_pathEnv(t *testing.T) { - dir, err := os.MkdirTemp("", "embedded-cluster") - if err != nil { - t.Fatal(err) - } - - oDefaultProvider := defaults.DefaultProvider - defaults.DefaultProvider = defaults.NewProvider(dir) - t.Cleanup(func() { - defaults.DefaultProvider = oDefaultProvider - }) - - binDir := defaults.DefaultProvider.EmbeddedClusterBinsSubDir() + provider := defaults.NewProvider(t.TempDir()) + binDir := provider.EmbeddedClusterBinsSubDir() type args struct { env []string @@ -177,7 +166,7 @@ func Test_pathEnv(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := pathEnv(tt.args.env) + got := pathEnv(tt.args.env, provider) gotMap := make(map[string]string) for _, e := range got { parts := strings.SplitN(e, "=", 2) diff --git a/pkg/preflights/template.go b/pkg/preflights/template.go index c852d13d5..df1a3b14e 100644 --- a/pkg/preflights/template.go +++ b/pkg/preflights/template.go @@ -13,6 +13,9 @@ type TemplateData struct { ProxyRegistryURL string AdminConsolePort int LocalArtifactMirrorPort int + DataDir string + K0sDataDir string + OpenEBSDataDir string SystemArchitecture string } diff --git a/pkg/support/materialize.go b/pkg/support/materialize.go new file mode 100644 index 000000000..49b9fbcd0 --- /dev/null +++ b/pkg/support/materialize.go @@ -0,0 +1,51 @@ +package support + +import ( + "bytes" + "fmt" + "os" + "text/template" + + "github.com/replicatedhq/embedded-cluster/pkg/defaults" +) + +type TemplateData struct { + DataDir string + K0sDataDir string + OpenEBSDataDir string +} + +func MaterializeSupportBundleSpec(provider *defaults.Provider) error { + data := TemplateData{ + DataDir: provider.EmbeddedClusterHomeDirectory(), + K0sDataDir: provider.EmbeddedClusterK0sSubDir(), + OpenEBSDataDir: provider.EmbeddedClusterOpenEBSLocalSubDir(), + } + path := provider.PathToEmbeddedClusterSupportFile("host-support-bundle.tmpl.yaml") + tmpl, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read support bundle template: %w", err) + } + contents, err := renderTemplate(string(tmpl), data) + if err != nil { + return fmt.Errorf("render support bundle template: %w", err) + } + path = provider.PathToEmbeddedClusterSupportFile("host-support-bundle.yaml") + if err := os.WriteFile(path, []byte(contents), 0644); err != nil { + return fmt.Errorf("write support bundle spec: %w", err) + } + return nil +} + +func renderTemplate(spec string, data TemplateData) (string, error) { + tmpl, err := template.New("preflight").Parse(spec) + if err != nil { + return "", fmt.Errorf("parse template: %w", err) + } + buf := bytes.NewBuffer(nil) + err = tmpl.Execute(buf, data) + if err != nil { + return "", fmt.Errorf("execute template: %w", err) + } + return buf.String(), nil +}