add killall as a subcommand

cmd/k3s/main.go

@@ -51,6 +51,7 @@ func main() {
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
 		cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
+		cmds.NewKillAllCommand(externalCLIAction("k3s-killall", dataDir)),
 		cmds.NewTokenCommands(
 			tokenCommand,
 			tokenCommand,

contrib/util/k3s-killall.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env sh
+
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+    [ -d "$bin" ] && export PATH=$PATH:$bin:$bin/aux
+done
+
+set -x
+
+for service in /etc/systemd/system/k3s*.service; do
+    [ -s "$service" ] && systemctl stop "$(basename "$service")"
+done
+
+for service in /etc/init.d/k3s*; do
+    [ -x "$service" ] && $service stop
+done
+
+pschildren() {
+    ps -e -o ppid= -o pid= | \
+    sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
+    grep -w "^$1" | \
+    cut -f2
+}
+
+pstree() {
+
+    for pid in "$@"; do
+        # Don't return the current process
+        if [ "$pid" != "$$" ]; then
+            echo "$pid"
+        fi
+        for child in $(pschildren "$pid"); do
+            pstree "$child"
+        done
+    done
+}
+
+killtree() {
+    kill -9 $(
+        { set +x; } 2>/dev/null;
+        pstree "$@";
+        set -x;
+    ) 2>/dev/null
+}
+
+remove_interfaces() {
+    # Delete network interface(s) that match 'master cni0'
+    ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
+        iface=${iface%%@*}
+        [ -z "$iface" ] || ip link delete "$iface"
+    done
+
+    # Delete cni related interfaces
+    ip link delete cni0
+    ip link delete flannel.1
+    ip link delete flannel-v6.1
+    ip link delete kube-ipvs0
+    ip link delete flannel-wg
+    ip link delete flannel-wg-v6
+
+    # Restart tailscale
+    if [ -n "$(command -v tailscale)" ]; then
+        tailscale set --advertise-routes=
+    fi
+}
+
+getshims() {
+    ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1
+}
+
+do_unmount_and_remove() {
+    set +x
+    while read -r _ path _; do
+        case "$path" in $1*) echo "$path" ;; esac
+    done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount -f "$0" && rm -rf "$0"'
+    set -x
+}
+
+killtree "$({ set +x; } 2>/dev/null; getshims; set -x)"
+
+do_unmount_and_remove '/run/k3s'
+do_unmount_and_remove '/var/lib/rancher/k3s'
+do_unmount_and_remove '/var/lib/kubelet/pods'
+do_unmount_and_remove '/var/lib/kubelet/plugins'
+do_unmount_and_remove '/run/netns/cni-'
+
+# Remove CNI namespaces
+ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
+
+remove_interfaces
+
+rm -rf /var/lib/cni/
+iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore
+ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore

pkg/cli/cmds/killall.go

@@ -0,0 +1,13 @@
+package cmds
+
+import "github.com/urfave/cli"
+
+func NewKillAllCommand(action func(*cli.Context) error) cli.Command {
+	return cli.Command{
+		Name:            "killall",
+		Usage:           "Kill all K3s and associated child processes",
+		SkipFlagParsing: true,
+		SkipArgReorder:  true,
+		Action:          action,
+	}
+}

scripts/package-cli

@@ -36,6 +36,7 @@ for i in "${cni_binaries[@]}"; do
 done
 
 cp contrib/util/check-config.sh bin/check-config
+cp contrib/util/k3s-killall.sh bin/k3s-killall
 
 rm -rf build/data
 mkdir -p build/data build/out

tests/e2e/startup/startup_test.go

@@ -133,8 +133,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 			_, _ = e2e.ParsePods(kubeConfigFile, true)
 		})
 		It("Kills the cluster", func() {
-			err := KillK3sCluster(append(serverNodeNames, agentNodeNames...))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(KillK3sCluster(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 		})
 	})
 	Context("Verify prefer-bundled-bin flag", func() {
@@ -177,8 +176,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 			_, _ = e2e.ParsePods(kubeConfigFile, true)
 		})
 		It("Kills the cluster", func() {
-			err := KillK3sCluster(append(serverNodeNames, agentNodeNames...))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(KillK3sCluster(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 		})
 	})
 	Context("Verify disable-agent and egress-selector-mode flags", func() {
@@ -248,8 +246,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 		})
 
 		It("Kills the cluster", func() {
-			err := KillK3sCluster(append(serverNodeNames, agentNodeNames...))
-			Expect(err).NotTo(HaveOccurred())
+			Expect(KillK3sCluster(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 		})
 	})
 	Context("Verify server fails to start with bootstrap token", func() {
@@ -265,9 +262,61 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 
 		})
 		It("Kills the cluster", func() {
-			err := KillK3sCluster(append(serverNodeNames, agentNodeNames...))
+			Expect(KillK3sCluster(append(serverNodeNames, agentNodeNames...))).To(Succeed())
+		})
+	})
+	Context("Verify k3s killall subcommand works on all nodes", func() {
+		It("Starts K3s with no issues", func() {
+			err := StartK3sCluster(append(serverNodeNames, agentNodeNames...), "", "")
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
+
+			fmt.Println("CLUSTER CONFIG")
+			fmt.Println("OS:", *nodeOS)
+			fmt.Println("Server Nodes:", serverNodeNames)
+			fmt.Println("Agent Nodes:", agentNodeNames)
+			kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 		})
+
+		It("Checks node and pod status", func() {
+			fmt.Printf("\nFetching node status\n")
+			Eventually(func(g Gomega) {
+				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
+				g.Expect(err).NotTo(HaveOccurred())
+				for _, node := range nodes {
+					g.Expect(node.Status).Should(Equal("Ready"))
+				}
+			}, "360s", "5s").Should(Succeed())
+			_, _ = e2e.ParseNodes(kubeConfigFile, true)
+
+			fmt.Printf("\nFetching pods status\n")
+			Eventually(func(g Gomega) {
+				pods, err := e2e.ParsePods(kubeConfigFile, false)
+				g.Expect(err).NotTo(HaveOccurred())
+				for _, pod := range pods {
+					if strings.Contains(pod.Name, "helm-install") {
+						g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
+					} else {
+						g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
+					}
+				}
+			}, "360s", "5s").Should(Succeed())
+			_, _ = e2e.ParsePods(kubeConfigFile, true)
+		})
+
+		It("Kills the cluster", func() {
+			for _, node := range append(serverNodeNames, agentNodeNames...) {
+				_, err := e2e.RunCmdOnNode("k3s killall", node)
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+		It("Checks that no k3s processes are running", func() {
+			Eventually(func(g Gomega) {
+				for _, node := range append(serverNodeNames, agentNodeNames...) {
+					g.Expect(e2e.RunCmdOnNode("ps x | grep k3s || true", node)).To(BeEmpty())
+				}
+			}, "20s", "5s").Should(Succeed())
+		})
 	})
 })
 

tests/integration/startup/startup_int_test.go

@@ -124,6 +124,7 @@ var _ = Describe("startup tests", Ordered, func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(apiInfo).To(ContainSubstring("11.11.22.22"))
 		})
+
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
 			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
@@ -263,7 +264,6 @@ var _ = Describe("startup tests", Ordered, func() {
 			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		})
 	})
-
 })
 
 var failed bool