From b2cbf0d843eb3aedd91e3b7795981b7de8e8cbad Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Tue, 7 Nov 2023 17:06:52 -0600 Subject: [PATCH] fix: k3s migration --- .github/workflows/e2e.yaml | 4 ++ Dockerfile | 4 +- charts/k3s/templates/statefulset.yaml | 2 +- pkg/k3s/k3s.go | 2 +- test/e2e_target_namespace/role.yaml | 35 +++++++++ test/e2e_target_namespace/targetNamespace.go | 75 +------------------- 6 files changed, 46 insertions(+), 76 deletions(-) create mode 100644 test/e2e_target_namespace/role.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b24115453..e681a723b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -187,6 +187,10 @@ jobs: extraArgs+=( -f ./test/multins_values.yaml ) fi + if [ ${{ matrix.test-suite-path }} == "./test/e2e_target_namespace" ]; then + kubectl apply -f ${{ matrix.test-suite-path }}/role.yaml + fi + sudo apt-get install -y sed sed -i "s|REPLACE_IMAGE_NAME|${{ env.IMAGE_NAME }}|g" ${{ matrix.test-suite-path }}/../commonValues.yaml diff --git a/Dockerfile b/Dockerfile index 91b5ed084..f086a0451 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,9 @@ RUN go generate -tags embed_charts ./... ENV HOME / # Build cmd -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -tags embed_charts -ldflags "-X github.com/loft-sh/vcluster/pkg/telemetry.SyncerVersion=$BUILD_VERSION -X github.com/loft-sh/vcluster/pkg/telemetry.telemetryPrivateKey=$TELEMETRY_PRIVATE_KEY" -o /vcluster cmd/vcluster/main.go +RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \ + --mount=type=cache,id=gobuild,target=/.cache/go-build \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -mod vendor -tags embed_charts -ldflags "-X github.com/loft-sh/vcluster/pkg/telemetry.SyncerVersion=$BUILD_VERSION -X github.com/loft-sh/vcluster/pkg/telemetry.telemetryPrivateKey=$TELEMETRY_PRIVATE_KEY" -o /vcluster cmd/vcluster/main.go # RUN useradd -u 12345 nonroot # USER nonroot diff --git a/charts/k3s/templates/statefulset.yaml b/charts/k3s/templates/statefulset.yaml index 49616ebff..48e55df08 100644 --- a/charts/k3s/templates/statefulset.yaml +++ b/charts/k3s/templates/statefulset.yaml @@ -319,7 +319,7 @@ spec: - name: VCLUSTER_COMMAND value: |- command: - {{ range $f := .Values.vcluster.command -}} + {{ range $f := .Values.vcluster.command }} - {{ $f }} {{- end }} args: diff --git a/pkg/k3s/k3s.go b/pkg/k3s/k3s.go index 30a2b6e0a..5edbb55a4 100644 --- a/pkg/k3s/k3s.go +++ b/pkg/k3s/k3s.go @@ -43,7 +43,7 @@ func StartK3S(ctx context.Context, serviceCIDR, k3sToken string) error { command.Args = append( command.Args, "--service-cidr", serviceCIDR, - "--token", k3sToken, + "--token", strings.TrimSpace(k3sToken), ) args := append(command.Command, command.Args...) diff --git a/test/e2e_target_namespace/role.yaml b/test/e2e_target_namespace/role.yaml new file mode 100644 index 000000000..23fbd3e89 --- /dev/null +++ b/test/e2e_target_namespace/role.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: vcluster-workload +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vc-workload-vcluster + namespace: vcluster-workload +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: vcluster-workload + namespace: vcluster-workload +rules: + - apiGroups: ["", "networking.k8s.io"] # "" indicates the core API group + resources: ["*"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vcluster-workload-binding + namespace: vcluster-workload +subjects: + - kind: ServiceAccount + name: vc-vcluster + namespace: vcluster +roleRef: + kind: Role + name: vcluster-workload + apiGroup: rbac.authorization.k8s.io + diff --git a/test/e2e_target_namespace/targetNamespace.go b/test/e2e_target_namespace/targetNamespace.go index fb5514c15..0dde1bb97 100644 --- a/test/e2e_target_namespace/targetNamespace.go +++ b/test/e2e_target_namespace/targetNamespace.go @@ -7,7 +7,6 @@ import ( "github.com/loft-sh/vcluster/test/framework" "github.com/onsi/ginkgo/v2" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ) @@ -19,76 +18,6 @@ var _ = ginkgo.Describe("Target Namespace", func() { } ginkgo.It("Create vcluster with target namespace", func() { - ginkgo.By("Create target namespace") - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vcluster-workload", - }, - } - _, err := f.HostClient.CoreV1().Namespaces().Create(f.Context, ns, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - err = wait.PollUntilContextTimeout(f.Context, time.Second, time.Minute*1, false, func(ctx context.Context) (done bool, err error) { - namespace, _ := f.HostClient.CoreV1().Namespaces().Get(ctx, ns.Name, metav1.GetOptions{}) - if namespace.Status.Phase == corev1.NamespaceActive { - return true, nil - } - return false, nil - }) - framework.ExpectNoError(err) - - ginkgo.By("Create service account, role and role binding in target namespace") - workloadSaName := "vc-workload-" + f.VclusterName - sa := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: workloadSaName, - Namespace: ns.Name, - }, - } - _, err = f.HostClient.CoreV1().ServiceAccounts(ns.Name).Create(f.Context, sa, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - role := &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vcluster-workload", - Namespace: ns.Name, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"", "networking.k8s.io"}, - Resources: []string{"*"}, - Verbs: []string{"*"}, - }, - }, - } - _, err = f.HostClient.RbacV1().Roles(ns.Name).Create(f.Context, role, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - vcSaName := "vc-" + f.VclusterName - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vcluster-workload", - Namespace: ns.Name, - Labels: map[string]string{ - "app": "vcluster-nginxa-app", - }, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: vcSaName, - Namespace: f.VclusterNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: role.Name, - }, - } - _, err = f.HostClient.RbacV1().RoleBindings(ns.Name).Create(f.Context, rb, metav1.CreateOptions{}) - framework.ExpectNoError(err) - ginkgo.By("Create workload in vcluster and verify if it's running in targeted namespace") pod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{ @@ -108,7 +37,7 @@ var _ = ginkgo.Describe("Target Namespace", func() { }, } - _, err = f.VclusterClient.CoreV1().Pods("default").Create(f.Context, pod, metav1.CreateOptions{}) + _, err := f.VclusterClient.CoreV1().Pods("default").Create(f.Context, pod, metav1.CreateOptions{}) framework.ExpectNoError(err) err = wait.PollUntilContextTimeout(f.Context, time.Second, time.Minute*2, false, func(ctx context.Context) (bool, error) { @@ -120,7 +49,7 @@ var _ = ginkgo.Describe("Target Namespace", func() { }) framework.ExpectNoError(err) - p, err := f.HostClient.CoreV1().Pods(ns.Name).List(f.Context, metav1.ListOptions{ + p, err := f.HostClient.CoreV1().Pods("vcluster-workload").List(f.Context, metav1.ListOptions{ LabelSelector: "vcluster.loft.sh/managed-by=" + f.VclusterName, }) framework.ExpectNoError(err)