diff --git a/.github/workflows/e2e-test-features.yaml b/.github/workflows/e2e-test-features.yaml new file mode 100644 index 000000000..e908819fd --- /dev/null +++ b/.github/workflows/e2e-test-features.yaml @@ -0,0 +1,262 @@ +name: E2E Test Features CI + +on: + schedule: + - cron: '* * * * *' + workflow_dispatch: + +env: + REPOSITORY_NAME: ghcr.io/${{ github.repository }}-ci + TAG_NAME: PR${{ github.event.number }} + VCLUSTER_SUFFIX: vcluster + VCLUSTER_NAME: vcluster + VCLUSTER_NAMESPACE: vcluster + +jobs: + build-and-push-syncer-image: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: Setup Just + uses: extractions/setup-just@v2 + - name: Setup Syft + uses: anchore/sbom-action/download-syft@v0.17.0 + - name: Setup GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + install-only: true + version: latest + - name: Build and save syncer image + run: | + set -x + TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster --clean --output ./vcluster + docker build -t "${{ env.REPOSITORY_NAME }}:${{ env.TAG_NAME }}" -f Dockerfile.release --build-arg TARGETARCH=amd64 --build-arg TARGETOS=linux . + docker save -o vcluster_syncer "${{ env.REPOSITORY_NAME }}:${{ env.TAG_NAME }}" + - name: Upload syncer image to artifact + uses: actions/upload-artifact@v4 + with: + name: vcluster_syncer + path: ./vcluster_syncer + retention-days: 7 + + build-vcluster-cli: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: Setup Just + uses: extractions/setup-just@v2 + - name: Setup Syft + uses: anchore/sbom-action/download-syft@v0.17.0 + - name: Setup GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + install-only: true + - name: Build vcluster cli + run: | + set -x + TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster-cli --clean --output ./vcluster + - name: Upload vcluster cli to artifact + uses: actions/upload-artifact@v4 + with: + name: vcluster + path: ./vcluster + retention-days: 7 + + build-tests: + name: Build tests binaries + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure git + run: git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ + env: + GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: go.mod + + - name: Build e2e binary + run: | + cd ./test + go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo build --require-suite -r --mod vendor $(ls -d ./features/* | jq -R . | jq -rcs '. | join(" \\\n")') + env: + GOWORK: off + + - name: Upload test binaries to artifacts + uses: actions/upload-artifact@v4 + with: + name: test-binaries + path: test/features/**/*.test + retention-days: 7 + + generate-matrix: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: List Go files + id: set-paths-matrix + run: | + cd ./test + set -x + sudo apt-get install -y jq + + paths=$(ls -d ./features/*/) + echo "matrix=$(printf '%s\n' "${paths}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT" + + outputs: + matrix: ${{ steps.set-paths-matrix.outputs.matrix }} + + execute-feature-tests: + needs: + - build-and-push-syncer-image + - build-vcluster-cli + - build-tests + - generate-matrix + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-suite-path: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + name: Setup Helm + with: + version: "v3.11.0" + + - name: Set up kind k8s cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.20.0" + image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e + + - name: Testing kind cluster set-up + run: | + set -x + kubectl cluster-info + kubectl get pods -n kube-system + echo "kubectl config current-context:" $(kubectl config current-context) + echo "KUBECONFIG env var:" ${KUBECONFIG} + + - name: Download vcluster cli + uses: actions/download-artifact@v4 + with: + name: vcluster + + - name: Download syncer image + uses: actions/download-artifact@v4 + with: + name: vcluster_syncer + + - name: Download test binaries + uses: actions/download-artifact@v4 + with: + name: test-binaries + path: ./test/features + + - name: Create vcluster + id: create-vcluster + run: | + set -x + + kind load image-archive vcluster_syncer + + chmod +x vcluster && sudo mv vcluster /usr/bin + + cd ./test + + sudo apt-get install -y sed + + sed -i "s|REPLACE_REPOSITORY_NAME|${{ env.REPOSITORY_NAME }}|g" ${{ matrix.test-suite-path }}commonValues.yaml + sed -i "s|REPLACE_TAG_NAME|${{ env.TAG_NAME }}|g" ${{ matrix.test-suite-path }}commonValues.yaml + + vcluster create ${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} \ + --create-namespace \ + --debug \ + --connect=false \ + --local-chart-dir ../chart \ + -f ${{ matrix.test-suite-path }}commonValues.yaml + continue-on-error: true + + - name: Wait until vcluster is ready + id: wait-until-vcluster-is-ready + if: steps.create-vcluster.outcome == 'success' + run: | + set -x + + ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} + + continue-on-error: true + + - name: Collect deployment information in case vcluster fails to start + if: steps.wait-until-vcluster-is-ready.outcome != 'success' + run: | + set -x + kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }} + echo "======================================================================================================================" + kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp' + echo "======================================================================================================================" + kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 + echo "======================================================================================================================" + kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }} + exit 1 + + # Skips NetworkPolicy tests because they require network plugin with support (e.g. Calico) + - name: Execute tests + id: execute-tests + run: | + set -x + + cd ./test/features + + cd $(echo "${{ matrix.test-suite-path }}" | sed -e 's#^./features/##' -e 's#/$##') + + sudo chmod +x $(echo "${{ matrix.test-suite-path }}" | sed -e 's#^./features/##' -e 's#/$##').test + + VCLUSTER_SUFFIX=${{ env.VCLUSTER_SUFFIX }} VCLUSTER_NAME=${{ env.VCLUSTER_NAME }} VCLUSTER_NAMESPACE=${{ env.VCLUSTER_NAMESPACE }} ./$(echo "${{ matrix.test-suite-path }}" | sed -e 's#^./features/##' -e 's#/$##').test -test.v --ginkgo.v --ginkgo.skip='.*NetworkPolicy.*' --ginkgo.fail-fast + + if kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p >/dev/null 2>/dev/null; then + echo "vCluster has restarted during testing, failing..." + exit 1 + fi + + continue-on-error: true + + - name: Print logs if tests fail + if: steps.execute-tests.outcome == 'failure' + run: | + set -x + kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }} + echo "======================================================================================================================" + kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp' + echo "======================================================================================================================" + kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 + echo "======================================================================================================================" + kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }} + exit 1 + diff --git a/test/deploy_changes/common/common_helpers.go b/test/deploy_changes/common/common_helpers.go index b61bbda6d..765e188df 100644 --- a/test/deploy_changes/common/common_helpers.go +++ b/test/deploy_changes/common/common_helpers.go @@ -2,6 +2,8 @@ package common import ( "bytes" + "context" + "fmt" "os" "os/exec" "strings" @@ -41,9 +43,13 @@ func DeployChangesToVClusterUsingCLI(f *framework.Framework) { func DisconnectFromVCluster(f *framework.Framework) { disconnectCmd := exec.Command("vcluster", "disconnect") - err := disconnectCmd.Run() - if err != nil && !strings.Contains(err.Error(), "not a virtual cluster context") { - framework.ExpectNoError(err, "Error disconnecting from vCluster") + output, err := disconnectCmd.CombinedOutput() + if err != nil { + if strings.Contains(string(output), "is not a virtual cluster context") { + fmt.Println("No virtual cluster context to disconnect from.") + } else { + framework.ExpectNoError(err, "Error disconnecting from vCluster") + } } } @@ -55,3 +61,19 @@ func VerifyClusterIsActive(f *framework.Framework) { return err == nil && strings.Contains(string(output), f.VclusterName) && strings.Contains(string(output), "Running") }).WithPolling(pollingInterval).WithTimeout(pollingDurationLong).Should(gomega.BeTrue()) } + +func DeleteVCluster(vClusterName string, f *framework.Framework) { + ctx, cancel := context.WithTimeout(context.Background(), pollingDurationLong) + defer cancel() + + deleteCmd := exec.CommandContext(ctx, "vcluster", "delete", vClusterName, "-n", f.VclusterNamespace) + var stdout, stderr bytes.Buffer + deleteCmd.Stdout = &stdout + deleteCmd.Stderr = &stderr + err := deleteCmd.Run() + if err != nil { + fmt.Println("stderr: ", stderr.String()) + framework.ExpectNoError(err, "Error executing vcluster delete command") + } + gomega.Expect(strings.Contains(stdout.String(), "Successfully deleted virtual cluster")).To(gomega.BeTrue()) +} diff --git a/test/features/duplicate_vcluster_namespace/commonValues.yaml b/test/features/duplicate_vcluster_namespace/commonValues.yaml new file mode 100644 index 000000000..234ef6b52 --- /dev/null +++ b/test/features/duplicate_vcluster_namespace/commonValues.yaml @@ -0,0 +1,35 @@ +controlPlane: + backingStore: + etcd: + deploy: + statefulSet: + resources: + requests: + cpu: "0" + statefulSet: + image: + registry: "" + repository: REPLACE_REPOSITORY_NAME + tag: REPLACE_TAG_NAME + env: + - name: DEBUG + value: "true" + resources: + requests: + cpu: "0" +# values for general test suite +networking: + replicateServices: + toHost: + - from: test/test + to: test + - from: test/nginx + to: nginx + fromHost: + - from: test/test + to: default/test + - from: test/nginx + to: default/nginx +experimental: + syncSettings: + setOwner: true diff --git a/test/features/duplicate_vcluster_namespace/helpers.go b/test/features/duplicate_vcluster_namespace/helpers.go new file mode 100644 index 000000000..a7cc79f13 --- /dev/null +++ b/test/features/duplicate_vcluster_namespace/helpers.go @@ -0,0 +1,28 @@ +package testfeatures + +import ( + "bytes" + "os/exec" + "strings" + + "github.com/loft-sh/vcluster/test/framework" + "github.com/onsi/gomega" +) + +func noDuplicateVirtualClusterInSameNamespace(vClusterName string, f *framework.Framework) { + deployCmd := exec.Command("vcluster", "create", vClusterName, "--namespace", f.VclusterNamespace, "-f", filePath) + output, err := deployCmd.CombinedOutput() + framework.ExpectError(err) + gomega.Expect(strings.Contains(string(output), "there is already a virtual cluster in namespace")).To(gomega.BeTrue()) +} + +func deploySecondVClusterInSameNamespace(vClusterName string, f *framework.Framework) { + gomega.Eventually(func() bool { + stdout := &bytes.Buffer{} + deployCmd := exec.Command("vcluster", "create", vClusterName, "--namespace", f.VclusterNamespace, "-f", filePath, "--reuse-namespace") + deployCmd.Stdout = stdout + err := deployCmd.Run() + framework.ExpectNoError(err) + return err == nil && strings.Contains(stdout.String(), "Switched active kube context to") + }).WithPolling(pollingInterval).WithTimeout(pollingDurationLong).Should(gomega.BeTrue()) +} diff --git a/test/features/duplicate_vcluster_namespace/test_dup_vcluster_namespace.go b/test/features/duplicate_vcluster_namespace/test_dup_vcluster_namespace.go new file mode 100644 index 000000000..51d077eb2 --- /dev/null +++ b/test/features/duplicate_vcluster_namespace/test_dup_vcluster_namespace.go @@ -0,0 +1,51 @@ +package testfeatures + +import ( + "time" + + "github.com/loft-sh/vcluster/pkg/platform/random" + "github.com/loft-sh/vcluster/test/deploy_changes/common" + "github.com/loft-sh/vcluster/test/framework" + "github.com/onsi/ginkgo/v2" + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + scheme = runtime.NewScheme() +) + +const ( + pollingInterval = time.Second * 2 + pollingDurationLong = time.Minute * 2 + filePath = "commonValues.yaml" + chartPath = "../../../chart" +) + +var _ = ginkgo.Describe("Test vCluster features", func() { + f := framework.DefaultFramework + ginkgo.BeforeEach(func() { + ginkgo.By("Disconnect from any vCluster") + common.DisconnectFromVCluster(f) + }) + + ginkgo.It("should verify no two virtual clusters can be deployed in same namespace ", func() { + vClusterName := "t-vc-" + random.String(6) + + ginkgo.By("Deploy vCluster in default namespace") + noDuplicateVirtualClusterInSameNamespace(vClusterName, f) + + }) + ginkgo.It("should verify two virtual clusters can be deployed in same namespace using the flag ", func() { + vClusterName := "t-vc-" + random.String(6) + + ginkgo.By("Deploy second vCluster in same namespace") + deploySecondVClusterInSameNamespace(vClusterName, f) + + ginkgo.By("Disconnect from any vCluster") + common.DisconnectFromVCluster(f) + + ginkgo.By("Delete vCluster") + common.DeleteVCluster(vClusterName, f) + + }) +}) diff --git a/test/features/duplicate_vcluster_namespace/vcluster_features_suite_test.go b/test/features/duplicate_vcluster_namespace/vcluster_features_suite_test.go new file mode 100644 index 000000000..52308f624 --- /dev/null +++ b/test/features/duplicate_vcluster_namespace/vcluster_features_suite_test.go @@ -0,0 +1,37 @@ +package testfeatures + +import ( + "context" + "testing" + + "github.com/loft-sh/log" + "github.com/loft-sh/vcluster/test/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + // Enable cloud provider auth + + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +func init() { + _ = clientgoscheme.AddToScheme(scheme) +} + +func TestRunDeployChangesSyncTests(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + err := framework.CreateFramework(context.Background(), scheme) + if err != nil { + log.GetInstance().Fatalf("Error setting up framework: %v", err) + } + + var _ = ginkgo.AfterSuite(func() { + err = framework.DefaultFramework.Cleanup() + if err != nil { + log.GetInstance().Warnf("Error executing testsuite cleanup: %v", err) + } + }) + + ginkgo.RunSpecs(t, "vCluster features test suite") +}