From 9d4c0a953c36fb5347abe15cdb42d670c2f6ba23 Mon Sep 17 00:00:00 2001 From: Fabian Fulga Date: Sun, 9 Jun 2024 23:06:01 +0300 Subject: [PATCH] Adding garm test suite --- .github/workflows/go-tests.yml | 2 +- Makefile | 4 +- test/integration/{e2e => }/client_utils.go | 22 +- test/integration/credentials_test.go | 233 ++++++++++++++++++ test/integration/e2e/client.go | 61 ----- test/integration/e2e/credentials.go | 206 ---------------- test/integration/e2e/e2e.go | 206 ---------------- test/integration/e2e/endpoints.go | 247 -------------------- test/integration/e2e/github_client_utils.go | 203 ---------------- test/integration/e2e/instances.go | 119 ---------- test/integration/e2e/jobs.go | 123 ---------- test/integration/e2e/organizations.go | 140 ----------- test/integration/e2e/pools.go | 54 ----- test/integration/e2e/repositories.go | 152 ------------ test/integration/endpoints.go | 48 ++++ test/integration/endpoints_test.go | 212 +++++++++++++++++ test/integration/external_provider_test.go | 171 ++++++++++++++ test/integration/gh_cleanup/main.go | 149 +++++++++++- test/integration/jobs_test.go | 168 +++++++++++++ test/integration/list_info_test.go | 72 ++++++ test/integration/main.go | 191 --------------- test/integration/organizations_test.go | 192 +++++++++++++++ test/integration/repositories_test.go | 208 +++++++++++++++++ test/integration/suite_test.go | 212 +++++++++++++++++ test/integration/{e2e => }/utils.go | 14 +- 25 files changed, 1672 insertions(+), 1737 deletions(-) rename test/integration/{e2e => }/client_utils.go (96%) create mode 100644 test/integration/credentials_test.go delete mode 100644 test/integration/e2e/client.go delete mode 100644 test/integration/e2e/credentials.go delete mode 100644 test/integration/e2e/e2e.go delete mode 100644 test/integration/e2e/endpoints.go delete mode 100644 test/integration/e2e/github_client_utils.go delete mode 100644 test/integration/e2e/instances.go delete mode 100644 test/integration/e2e/jobs.go delete mode 100644 test/integration/e2e/organizations.go delete mode 100644 test/integration/e2e/pools.go delete mode 100644 test/integration/e2e/repositories.go create mode 100644 test/integration/endpoints.go create mode 100644 test/integration/endpoints_test.go create mode 100644 test/integration/external_provider_test.go create mode 100644 test/integration/jobs_test.go create mode 100644 test/integration/list_info_test.go delete mode 100644 test/integration/main.go create mode 100644 test/integration/organizations_test.go create mode 100644 test/integration/repositories_test.go create mode 100644 test/integration/suite_test.go rename test/integration/{e2e => }/utils.go (56%) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index cdb219b9..7c6f6530 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -31,7 +31,7 @@ jobs: go-version: 'stable' - uses: actions/checkout@v3 - name: make lint - run: make golangci-lint && GOLANGCI_LINT_EXTRA_ARGS="--timeout=8m --build-tags testing" make lint + run: make golangci-lint && GOLANGCI_LINT_EXTRA_ARGS="--timeout=8m --build-tags=testing,integration" make lint - name: Verify go vendor, go modules and gofmt run: | sudo apt-get install -y jq diff --git a/Makefile b/Makefile index 0eb6e7e3..92563387 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ release: build-static create-release-files ## Create a release ##@ Lint / Verify .PHONY: lint lint: golangci-lint $(GOLANGCI_LINT) ## Run linting. - $(GOLANGCI_LINT) run -v --build-tags testing $(GOLANGCI_LINT_EXTRA_ARGS) + $(GOLANGCI_LINT) run -v --build-tags=testing,integration $(GOLANGCI_LINT_EXTRA_ARGS) .PHONY: lint-fix lint-fix: golangci-lint $(GOLANGCI_LINT) ## Lint the codebase and run auto-fixers if supported by the linte @@ -84,7 +84,7 @@ integration: build ## Run integration tests } trap cleanup EXIT @./test/integration/scripts/setup-garm.sh - @$(GO) run ./test/integration/main.go + @$(GO) test -v ./test/integration/. -timeout=30m -tags=integration ##@ Development diff --git a/test/integration/e2e/client_utils.go b/test/integration/client_utils.go similarity index 96% rename from test/integration/e2e/client_utils.go rename to test/integration/client_utils.go index 43a28ee7..a0f17893 100644 --- a/test/integration/e2e/client_utils.go +++ b/test/integration/client_utils.go @@ -1,4 +1,4 @@ -package e2e +package integration import ( "github.com/go-openapi/runtime" @@ -67,16 +67,6 @@ func deleteGithubCredentials(apiCli *client.GarmAPI, apiAuthToken runtime.Client apiAuthToken) } -func getGithubCredential(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, credentialsID int64) (*params.GithubCredentials, error) { - getCredentialsResponse, err := apiCli.Credentials.GetCredentials( - clientCredentials.NewGetCredentialsParams().WithID(credentialsID), - apiAuthToken) - if err != nil { - return nil, err - } - return &getCredentialsResponse.Payload, nil -} - func updateGithubCredentials(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, credentialsID int64, credentialsParams params.UpdateGithubCredentialsParams) (*params.GithubCredentials, error) { updateCredentialsResponse, err := apiCli.Credentials.UpdateCredentials( clientCredentials.NewUpdateCredentialsParams().WithID(credentialsID).WithBody(credentialsParams), @@ -501,16 +491,6 @@ func updatePool(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrite return &updatePoolResponse.Payload, nil } -func listPoolInstances(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, poolID string) (params.Instances, error) { - listPoolInstancesResponse, err := apiCli.Instances.ListPoolInstances( - clientInstances.NewListPoolInstancesParams().WithPoolID(poolID), - apiAuthToken) - if err != nil { - return nil, err - } - return listPoolInstancesResponse.Payload, nil -} - func deletePool(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, poolID string) error { return apiCli.Pools.DeletePool( clientPools.NewDeletePoolParams().WithPoolID(poolID), diff --git a/test/integration/credentials_test.go b/test/integration/credentials_test.go new file mode 100644 index 00000000..b83f131f --- /dev/null +++ b/test/integration/credentials_test.go @@ -0,0 +1,233 @@ +//go:build integration +// +build integration + +package integration + +import ( + "github.com/cloudbase/garm/params" +) + +const ( + defaultEndpointName string = "github.com" + dummyCredentialsName string = "dummy" +) + +func (suite *GarmSuite) TestGithubCredentialsErrorOnDuplicateCredentialsName() { + t := suite.T() + t.Log("Testing error on duplicate credentials name") + creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName) + suite.NoError(err) + t.Cleanup(func() { + suite.DeleteGithubCredential(int64(creds.ID)) + }) + + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: defaultEndpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypePAT, + PAT: params.GithubPAT{ + OAuth2Token: "dummy", + }, + } + _, err = createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with duplicate name") +} + +func (suite *GarmSuite) TestGithubCredentialsFailsToDeleteWhenInUse() { + t := suite.T() + t.Log("Testing error when deleting credentials in use") + creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName) + suite.NoError(err) + + orgName := "dummy-owner" + repoName := "dummy-repo" + createParams := params.CreateRepoParams{ + Owner: orgName, + Name: repoName, + CredentialsName: creds.Name, + WebhookSecret: "superSecret@123BlaBla", + } + + t.Logf("Create repository with owner_name: %s, repo_name: %s", orgName, repoName) + repo, err := createRepo(suite.cli, suite.authToken, createParams) + suite.NoError(err) + t.Cleanup(func() { + deleteRepo(suite.cli, suite.authToken, repo.ID) + deleteGithubCredentials(suite.cli, suite.authToken, int64(creds.ID)) + }) + + err = deleteGithubCredentials(suite.cli, suite.authToken, int64(creds.ID)) + suite.Error(err, "expected error when deleting credentials in use") +} + +func (suite *GarmSuite) TestGithubCredentialsFailsOnInvalidAuthType() { + t := suite.T() + t.Log("Testing error on invalid auth type") + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: defaultEndpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthType("invalid"), + PAT: params.GithubPAT{ + OAuth2Token: "dummy", + }, + } + _, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with invalid auth type") + expectAPIStatusCode(err, 400) +} + +func (suite *GarmSuite) TestGithubCredentialsFailsWhenAuthTypeParamsAreIncorrect() { + t := suite.T() + t.Log("Testing error when auth type params are incorrect") + privateKeyBytes, err := getTestFileContents("certs/srv-key.pem") + suite.NoError(err) + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: defaultEndpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypePAT, + App: params.GithubApp{ + AppID: 123, + InstallationID: 456, + PrivateKeyBytes: privateKeyBytes, + }, + } + _, err = createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with invalid auth type params") + + expectAPIStatusCode(err, 400) +} + +func (suite *GarmSuite) TestGithubCredentialsFailsWhenAuthTypeParamsAreMissing() { + t := suite.T() + t.Log("Testing error when auth type params are missing") + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: defaultEndpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypeApp, + } + _, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with missing auth type params") + expectAPIStatusCode(err, 400) +} + +func (suite *GarmSuite) TestGithubCredentialsUpdateFailsWhenBothPATAndAppAreSupplied() { + t := suite.T() + t.Log("Testing error when both PAT and App are supplied") + creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName) + suite.NoError(err) + t.Cleanup(func() { + suite.DeleteGithubCredential(int64(creds.ID)) + }) + + privateKeyBytes, err := getTestFileContents("certs/srv-key.pem") + suite.NoError(err) + updateCredsParams := params.UpdateGithubCredentialsParams{ + PAT: ¶ms.GithubPAT{ + OAuth2Token: "dummy", + }, + App: ¶ms.GithubApp{ + AppID: 123, + InstallationID: 456, + PrivateKeyBytes: privateKeyBytes, + }, + } + _, err = updateGithubCredentials(suite.cli, suite.authToken, int64(creds.ID), updateCredsParams) + suite.Error(err, "expected error when updating credentials with both PAT and App") + expectAPIStatusCode(err, 400) +} + +func (suite *GarmSuite) TestGithubCredentialsFailWhenAppKeyIsInvalid() { + t := suite.T() + t.Log("Testing error when app key is invalid") + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: defaultEndpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypeApp, + App: params.GithubApp{ + AppID: 123, + InstallationID: 456, + PrivateKeyBytes: []byte("invalid"), + }, + } + _, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with invalid app key") + expectAPIStatusCode(err, 400) +} + +func (suite *GarmSuite) TestGithubCredentialsFailWhenEndpointDoesntExist() { + t := suite.T() + t.Log("Testing error when endpoint doesn't exist") + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: "iDontExist.example.com", + Description: "GARM test credentials", + AuthType: params.GithubAuthTypePAT, + PAT: params.GithubPAT{ + OAuth2Token: "dummy", + }, + } + _, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with invalid endpoint") + expectAPIStatusCode(err, 404) +} + +func (suite *GarmSuite) TestGithubCredentialsFailsOnDuplicateName() { + t := suite.T() + t.Log("Testing error on duplicate credentials name") + creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName) + suite.NoError(err) + t.Cleanup(func() { + suite.DeleteGithubCredential(int64(creds.ID)) + }) + + createCredsParams := params.CreateGithubCredentialsParams{ + Name: dummyCredentialsName, + Endpoint: defaultEndpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypePAT, + PAT: params.GithubPAT{ + OAuth2Token: "dummy", + }, + } + _, err = createGithubCredentials(suite.cli, suite.authToken, createCredsParams) + suite.Error(err, "expected error when creating credentials with duplicate name") + expectAPIStatusCode(err, 409) +} + +func (suite *GarmSuite) createDummyCredentials(name, endpointName string) (*params.GithubCredentials, error) { + createCredsParams := params.CreateGithubCredentialsParams{ + Name: name, + Endpoint: endpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypePAT, + PAT: params.GithubPAT{ + OAuth2Token: "dummy", + }, + } + return suite.CreateGithubCredentials(createCredsParams) +} + +func (suite *GarmSuite) CreateGithubCredentials(credentialsParams params.CreateGithubCredentialsParams) (*params.GithubCredentials, error) { + t := suite.T() + t.Log("Create GitHub credentials") + credentials, err := createGithubCredentials(suite.cli, suite.authToken, credentialsParams) + if err != nil { + return nil, err + } + + return credentials, nil +} + +func (suite *GarmSuite) DeleteGithubCredential(id int64) error { + t := suite.T() + t.Log("Delete GitHub credential") + if err := deleteGithubCredentials(suite.cli, suite.authToken, id); err != nil { + return err + } + return nil +} diff --git a/test/integration/e2e/client.go b/test/integration/e2e/client.go deleted file mode 100644 index db841983..00000000 --- a/test/integration/e2e/client.go +++ /dev/null @@ -1,61 +0,0 @@ -package e2e - -import ( - "log/slog" - "net/url" - - "github.com/go-openapi/runtime" - openapiRuntimeClient "github.com/go-openapi/runtime/client" - - "github.com/cloudbase/garm/client" - "github.com/cloudbase/garm/params" -) - -var ( - cli *client.GarmAPI - authToken runtime.ClientAuthInfoWriter -) - -func InitClient(baseURL string) { - garmURL, err := url.Parse(baseURL) - if err != nil { - panic(err) - } - apiPath, err := url.JoinPath(garmURL.Path, client.DefaultBasePath) - if err != nil { - panic(err) - } - transportCfg := client.DefaultTransportConfig(). - WithHost(garmURL.Host). - WithBasePath(apiPath). - WithSchemes([]string{garmURL.Scheme}) - cli = client.NewHTTPClientWithConfig(nil, transportCfg) -} - -func FirstRun(adminUsername, adminPassword, adminFullName, adminEmail string) *params.User { - slog.Info("First run") - newUser := params.NewUserParams{ - Username: adminUsername, - Password: adminPassword, - FullName: adminFullName, - Email: adminEmail, - } - user, err := firstRun(cli, newUser) - if err != nil { - panic(err) - } - return &user -} - -func Login(username, password string) { - slog.Info("Login") - loginParams := params.PasswordLoginParams{ - Username: username, - Password: password, - } - token, err := login(cli, loginParams) - if err != nil { - panic(err) - } - authToken = openapiRuntimeClient.BearerToken(token) -} diff --git a/test/integration/e2e/credentials.go b/test/integration/e2e/credentials.go deleted file mode 100644 index 67e1d35c..00000000 --- a/test/integration/e2e/credentials.go +++ /dev/null @@ -1,206 +0,0 @@ -package e2e - -import ( - "fmt" - "log/slog" - - "github.com/cloudbase/garm/params" -) - -func EnsureTestCredentials(name string, oauthToken string, endpointName string) { - slog.Info("Ensuring test credentials exist") - createCredsParams := params.CreateGithubCredentialsParams{ - Name: name, - Endpoint: endpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypePAT, - PAT: params.GithubPAT{ - OAuth2Token: oauthToken, - }, - } - CreateGithubCredentials(createCredsParams) - - createCredsParams.Name = fmt.Sprintf("%s-clone", name) - CreateGithubCredentials(createCredsParams) -} - -func createDummyCredentials(name, endpointName string) *params.GithubCredentials { - createCredsParams := params.CreateGithubCredentialsParams{ - Name: name, - Endpoint: endpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypePAT, - PAT: params.GithubPAT{ - OAuth2Token: "dummy", - }, - } - return CreateGithubCredentials(createCredsParams) -} - -func TestGithubCredentialsErrorOnDuplicateCredentialsName() { - slog.Info("Testing error on duplicate credentials name") - creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName) - defer DeleteGithubCredential(int64(creds.ID)) - - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: defaultEndpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypePAT, - PAT: params.GithubPAT{ - OAuth2Token: "dummy", - }, - } - if _, err := createGithubCredentials(cli, authToken, createCredsParams); err == nil { - panic("expected error when creating credentials with duplicate name") - } -} - -func TestGithubCredentialsFailsToDeleteWhenInUse() { - slog.Info("Testing error when deleting credentials in use") - creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName) - - repo := CreateRepo("dummy-owner", "dummy-repo", creds.Name, "superSecret@123BlaBla") - defer func() { - deleteRepo(cli, authToken, repo.ID) - deleteGithubCredentials(cli, authToken, int64(creds.ID)) - }() - - if err := deleteGithubCredentials(cli, authToken, int64(creds.ID)); err == nil { - panic("expected error when deleting credentials in use") - } -} - -func TestGithubCredentialsFailsOnInvalidAuthType() { - slog.Info("Testing error on invalid auth type") - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: defaultEndpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthType("invalid"), - PAT: params.GithubPAT{ - OAuth2Token: "dummy", - }, - } - _, err := createGithubCredentials(cli, authToken, createCredsParams) - if err == nil { - panic("expected error when creating credentials with invalid auth type") - } - expectAPIStatusCode(err, 400) -} - -func TestGithubCredentialsFailsWhenAuthTypeParamsAreIncorrect() { - slog.Info("Testing error when auth type params are incorrect") - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: defaultEndpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypePAT, - App: params.GithubApp{ - AppID: 123, - InstallationID: 456, - PrivateKeyBytes: getTestFileContents("certs/srv-key.pem"), - }, - } - _, err := createGithubCredentials(cli, authToken, createCredsParams) - if err == nil { - panic("expected error when creating credentials with invalid auth type params") - } - expectAPIStatusCode(err, 400) -} - -func TestGithubCredentialsFailsWhenAuthTypeParamsAreMissing() { - slog.Info("Testing error when auth type params are missing") - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: defaultEndpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypeApp, - } - _, err := createGithubCredentials(cli, authToken, createCredsParams) - if err == nil { - panic("expected error when creating credentials with missing auth type params") - } - expectAPIStatusCode(err, 400) -} - -func TestGithubCredentialsUpdateFailsWhenBothPATAndAppAreSupplied() { - slog.Info("Testing error when both PAT and App are supplied") - creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName) - defer DeleteGithubCredential(int64(creds.ID)) - - updateCredsParams := params.UpdateGithubCredentialsParams{ - PAT: ¶ms.GithubPAT{ - OAuth2Token: "dummy", - }, - App: ¶ms.GithubApp{ - AppID: 123, - InstallationID: 456, - PrivateKeyBytes: getTestFileContents("certs/srv-key.pem"), - }, - } - _, err := updateGithubCredentials(cli, authToken, int64(creds.ID), updateCredsParams) - if err == nil { - panic("expected error when updating credentials with both PAT and App") - } - expectAPIStatusCode(err, 400) -} - -func TestGithubCredentialsFailWhenAppKeyIsInvalid() { - slog.Info("Testing error when app key is invalid") - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: defaultEndpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypeApp, - App: params.GithubApp{ - AppID: 123, - InstallationID: 456, - PrivateKeyBytes: []byte("invalid"), - }, - } - _, err := createGithubCredentials(cli, authToken, createCredsParams) - if err == nil { - panic("expected error when creating credentials with invalid app key") - } - expectAPIStatusCode(err, 400) -} - -func TestGithubCredentialsFailWhenEndpointDoesntExist() { - slog.Info("Testing error when endpoint doesn't exist") - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: "iDontExist.example.com", - Description: "GARM test credentials", - AuthType: params.GithubAuthTypePAT, - PAT: params.GithubPAT{ - OAuth2Token: "dummy", - }, - } - _, err := createGithubCredentials(cli, authToken, createCredsParams) - if err == nil { - panic("expected error when creating credentials with invalid endpoint") - } - expectAPIStatusCode(err, 404) -} - -func TestGithubCredentialsFailsOnDuplicateName() { - slog.Info("Testing error on duplicate credentials name") - creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName) - defer DeleteGithubCredential(int64(creds.ID)) - - createCredsParams := params.CreateGithubCredentialsParams{ - Name: dummyCredentialsName, - Endpoint: defaultEndpointName, - Description: "GARM test credentials", - AuthType: params.GithubAuthTypePAT, - PAT: params.GithubPAT{ - OAuth2Token: "dummy", - }, - } - _, err := createGithubCredentials(cli, authToken, createCredsParams) - if err == nil { - panic("expected error when creating credentials with duplicate name") - } - expectAPIStatusCode(err, 409) -} diff --git a/test/integration/e2e/e2e.go b/test/integration/e2e/e2e.go deleted file mode 100644 index 84742a67..00000000 --- a/test/integration/e2e/e2e.go +++ /dev/null @@ -1,206 +0,0 @@ -package e2e - -import ( - "fmt" - "log/slog" - "os" - "time" - - "github.com/cloudbase/garm/params" -) - -func ListCredentials() params.Credentials { - slog.Info("List credentials") - credentials, err := listCredentials(cli, authToken) - if err != nil { - panic(err) - } - return credentials -} - -func CreateGithubCredentials(credentialsParams params.CreateGithubCredentialsParams) *params.GithubCredentials { - slog.Info("Create GitHub credentials") - credentials, err := createGithubCredentials(cli, authToken, credentialsParams) - if err != nil { - panic(err) - } - return credentials -} - -func GetGithubCredential(id int64) *params.GithubCredentials { - slog.Info("Get GitHub credential") - credentials, err := getGithubCredential(cli, authToken, id) - if err != nil { - panic(err) - } - return credentials -} - -func DeleteGithubCredential(id int64) { - slog.Info("Delete GitHub credential") - if err := deleteGithubCredentials(cli, authToken, id); err != nil { - panic(err) - } -} - -func CreateGithubEndpoint(endpointParams params.CreateGithubEndpointParams) *params.GithubEndpoint { - slog.Info("Create GitHub endpoint") - endpoint, err := createGithubEndpoint(cli, authToken, endpointParams) - if err != nil { - panic(err) - } - return endpoint -} - -func ListGithubEndpoints() params.GithubEndpoints { - slog.Info("List GitHub endpoints") - endpoints, err := listGithubEndpoints(cli, authToken) - if err != nil { - panic(err) - } - return endpoints -} - -func GetGithubEndpoint(name string) *params.GithubEndpoint { - slog.Info("Get GitHub endpoint") - endpoint, err := getGithubEndpoint(cli, authToken, name) - if err != nil { - panic(err) - } - return endpoint -} - -func DeleteGithubEndpoint(name string) { - slog.Info("Delete GitHub endpoint") - if err := deleteGithubEndpoint(cli, authToken, name); err != nil { - panic(err) - } -} - -func UpdateGithubEndpoint(name string, updateParams params.UpdateGithubEndpointParams) *params.GithubEndpoint { - slog.Info("Update GitHub endpoint") - updated, err := updateGithubEndpoint(cli, authToken, name, updateParams) - if err != nil { - panic(err) - } - return updated -} - -func ListProviders() params.Providers { - slog.Info("List providers") - providers, err := listProviders(cli, authToken) - if err != nil { - panic(err) - } - return providers -} - -func GetMetricsToken() { - slog.Info("Get metrics token") - _, err := getMetricsToken(cli, authToken) - if err != nil { - panic(err) - } -} - -func GetControllerInfo() *params.ControllerInfo { - slog.Info("Get controller info") - controllerInfo, err := getControllerInfo(cli, authToken) - if err != nil { - panic(err) - } - if err := appendCtrlInfoToGitHubEnv(&controllerInfo); err != nil { - panic(err) - } - if err := printJSONResponse(controllerInfo); err != nil { - panic(err) - } - return &controllerInfo -} - -func GracefulCleanup() { - slog.Info("Graceful cleanup") - // disable all the pools - pools, err := listPools(cli, authToken) - if err != nil { - panic(err) - } - enabled := false - poolParams := params.UpdatePoolParams{Enabled: &enabled} - for _, pool := range pools { - if _, err := updatePool(cli, authToken, pool.ID, poolParams); err != nil { - panic(err) - } - slog.Info("Pool disabled", "pool_id", pool.ID, "stage", "graceful_cleanup") - } - - // delete all the instances - for _, pool := range pools { - poolInstances, err := listPoolInstances(cli, authToken, pool.ID) - if err != nil { - panic(err) - } - for _, instance := range poolInstances { - if err := deleteInstance(cli, authToken, instance.Name, false, false); err != nil { - panic(err) - } - slog.Info("Instance deletion initiated", "instance", instance.Name, "stage", "graceful_cleanup") - } - } - - // wait for all instances to be deleted - for _, pool := range pools { - if err := waitPoolNoInstances(pool.ID, 3*time.Minute); err != nil { - panic(err) - } - } - - // delete all the pools - for _, pool := range pools { - if err := deletePool(cli, authToken, pool.ID); err != nil { - panic(err) - } - slog.Info("Pool deleted", "pool_id", pool.ID, "stage", "graceful_cleanup") - } - - // delete all the repositories - repos, err := listRepos(cli, authToken) - if err != nil { - panic(err) - } - for _, repo := range repos { - if err := deleteRepo(cli, authToken, repo.ID); err != nil { - panic(err) - } - slog.Info("Repo deleted", "repo_id", repo.ID, "stage", "graceful_cleanup") - } - - // delete all the organizations - orgs, err := listOrgs(cli, authToken) - if err != nil { - panic(err) - } - for _, org := range orgs { - if err := deleteOrg(cli, authToken, org.ID); err != nil { - panic(err) - } - slog.Info("Org deleted", "org_id", org.ID, "stage", "graceful_cleanup") - } -} - -func appendCtrlInfoToGitHubEnv(controllerInfo *params.ControllerInfo) error { - envFile, found := os.LookupEnv("GITHUB_ENV") - if !found { - slog.Info("GITHUB_ENV not set, skipping appending controller info") - return nil - } - file, err := os.OpenFile(envFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) - if err != nil { - return err - } - defer file.Close() - if _, err := file.WriteString(fmt.Sprintf("export GARM_CONTROLLER_ID=%s\n", controllerInfo.ControllerID)); err != nil { - return err - } - return nil -} diff --git a/test/integration/e2e/endpoints.go b/test/integration/e2e/endpoints.go deleted file mode 100644 index 9a276627..00000000 --- a/test/integration/e2e/endpoints.go +++ /dev/null @@ -1,247 +0,0 @@ -package e2e - -import ( - "log/slog" - "os" - "path/filepath" - - "github.com/cloudbase/garm/params" -) - -const ( - defaultEndpointName string = "github.com" - dummyCredentialsName string = "dummy" -) - -func MustDefaultGithubEndpoint() { - ep := GetGithubEndpoint("github.com") - if ep == nil { - panic("Default GitHub endpoint not found") - } - - if ep.Name != "github.com" { - panic("Default GitHub endpoint name mismatch") - } -} - -func checkEndpointParamsAreEqual(a, b params.GithubEndpoint) { - if a.Name != b.Name { - panic("Endpoint name mismatch") - } - - if a.Description != b.Description { - panic("Endpoint description mismatch") - } - - if a.BaseURL != b.BaseURL { - panic("Endpoint base URL mismatch") - } - - if a.APIBaseURL != b.APIBaseURL { - panic("Endpoint API base URL mismatch") - } - - if a.UploadBaseURL != b.UploadBaseURL { - panic("Endpoint upload base URL mismatch") - } - - if string(a.CACertBundle) != string(b.CACertBundle) { - panic("Endpoint CA cert bundle mismatch") - } -} - -func getTestFileContents(relPath string) []byte { - baseDir := os.Getenv("GARM_CHECKOUT_DIR") - if baseDir == "" { - panic("GARM_CHECKOUT_DIR not set") - } - contents, err := os.ReadFile(filepath.Join(baseDir, "testdata", relPath)) - if err != nil { - panic(err) - } - return contents -} - -func TestGithubEndpointOperations() { - slog.Info("Testing endpoint operations") - MustDefaultGithubEndpoint() - - caBundle := getTestFileContents("certs/srv-pub.pem") - - endpointParams := params.CreateGithubEndpointParams{ - Name: "test-endpoint", - Description: "Test endpoint", - BaseURL: "https://ghes.example.com", - APIBaseURL: "https://api.ghes.example.com/", - UploadBaseURL: "https://uploads.ghes.example.com/", - CACertBundle: caBundle, - } - - endpoint := CreateGithubEndpoint(endpointParams) - if endpoint.Name != endpointParams.Name { - panic("Endpoint name mismatch") - } - - if endpoint.Description != endpointParams.Description { - panic("Endpoint description mismatch") - } - - if endpoint.BaseURL != endpointParams.BaseURL { - panic("Endpoint base URL mismatch") - } - - if endpoint.APIBaseURL != endpointParams.APIBaseURL { - panic("Endpoint API base URL mismatch") - } - - if endpoint.UploadBaseURL != endpointParams.UploadBaseURL { - panic("Endpoint upload base URL mismatch") - } - - if string(endpoint.CACertBundle) != string(caBundle) { - panic("Endpoint CA cert bundle mismatch") - } - - endpoint2 := GetGithubEndpoint(endpointParams.Name) - if endpoint == nil || endpoint2 == nil { - panic("endpoint is nil") - } - checkEndpointParamsAreEqual(*endpoint, *endpoint2) - - endpoints := ListGithubEndpoints() - var found bool - for _, ep := range endpoints { - if ep.Name == endpointParams.Name { - checkEndpointParamsAreEqual(*endpoint, ep) - found = true - break - } - } - if !found { - panic("Endpoint not found in list") - } - - DeleteGithubEndpoint(endpoint.Name) -} - -func TestGithubEndpointMustFailToDeleteDefaultGithubEndpoint() { - slog.Info("Testing error when deleting default github.com endpoint") - if err := deleteGithubEndpoint(cli, authToken, "github.com"); err == nil { - panic("expected error when attempting to delete the default github.com endpoint") - } -} - -func TestGithubEndpointFailsOnInvalidCABundle() { - slog.Info("Testing endpoint creation with invalid CA cert bundle") - badCABundle := getTestFileContents("certs/srv-key.pem") - - endpointParams := params.CreateGithubEndpointParams{ - Name: "dummy", - Description: "Dummy endpoint", - BaseURL: "https://ghes.example.com", - APIBaseURL: "https://api.ghes.example.com/", - UploadBaseURL: "https://uploads.ghes.example.com/", - CACertBundle: badCABundle, - } - - if _, err := createGithubEndpoint(cli, authToken, endpointParams); err == nil { - panic("expected error when creating endpoint with invalid CA cert bundle") - } -} - -func TestGithubEndpointDeletionFailsWhenCredentialsExist() { - slog.Info("Testing endpoint deletion when credentials exist") - endpointParams := params.CreateGithubEndpointParams{ - Name: "dummy", - Description: "Dummy endpoint", - BaseURL: "https://ghes.example.com", - APIBaseURL: "https://api.ghes.example.com/", - UploadBaseURL: "https://uploads.ghes.example.com/", - } - - endpoint := CreateGithubEndpoint(endpointParams) - creds := createDummyCredentials("test-creds", endpoint.Name) - - if err := deleteGithubEndpoint(cli, authToken, endpoint.Name); err == nil { - panic("expected error when deleting endpoint with credentials") - } - - DeleteGithubCredential(int64(creds.ID)) - DeleteGithubEndpoint(endpoint.Name) -} - -func TestGithubEndpointFailsOnDuplicateName() { - slog.Info("Testing endpoint creation with duplicate name") - endpointParams := params.CreateGithubEndpointParams{ - Name: "github.com", - Description: "Dummy endpoint", - BaseURL: "https://ghes.example.com", - APIBaseURL: "https://api.ghes.example.com/", - UploadBaseURL: "https://uploads.ghes.example.com/", - } - - if _, err := createGithubEndpoint(cli, authToken, endpointParams); err == nil { - panic("expected error when creating endpoint with duplicate name") - } -} - -func TestGithubEndpointUpdateEndpoint() { - slog.Info("Testing endpoint update") - endpoint := createDummyEndpoint("dummy") - defer DeleteGithubEndpoint(endpoint.Name) - - newDescription := "Updated description" - newBaseURL := "https://ghes2.example.com" - newAPIBaseURL := "https://api.ghes2.example.com/" - newUploadBaseURL := "https://uploads.ghes2.example.com/" - newCABundle := getTestFileContents("certs/srv-pub.pem") - - updateParams := params.UpdateGithubEndpointParams{ - Description: &newDescription, - BaseURL: &newBaseURL, - APIBaseURL: &newAPIBaseURL, - UploadBaseURL: &newUploadBaseURL, - CACertBundle: newCABundle, - } - - updated, err := updateGithubEndpoint(cli, authToken, endpoint.Name, updateParams) - if err != nil { - panic(err) - } - - if updated.Name != endpoint.Name { - panic("Endpoint name mismatch") - } - - if updated.Description != newDescription { - panic("Endpoint description mismatch") - } - - if updated.BaseURL != newBaseURL { - panic("Endpoint base URL mismatch") - } - - if updated.APIBaseURL != newAPIBaseURL { - panic("Endpoint API base URL mismatch") - } - - if updated.UploadBaseURL != newUploadBaseURL { - panic("Endpoint upload base URL mismatch") - } - - if string(updated.CACertBundle) != string(newCABundle) { - panic("Endpoint CA cert bundle mismatch") - } -} - -func createDummyEndpoint(name string) *params.GithubEndpoint { - endpointParams := params.CreateGithubEndpointParams{ - Name: name, - Description: "Dummy endpoint", - BaseURL: "https://ghes.example.com", - APIBaseURL: "https://api.ghes.example.com/", - UploadBaseURL: "https://uploads.ghes.example.com/", - } - - return CreateGithubEndpoint(endpointParams) -} diff --git a/test/integration/e2e/github_client_utils.go b/test/integration/e2e/github_client_utils.go deleted file mode 100644 index 81ced630..00000000 --- a/test/integration/e2e/github_client_utils.go +++ /dev/null @@ -1,203 +0,0 @@ -package e2e - -import ( - "context" - "fmt" - "log/slog" - - "github.com/google/go-github/v57/github" - "golang.org/x/oauth2" -) - -func TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, labelName string) { - slog.Info("Trigger workflow", "label", labelName) - - client := getGithubClient(ghToken) - eventReq := github.CreateWorkflowDispatchEventRequest{ - Ref: "main", - Inputs: map[string]interface{}{ - "sleep_time": "50", - "runner_label": labelName, - }, - } - if _, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), orgName, repoName, workflowFileName, eventReq); err != nil { - panic(err) - } -} - -func GhOrgRunnersCleanup(ghToken, orgName, controllerID string) error { - slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName) - - client := getGithubClient(ghToken) - ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil) - if err != nil { - return err - } - - // Remove organization runners - controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID) - for _, orgRunner := range ghOrgRunners.Runners { - for _, label := range orgRunner.Labels { - if label.GetName() == controllerLabel { - if _, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID()); err != nil { - // We don't fail if we can't remove a single runner. This - // is a best effort to try and remove all the orphan runners. - slog.With(slog.Any("error", err)).Info("Failed to remove organization runner", "org_runner", orgRunner.GetName()) - break - } - slog.Info("Removed organization runner", "org_runner", orgRunner.GetName()) - break - } - } - } - - return nil -} - -func GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID string) error { - slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName, "repo_name", repoName) - - client := getGithubClient(ghToken) - ghRepoRunners, _, err := client.Actions.ListRunners(context.Background(), orgName, repoName, nil) - if err != nil { - return err - } - - // Remove repository runners - controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID) - for _, repoRunner := range ghRepoRunners.Runners { - for _, label := range repoRunner.Labels { - if label.GetName() == controllerLabel { - if _, err := client.Actions.RemoveRunner(context.Background(), orgName, repoName, repoRunner.GetID()); err != nil { - // We don't fail if we can't remove a single runner. This - // is a best effort to try and remove all the orphan runners. - slog.With(slog.Any("error", err)).Error("Failed to remove repository runner", "runner_name", repoRunner.GetName()) - break - } - slog.Info("Removed repository runner", "runner_name", repoRunner.GetName()) - break - } - } - } - - return nil -} - -func ValidateOrgWebhookInstalled(ghToken, url, orgName string) { - hook, err := getGhOrgWebhook(url, ghToken, orgName) - if err != nil { - panic(err) - } - if hook == nil { - panic(fmt.Errorf("github webhook with url %s, for org %s was not properly installed", url, orgName)) - } -} - -func ValidateOrgWebhookUninstalled(ghToken, url, orgName string) { - hook, err := getGhOrgWebhook(url, ghToken, orgName) - if err != nil { - panic(err) - } - if hook != nil { - panic(fmt.Errorf("github webhook with url %s, for org %s was not properly uninstalled", url, orgName)) - } -} - -func ValidateRepoWebhookInstalled(ghToken, url, orgName, repoName string) { - hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName) - if err != nil { - panic(err) - } - if hook == nil { - panic(fmt.Errorf("github webhook with url %s, for repo %s/%s was not properly installed", url, orgName, repoName)) - } -} - -func ValidateRepoWebhookUninstalled(ghToken, url, orgName, repoName string) { - hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName) - if err != nil { - panic(err) - } - if hook != nil { - panic(fmt.Errorf("github webhook with url %s, for repo %s/%s was not properly uninstalled", url, orgName, repoName)) - } -} - -func GhOrgWebhookCleanup(ghToken, webhookURL, orgName string) error { - slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName) - hook, err := getGhOrgWebhook(webhookURL, ghToken, orgName) - if err != nil { - return err - } - - // Remove organization webhook - if hook != nil { - client := getGithubClient(ghToken) - if _, err := client.Organizations.DeleteHook(context.Background(), orgName, hook.GetID()); err != nil { - return err - } - slog.Info("Github webhook removed", "webhook_url", webhookURL, "org_name", orgName) - } - - return nil -} - -func GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName string) error { - slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName) - - hook, err := getGhRepoWebhook(webhookURL, ghToken, orgName, repoName) - if err != nil { - return err - } - - // Remove repository webhook - if hook != nil { - client := getGithubClient(ghToken) - if _, err := client.Repositories.DeleteHook(context.Background(), orgName, repoName, hook.GetID()); err != nil { - return err - } - slog.Info("Github webhook with", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName) - } - - return nil -} - -func getGhOrgWebhook(url, ghToken, orgName string) (*github.Hook, error) { - client := getGithubClient(ghToken) - ghOrgHooks, _, err := client.Organizations.ListHooks(context.Background(), orgName, nil) - if err != nil { - return nil, err - } - - for _, hook := range ghOrgHooks { - hookURL, ok := hook.Config["url"].(string) - if ok && hookURL == url { - return hook, nil - } - } - - return nil, nil -} - -func getGhRepoWebhook(url, ghToken, orgName, repoName string) (*github.Hook, error) { - client := getGithubClient(ghToken) - ghRepoHooks, _, err := client.Repositories.ListHooks(context.Background(), orgName, repoName, nil) - if err != nil { - return nil, err - } - - for _, hook := range ghRepoHooks { - hookURL, ok := hook.Config["url"].(string) - if ok && hookURL == url { - return hook, nil - } - } - - return nil, nil -} - -func getGithubClient(oauthToken string) *github.Client { - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken}) - tc := oauth2.NewClient(context.Background(), ts) - return github.NewClient(tc) -} diff --git a/test/integration/e2e/instances.go b/test/integration/e2e/instances.go deleted file mode 100644 index 2f33e79d..00000000 --- a/test/integration/e2e/instances.go +++ /dev/null @@ -1,119 +0,0 @@ -package e2e - -import ( - "fmt" - "log/slog" - "time" - - commonParams "github.com/cloudbase/garm-provider-common/params" - "github.com/cloudbase/garm/params" -) - -func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) (*params.Instance, error) { - var timeWaited time.Duration // default is 0 - var instance *params.Instance - var err error - - slog.Info("Waiting for instance to reach desired status", "instance", name, "desired_status", status, "desired_runner_status", runnerStatus) - for timeWaited < timeout { - instance, err = getInstance(cli, authToken, name) - if err != nil { - return nil, err - } - slog.Info("Instance status", "instance_name", name, "status", instance.Status, "runner_status", instance.RunnerStatus) - if instance.Status == status && instance.RunnerStatus == runnerStatus { - return instance, nil - } - time.Sleep(5 * time.Second) - timeWaited += 5 * time.Second - } - - if err := printJSONResponse(*instance); err != nil { - return nil, err - } - return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus) -} - -func DeleteInstance(name string, forceRemove, bypassGHUnauthorized bool) { - slog.Info("Delete instance", "instance_name", name, "force_remove", forceRemove) - if err := deleteInstance(cli, authToken, name, forceRemove, bypassGHUnauthorized); err != nil { - slog.Error("Failed to delete instance", "instance_name", name, "error", err) - panic(err) - } - slog.Info("Instance deletion initiated", "instance_name", name) -} - -func WaitInstanceToBeRemoved(name string, timeout time.Duration) error { - var timeWaited time.Duration // default is 0 - var instance *params.Instance - - slog.Info("Waiting for instance to be removed", "instance_name", name) - for timeWaited < timeout { - instances, err := listInstances(cli, authToken) - if err != nil { - return err - } - - instance = nil - for k, v := range instances { - if v.Name == name { - instance = &instances[k] - break - } - } - if instance == nil { - // The instance is not found in the list. We can safely assume - // that it is removed - return nil - } - - time.Sleep(5 * time.Second) - timeWaited += 5 * time.Second - } - - if err := printJSONResponse(*instance); err != nil { - return err - } - return fmt.Errorf("instance %s was not removed within the timeout", name) -} - -func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error { - var timeWaited time.Duration // default is 0 - - pool, err := getPool(cli, authToken, poolID) - if err != nil { - return err - } - - slog.Info("Waiting for pool instances to reach desired status", "pool_id", poolID, "desired_status", status, "desired_runner_status", runnerStatus) - for timeWaited < timeout { - poolInstances, err := listPoolInstances(cli, authToken, poolID) - if err != nil { - return err - } - - instancesCount := 0 - for _, instance := range poolInstances { - if instance.Status == status && instance.RunnerStatus == runnerStatus { - instancesCount++ - } - } - - slog.Info( - "Pool instance reached status", - "pool_id", poolID, - "status", status, - "runner_status", runnerStatus, - "desired_instance_count", instancesCount, - "pool_instance_count", len(poolInstances)) - if int(pool.MinIdleRunners) == instancesCount { - return nil - } - time.Sleep(5 * time.Second) - timeWaited += 5 * time.Second - } - - _ = dumpPoolInstancesDetails(pool.ID) - - return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus) -} diff --git a/test/integration/e2e/jobs.go b/test/integration/e2e/jobs.go deleted file mode 100644 index bc93ff0a..00000000 --- a/test/integration/e2e/jobs.go +++ /dev/null @@ -1,123 +0,0 @@ -package e2e - -import ( - "fmt" - "log/slog" - "time" - - commonParams "github.com/cloudbase/garm-provider-common/params" - "github.com/cloudbase/garm/params" -) - -func ValidateJobLifecycle(label string) { - slog.Info("Validate GARM job lifecycle", "label", label) - - // wait for job list to be updated - job, err := waitLabelledJob(label, 4*time.Minute) - if err != nil { - panic(err) - } - - // check expected job status - job, err = waitJobStatus(job.ID, params.JobStatusQueued, 4*time.Minute) - if err != nil { - panic(err) - } - job, err = waitJobStatus(job.ID, params.JobStatusInProgress, 4*time.Minute) - if err != nil { - panic(err) - } - - // check expected instance status - instance, err := waitInstanceStatus(job.RunnerName, commonParams.InstanceRunning, params.RunnerActive, 5*time.Minute) - if err != nil { - panic(err) - } - - // wait for job to be completed - _, err = waitJobStatus(job.ID, params.JobStatusCompleted, 4*time.Minute) - if err != nil { - panic(err) - } - - // wait for instance to be removed - err = WaitInstanceToBeRemoved(instance.Name, 5*time.Minute) - if err != nil { - panic(err) - } - - // wait for GARM to rebuild the pool running idle instances - err = WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute) - if err != nil { - panic(err) - } -} - -func waitLabelledJob(label string, timeout time.Duration) (*params.Job, error) { - var timeWaited time.Duration // default is 0 - var jobs params.Jobs - var err error - - slog.Info("Waiting for job", "label", label) - for timeWaited < timeout { - jobs, err = listJobs(cli, authToken) - if err != nil { - return nil, err - } - for _, job := range jobs { - for _, jobLabel := range job.Labels { - if jobLabel == label { - return &job, err - } - } - } - time.Sleep(5 * time.Second) - timeWaited += 5 * time.Second - } - - if err := printJSONResponse(jobs); err != nil { - return nil, err - } - return nil, fmt.Errorf("failed to wait job with label %s", label) -} - -func waitJobStatus(id int64, status params.JobStatus, timeout time.Duration) (*params.Job, error) { - var timeWaited time.Duration // default is 0 - var job *params.Job - - slog.Info("Waiting for job to reach status", "job_id", id, "status", status) - for timeWaited < timeout { - jobs, err := listJobs(cli, authToken) - if err != nil { - return nil, err - } - - job = nil - for k, v := range jobs { - if v.ID == id { - job = &jobs[k] - break - } - } - - if job == nil { - if status == params.JobStatusCompleted { - // The job is not found in the list. We can safely assume - // that it is completed - return nil, nil - } - // if the job is not found, and expected status is not "completed", - // we need to error out. - return nil, fmt.Errorf("job %d not found, expected to be found in status %s", id, status) - } else if job.Status == string(status) { - return job, nil - } - time.Sleep(5 * time.Second) - timeWaited += 5 * time.Second - } - - if err := printJSONResponse(*job); err != nil { - return nil, err - } - return nil, fmt.Errorf("timeout waiting for job %d to reach status %s", id, status) -} diff --git a/test/integration/e2e/organizations.go b/test/integration/e2e/organizations.go deleted file mode 100644 index 7a7d0e0d..00000000 --- a/test/integration/e2e/organizations.go +++ /dev/null @@ -1,140 +0,0 @@ -package e2e - -import ( - "log/slog" - "time" - - commonParams "github.com/cloudbase/garm-provider-common/params" - "github.com/cloudbase/garm/params" -) - -func CreateOrg(orgName, credentialsName, orgWebhookSecret string) *params.Organization { - slog.Info("Create org", "org_name", orgName) - orgParams := params.CreateOrgParams{ - Name: orgName, - CredentialsName: credentialsName, - WebhookSecret: orgWebhookSecret, - } - org, err := createOrg(cli, authToken, orgParams) - if err != nil { - panic(err) - } - return org -} - -func UpdateOrg(id, credentialsName string) *params.Organization { - slog.Info("Update org", "org_id", id) - updateParams := params.UpdateEntityParams{ - CredentialsName: credentialsName, - } - org, err := updateOrg(cli, authToken, id, updateParams) - if err != nil { - panic(err) - } - return org -} - -func InstallOrgWebhook(id string) *params.HookInfo { - slog.Info("Install org webhook", "org_id", id) - webhookParams := params.InstallWebhookParams{ - WebhookEndpointType: params.WebhookEndpointDirect, - } - _, err := installOrgWebhook(cli, authToken, id, webhookParams) - if err != nil { - panic(err) - } - webhookInfo, err := getOrgWebhook(cli, authToken, id) - if err != nil { - panic(err) - } - return webhookInfo -} - -func UninstallOrgWebhook(id string) { - slog.Info("Uninstall org webhook", "org_id", id) - if err := uninstallOrgWebhook(cli, authToken, id); err != nil { - panic(err) - } -} - -func CreateOrgPool(orgID string, poolParams params.CreatePoolParams) *params.Pool { - slog.Info("Create org pool", "org_id", orgID) - pool, err := createOrgPool(cli, authToken, orgID, poolParams) - if err != nil { - panic(err) - } - return pool -} - -func GetOrgPool(orgID, orgPoolID string) *params.Pool { - slog.Info("Get org pool", "org_id", orgID, "pool_id", orgPoolID) - pool, err := getOrgPool(cli, authToken, orgID, orgPoolID) - if err != nil { - panic(err) - } - return pool -} - -func UpdateOrgPool(orgID, orgPoolID string, maxRunners, minIdleRunners uint) *params.Pool { - slog.Info("Update org pool", "org_id", orgID, "pool_id", orgPoolID) - poolParams := params.UpdatePoolParams{ - MinIdleRunners: &minIdleRunners, - MaxRunners: &maxRunners, - } - pool, err := updateOrgPool(cli, authToken, orgID, orgPoolID, poolParams) - if err != nil { - panic(err) - } - return pool -} - -func DeleteOrgPool(orgID, orgPoolID string) { - slog.Info("Delete org pool", "org_id", orgID, "pool_id", orgPoolID) - if err := deleteOrgPool(cli, authToken, orgID, orgPoolID); err != nil { - panic(err) - } -} - -func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) { - orgPools, err := listOrgPools(cli, authToken, orgID) - if err != nil { - panic(err) - } - for _, pool := range orgPools { - err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout) - if err != nil { - _ = dumpOrgInstancesDetails(orgID) - panic(err) - } - } -} - -func dumpOrgInstancesDetails(orgID string) error { - // print org details - slog.Info("Dumping org details", "org_id", orgID) - org, err := getOrg(cli, authToken, orgID) - if err != nil { - return err - } - if err := printJSONResponse(org); err != nil { - return err - } - - // print org instances details - slog.Info("Dumping org instances details", "org_id", orgID) - instances, err := listOrgInstances(cli, authToken, orgID) - if err != nil { - return err - } - for _, instance := range instances { - instance, err := getInstance(cli, authToken, instance.Name) - if err != nil { - return err - } - slog.Info("Instance info", "instance_name", instance.Name) - if err := printJSONResponse(instance); err != nil { - return err - } - } - return nil -} diff --git a/test/integration/e2e/pools.go b/test/integration/e2e/pools.go deleted file mode 100644 index b4371e29..00000000 --- a/test/integration/e2e/pools.go +++ /dev/null @@ -1,54 +0,0 @@ -package e2e - -import ( - "fmt" - "log/slog" - "time" - - "github.com/cloudbase/garm/params" -) - -func waitPoolNoInstances(id string, timeout time.Duration) error { - var timeWaited time.Duration // default is 0 - var pool *params.Pool - var err error - - slog.Info("Wait until pool has no instances", "pool_id", id) - for timeWaited < timeout { - pool, err = getPool(cli, authToken, id) - if err != nil { - return err - } - slog.Info("Current pool instances", "instance_count", len(pool.Instances)) - if len(pool.Instances) == 0 { - return nil - } - time.Sleep(5 * time.Second) - timeWaited += 5 * time.Second - } - - _ = dumpPoolInstancesDetails(pool.ID) - - return fmt.Errorf("failed to wait for pool %s to have no instances", pool.ID) -} - -func dumpPoolInstancesDetails(poolID string) error { - pool, err := getPool(cli, authToken, poolID) - if err != nil { - return err - } - if err := printJSONResponse(pool); err != nil { - return err - } - for _, instance := range pool.Instances { - instanceDetails, err := getInstance(cli, authToken, instance.Name) - if err != nil { - return err - } - slog.Info("Instance details", "instance_name", instance.Name) - if err := printJSONResponse(instanceDetails); err != nil { - return err - } - } - return nil -} diff --git a/test/integration/e2e/repositories.go b/test/integration/e2e/repositories.go deleted file mode 100644 index 6744e53b..00000000 --- a/test/integration/e2e/repositories.go +++ /dev/null @@ -1,152 +0,0 @@ -package e2e - -import ( - "log/slog" - "time" - - commonParams "github.com/cloudbase/garm-provider-common/params" - "github.com/cloudbase/garm/params" -) - -func CreateRepo(orgName, repoName, credentialsName, repoWebhookSecret string) *params.Repository { - slog.Info("Create repository", "owner_name", orgName, "repo_name", repoName) - createParams := params.CreateRepoParams{ - Owner: orgName, - Name: repoName, - CredentialsName: credentialsName, - WebhookSecret: repoWebhookSecret, - } - repo, err := createRepo(cli, authToken, createParams) - if err != nil { - panic(err) - } - return repo -} - -func UpdateRepo(id, credentialsName string) *params.Repository { - slog.Info("Update repo", "repo_id", id) - updateParams := params.UpdateEntityParams{ - CredentialsName: credentialsName, - } - repo, err := updateRepo(cli, authToken, id, updateParams) - if err != nil { - panic(err) - } - return repo -} - -func InstallRepoWebhook(id string) *params.HookInfo { - slog.Info("Install repo webhook", "repo_id", id) - webhookParams := params.InstallWebhookParams{ - WebhookEndpointType: params.WebhookEndpointDirect, - } - _, err := installRepoWebhook(cli, authToken, id, webhookParams) - if err != nil { - slog.Error("Failed to install repo webhook", "error", err) - panic(err) - } - webhookInfo, err := getRepoWebhook(cli, authToken, id) - if err != nil { - panic(err) - } - return webhookInfo -} - -func UninstallRepoWebhook(id string) { - slog.Info("Uninstall repo webhook", "repo_id", id) - if err := uninstallRepoWebhook(cli, authToken, id); err != nil { - panic(err) - } -} - -func CreateRepoPool(repoID string, poolParams params.CreatePoolParams) *params.Pool { - slog.Info("Create repo pool", "repo_id", repoID, "pool_params", poolParams) - pool, err := createRepoPool(cli, authToken, repoID, poolParams) - if err != nil { - slog.Error("Failed to create repo pool", "error", err) - panic(err) - } - return pool -} - -func GetRepoPool(repoID, repoPoolID string) *params.Pool { - slog.Info("Get repo pool", "repo_id", repoID, "pool_id", repoPoolID) - pool, err := getRepoPool(cli, authToken, repoID, repoPoolID) - if err != nil { - panic(err) - } - return pool -} - -func UpdateRepoPool(repoID, repoPoolID string, maxRunners, minIdleRunners uint) *params.Pool { - slog.Info("Update repo pool", "repo_id", repoID, "pool_id", repoPoolID) - poolParams := params.UpdatePoolParams{ - MinIdleRunners: &minIdleRunners, - MaxRunners: &maxRunners, - } - pool, err := updateRepoPool(cli, authToken, repoID, repoPoolID, poolParams) - if err != nil { - panic(err) - } - return pool -} - -func DeleteRepoPool(repoID, repoPoolID string) { - slog.Info("Delete repo pool", "repo_id", repoID, "pool_id", repoPoolID) - if err := deleteRepoPool(cli, authToken, repoID, repoPoolID); err != nil { - panic(err) - } -} - -func DisableRepoPool(repoID, repoPoolID string) { - slog.Info("Disable repo pool", "repo_id", repoID, "pool_id", repoPoolID) - enabled := false - poolParams := params.UpdatePoolParams{Enabled: &enabled} - if _, err := updateRepoPool(cli, authToken, repoID, repoPoolID, poolParams); err != nil { - panic(err) - } -} - -func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) { - repoPools, err := listRepoPools(cli, authToken, repoID) - if err != nil { - panic(err) - } - for _, pool := range repoPools { - err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout) - if err != nil { - _ = dumpRepoInstancesDetails(repoID) - panic(err) - } - } -} - -func dumpRepoInstancesDetails(repoID string) error { - // print repo details - slog.Info("Dumping repo details", "repo_id", repoID) - repo, err := getRepo(cli, authToken, repoID) - if err != nil { - return err - } - if err := printJSONResponse(repo); err != nil { - return err - } - - // print repo instances details - slog.Info("Dumping repo instances details", "repo_id", repoID) - instances, err := listRepoInstances(cli, authToken, repoID) - if err != nil { - return err - } - for _, instance := range instances { - instance, err := getInstance(cli, authToken, instance.Name) - if err != nil { - return err - } - slog.Info("Instance info", "instance_name", instance.Name) - if err := printJSONResponse(instance); err != nil { - return err - } - } - return nil -} diff --git a/test/integration/endpoints.go b/test/integration/endpoints.go new file mode 100644 index 00000000..9e47d854 --- /dev/null +++ b/test/integration/endpoints.go @@ -0,0 +1,48 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/cloudbase/garm/params" +) + +func checkEndpointParamsAreEqual(a, b params.GithubEndpoint) error { + if a.Name != b.Name { + return fmt.Errorf("endpoint name mismatch") + } + + if a.Description != b.Description { + return fmt.Errorf("endpoint description mismatch") + } + + if a.BaseURL != b.BaseURL { + return fmt.Errorf("endpoint base URL mismatch") + } + + if a.APIBaseURL != b.APIBaseURL { + return fmt.Errorf("endpoint API base URL mismatch") + } + + if a.UploadBaseURL != b.UploadBaseURL { + return fmt.Errorf("endpoint upload base URL mismatch") + } + + if string(a.CACertBundle) != string(b.CACertBundle) { + return fmt.Errorf("endpoint CA cert bundle mismatch") + } + return nil +} + +func getTestFileContents(relPath string) ([]byte, error) { + baseDir := os.Getenv("GARM_CHECKOUT_DIR") + if baseDir == "" { + return nil, fmt.Errorf("ariable GARM_CHECKOUT_DIR not set") + } + contents, err := os.ReadFile(filepath.Join(baseDir, "testdata", relPath)) + if err != nil { + return nil, err + } + return contents, nil +} diff --git a/test/integration/endpoints_test.go b/test/integration/endpoints_test.go new file mode 100644 index 00000000..c6295349 --- /dev/null +++ b/test/integration/endpoints_test.go @@ -0,0 +1,212 @@ +//go:build integration +// +build integration + +package integration + +import ( + "github.com/cloudbase/garm/params" +) + +func (suite *GarmSuite) TestGithubEndpointOperations() { + t := suite.T() + t.Log("Testing endpoint operations") + suite.MustDefaultGithubEndpoint() + + caBundle, err := getTestFileContents("certs/srv-pub.pem") + suite.NoError(err) + + endpointParams := params.CreateGithubEndpointParams{ + Name: "test-endpoint", + Description: "Test endpoint", + BaseURL: "https://ghes.example.com", + APIBaseURL: "https://api.ghes.example.com/", + UploadBaseURL: "https://uploads.ghes.example.com/", + CACertBundle: caBundle, + } + + endpoint, err := suite.CreateGithubEndpoint(endpointParams) + suite.NoError(err) + suite.Equal(endpoint.Name, endpointParams.Name, "Endpoint name mismatch") + suite.Equal(endpoint.Description, endpointParams.Description, "Endpoint description mismatch") + suite.Equal(endpoint.BaseURL, endpointParams.BaseURL, "Endpoint base URL mismatch") + suite.Equal(endpoint.APIBaseURL, endpointParams.APIBaseURL, "Endpoint API base URL mismatch") + suite.Equal(endpoint.UploadBaseURL, endpointParams.UploadBaseURL, "Endpoint upload base URL mismatch") + suite.Equal(string(endpoint.CACertBundle), string(caBundle), "Endpoint CA cert bundle mismatch") + + endpoint2 := suite.GetGithubEndpoint(endpointParams.Name) + suite.NotNil(endpoint, "endpoint is nil") + suite.NotNil(endpoint2, "endpoint2 is nil") + + err = checkEndpointParamsAreEqual(*endpoint, *endpoint2) + suite.NoError(err, "endpoint params are not equal") + endpoints := suite.ListGithubEndpoints() + suite.NoError(err, "error listing github endpoints") + var found bool + for _, ep := range endpoints { + if ep.Name == endpointParams.Name { + checkEndpointParamsAreEqual(*endpoint, ep) + found = true + break + } + } + suite.Equal(found, true, "endpoint not found in list") + + err = suite.DeleteGithubEndpoint(endpoint.Name) + suite.NoError(err, "error deleting github endpoint") +} + +func (suite *GarmSuite) TestGithubEndpointMustFailToDeleteDefaultGithubEndpoint() { + t := suite.T() + t.Log("Testing error when deleting default github.com endpoint") + err := deleteGithubEndpoint(suite.cli, suite.authToken, "github.com") + suite.Error(err, "expected error when attempting to delete the default github.com endpoint") +} + +func (suite *GarmSuite) TestGithubEndpointFailsOnInvalidCABundle() { + t := suite.T() + t.Log("Testing endpoint creation with invalid CA cert bundle") + badCABundle, err := getTestFileContents("certs/srv-key.pem") + suite.NoError(err, "error reading CA cert bundle") + + endpointParams := params.CreateGithubEndpointParams{ + Name: "dummy", + Description: "Dummy endpoint", + BaseURL: "https://ghes.example.com", + APIBaseURL: "https://api.ghes.example.com/", + UploadBaseURL: "https://uploads.ghes.example.com/", + CACertBundle: badCABundle, + } + + _, err = createGithubEndpoint(suite.cli, suite.authToken, endpointParams) + suite.Error(err, "expected error when creating endpoint with invalid CA cert bundle") +} + +func (suite *GarmSuite) TestGithubEndpointDeletionFailsWhenCredentialsExist() { + t := suite.T() + t.Log("Testing endpoint deletion when credentials exist") + endpointParams := params.CreateGithubEndpointParams{ + Name: "dummy", + Description: "Dummy endpoint", + BaseURL: "https://ghes.example.com", + APIBaseURL: "https://api.ghes.example.com/", + UploadBaseURL: "https://uploads.ghes.example.com/", + } + + endpoint, err := suite.CreateGithubEndpoint(endpointParams) + suite.NoError(err, "error creating github endpoint") + creds, err := suite.createDummyCredentials("test-creds", endpoint.Name) + suite.NoError(err, "error creating dummy credentials") + + err = deleteGithubEndpoint(suite.cli, suite.authToken, endpoint.Name) + suite.Error(err, "expected error when deleting endpoint with credentials") + + err = suite.DeleteGithubCredential(int64(creds.ID)) + suite.NoError(err, "error deleting credentials") + err = suite.DeleteGithubEndpoint(endpoint.Name) + suite.NoError(err, "error deleting endpoint") +} + +func (suite *GarmSuite) TestGithubEndpointFailsOnDuplicateName() { + t := suite.T() + t.Log("Testing endpoint creation with duplicate name") + endpointParams := params.CreateGithubEndpointParams{ + Name: "github.com", + Description: "Dummy endpoint", + BaseURL: "https://ghes.example.com", + APIBaseURL: "https://api.ghes.example.com/", + UploadBaseURL: "https://uploads.ghes.example.com/", + } + + _, err := createGithubEndpoint(suite.cli, suite.authToken, endpointParams) + suite.Error(err, "expected error when creating endpoint with duplicate name") +} + +func (suite *GarmSuite) TestGithubEndpointUpdateEndpoint() { + t := suite.T() + t.Log("Testing endpoint update") + endpoint, err := suite.createDummyEndpoint("dummy") + suite.NoError(err, "error creating dummy endpoint") + t.Cleanup(func() { + suite.DeleteGithubEndpoint(endpoint.Name) + }) + + newDescription := "Updated description" + newBaseURL := "https://ghes2.example.com" + newAPIBaseURL := "https://api.ghes2.example.com/" + newUploadBaseURL := "https://uploads.ghes2.example.com/" + newCABundle, err := getTestFileContents("certs/srv-pub.pem") + suite.NoError(err, "error reading CA cert bundle") + + updateParams := params.UpdateGithubEndpointParams{ + Description: &newDescription, + BaseURL: &newBaseURL, + APIBaseURL: &newAPIBaseURL, + UploadBaseURL: &newUploadBaseURL, + CACertBundle: newCABundle, + } + + updated, err := updateGithubEndpoint(suite.cli, suite.authToken, endpoint.Name, updateParams) + suite.NoError(err, "error updating github endpoint") + + suite.Equal(updated.Name, endpoint.Name, "Endpoint name mismatch") + suite.Equal(updated.Description, newDescription, "Endpoint description mismatch") + suite.Equal(updated.BaseURL, newBaseURL, "Endpoint base URL mismatch") + suite.Equal(updated.APIBaseURL, newAPIBaseURL, "Endpoint API base URL mismatch") + suite.Equal(updated.UploadBaseURL, newUploadBaseURL, "Endpoint upload base URL mismatch") + suite.Equal(string(updated.CACertBundle), string(newCABundle), "Endpoint CA cert bundle mismatch") +} + +func (suite *GarmSuite) MustDefaultGithubEndpoint() { + ep := suite.GetGithubEndpoint("github.com") + + suite.NotNil(ep, "default GitHub endpoint not found") + suite.Equal(ep.Name, "github.com", "default GitHub endpoint name mismatch") +} + +func (suite *GarmSuite) GetGithubEndpoint(name string) *params.GithubEndpoint { + t := suite.T() + t.Log("Get GitHub endpoint") + endpoint, err := getGithubEndpoint(suite.cli, suite.authToken, name) + suite.NoError(err, "error getting GitHub endpoint") + + return endpoint +} + +func (suite *GarmSuite) CreateGithubEndpoint(params params.CreateGithubEndpointParams) (*params.GithubEndpoint, error) { + t := suite.T() + t.Log("Create GitHub endpoint") + endpoint, err := createGithubEndpoint(suite.cli, suite.authToken, params) + suite.NoError(err, "error creating GitHub endpoint") + + return endpoint, nil +} + +func (suite *GarmSuite) DeleteGithubEndpoint(name string) error { + t := suite.T() + t.Log("Delete GitHub endpoint") + err := deleteGithubEndpoint(suite.cli, suite.authToken, name) + suite.NoError(err, "error deleting GitHub endpoint") + + return nil +} + +func (suite *GarmSuite) ListGithubEndpoints() params.GithubEndpoints { + t := suite.T() + t.Log("List GitHub endpoints") + endpoints, err := listGithubEndpoints(suite.cli, suite.authToken) + suite.NoError(err, "error listing GitHub endpoints") + + return endpoints +} + +func (suite *GarmSuite) createDummyEndpoint(name string) (*params.GithubEndpoint, error) { + endpointParams := params.CreateGithubEndpointParams{ + Name: name, + Description: "Dummy endpoint", + BaseURL: "https://ghes.example.com", + APIBaseURL: "https://api.ghes.example.com/", + UploadBaseURL: "https://uploads.ghes.example.com/", + } + + return suite.CreateGithubEndpoint(endpointParams) +} diff --git a/test/integration/external_provider_test.go b/test/integration/external_provider_test.go new file mode 100644 index 00000000..bd0d1d05 --- /dev/null +++ b/test/integration/external_provider_test.go @@ -0,0 +1,171 @@ +//go:build integration +// +build integration + +package integration + +import ( + "fmt" + "time" + + "github.com/go-openapi/runtime" + + commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm/client" + clientInstances "github.com/cloudbase/garm/client/instances" + "github.com/cloudbase/garm/params" +) + +func (suite *GarmSuite) TestExternalProvider() { + t := suite.T() + t.Log("Testing external provider") + repoPoolParams2 := params.CreatePoolParams{ + MaxRunners: 2, + MinIdleRunners: 0, + Flavor: "default", + Image: "ubuntu:22.04", + OSType: commonParams.Linux, + OSArch: commonParams.Amd64, + ProviderName: "test_external", + Tags: []string{"repo-runner-2"}, + Enabled: true, + } + repoPool2 := suite.CreateRepoPool(suite.repo.ID, repoPoolParams2) + newParams := suite.UpdateRepoPool(suite.repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1) + t.Logf("Updated repo pool with pool_id %s with new_params %+v", repoPool2.ID, newParams) + + err := suite.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute) + suite.NoError(err, "error waiting for pool instances to be running") + repoPool2 = suite.GetRepoPool(suite.repo.ID, repoPool2.ID) + suite.DisableRepoPool(suite.repo.ID, repoPool2.ID) + suite.DeleteInstance(repoPool2.Instances[0].Name, false, false) + err = suite.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute) + suite.NoError(err, "error waiting for pool instances to be pending delete") + suite.DeleteInstance(repoPool2.Instances[0].Name, true, false) // delete instance with forceRemove + err = suite.WaitInstanceToBeRemoved(repoPool2.Instances[0].Name, 1*time.Minute) + suite.NoError(err, "error waiting for instance to be removed") + suite.DeleteRepoPool(suite.repo.ID, repoPool2.ID) +} + +func (suite *GarmSuite) WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error { + t := suite.T() + var timeWaited time.Duration // default is 0 + + pool, err := getPool(suite.cli, suite.authToken, poolID) + if err != nil { + return err + } + + t.Logf("Waiting for pool instances with pool_id %s to reach desired status %v and desired_runner_status %v", poolID, status, runnerStatus) + for timeWaited < timeout { + poolInstances, err := listPoolInstances(suite.cli, suite.authToken, poolID) + if err != nil { + return err + } + + instancesCount := 0 + for _, instance := range poolInstances { + if instance.Status == status && instance.RunnerStatus == runnerStatus { + instancesCount++ + } + } + + t.Logf( + "Pool instance with pool_id %s reached status %v and runner_status %v, desired_instance_count %d, pool_instance_count %d", + poolID, status, runnerStatus, instancesCount, + len(poolInstances)) + if int(pool.MinIdleRunners) == instancesCount { + return nil + } + time.Sleep(5 * time.Second) + timeWaited += 5 * time.Second + } + + err = suite.dumpPoolInstancesDetails(pool.ID) + suite.NoError(err, "error dumping pool instances details") + + return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus) +} + +func (suite *GarmSuite) dumpPoolInstancesDetails(poolID string) error { + t := suite.T() + pool, err := getPool(suite.cli, suite.authToken, poolID) + if err != nil { + return err + } + if err := printJSONResponse(pool); err != nil { + return err + } + for _, instance := range pool.Instances { + instanceDetails, err := getInstance(suite.cli, suite.authToken, instance.Name) + if err != nil { + return err + } + t.Logf("Instance details: instance_name %s", instance.Name) + if err := printJSONResponse(instanceDetails); err != nil { + return err + } + } + return nil +} + +func (suite *GarmSuite) DisableRepoPool(repoID, repoPoolID string) { + t := suite.T() + t.Logf("Disable repo pool with repo_id %s and pool_id %s", repoID, repoPoolID) + enabled := false + poolParams := params.UpdatePoolParams{Enabled: &enabled} + _, err := updateRepoPool(suite.cli, suite.authToken, repoID, repoPoolID, poolParams) + suite.NoError(err, "error disabling repository pool") +} + +func (suite *GarmSuite) DeleteInstance(name string, forceRemove, bypassGHUnauthorized bool) { + t := suite.T() + t.Logf("Delete instance %s with force_remove %t", name, forceRemove) + err := deleteInstance(suite.cli, suite.authToken, name, forceRemove, bypassGHUnauthorized) + suite.NoError(err, "error deleting instance", name) + t.Logf("Instance deletion initiated for instance %s", name) +} + +func (suite *GarmSuite) WaitInstanceToBeRemoved(name string, timeout time.Duration) error { + t := suite.T() + var timeWaited time.Duration // default is 0 + var instance *params.Instance + + t.Logf("Waiting for instance %s to be removed", name) + for timeWaited < timeout { + instances, err := listInstances(suite.cli, suite.authToken) + if err != nil { + return err + } + + instance = nil + for k, v := range instances { + if v.Name == name { + instance = &instances[k] + break + } + } + if instance == nil { + // The instance is not found in the list. We can safely assume + // that it is removed + return nil + } + + time.Sleep(5 * time.Second) + timeWaited += 5 * time.Second + } + + if err := printJSONResponse(*instance); err != nil { + return err + } + return fmt.Errorf("instance %s was not removed within the timeout", name) +} + +func listPoolInstances(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, poolID string) (params.Instances, error) { + listPoolInstancesResponse, err := apiCli.Instances.ListPoolInstances( + clientInstances.NewListPoolInstancesParams().WithPoolID(poolID), + apiAuthToken) + if err != nil { + return nil, err + } + return listPoolInstancesResponse.Payload, nil +} diff --git a/test/integration/gh_cleanup/main.go b/test/integration/gh_cleanup/main.go index 2c9c3735..0095dba8 100644 --- a/test/integration/gh_cleanup/main.go +++ b/test/integration/gh_cleanup/main.go @@ -1,11 +1,13 @@ package main import ( + "context" "fmt" "log/slog" "os" - "github.com/cloudbase/garm/test/integration/e2e" + "github.com/google/go-github/v57/github" + "golang.org/x/oauth2" ) var ( @@ -18,8 +20,8 @@ var ( func main() { controllerID, ctrlIDFound := os.LookupEnv("GARM_CONTROLLER_ID") if ctrlIDFound { - _ = e2e.GhOrgRunnersCleanup(ghToken, orgName, controllerID) - _ = e2e.GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID) + _ = GhOrgRunnersCleanup(ghToken, orgName, controllerID) + _ = GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID) } else { slog.Warn("Env variable GARM_CONTROLLER_ID is not set, skipping GitHub runners cleanup") } @@ -27,9 +29,146 @@ func main() { baseURL, baseURLFound := os.LookupEnv("GARM_BASE_URL") if ctrlIDFound && baseURLFound { webhookURL := fmt.Sprintf("%s/webhooks/%s", baseURL, controllerID) - _ = e2e.GhOrgWebhookCleanup(ghToken, webhookURL, orgName) - _ = e2e.GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName) + _ = GhOrgWebhookCleanup(ghToken, webhookURL, orgName) + _ = GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName) } else { slog.Warn("Env variables GARM_CONTROLLER_ID & GARM_BASE_URL are not set, skipping webhooks cleanup") } } + +func GhOrgRunnersCleanup(ghToken, orgName, controllerID string) error { + slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName) + + client := getGithubClient(ghToken) + ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil) + if err != nil { + return err + } + + // Remove organization runners + controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID) + for _, orgRunner := range ghOrgRunners.Runners { + for _, label := range orgRunner.Labels { + if label.GetName() == controllerLabel { + if _, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID()); err != nil { + // We don't fail if we can't remove a single runner. This + // is a best effort to try and remove all the orphan runners. + slog.With(slog.Any("error", err)).Info("Failed to remove organization runner", "org_runner", orgRunner.GetName()) + break + } + slog.Info("Removed organization runner", "org_runner", orgRunner.GetName()) + break + } + } + } + + return nil +} + +func GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID string) error { + slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName, "repo_name", repoName) + + client := getGithubClient(ghToken) + ghRepoRunners, _, err := client.Actions.ListRunners(context.Background(), orgName, repoName, nil) + if err != nil { + return err + } + + // Remove repository runners + controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID) + for _, repoRunner := range ghRepoRunners.Runners { + for _, label := range repoRunner.Labels { + if label.GetName() == controllerLabel { + if _, err := client.Actions.RemoveRunner(context.Background(), orgName, repoName, repoRunner.GetID()); err != nil { + // We don't fail if we can't remove a single runner. This + // is a best effort to try and remove all the orphan runners. + slog.With(slog.Any("error", err)).Error("Failed to remove repository runner", "runner_name", repoRunner.GetName()) + break + } + slog.Info("Removed repository runner", "runner_name", repoRunner.GetName()) + break + } + } + } + + return nil +} + +func GhOrgWebhookCleanup(ghToken, webhookURL, orgName string) error { + slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName) + hook, err := getGhOrgWebhook(webhookURL, ghToken, orgName) + if err != nil { + return err + } + + // Remove organization webhook + if hook != nil { + client := getGithubClient(ghToken) + if _, err := client.Organizations.DeleteHook(context.Background(), orgName, hook.GetID()); err != nil { + return err + } + slog.Info("Github webhook removed", "webhook_url", webhookURL, "org_name", orgName) + } + + return nil +} + +func GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName string) error { + slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName) + + hook, err := getGhRepoWebhook(webhookURL, ghToken, orgName, repoName) + if err != nil { + return err + } + + // Remove repository webhook + if hook != nil { + client := getGithubClient(ghToken) + if _, err := client.Repositories.DeleteHook(context.Background(), orgName, repoName, hook.GetID()); err != nil { + return err + } + slog.Info("Github webhook with", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName) + } + + return nil +} + +func getGhOrgWebhook(url, ghToken, orgName string) (*github.Hook, error) { + client := getGithubClient(ghToken) + ghOrgHooks, _, err := client.Organizations.ListHooks(context.Background(), orgName, nil) + if err != nil { + return nil, err + } + + for _, hook := range ghOrgHooks { + hookURL, ok := hook.Config["url"].(string) + if ok && hookURL == url { + return hook, nil + } + } + + return nil, nil +} + +func getGhRepoWebhook(url, ghToken, orgName, repoName string) (*github.Hook, error) { + client := getGithubClient(ghToken) + ghRepoHooks, _, err := client.Repositories.ListHooks(context.Background(), orgName, repoName, nil) + if err != nil { + return nil, err + } + + for _, hook := range ghRepoHooks { + hookURL, ok := hook.Config["url"].(string) + if ok && hookURL == url { + return hook, nil + } + } + + return nil, nil +} + +func getGithubClient(oauthToken string) *github.Client { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken}) + tc := oauth2.NewClient(context.Background(), ts) + return github.NewClient(tc) +} diff --git a/test/integration/jobs_test.go b/test/integration/jobs_test.go new file mode 100644 index 00000000..e9483e17 --- /dev/null +++ b/test/integration/jobs_test.go @@ -0,0 +1,168 @@ +//go:build integration +// +build integration + +package integration + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-github/v57/github" + + commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm/params" +) + +func (suite *GarmSuite) TestWorkflowJobs() { + suite.TriggerWorkflow(suite.ghToken, orgName, repoName, workflowFileName, "org-runner") + suite.ValidateJobLifecycle("org-runner") + + suite.TriggerWorkflow(suite.ghToken, orgName, repoName, workflowFileName, "repo-runner") + suite.ValidateJobLifecycle("repo-runner") +} + +func (suite *GarmSuite) TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, labelName string) { + t := suite.T() + t.Logf("Trigger workflow with label %s", labelName) + + client := getGithubClient(ghToken) + eventReq := github.CreateWorkflowDispatchEventRequest{ + Ref: "main", + Inputs: map[string]interface{}{ + "sleep_time": "50", + "runner_label": labelName, + }, + } + _, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), orgName, repoName, workflowFileName, eventReq) + suite.NoError(err, "error triggering workflow") +} + +func (suite *GarmSuite) ValidateJobLifecycle(label string) { + t := suite.T() + t.Logf("Validate GARM job lifecycle with label %s", label) + + // wait for job list to be updated + job, err := suite.waitLabelledJob(label, 4*time.Minute) + suite.NoError(err, "error waiting for job to be created") + + // check expected job status + job, err = suite.waitJobStatus(job.ID, params.JobStatusQueued, 4*time.Minute) + suite.NoError(err, "error waiting for job to be queued") + + job, err = suite.waitJobStatus(job.ID, params.JobStatusInProgress, 4*time.Minute) + suite.NoError(err, "error waiting for job to be in progress") + + // check expected instance status + instance, err := suite.waitInstanceStatus(job.RunnerName, commonParams.InstanceRunning, params.RunnerActive, 5*time.Minute) + suite.NoError(err, "error waiting for instance to be running") + + // wait for job to be completed + _, err = suite.waitJobStatus(job.ID, params.JobStatusCompleted, 4*time.Minute) + suite.NoError(err, "error waiting for job to be completed") + + // wait for instance to be removed + err = suite.WaitInstanceToBeRemoved(instance.Name, 5*time.Minute) + suite.NoError(err, "error waiting for instance to be removed") + + // wait for GARM to rebuild the pool running idle instances + err = suite.WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute) + suite.NoError(err, "error waiting for pool instances to be running idle") +} + +func (suite *GarmSuite) waitLabelledJob(label string, timeout time.Duration) (*params.Job, error) { + t := suite.T() + var timeWaited time.Duration // default is 0 + var jobs params.Jobs + var err error + + t.Logf("Waiting for job with label %s", label) + for timeWaited < timeout { + jobs, err = listJobs(suite.cli, suite.authToken) + if err != nil { + return nil, err + } + for _, job := range jobs { + for _, jobLabel := range job.Labels { + if jobLabel == label { + return &job, err + } + } + } + time.Sleep(5 * time.Second) + timeWaited += 5 * time.Second + } + + if err := printJSONResponse(jobs); err != nil { + return nil, err + } + return nil, fmt.Errorf("failed to wait job with label %s", label) +} + +func (suite *GarmSuite) waitJobStatus(id int64, status params.JobStatus, timeout time.Duration) (*params.Job, error) { + t := suite.T() + var timeWaited time.Duration // default is 0 + var job *params.Job + + t.Logf("Waiting for job %d to reach status %v", id, status) + for timeWaited < timeout { + jobs, err := listJobs(suite.cli, suite.authToken) + if err != nil { + return nil, err + } + + job = nil + for k, v := range jobs { + if v.ID == id { + job = &jobs[k] + break + } + } + + if job == nil { + if status == params.JobStatusCompleted { + // The job is not found in the list. We can safely assume + // that it is completed + return nil, nil + } + // if the job is not found, and expected status is not "completed", + // we need to error out. + return nil, fmt.Errorf("job %d not found, expected to be found in status %s", id, status) + } else if job.Status == string(status) { + return job, nil + } + time.Sleep(5 * time.Second) + timeWaited += 5 * time.Second + } + + if err := printJSONResponse(*job); err != nil { + return nil, err + } + return nil, fmt.Errorf("timeout waiting for job %d to reach status %s", id, status) +} + +func (suite *GarmSuite) waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) (*params.Instance, error) { + t := suite.T() + var timeWaited time.Duration // default is 0 + var instance *params.Instance + var err error + + t.Logf("Waiting for instance %s to reach desired status %v and desired runner status %v", name, status, runnerStatus) + for timeWaited < timeout { + instance, err = getInstance(suite.cli, suite.authToken, name) + if err != nil { + return nil, err + } + t.Logf("Instance %s has status %v and runner status %v", name, instance.Status, instance.RunnerStatus) + if instance.Status == status && instance.RunnerStatus == runnerStatus { + return instance, nil + } + time.Sleep(5 * time.Second) + timeWaited += 5 * time.Second + } + + if err := printJSONResponse(*instance); err != nil { + return nil, err + } + return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus) +} diff --git a/test/integration/list_info_test.go b/test/integration/list_info_test.go new file mode 100644 index 00000000..1eef816e --- /dev/null +++ b/test/integration/list_info_test.go @@ -0,0 +1,72 @@ +//go:build integration +// +build integration + +package integration + +import ( + "fmt" + "os" + + "github.com/cloudbase/garm/params" +) + +func (suite *GarmSuite) TestGetControllerInfo() { + controllerInfo := suite.GetControllerInfo() + suite.NotEmpty(controllerInfo.ControllerID, "controller ID is empty") +} + +func (suite *GarmSuite) GetMetricsToken() { + t := suite.T() + t.Log("Get metrics token") + metricsToken, err := getMetricsToken(suite.cli, suite.authToken) + suite.NoError(err, "error getting metrics token") + suite.NotEmpty(metricsToken, "metrics token is empty") +} + +func (suite *GarmSuite) GetControllerInfo() *params.ControllerInfo { + t := suite.T() + t.Log("Get controller info") + controllerInfo, err := getControllerInfo(suite.cli, suite.authToken) + suite.NoError(err, "error getting controller info") + err = suite.appendCtrlInfoToGitHubEnv(&controllerInfo) + suite.NoError(err, "error appending controller info to GitHub env") + err = printJSONResponse(controllerInfo) + suite.NoError(err, "error printing controller info") + return &controllerInfo +} + +func (suite *GarmSuite) TestListCredentials() { + t := suite.T() + t.Log("List credentials") + credentials, err := listCredentials(suite.cli, suite.authToken) + suite.NoError(err, "error listing credentials") + suite.NotEmpty(credentials, "credentials list is empty") +} + +func (suite *GarmSuite) TestListProviders() { + t := suite.T() + t.Log("List providers") + providers, err := listProviders(suite.cli, suite.authToken) + suite.NoError(err, "error listing providers") + suite.NotEmpty(providers, "providers list is empty") +} + +func (suite *GarmSuite) appendCtrlInfoToGitHubEnv(controllerInfo *params.ControllerInfo) error { + t := suite.T() + envFile, found := os.LookupEnv("GITHUB_ENV") + if !found { + t.Log("GITHUB_ENV not set, skipping appending controller info") + return nil + } + file, err := os.OpenFile(envFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + return err + } + t.Cleanup(func() { + file.Close() + }) + if _, err := file.WriteString(fmt.Sprintf("export GARM_CONTROLLER_ID=%s\n", controllerInfo.ControllerID)); err != nil { + return err + } + return nil +} diff --git a/test/integration/main.go b/test/integration/main.go deleted file mode 100644 index c1ab0de3..00000000 --- a/test/integration/main.go +++ /dev/null @@ -1,191 +0,0 @@ -package main - -import ( - "fmt" - "log/slog" - "os" - "time" - - commonParams "github.com/cloudbase/garm-provider-common/params" - "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/test/integration/e2e" -) - -var ( - adminPassword = os.Getenv("GARM_PASSWORD") - adminUsername = os.Getenv("GARM_ADMIN_USERNAME") - adminFullName = "GARM Admin" - adminEmail = "admin@example.com" - - baseURL = os.Getenv("GARM_BASE_URL") - credentialsName = os.Getenv("CREDENTIALS_NAME") - - repoName = os.Getenv("REPO_NAME") - repoWebhookSecret = os.Getenv("REPO_WEBHOOK_SECRET") - repoPoolParams = params.CreatePoolParams{ - MaxRunners: 2, - MinIdleRunners: 0, - Flavor: "default", - Image: "ubuntu:22.04", - OSType: commonParams.Linux, - OSArch: commonParams.Amd64, - ProviderName: "lxd_local", - Tags: []string{"repo-runner"}, - Enabled: true, - } - repoPoolParams2 = params.CreatePoolParams{ - MaxRunners: 2, - MinIdleRunners: 0, - Flavor: "default", - Image: "ubuntu:22.04", - OSType: commonParams.Linux, - OSArch: commonParams.Amd64, - ProviderName: "test_external", - Tags: []string{"repo-runner-2"}, - Enabled: true, - } - - orgName = os.Getenv("ORG_NAME") - orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET") - orgPoolParams = params.CreatePoolParams{ - MaxRunners: 2, - MinIdleRunners: 0, - Flavor: "default", - Image: "ubuntu:22.04", - OSType: commonParams.Linux, - OSArch: commonParams.Amd64, - ProviderName: "lxd_local", - Tags: []string{"org-runner"}, - Enabled: true, - } - - ghToken = os.Getenv("GH_TOKEN") - workflowFileName = os.Getenv("WORKFLOW_FILE_NAME") -) - -func main() { - ///////////// - // Cleanup // - ///////////// - defer e2e.GracefulCleanup() - - /////////////// - // garm init // - /////////////// - e2e.InitClient(baseURL) - e2e.FirstRun(adminUsername, adminPassword, adminFullName, adminEmail) - e2e.Login(adminUsername, adminPassword) - - // Test endpoint operations - e2e.TestGithubEndpointOperations() - e2e.TestGithubEndpointFailsOnInvalidCABundle() - e2e.TestGithubEndpointDeletionFailsWhenCredentialsExist() - e2e.TestGithubEndpointFailsOnDuplicateName() - e2e.TestGithubEndpointMustFailToDeleteDefaultGithubEndpoint() - - // Create test credentials - e2e.EnsureTestCredentials(credentialsName, ghToken, "github.com") - e2e.TestGithubCredentialsErrorOnDuplicateCredentialsName() - e2e.TestGithubCredentialsFailsToDeleteWhenInUse() - e2e.TestGithubCredentialsFailsOnInvalidAuthType() - e2e.TestGithubCredentialsFailsWhenAuthTypeParamsAreIncorrect() - e2e.TestGithubCredentialsFailsWhenAuthTypeParamsAreMissing() - e2e.TestGithubCredentialsUpdateFailsWhenBothPATAndAppAreSupplied() - e2e.TestGithubCredentialsFailWhenAppKeyIsInvalid() - e2e.TestGithubCredentialsFailWhenEndpointDoesntExist() - e2e.TestGithubCredentialsFailsOnDuplicateName() - - // ////////////////// - // controller info // - // ////////////////// - e2e.GetControllerInfo() - - // //////////////////////////// - // credentials and providers // - // //////////////////////////// - e2e.ListCredentials() - e2e.ListProviders() - - //////////////////// - /// metrics token // - //////////////////// - e2e.GetMetricsToken() - - ////////////////// - // repositories // - ////////////////// - repo := e2e.CreateRepo(orgName, repoName, credentialsName, repoWebhookSecret) - repo = e2e.UpdateRepo(repo.ID, fmt.Sprintf("%s-clone", credentialsName)) - hookRepoInfo := e2e.InstallRepoWebhook(repo.ID) - e2e.ValidateRepoWebhookInstalled(ghToken, hookRepoInfo.URL, orgName, repoName) - e2e.UninstallRepoWebhook(repo.ID) - e2e.ValidateRepoWebhookUninstalled(ghToken, hookRepoInfo.URL, orgName, repoName) - _ = e2e.InstallRepoWebhook(repo.ID) - e2e.ValidateRepoWebhookInstalled(ghToken, hookRepoInfo.URL, orgName, repoName) - - repoPool := e2e.CreateRepoPool(repo.ID, repoPoolParams) - repoPool = e2e.GetRepoPool(repo.ID, repoPool.ID) - e2e.DeleteRepoPool(repo.ID, repoPool.ID) - - repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams) - _ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1) - - ///////////////////////////// - // Test external provider /// - ///////////////////////////// - slog.Info("Testing external provider") - repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2) - newParams := e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1) - slog.Info("Updated repo pool", "new_params", newParams) - err := e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute) - if err != nil { - slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be running", "pool_id", repoPool2.ID, "provider_name", repoPoolParams2.ProviderName) - } - repoPool2 = e2e.GetRepoPool(repo.ID, repoPool2.ID) - e2e.DisableRepoPool(repo.ID, repoPool2.ID) - e2e.DeleteInstance(repoPool2.Instances[0].Name, false, false) - err = e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute) - if err != nil { - slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be running") - } - e2e.DeleteInstance(repoPool2.Instances[0].Name, true, false) // delete instance with forceRemove - err = e2e.WaitInstanceToBeRemoved(repoPool2.Instances[0].Name, 1*time.Minute) - if err != nil { - slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be removed") - } - e2e.DeleteRepoPool(repo.ID, repoPool2.ID) - - /////////////////// - // organizations // - /////////////////// - org := e2e.CreateOrg(orgName, credentialsName, orgWebhookSecret) - org = e2e.UpdateOrg(org.ID, fmt.Sprintf("%s-clone", credentialsName)) - orgHookInfo := e2e.InstallOrgWebhook(org.ID) - e2e.ValidateOrgWebhookInstalled(ghToken, orgHookInfo.URL, orgName) - e2e.UninstallOrgWebhook(org.ID) - e2e.ValidateOrgWebhookUninstalled(ghToken, orgHookInfo.URL, orgName) - _ = e2e.InstallOrgWebhook(org.ID) - e2e.ValidateOrgWebhookInstalled(ghToken, orgHookInfo.URL, orgName) - - orgPool := e2e.CreateOrgPool(org.ID, orgPoolParams) - orgPool = e2e.GetOrgPool(org.ID, orgPool.ID) - e2e.DeleteOrgPool(org.ID, orgPool.ID) - - orgPool = e2e.CreateOrgPool(org.ID, orgPoolParams) - _ = e2e.UpdateOrgPool(org.ID, orgPool.ID, orgPoolParams.MaxRunners, 1) - - /////////////// - // instances // - /////////////// - e2e.WaitRepoRunningIdleInstances(repo.ID, 6*time.Minute) - e2e.WaitOrgRunningIdleInstances(org.ID, 6*time.Minute) - - ////////// - // jobs // - ////////// - e2e.TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, "org-runner") - e2e.ValidateJobLifecycle("org-runner") - - e2e.TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, "repo-runner") - e2e.ValidateJobLifecycle("repo-runner") -} diff --git a/test/integration/organizations_test.go b/test/integration/organizations_test.go new file mode 100644 index 00000000..b024cade --- /dev/null +++ b/test/integration/organizations_test.go @@ -0,0 +1,192 @@ +//go:build integration +// +build integration + +package integration + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-github/v57/github" + + commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm/params" +) + +func (suite *GarmSuite) TestOrganizations() { + organization := suite.CreateOrg(orgName, suite.credentialsName, orgWebhookSecret) + org := suite.UpdateOrg(organization.ID, fmt.Sprintf("%s-clone", suite.credentialsName)) + suite.NotEqual(organization, org, "organization not updated") + orgHookInfo := suite.InstallOrgWebhook(org.ID) + suite.ValidateOrgWebhookInstalled(suite.ghToken, orgHookInfo.URL, orgName) + suite.UninstallOrgWebhook(org.ID) + suite.ValidateOrgWebhookUninstalled(suite.ghToken, orgHookInfo.URL, orgName) + _ = suite.InstallOrgWebhook(org.ID) + suite.ValidateOrgWebhookInstalled(suite.ghToken, orgHookInfo.URL, orgName) + + orgPoolParams := params.CreatePoolParams{ + MaxRunners: 2, + MinIdleRunners: 0, + Flavor: "default", + Image: "ubuntu:22.04", + OSType: commonParams.Linux, + OSArch: commonParams.Amd64, + ProviderName: "lxd_local", + Tags: []string{"org-runner"}, + Enabled: true, + } + orgPool := suite.CreateOrgPool(org.ID, orgPoolParams) + orgPoolGot := suite.GetOrgPool(org.ID, orgPool.ID) + suite.Equal(orgPool, orgPoolGot, "organization pool mismatch") + suite.DeleteOrgPool(org.ID, orgPool.ID) + + orgPool = suite.CreateOrgPool(org.ID, orgPoolParams) + orgPoolUpdated := suite.UpdateOrgPool(org.ID, orgPool.ID, orgPoolParams.MaxRunners, 1) + suite.NotEqual(orgPool, orgPoolUpdated, "organization pool not updated") + + suite.WaitOrgRunningIdleInstances(org.ID, 6*time.Minute) +} + +func (suite *GarmSuite) CreateOrg(orgName, credentialsName, orgWebhookSecret string) *params.Organization { + t := suite.T() + t.Logf("Create org with org_name %s", orgName) + orgParams := params.CreateOrgParams{ + Name: orgName, + CredentialsName: credentialsName, + WebhookSecret: orgWebhookSecret, + } + org, err := createOrg(suite.cli, suite.authToken, orgParams) + suite.NoError(err, "error creating organization") + return org +} + +func (suite *GarmSuite) UpdateOrg(id, credentialsName string) *params.Organization { + t := suite.T() + t.Logf("Update org with org_id %s", id) + updateParams := params.UpdateEntityParams{ + CredentialsName: credentialsName, + } + org, err := updateOrg(suite.cli, suite.authToken, id, updateParams) + suite.NoError(err, "error updating organization") + return org +} + +func (suite *GarmSuite) InstallOrgWebhook(id string) *params.HookInfo { + t := suite.T() + t.Logf("Install org webhook with org_id %s", id) + webhookParams := params.InstallWebhookParams{ + WebhookEndpointType: params.WebhookEndpointDirect, + } + _, err := installOrgWebhook(suite.cli, suite.authToken, id, webhookParams) + suite.NoError(err, "error installing organization webhook") + webhookInfo, err := getOrgWebhook(suite.cli, suite.authToken, id) + suite.NoError(err, "error getting organization webhook") + return webhookInfo +} + +func (suite *GarmSuite) ValidateOrgWebhookInstalled(ghToken, url, orgName string) { + hook, err := getGhOrgWebhook(url, ghToken, orgName) + suite.NoError(err, "error getting github webhook") + suite.NotNil(hook, "github webhook with url %s, for org %s was not properly installed", url, orgName) +} + +func getGhOrgWebhook(url, ghToken, orgName string) (*github.Hook, error) { + client := getGithubClient(ghToken) + ghOrgHooks, _, err := client.Organizations.ListHooks(context.Background(), orgName, nil) + if err != nil { + return nil, err + } + + for _, hook := range ghOrgHooks { + hookURL, ok := hook.Config["url"].(string) + if ok && hookURL == url { + return hook, nil + } + } + + return nil, nil +} + +func (suite *GarmSuite) UninstallOrgWebhook(id string) { + t := suite.T() + t.Logf("Uninstall org webhook with org_id %s", id) + err := uninstallOrgWebhook(suite.cli, suite.authToken, id) + suite.NoError(err, "error uninstalling organization webhook") +} + +func (suite *GarmSuite) ValidateOrgWebhookUninstalled(ghToken, url, orgName string) { + hook, err := getGhOrgWebhook(url, ghToken, orgName) + suite.NoError(err, "error getting github webhook") + suite.Nil(hook, "github webhook with url %s, for org %s was not properly uninstalled", url, orgName) +} + +func (suite *GarmSuite) CreateOrgPool(orgID string, poolParams params.CreatePoolParams) *params.Pool { + t := suite.T() + t.Logf("Create org pool with org_id %s", orgID) + pool, err := createOrgPool(suite.cli, suite.authToken, orgID, poolParams) + suite.NoError(err, "error creating organization pool") + return pool +} + +func (suite *GarmSuite) GetOrgPool(orgID, orgPoolID string) *params.Pool { + t := suite.T() + t.Logf("Get org pool with org_id %s and pool_id %s", orgID, orgPoolID) + pool, err := getOrgPool(suite.cli, suite.authToken, orgID, orgPoolID) + suite.NoError(err, "error getting organization pool") + return pool +} + +func (suite *GarmSuite) DeleteOrgPool(orgID, orgPoolID string) { + t := suite.T() + t.Logf("Delete org pool with org_id %s and pool_id %s", orgID, orgPoolID) + err := deleteOrgPool(suite.cli, suite.authToken, orgID, orgPoolID) + suite.NoError(err, "error deleting organization pool") +} + +func (suite *GarmSuite) UpdateOrgPool(orgID, orgPoolID string, maxRunners, minIdleRunners uint) *params.Pool { + t := suite.T() + t.Logf("Update org pool with org_id %s and pool_id %s", orgID, orgPoolID) + poolParams := params.UpdatePoolParams{ + MinIdleRunners: &minIdleRunners, + MaxRunners: &maxRunners, + } + pool, err := updateOrgPool(suite.cli, suite.authToken, orgID, orgPoolID, poolParams) + suite.NoError(err, "error updating organization pool") + return pool +} + +func (suite *GarmSuite) WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) { + t := suite.T() + orgPools, err := listOrgPools(suite.cli, suite.authToken, orgID) + suite.NoError(err, "error listing organization pools") + for _, pool := range orgPools { + err := suite.WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout) + if err != nil { + suite.dumpOrgInstancesDetails(orgID) + t.Errorf("timeout waiting for organization %s instances to reach status: %s and runner status: %s", orgID, commonParams.InstanceRunning, params.RunnerIdle) + } + } +} + +func (suite *GarmSuite) dumpOrgInstancesDetails(orgID string) { + t := suite.T() + // print org details + t.Logf("Dumping org details with org_id %s", orgID) + org, err := getOrg(suite.cli, suite.authToken, orgID) + suite.NoError(err, "error getting organization") + err = printJSONResponse(org) + suite.NoError(err, "error printing organization") + + // print org instances details + t.Logf("Dumping org instances details for org %s", orgID) + instances, err := listOrgInstances(suite.cli, suite.authToken, orgID) + suite.NoError(err, "error listing organization instances") + for _, instance := range instances { + instance, err := getInstance(suite.cli, suite.authToken, instance.Name) + suite.NoError(err, "error getting instance") + t.Logf("Instance info for instace %s", instance.Name) + err = printJSONResponse(instance) + suite.NoError(err, "error printing instance") + } +} diff --git a/test/integration/repositories_test.go b/test/integration/repositories_test.go new file mode 100644 index 00000000..167778f1 --- /dev/null +++ b/test/integration/repositories_test.go @@ -0,0 +1,208 @@ +//go:build integration +// +build integration + +package integration + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-github/v57/github" + "golang.org/x/oauth2" + + commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm/params" +) + +func (suite *GarmSuite) EnsureTestCredentials(name string, oauthToken string, endpointName string) { + t := suite.T() + t.Log("Ensuring test credentials exist") + createCredsParams := params.CreateGithubCredentialsParams{ + Name: name, + Endpoint: endpointName, + Description: "GARM test credentials", + AuthType: params.GithubAuthTypePAT, + PAT: params.GithubPAT{ + OAuth2Token: oauthToken, + }, + } + suite.CreateGithubCredentials(createCredsParams) + + createCredsParams.Name = fmt.Sprintf("%s-clone", name) + suite.CreateGithubCredentials(createCredsParams) +} + +func (suite *GarmSuite) TestRepositories() { + t := suite.T() + + t.Logf("Update repo with repo_id %s", suite.repo.ID) + updateParams := params.UpdateEntityParams{ + CredentialsName: fmt.Sprintf("%s-clone", suite.credentialsName), + } + repo, err := updateRepo(suite.cli, suite.authToken, suite.repo.ID, updateParams) + suite.NoError(err, "error updating repository") + suite.Equal(fmt.Sprintf("%s-clone", suite.credentialsName), repo.CredentialsName, "credentials name mismatch") + suite.repo = repo + + hookRepoInfo := suite.InstallRepoWebhook(suite.repo.ID) + suite.ValidateRepoWebhookInstalled(suite.ghToken, hookRepoInfo.URL, orgName, repoName) + suite.UninstallRepoWebhook(suite.repo.ID) + suite.ValidateRepoWebhookUninstalled(suite.ghToken, hookRepoInfo.URL, orgName, repoName) + + suite.InstallRepoWebhook(suite.repo.ID) + suite.ValidateRepoWebhookInstalled(suite.ghToken, hookRepoInfo.URL, orgName, repoName) + + repoPoolParams := params.CreatePoolParams{ + MaxRunners: 2, + MinIdleRunners: 0, + Flavor: "default", + Image: "ubuntu:22.04", + OSType: commonParams.Linux, + OSArch: commonParams.Amd64, + ProviderName: "lxd_local", + Tags: []string{"repo-runner"}, + Enabled: true, + } + + repoPool := suite.CreateRepoPool(suite.repo.ID, repoPoolParams) + suite.Equal(repoPool.MaxRunners, repoPoolParams.MaxRunners, "max runners mismatch") + suite.Equal(repoPool.MinIdleRunners, repoPoolParams.MinIdleRunners, "min idle runners mismatch") + + repoPoolGet := suite.GetRepoPool(suite.repo.ID, repoPool.ID) + suite.Equal(*repoPool, *repoPoolGet, "pool get mismatch") + + suite.DeleteRepoPool(suite.repo.ID, repoPool.ID) + + repoPool = suite.CreateRepoPool(suite.repo.ID, repoPoolParams) + updatedRepoPool := suite.UpdateRepoPool(suite.repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1) + suite.NotEqual(updatedRepoPool.MinIdleRunners, repoPool.MinIdleRunners, "min idle runners mismatch") + + suite.WaitRepoRunningIdleInstances(suite.repo.ID, 6*time.Minute) +} + +func (suite *GarmSuite) InstallRepoWebhook(id string) *params.HookInfo { + t := suite.T() + t.Logf("Install repo webhook with repo_id %s", id) + webhookParams := params.InstallWebhookParams{ + WebhookEndpointType: params.WebhookEndpointDirect, + } + _, err := installRepoWebhook(suite.cli, suite.authToken, id, webhookParams) + suite.NoError(err, "error installing repository webhook") + + webhookInfo, err := getRepoWebhook(suite.cli, suite.authToken, id) + suite.NoError(err, "error getting repository webhook") + return webhookInfo +} + +func (suite *GarmSuite) ValidateRepoWebhookInstalled(ghToken, url, orgName, repoName string) { + hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName) + suite.NoError(err, "error getting github webhook") + suite.NotNil(hook, "github webhook with url %s, for repo %s/%s was not properly installed", url, orgName, repoName) +} + +func getGhRepoWebhook(url, ghToken, orgName, repoName string) (*github.Hook, error) { + client := getGithubClient(ghToken) + ghRepoHooks, _, err := client.Repositories.ListHooks(context.Background(), orgName, repoName, nil) + if err != nil { + return nil, err + } + + for _, hook := range ghRepoHooks { + hookURL, ok := hook.Config["url"].(string) + if ok && hookURL == url { + return hook, nil + } + } + + return nil, nil +} + +func getGithubClient(oauthToken string) *github.Client { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken}) + tc := oauth2.NewClient(context.Background(), ts) + return github.NewClient(tc) +} + +func (suite *GarmSuite) UninstallRepoWebhook(id string) { + t := suite.T() + t.Logf("Uninstall repo webhook with repo_id %s", id) + err := uninstallRepoWebhook(suite.cli, suite.authToken, id) + suite.NoError(err, "error uninstalling repository webhook") +} + +func (suite *GarmSuite) ValidateRepoWebhookUninstalled(ghToken, url, orgName, repoName string) { + hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName) + suite.NoError(err, "error getting github webhook") + suite.Nil(hook, "github webhook with url %s, for repo %s/%s was not properly uninstalled", url, orgName, repoName) +} + +func (suite *GarmSuite) CreateRepoPool(repoID string, poolParams params.CreatePoolParams) *params.Pool { + t := suite.T() + t.Logf("Create repo pool with repo_id %s and pool_params %+v", repoID, poolParams) + pool, err := createRepoPool(suite.cli, suite.authToken, repoID, poolParams) + suite.NoError(err, "error creating repository pool") + return pool +} + +func (suite *GarmSuite) GetRepoPool(repoID, repoPoolID string) *params.Pool { + t := suite.T() + t.Logf("Get repo pool repo_id %s and pool_id %s", repoID, repoPoolID) + pool, err := getRepoPool(suite.cli, suite.authToken, repoID, repoPoolID) + suite.NoError(err, "error getting repository pool") + return pool +} + +func (suite *GarmSuite) DeleteRepoPool(repoID, repoPoolID string) { + t := suite.T() + t.Logf("Delete repo pool with repo_id %s and pool_id %s", repoID, repoPoolID) + err := deleteRepoPool(suite.cli, suite.authToken, repoID, repoPoolID) + suite.NoError(err, "error deleting repository pool") +} + +func (suite *GarmSuite) UpdateRepoPool(repoID, repoPoolID string, maxRunners, minIdleRunners uint) *params.Pool { + t := suite.T() + t.Logf("Update repo pool with repo_id %s and pool_id %s", repoID, repoPoolID) + poolParams := params.UpdatePoolParams{ + MinIdleRunners: &minIdleRunners, + MaxRunners: &maxRunners, + } + pool, err := updateRepoPool(suite.cli, suite.authToken, repoID, repoPoolID, poolParams) + suite.NoError(err, "error updating repository pool") + return pool +} + +func (suite *GarmSuite) WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) { + t := suite.T() + repoPools, err := listRepoPools(suite.cli, suite.authToken, repoID) + suite.NoError(err, "error listing repo pools") + for _, pool := range repoPools { + err := suite.WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout) + if err != nil { + suite.dumpRepoInstancesDetails(repoID) + t.Errorf("error waiting for pool instances to be running idle: %v", err) + } + } +} + +func (suite *GarmSuite) dumpRepoInstancesDetails(repoID string) { + t := suite.T() + // print repo details + t.Logf("Dumping repo details for repo %s", repoID) + repo, err := getRepo(suite.cli, suite.authToken, repoID) + suite.NoError(err, "error getting repo") + err = printJSONResponse(repo) + suite.NoError(err, "error printing repo") + + // print repo instances details + t.Logf("Dumping repo instances details for repo %s", repoID) + instances, err := listRepoInstances(suite.cli, suite.authToken, repoID) + suite.NoError(err, "error listing repo instances") + for _, instance := range instances { + instance, err := getInstance(suite.cli, suite.authToken, instance.Name) + suite.NoError(err, "error getting instance") + t.Logf("Instance info for instance %s", instance.Name) + err = printJSONResponse(instance) + suite.NoError(err, "error printing instance") + } +} diff --git a/test/integration/suite_test.go b/test/integration/suite_test.go new file mode 100644 index 00000000..c2f4bd5f --- /dev/null +++ b/test/integration/suite_test.go @@ -0,0 +1,212 @@ +//go:build integration +// +build integration + +package integration + +import ( + "context" + "fmt" + "net/url" + "os" + "testing" + "time" + + "github.com/go-openapi/runtime" + openapiRuntimeClient "github.com/go-openapi/runtime/client" + "github.com/stretchr/testify/suite" + + "github.com/cloudbase/garm/client" + "github.com/cloudbase/garm/params" +) + +var ( + orgName string + repoName string + orgWebhookSecret string + workflowFileName string +) + +type GarmSuite struct { + suite.Suite + cli *client.GarmAPI + authToken runtime.ClientAuthInfoWriter + ghToken string + credentialsName string + repo *params.Repository +} + +func (suite *GarmSuite) SetupSuite() { + t := suite.T() + suite.ghToken = os.Getenv("GH_TOKEN") + orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET") + workflowFileName = os.Getenv("WORKFLOW_FILE_NAME") + baseURL := os.Getenv("GARM_BASE_URL") + adminPassword := os.Getenv("GARM_PASSWORD") + adminUsername := os.Getenv("GARM_ADMIN_USERNAME") + adminFullName := "GARM Admin" + adminEmail := "admin@example.com" + garmURL, err := url.Parse(baseURL) + suite.NoError(err, "error parsing GARM_BASE_URL") + + apiPath, err := url.JoinPath(garmURL.Path, client.DefaultBasePath) + suite.NoError(err, "error joining path") + + transportCfg := client.DefaultTransportConfig(). + WithHost(garmURL.Host). + WithBasePath(apiPath). + WithSchemes([]string{garmURL.Scheme}) + suite.cli = client.NewHTTPClientWithConfig(nil, transportCfg) + + t.Log("First run") + newUser := params.NewUserParams{ + Username: adminUsername, + Password: adminPassword, + FullName: adminFullName, + Email: adminEmail, + } + _, err = firstRun(suite.cli, newUser) + suite.NoError(err, "error at first run") + + t.Log("Login") + loginParams := params.PasswordLoginParams{ + Username: adminUsername, + Password: adminPassword, + } + token, err := login(suite.cli, loginParams) + suite.NoError(err, "error at login") + suite.authToken = openapiRuntimeClient.BearerToken(token) + t.Log("Log in successful") + + suite.credentialsName = os.Getenv("CREDENTIALS_NAME") + suite.EnsureTestCredentials(suite.credentialsName, suite.ghToken, "github.com") + + t.Log("Create repository") + orgName = os.Getenv("ORG_NAME") + repoName = os.Getenv("REPO_NAME") + repoWebhookSecret := os.Getenv("REPO_WEBHOOK_SECRET") + createParams := params.CreateRepoParams{ + Owner: orgName, + Name: repoName, + CredentialsName: suite.credentialsName, + WebhookSecret: repoWebhookSecret, + } + suite.repo, err = createRepo(suite.cli, suite.authToken, createParams) + suite.NoError(err, "error creating repository") + suite.Equal(orgName, suite.repo.Owner, "owner name mismatch") + suite.Equal(repoName, suite.repo.Name, "repo name mismatch") + suite.Equal(suite.credentialsName, suite.repo.CredentialsName, "credentials name mismatch") +} + +func (suite *GarmSuite) TearDownSuite() { + t := suite.T() + t.Log("Graceful cleanup") + // disable all the pools + pools, err := listPools(suite.cli, suite.authToken) + suite.NoError(err, "error listing pools") + enabled := false + poolParams := params.UpdatePoolParams{Enabled: &enabled} + for _, pool := range pools { + _, err := updatePool(suite.cli, suite.authToken, pool.ID, poolParams) + suite.NoError(err, "error disabling pool") + t.Logf("Pool %s disabled during stage graceful_cleanup", pool.ID) + } + + // delete all the instances + for _, pool := range pools { + poolInstances, err := listPoolInstances(suite.cli, suite.authToken, pool.ID) + suite.NoError(err, "error listing pool instances") + for _, instance := range poolInstances { + err := deleteInstance(suite.cli, suite.authToken, instance.Name, false, false) + suite.NoError(err, "error deleting instance") + t.Logf("Instance deletion initiated for instace %s during stage graceful_cleanup", instance.Name) + } + } + + // wait for all instances to be deleted + for _, pool := range pools { + err := suite.waitPoolNoInstances(pool.ID, 3*time.Minute) + suite.NoError(err, "error waiting for pool to have no instances") + } + + // delete all the pools + for _, pool := range pools { + err := deletePool(suite.cli, suite.authToken, pool.ID) + suite.NoError(err, "error deleting pool") + t.Logf("Pool %s deleted during stage graceful_cleanup", pool.ID) + } + + // delete all the repositories + repos, err := listRepos(suite.cli, suite.authToken) + suite.NoError(err, "error listing repositories") + for _, repo := range repos { + err := deleteRepo(suite.cli, suite.authToken, repo.ID) + suite.NoError(err, "error deleting repository") + t.Logf("Repo %s deleted during stage graceful_cleanup", repo.ID) + } + + // delete all the organizations + orgs, err := listOrgs(suite.cli, suite.authToken) + suite.NoError(err, "error listing organizations") + for _, org := range orgs { + err := deleteOrg(suite.cli, suite.authToken, org.ID) + suite.NoError(err, "error deleting organization") + t.Logf("Org %s deleted during stage graceful_cleanup", org.ID) + } +} + +func TestGarmTestSuite(t *testing.T) { + suite.Run(t, new(GarmSuite)) +} + +func (suite *GarmSuite) waitPoolNoInstances(id string, timeout time.Duration) error { + t := suite.T() + var timeWaited time.Duration // default is 0 + var pool *params.Pool + var err error + + t.Logf("Wait until pool with id %s has no instances", id) + for timeWaited < timeout { + pool, err = getPool(suite.cli, suite.authToken, id) + suite.NoError(err, "error getting pool") + t.Logf("Current pool has %d instances", len(pool.Instances)) + if len(pool.Instances) == 0 { + return nil + } + time.Sleep(5 * time.Second) + timeWaited += 5 * time.Second + } + + err = suite.dumpPoolInstancesDetails(pool.ID) + suite.NoError(err, "error dumping pool instances details") + + return fmt.Errorf("failed to wait for pool %s to have no instances", pool.ID) +} + +func (suite *GarmSuite) GhOrgRunnersCleanup(ghToken, orgName, controllerID string) error { + t := suite.T() + t.Logf("Cleanup Github runners for controller %s and org %s", controllerID, orgName) + + client := getGithubClient(ghToken) + ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil) + if err != nil { + return err + } + + // Remove organization runners + controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID) + for _, orgRunner := range ghOrgRunners.Runners { + for _, label := range orgRunner.Labels { + if label.GetName() == controllerLabel { + if _, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID()); err != nil { + // We don't fail if we can't remove a single runner. This + // is a best effort to try and remove all the orphan runners. + t.Logf("Failed to remove organization runner %s: %v", orgRunner.GetName(), err) + break + } + t.Logf("Removed organization runner %s", orgRunner.GetName()) + break + } + } + } + return nil +} diff --git a/test/integration/e2e/utils.go b/test/integration/utils.go similarity index 56% rename from test/integration/e2e/utils.go rename to test/integration/utils.go index 3c449ddc..24e97b7f 100644 --- a/test/integration/e2e/utils.go +++ b/test/integration/utils.go @@ -1,8 +1,8 @@ -package e2e +package integration import ( "encoding/json" - "log" + "fmt" "log/slog" ) @@ -19,15 +19,17 @@ type apiCodeGetter interface { IsCode(code int) bool } -func expectAPIStatusCode(err error, expectedCode int) { +func expectAPIStatusCode(err error, expectedCode int) error { if err == nil { - panic("expected error") + return fmt.Errorf("expected error, got nil") } apiErr, ok := err.(apiCodeGetter) if !ok { - log.Fatalf("expected API error, got %v (%T)", err, err) + return fmt.Errorf("expected API error, got %v (%T)", err, err) } if !apiErr.IsCode(expectedCode) { - log.Fatalf("expected status code %d: %v", expectedCode, err) + return fmt.Errorf("expected status code %d: %v", expectedCode, err) } + + return nil }