From 65ed2b5609a381a8227a1c4a5832caf456aa3297 Mon Sep 17 00:00:00 2001 From: Mohamed Habib Date: Fri, 2 Feb 2024 17:41:43 +0000 Subject: [PATCH] always load digger.yml from branch, not from database repo (#1116) * always load digger.yml from repo branch, not from database repo --- backend/controllers/github.go | 75 +++++++++++---------------------- backend/templates/repo_add.tmpl | 13 +++--- backend/utils/github.go | 10 +++-- backend/utils/github_test.go | 6 +-- go.work.sum | 8 ++-- 5 files changed, 44 insertions(+), 68 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 6f272db2c..b02b67b23 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -391,13 +391,14 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) e log.Printf("Error getting github service: %v", err) return fmt.Errorf("error getting github service") } - utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) { + utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error { dat, err := os.ReadFile(path.Join(dir, "digger.yml")) //TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that if err != nil { log.Printf("ERROR fetching digger.yml file: %v", err) } models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo) + return nil }) } @@ -412,7 +413,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR cloneURL := *payload.Repo.CloneURL prNumber := *payload.PullRequest.Number - ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber) + diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber) if err != nil { log.Printf("getDiggerConfig error: %v", err) return fmt.Errorf("error getting digger config") @@ -466,14 +467,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR impactedJobsMap[j.ProjectName] = j } - repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName) - if err != nil { - log.Printf("GetRepoByInstallationId error: %v", err) - utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: GetRepoByInstallationId error: %v", err)) - return fmt.Errorf("error converting jobs, GetRepoByInstallationId error: %v", err) - } batchType := getBatchType(jobsForImpactedProjects) - batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, repo.DiggerConfig, batchType) + batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -490,54 +485,39 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return nil } -func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (*dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) { +func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) { ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) if err != nil { log.Printf("Error getting github service: %v", err) - return nil, nil, nil, nil, fmt.Errorf("error getting github service") + return "", nil, nil, nil, nil, fmt.Errorf("error getting github service") } var prBranch string prBranch, err = ghService.GetBranchName(prNumber) if err != nil { log.Printf("Error getting branch name: %v", err) - return nil, nil, nil, nil, fmt.Errorf("error getting branch name") - } - - repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName) - if err != nil { - return nil, nil, nil, nil, err - } - if err != nil { - log.Printf("Error getting repo: %v", err) - return nil, nil, nil, nil, fmt.Errorf("error getting repo") - } - - configYaml, err := dg_configuration.LoadDiggerConfigYamlFromString(repo.DiggerConfig) - if err != nil { - log.Printf("Error loading digger config: %v", err) - return nil, nil, nil, nil, fmt.Errorf("error loading digger config") + return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name") } - log.Printf("Digger config loadded successfully\n") - - if configYaml.GenerateProjectsConfig != nil { - err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) { - dg_configuration.HandleYamlProjectGeneration(configYaml, dir) - }) + var config *dg_configuration.DiggerConfig + var diggerYmlStr string + var dependencyGraph graph.Graph[string, dg_configuration.Project] + err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) error { + diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) + diggerYmlStr = string(diggerYmlBytes) + config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir) if err != nil { - log.Printf("Error generating projects: %v", err) - return nil, nil, nil, nil, fmt.Errorf("error generating projects") + log.Printf("Error loading digger config: %v", err) + return err } - } - - config, dependencyGraph, err := loadDiggerConfig(configYaml) - + return nil + }) if err != nil { - log.Printf("Error loading digger config: %v", err) - return nil, nil, nil, nil, fmt.Errorf("error loading digger config") + log.Printf("Error generating projects: %v", err) + return "", nil, nil, nil, nil, fmt.Errorf("error generating projects") } - log.Printf("Digger config parsed successfully\n") - return ghService, config, dependencyGraph, &prBranch, nil + + log.Printf("Digger config loadded successfully\n") + return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, nil } func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) { @@ -581,7 +561,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu return nil } - ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber) + diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber) if err != nil { log.Printf("getDiggerConfig error: %v", err) return fmt.Errorf("error getting digger config") @@ -649,13 +629,8 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu impactedProjectsJobMap[j.ProjectName] = j } - repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName) - if err != nil { - log.Printf("GetRepoByInstallationId error: %v", err) - return fmt.Errorf("error converting jobs, GetRepoByInstallationId error: %v", err) - } batchType := getBatchType(jobs) - batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, repo.DiggerConfig, batchType) + batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/backend/templates/repo_add.tmpl b/backend/templates/repo_add.tmpl index b5de048da..f868ba7d9 100644 --- a/backend/templates/repo_add.tmpl +++ b/backend/templates/repo_add.tmpl @@ -9,16 +9,13 @@
{{template "notifications" . }} -
-
-
-
- -
+
+
+
+
-
- +
diff --git a/backend/utils/github.go b/backend/utils/github.go index 37a035da8..9935c7375 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -27,7 +27,7 @@ func createTempDir() string { return tempDir } -type action func(string) +type action func(string) error func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action action) error { dir := createTempDir() @@ -50,10 +50,14 @@ func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action log.Printf("PlainClone error: %v\n", err) return err } + defer os.RemoveAll(dir) - action(dir) + err = action(dir) + if err != nil { + log.Printf("error performing action: %v", err) + return err + } - defer os.RemoveAll(dir) return nil } diff --git a/backend/utils/github_test.go b/backend/utils/github_test.go index f94d715b0..e369c3cdb 100644 --- a/backend/utils/github_test.go +++ b/backend/utils/github_test.go @@ -13,21 +13,21 @@ func init() { } func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) { - f := func(d string) {} + f := func(d string) error { return nil } err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "invalid-token", f) assert.NotNil(t, err) } func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") - f := func(d string) {} + f := func(d string) error { return nil } err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", token, f) assert.Nil(t, err) } func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") - f := func(d string) {} + f := func(d string) error { return nil } err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", token, f) assert.NotNil(t, err) } diff --git a/go.work.sum b/go.work.sum index dcccfc190..3ce064a6a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -510,8 +510,6 @@ github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbf github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/kingpin/v2 v2.3.1 h1:ANLJcKmQm4nIaog7xdr/id6FM6zm5hHnfZrvtKPxqGg= github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE= -github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4= -github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= @@ -801,6 +799,8 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk= github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= @@ -955,7 +955,6 @@ github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiN github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= @@ -981,7 +980,6 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s= @@ -1005,6 +1003,8 @@ github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbm github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig= github.com/stuart-warren/yamlfmt v0.1.2/go.mod h1:X5TuPH+hf4O0U1KBvNqygvHbvAnoi9Wyl9BbtPv8SZk= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=