Skip to content

Commit

Permalink
always load digger.yml from branch, not from database repo (#1116)
Browse files Browse the repository at this point in the history 
* always load digger.yml from repo branch, not from database repo
  • Loading branch information
@motatoes
motatoes authored Feb 2, 2024
1 parent 5dbcbfd commit 65ed2b5
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 68 deletions.
75 changes: 25 additions & 50 deletions backend/controllers/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,13 +391,14 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) e
log.Printf("Error getting github service: %v", err)
return fmt.Errorf("error getting github service")
}
utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) {
utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error {
dat, err := os.ReadFile(path.Join(dir, "digger.yml"))
//TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that
if err != nil {
log.Printf("ERROR fetching digger.yml file: %v", err)
}
models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo)
return nil
})
}

Expand All @@ -412,7 +413,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
cloneURL := *payload.Repo.CloneURL
prNumber := *payload.PullRequest.Number

ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
if err != nil {
log.Printf("getDiggerConfig error: %v", err)
return fmt.Errorf("error getting digger config")
Expand Down Expand Up @@ -466,14 +467,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
impactedJobsMap[j.ProjectName] = j
}

repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName)
if err != nil {
log.Printf("GetRepoByInstallationId error: %v", err)
utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: GetRepoByInstallationId error: %v", err))
return fmt.Errorf("error converting jobs, GetRepoByInstallationId error: %v", err)
}
batchType := getBatchType(jobsForImpactedProjects)
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, repo.DiggerConfig, batchType)
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand All @@ -490,54 +485,39 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
return nil
}

func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (*dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
func getDiggerConfig(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
if err != nil {
log.Printf("Error getting github service: %v", err)
return nil, nil, nil, nil, fmt.Errorf("error getting github service")
return "", nil, nil, nil, nil, fmt.Errorf("error getting github service")
}
var prBranch string
prBranch, err = ghService.GetBranchName(prNumber)
if err != nil {
log.Printf("Error getting branch name: %v", err)
return nil, nil, nil, nil, fmt.Errorf("error getting branch name")
}

repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName)
if err != nil {
return nil, nil, nil, nil, err
}
if err != nil {
log.Printf("Error getting repo: %v", err)
return nil, nil, nil, nil, fmt.Errorf("error getting repo")
}

configYaml, err := dg_configuration.LoadDiggerConfigYamlFromString(repo.DiggerConfig)
if err != nil {
log.Printf("Error loading digger config: %v", err)
return nil, nil, nil, nil, fmt.Errorf("error loading digger config")
return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name")
}

log.Printf("Digger config loadded successfully\n")

if configYaml.GenerateProjectsConfig != nil {
err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) {
dg_configuration.HandleYamlProjectGeneration(configYaml, dir)
})
var config *dg_configuration.DiggerConfig
var diggerYmlStr string
var dependencyGraph graph.Graph[string, dg_configuration.Project]
err = utils.CloneGitRepoAndDoAction(cloneUrl, prBranch, *token, func(dir string) error {
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
diggerYmlStr = string(diggerYmlBytes)
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir)
if err != nil {
log.Printf("Error generating projects: %v", err)
return nil, nil, nil, nil, fmt.Errorf("error generating projects")
log.Printf("Error loading digger config: %v", err)
return err
}
}

config, dependencyGraph, err := loadDiggerConfig(configYaml)

return nil
})
if err != nil {
log.Printf("Error loading digger config: %v", err)
return nil, nil, nil, nil, fmt.Errorf("error loading digger config")
log.Printf("Error generating projects: %v", err)
return "", nil, nil, nil, nil, fmt.Errorf("error generating projects")
}
log.Printf("Digger config parsed successfully\n")
return ghService, config, dependencyGraph, &prBranch, nil

log.Printf("Digger config loadded successfully\n")
return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, nil
}

func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {
Expand Down Expand Up @@ -581,7 +561,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
return nil
}

ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfig(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
if err != nil {
log.Printf("getDiggerConfig error: %v", err)
return fmt.Errorf("error getting digger config")
Expand Down Expand Up @@ -649,13 +629,8 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
impactedProjectsJobMap[j.ProjectName] = j
}

repo, err := GetRepoByInstllationId(installationId, repoOwner, repoName)
if err != nil {
log.Printf("GetRepoByInstallationId error: %v", err)
return fmt.Errorf("error converting jobs, GetRepoByInstallationId error: %v", err)
}
batchType := getBatchType(jobs)
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, repo.DiggerConfig, batchType)
batchId, _, err := utils.ConvertJobsToDiggerJobs(impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down
13 changes: 5 additions & 8 deletions backend/templates/repo_add.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,13 @@
<div class="card-body">

{{template "notifications" . }}
<form method="POST">
<div class="row">
<div class="col">
<div class="mb-3"><label class="form-label" ><strong>Digger Config</strong></label>
<textarea class="form-control prism-live language-yaml" type="text" id="diggerconfig" name="diggerconfig">{{.Repo.DiggerConfig}}</textarea>
</div>
<div class="row">
<div class="col">
<div class="mb-3"><label class="form-label" ><strong>Digger Config</strong></label>
<textarea class="form-control prism-live language-yaml" type="text" id="diggerconfig" name="diggerconfig" readonly>{{.Repo.DiggerConfig}}</textarea>
</div>
</div>
<div class="mb-3"><button class="btn btn-primary btn-sm" type="submit">Update</button></div>
</form>
</div>
</div>
</div>
</div>
Expand Down
10 changes: 7 additions & 3 deletions backend/utils/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func createTempDir() string {
return tempDir
}

type action func(string)
type action func(string) error

func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action action) error {
dir := createTempDir()
Expand All @@ -50,10 +50,14 @@ func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action
log.Printf("PlainClone error: %v\n", err)
return err
}
defer os.RemoveAll(dir)

action(dir)
err = action(dir)
if err != nil {
log.Printf("error performing action: %v", err)
return err
}

defer os.RemoveAll(dir)
return nil

}
Expand Down
6 changes: 3 additions & 3 deletions backend/utils/github_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,21 @@ func init() {
}

func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) {
f := func(d string) {}
f := func(d string) error { return nil }
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "invalid-token", f)
assert.NotNil(t, err)
}

func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) {
token := os.Getenv("GITHUB_PAT_TOKEN")
f := func(d string) {}
f := func(d string) error { return nil }
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", token, f)
assert.Nil(t, err)
}

func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) {
token := os.Getenv("GITHUB_PAT_TOKEN")
f := func(d string) {}
f := func(d string) error { return nil }
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", token, f)
assert.NotNil(t, err)
}
8 changes: 4 additions & 4 deletions go.work.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -510,8 +510,6 @@ github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbf
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
github.com/alecthomas/kingpin/v2 v2.3.1 h1:ANLJcKmQm4nIaog7xdr/id6FM6zm5hHnfZrvtKPxqGg=
github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
Expand Down Expand Up @@ -801,6 +799,8 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI=
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk=
github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw=
github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA=
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
Expand Down Expand Up @@ -955,7 +955,6 @@ github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiN
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs=
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
Expand All @@ -981,7 +980,6 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s=
Expand All @@ -1005,6 +1003,8 @@ github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbm
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig=
github.com/stuart-warren/yamlfmt v0.1.2/go.mod h1:X5TuPH+hf4O0U1KBvNqygvHbvAnoi9Wyl9BbtPv8SZk=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
Expand Down

0 comments on commit 65ed2b5

Please sign in to comment.