diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 8d1e89029..2c4557270 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -39,9 +39,12 @@ import ( "golang.org/x/oauth2" ) +type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error + type DiggerController struct { - CiBackendProvider ci_backends.CiBackendProvider - GithubClientProvider utils.GithubClientProvider + CiBackendProvider ci_backends.CiBackendProvider + GithubClientProvider utils.GithubClientProvider + GithubWebhookPostIssueCommentHooks []IssueCommentHook } func (d DiggerController) GithubAppWebHook(c *gin.Context) { @@ -89,6 +92,16 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { c.String(http.StatusInternalServerError, err.Error()) return } + + log.Printf("executing issue comment event post hooks:") + for _, hook := range d.GithubWebhookPostIssueCommentHooks { + err := hook(gh, event, d.CiBackendProvider) + if err != nil { + log.Printf("handleIssueCommentEvent post hook error: %v", err) + c.String(http.StatusInternalServerError, err.Error()) + return + } + } case *github.PullRequestEvent: log.Printf("Got pull request event for %d", *event.PullRequest.ID) err := handlePullRequestEvent(gh, event, d.CiBackendProvider) @@ -599,7 +612,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return nil } -func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) { +func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) { ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) if err != nil { log.Printf("Error getting github service: %v", err) @@ -610,11 +623,6 @@ func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 var diggerYmlStr string var dependencyGraph graph.Graph[string, dg_configuration.Project] - changedFiles, err := ghService.GetChangedFiles(prNumber) - if err != nil { - log.Printf("Error getting changed files: %v", err) - return "", nil, nil, nil, fmt.Errorf("error getting changed files") - } err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) @@ -649,7 +657,13 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting branch name") } - diggerYmlStr, ghService, config, dependencyGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, prNumber) + changedFiles, err := ghService.GetChangedFiles(prNumber) + if err != nil { + log.Printf("Error getting changed files: %v", err) + return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting changed files") + } + + diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles) if err != nil { log.Printf("Error loading digger.yml: %v", err) return "", nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml") diff --git a/backend/controllers/projects_test.go b/backend/controllers/projects_test.go index 2ab478e0a..a4ace792b 100644 --- a/backend/controllers/projects_test.go +++ b/backend/controllers/projects_test.go @@ -17,6 +17,12 @@ func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) { defer teardownSuite(t) isMergeCalled := false mockedHTTPClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch( + mock.GetReposIssuesByOwnerByRepoByIssueNumber, + github.Issue{ + Number: github.Int(2), + PullRequestLinks: &github.PullRequestLinks{}, + }), mock.WithRequestMatch( mock.GetReposPullsByOwnerByRepoByPullNumber, github.PullRequest{ diff --git a/backend/main.go b/backend/main.go index 6d8e9d981..4c3baaf45 100644 --- a/backend/main.go +++ b/backend/main.go @@ -15,8 +15,9 @@ var templates embed.FS func main() { ghController := controllers.DiggerController{ - CiBackendProvider: ci_backends.DefaultBackendProvider{}, - GithubClientProvider: utils.DiggerGithubRealClientProvider{}, + CiBackendProvider: ci_backends.DefaultBackendProvider{}, + GithubClientProvider: utils.DiggerGithubRealClientProvider{}, + GithubWebhookPostIssueCommentHooks: make([]controllers.IssueCommentHook, 0), } r := bootstrap.Bootstrap(templates, ghController) r.GET("/", controllers.Home) diff --git a/ee/backend/hooks/github.go b/ee/backend/hooks/github.go new file mode 100644 index 000000000..fbe8afb7b --- /dev/null +++ b/ee/backend/hooks/github.go @@ -0,0 +1,191 @@ +package hooks + +import ( + "fmt" + "github.com/diggerhq/digger/backend/ci_backends" + ce_controllers "github.com/diggerhq/digger/backend/controllers" + "github.com/diggerhq/digger/backend/locking" + "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/ci/generic" + dg_github "github.com/diggerhq/digger/libs/ci/github" + comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting" + "github.com/diggerhq/digger/libs/digger_config" + dg_locking "github.com/diggerhq/digger/libs/locking" + "github.com/diggerhq/digger/libs/scheduler" + "github.com/google/go-github/v61/github" + "github.com/samber/lo" + "log" + "regexp" + "strconv" + "strings" +) + +var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error { + log.Printf("handling the drift reconcilliation hook") + installationId := *payload.Installation.ID + repoName := *payload.Repo.Name + repoOwner := *payload.Repo.Owner.Login + repoFullName := *payload.Repo.FullName + cloneURL := *payload.Repo.CloneURL + issueTitle := *payload.Issue.Title + issueNumber := *payload.Issue.Number + userCommentId := *payload.GetComment().ID + actor := *payload.Sender.Login + commentBody := *payload.Comment.Body + defaultBranch := *payload.Repo.DefaultBranch + isPullRequest := payload.Issue.IsPullRequest() + + if isPullRequest { + log.Printf("Comment is not an issue, ignoring") + return nil + } + + // checking that the title of the issue matches regex + var projectName string + re := regexp.MustCompile(`^Drift detected in project:\s*(\S+)`) + matches := re.FindStringSubmatch(issueTitle) + if len(matches) > 1 { + projectName = matches[1] + } else { + log.Printf("does not look like a drift issue, ignoring") + } + + link, err := models.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + log.Printf("Error getting GetGithubAppInstallationLink: %v", err) + return fmt.Errorf("error getting github app link") + } + orgId := link.OrganisationId + + if *payload.Action != "created" { + log.Printf("comment is not of type 'created', ignoring") + return nil + } + + allowedCommands := []string{"digger apply", "digger unlock"} + if !lo.Contains(allowedCommands, strings.TrimSpace(*payload.Comment.Body)) { + log.Printf("comment is not in allowed commands, ignoring") + log.Printf("allowed commands: %v", allowedCommands) + return nil + } + + diggerYmlStr, ghService, config, projectsGraph, err := ce_controllers.GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, defaultBranch, nil) + if err != nil { + log.Printf("Error loading digger.yml: %v", err) + return fmt.Errorf("error loading digger.yml") + } + + commentIdStr := strconv.FormatInt(userCommentId, 10) + err = ghService.CreateCommentReaction(commentIdStr, string(dg_github.GithubCommentEyesReaction)) + if err != nil { + log.Printf("CreateCommentReaction error: %v", err) + } + + diggerCommand, err := scheduler.GetCommandFromComment(*payload.Comment.Body) + if err != nil { + log.Printf("unkown digger command in comment: %v", *payload.Comment.Body) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Could not recognise comment, error: %v", err)) + return fmt.Errorf("unkown digger command in comment %v", err) + } + + // attempting to lock for performing drift apply command + prLock := dg_locking.PullRequestLock{ + InternalLock: locking.BackendDBLock{ + OrgId: orgId, + }, + CIService: ghService, + Reporter: comment_updater.NoopReporter{}, + ProjectName: projectName, + ProjectNamespace: repoFullName, + PrNumber: issueNumber, + } + err = dg_locking.PerformLockingActionFromCommand(prLock, *diggerCommand) + if err != nil { + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Failed perform lock action on project: %v %v", projectName, err)) + return fmt.Errorf("failed perform lock action on project: %v %v", projectName, err) + } + + if *diggerCommand == scheduler.DiggerCommandUnlock { + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":white_check_mark: Command %v completed successfully", *diggerCommand)) + return nil + } + + // === if we get here its a "digger apply command and we are already locked for this project ==== + // perform apply here then unlock the project + commentReporter, err := utils.InitCommentReporter(ghService, issueNumber, ":construction_worker: Digger starting....") + if err != nil { + log.Printf("Error initializing comment reporter: %v", err) + return fmt.Errorf("error initializing comment reporter") + } + + impactedProjects := config.GetProjects(projectName) + jobs, _, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, nil, config.Workflows, defaultBranch, defaultBranch) + if err != nil { + log.Printf("Error converting event to jobs: %v", err) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Error converting event to jobs: %v", err)) + return fmt.Errorf("error converting event to jobs") + } + log.Printf("GitHub IssueComment event converted to Jobs successfully\n") + + err = utils.ReportInitialJobsStatus(commentReporter, jobs) + if err != nil { + log.Printf("Failed to comment initial status for jobs: %v", err) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Failed to comment initial status for jobs: %v", err)) + return fmt.Errorf("failed to comment initial status for jobs") + } + + impactedProjectsMap := make(map[string]digger_config.Project) + for _, p := range impactedProjects { + impactedProjectsMap[p.Name] = p + } + + impactedProjectsJobMap := make(map[string]scheduler.Job) + for _, j := range jobs { + impactedProjectsJobMap[j.ProjectName] = j + } + + reporterCommentId, err := strconv.ParseInt(commentReporter.CommentId, 10, 64) + if err != nil { + log.Printf("strconv.ParseInt error: %v", err) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) + } + + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0) + if err != nil { + log.Printf("ConvertJobsToDiggerJobs error: %v", err) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) + return fmt.Errorf("error convertingjobs") + } + + ciBackend, err := ciBackendProvider.GetCiBackend( + ci_backends.CiBackendOptions{ + GithubClientProvider: gh, + GithubInstallationId: installationId, + RepoName: repoName, + RepoOwner: repoOwner, + RepoFullName: repoFullName, + }, + ) + if err != nil { + log.Printf("GetCiBackend error: %v", err) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: GetCiBackend error: %v", err)) + return fmt.Errorf("error fetching ci backed %v", err) + } + + err = ce_controllers.TriggerDiggerJobs(ciBackend, repoFullName, repoOwner, repoName, batchId, issueNumber, ghService, gh) + if err != nil { + log.Printf("TriggerDiggerJobs error: %v", err) + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err)) + return fmt.Errorf("error triggerring Digger Jobs") + } + + // === now unlocking the project === + err = dg_locking.PerformLockingActionFromCommand(prLock, scheduler.DiggerCommandUnlock) + if err != nil { + utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Failed perform lock action on project: %v %v", projectName, err)) + return fmt.Errorf("failed perform lock action on project: %v %v", projectName, err) + } + + return nil +} diff --git a/ee/backend/main.go b/ee/backend/main.go index 8719bc968..f3f9baf09 100644 --- a/ee/backend/main.go +++ b/ee/backend/main.go @@ -10,6 +10,7 @@ import ( "github.com/diggerhq/digger/backend/utils" ci_backends2 "github.com/diggerhq/digger/ee/backend/ci_backends" "github.com/diggerhq/digger/ee/backend/controllers" + "github.com/diggerhq/digger/ee/backend/hooks" "github.com/diggerhq/digger/ee/backend/providers/github" "github.com/diggerhq/digger/libs/license" "github.com/gin-gonic/gin" @@ -31,8 +32,9 @@ func main() { os.Exit(1) } diggerController := ce_controllers.DiggerController{ - CiBackendProvider: ci_backends2.EEBackendProvider{}, - GithubClientProvider: github.DiggerGithubEEClientProvider{}, + CiBackendProvider: ci_backends2.EEBackendProvider{}, + GithubClientProvider: github.DiggerGithubEEClientProvider{}, + GithubWebhookPostIssueCommentHooks: []ce_controllers.IssueCommentHook{hooks.DriftReconcilliationHook}, } r := bootstrap.Bootstrap(templates, diggerController) @@ -92,7 +94,7 @@ func main() { jobArtefactsGroup.Use(middleware.GetApiMiddleware()) jobArtefactsGroup.PUT("/", controllers.SetJobArtefact) jobArtefactsGroup.GET("/", controllers.DownloadJobArtefact) - + port := config.GetPort() r.Run(fmt.Sprintf(":%d", port)) } diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index 82757cdd1..fc45c51bf 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -225,14 +225,34 @@ func (svc GithubService) CreateCommentReaction(id string, reaction string) error return nil } +func (svc GithubService) IsPullRequest(PrNumber int) (bool, error) { + issue, _, err := svc.Client.Issues.Get(context.Background(), svc.Owner, svc.RepoName, PrNumber) + if err != nil { + log.Printf("error getting pull request (as issue): %v", err) + return false, fmt.Errorf("error getting pull request (as issue): %v", err) + } + return issue.IsPullRequest(), nil +} + func (svc GithubService) SetStatus(prNumber int, status string, statusContext string) error { + // we have to check if prNumber is an issue or not + isPullRequest, err := svc.IsPullRequest(prNumber) + if err != nil { + log.Printf("error checking if pull request is issue: %v", err) + return fmt.Errorf("error checking if pull request is issue: %v", err) + + } + if !isPullRequest { + log.Printf("issue is not of type pull request, ignoring") + return nil + } + pr, _, err := svc.Client.PullRequests.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) if err != nil { log.Printf("error getting pull request : %v", err) return fmt.Errorf("error getting pull request : %v", err) } - _, _, err = svc.Client.Repositories.CreateStatus(context.Background(), svc.Owner, svc.RepoName, *pr.Head.SHA, &github.RepoStatus{ State: &status, Context: &statusContext, @@ -259,6 +279,28 @@ func (svc GithubService) GetCombinedPullRequestStatus(prNumber int) (string, err } func (svc GithubService) MergePullRequest(prNumber int) error { + isPullRequest, err := svc.IsPullRequest(prNumber) + if err != nil { + log.Printf("error checking if PR is issue: %v", err) + return fmt.Errorf("error checking if PR is issue: %v", err) + } + + // if it is an issue, close it + if !isPullRequest { + closedState := "closed" + issueRequest := &github.IssueRequest{ + State: &closedState, + } + + _, _, err := svc.Client.Issues.Edit(context.Background(), svc.Owner, svc.RepoName, prNumber, issueRequest) + if err != nil { + log.Printf("error closing issue (merging): %v", err) + return fmt.Errorf("error closing issue (merging): %v", err) + } + return nil + + } + pr, _, err := svc.Client.PullRequests.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) if err != nil { log.Printf("error getting pull request: %v", err) @@ -288,6 +330,17 @@ func isMergeableState(mergeableState string) bool { } func (svc GithubService) IsMergeable(prNumber int) (bool, error) { + isPullRequest, err := svc.IsPullRequest(prNumber) + if err != nil { + log.Printf("could not get pull request type: %v", err) + return false, fmt.Errorf("could not get pull request type: %v", err) + } + + // if this is an issue it will always be merable (closable + if !isPullRequest { + return true, nil + } + pr, _, err := svc.Client.PullRequests.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) if err != nil { log.Printf("error getting pull request: %v", err) @@ -297,6 +350,18 @@ func (svc GithubService) IsMergeable(prNumber int) (bool, error) { } func (svc GithubService) IsMerged(prNumber int) (bool, error) { + // we have to check if prNumber is an issue or not + issue, _, err := svc.Client.Issues.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) + if err != nil { + log.Printf("error getting pull request (as issue): %v", err) + return false, fmt.Errorf("error getting pull request (as issue): %v", err) + } + + // if it is an issue, we check if it is "closed" instead of "merged" + if !issue.IsPullRequest() { + return issue.GetState() == "closed", nil + } + pr, _, err := svc.Client.PullRequests.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) if err != nil { log.Printf("error getting pull request: %v", err)