From eab48a7bcead56e36bffb5d82621e29b6014c999 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 26 Sep 2024 18:10:36 +0100 Subject: [PATCH] load projects from repos --- ee/drift/controllers/github.go | 61 ++++++++++++++++++++++++ ee/drift/dbmodels/github.go | 17 +++++++ ee/drift/dbmodels/projects.go | 51 ++++++++++++++++++++ ee/drift/dbmodels/repos.go | 53 +++++++++++++++++++++ ee/drift/dbmodels/storage.go | 17 +++++++ ee/drift/main.go | 1 + ee/drift/tasks/github.go | 59 +++++++++++++++++++++++ ee/drift/utils/github.go | 87 ++++++++++++++++++++++++++++++++++ 8 files changed, 346 insertions(+) create mode 100644 ee/drift/dbmodels/projects.go create mode 100644 ee/drift/dbmodels/repos.go create mode 100644 ee/drift/tasks/github.go create mode 100644 ee/drift/utils/github.go diff --git a/ee/drift/controllers/github.go b/ee/drift/controllers/github.go index 218117433..c87d745f8 100644 --- a/ee/drift/controllers/github.go +++ b/ee/drift/controllers/github.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "fmt" + "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/ee/drift/dbmodels" "github.com/diggerhq/digger/ee/drift/middleware" "github.com/diggerhq/digger/ee/drift/model" + "github.com/diggerhq/digger/ee/drift/tasks" next_utils "github.com/diggerhq/digger/next/utils" "github.com/gin-gonic/gin" "github.com/google/go-github/v61/github" @@ -14,10 +16,65 @@ import ( "log" "net/http" "os" + "reflect" "strconv" "strings" ) +func (mc MainController) GithubAppWebHook(c *gin.Context) { + c.Header("Content-Type", "application/json") + gh := mc.GithubClientProvider + log.Printf("GithubAppWebHook") + + payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET"))) + if err != nil { + log.Printf("Error validating github app webhook's payload: %v", err) + c.String(http.StatusBadRequest, "Error validating github app webhook's payload") + return + } + + webhookType := github.WebHookType(c.Request) + event, err := github.ParseWebHook(webhookType, payload) + if err != nil { + log.Printf("Failed to parse Github Event. :%v\n", err) + c.String(http.StatusInternalServerError, "Failed to parse Github Event") + return + } + + log.Printf("github event type: %v\n", reflect.TypeOf(event)) + + switch event := event.(type) { + case *github.PushEvent: + log.Printf("Got push event for %d", event.Repo.URL) + err := handlePushEvent(gh, event) + if err != nil { + log.Printf("handlePushEvent error: %v", err) + c.String(http.StatusInternalServerError, err.Error()) + return + } + default: + log.Printf("Unhandled event, event type %v", reflect.TypeOf(event)) + } + + c.JSON(200, "ok") +} + +func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) error { + installationId := *payload.Installation.ID + repoName := *payload.Repo.Name + repoFullName := *payload.Repo.FullName + repoOwner := *payload.Repo.Owner.Login + cloneURL := *payload.Repo.CloneURL + ref := *payload.Ref + defaultBranch := *payload.Repo.DefaultBranch + + if strings.HasSuffix(ref, defaultBranch) { + go tasks.LoadProjectsFromGithubRepo(gh, strconv.FormatInt(installationId, 10), repoFullName, repoOwner, repoName, cloneURL, defaultBranch) + } + + return nil +} + func (mc MainController) GithubAppCallbackPage(c *gin.Context) { installationId := c.Request.URL.Query()["installation_id"][0] //setupAction := c.Request.URL.Query()["setup_action"][0] @@ -89,6 +146,8 @@ func (mc MainController) GithubAppCallbackPage(c *gin.Context) { // here we mark repos that are available one by one for _, repo := range repos { + cloneUrl := *repo.CloneURL + defaultBranch := *repo.DefaultBranch repoFullName := *repo.FullName repoOwner := strings.Split(*repo.FullName, "/")[0] repoName := *repo.Name @@ -100,6 +159,8 @@ func (mc MainController) GithubAppCallbackPage(c *gin.Context) { c.String(http.StatusInternalServerError, "createOrGetDiggerRepoForGithubRepo error: %v", err) return } + + go tasks.LoadProjectsFromGithubRepo(mc.GithubClientProvider, installationId, repoFullName, repoOwner, repoName, cloneUrl, defaultBranch) } c.String(http.StatusOK, "success", gin.H{}) diff --git a/ee/drift/dbmodels/github.go b/ee/drift/dbmodels/github.go index 376d77045..dbcc55b56 100644 --- a/ee/drift/dbmodels/github.go +++ b/ee/drift/dbmodels/github.go @@ -71,3 +71,20 @@ func CreateOrGetDiggerRepoForGithubRepo(ghRepoFullName string, ghRepoOrganisatio log.Printf("Created digger repo: %v", repo) return repo, org, nil } + +// GetGithubAppInstallationLink repoFullName should be in the following format: org/repo_name, for example "diggerhq/github-job-scheduler" +func (db *Database) GetGithubAppInstallationLink(installationId string) (*model.GithubAppInstallationLink, error) { + var link model.GithubAppInstallationLink + result := db.GormDB.Where("github_installation_id = ? AND status=?", installationId, GithubAppInstallationLinkActive).Find(&link) + if result.Error != nil { + if !errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, result.Error + } + } + + // If not found, the values will be default values, which means ID will be 0 + if link.ID == "" { + return nil, nil + } + return &link, nil +} diff --git a/ee/drift/dbmodels/projects.go b/ee/drift/dbmodels/projects.go new file mode 100644 index 000000000..c7ceaf344 --- /dev/null +++ b/ee/drift/dbmodels/projects.go @@ -0,0 +1,51 @@ +package dbmodels + +import ( + "errors" + "github.com/diggerhq/digger/ee/drift/model" + "gorm.io/gorm" + "log" +) + +type DriftStatus string + +var DriftStatusNewDrift = "new drift" +var DriftStatusNoDrift = "no drift" +var DriftStatusAcknowledgeDrift = "acknowledged drift" + +// GetProjectByName return project for specified org and repo +// if record doesn't exist return nil +func (db *Database) GetProjectByName(orgId any, repo *model.Repo, name string) (*model.Project, error) { + log.Printf("GetProjectByName, org id: %v, project name: %v\n", orgId, name) + var project model.Project + + err := db.GormDB. + Joins("INNER JOIN repos ON projects.repo_id = repos.id"). + Where("repos.id = ?", repo.ID). + Where("projects.name = ?", name).First(&project).Error + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + log.Printf("Unknown error occurred while fetching database, %v\n", err) + return nil, err + } + + return &project, nil +} + +func (db *Database) CreateProject(name string, repo *model.Repo) (*model.Project, error) { + project := &model.Project{ + Name: name, + RepoID: repo.ID, + DriftStatus: DriftStatusNewDrift, + } + result := db.GormDB.Save(project) + if result.Error != nil { + log.Printf("Failed to create project: %v, error: %v\n", name, result.Error) + return nil, result.Error + } + log.Printf("Project %s, (id: %v) has been created successfully\n", name, project.ID) + return project, nil +} diff --git a/ee/drift/dbmodels/repos.go b/ee/drift/dbmodels/repos.go new file mode 100644 index 000000000..88f8e23c4 --- /dev/null +++ b/ee/drift/dbmodels/repos.go @@ -0,0 +1,53 @@ +package dbmodels + +import ( + "errors" + "fmt" + "github.com/diggerhq/digger/ee/drift/model" + configuration "github.com/diggerhq/digger/libs/digger_config" + "gorm.io/gorm" + "log" +) + +// GetRepo returns digger repo by organisationId and repo name (diggerhq-digger) +// it will return an empty object if record doesn't exist in database +func (db *Database) GetRepo(orgIdKey any, repoName string) (*model.Repo, error) { + var repo model.Repo + + err := db.GormDB.Where("organisation_id = ? AND repos.name=?", orgIdKey, repoName).First(&repo).Error + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + log.Printf("Failed to find digger repo for orgId: %v, and repoName: %v, error: %v\n", orgIdKey, repoName, err) + return nil, err + } + return &repo, nil +} + +func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.DiggerConfigYaml, repo *model.Repo) error { + log.Printf("UpdateRepoDiggerConfig, repo: %v\n", repo) + + err := db.GormDB.Transaction(func(tx *gorm.DB) error { + for _, dc := range config.Projects { + projectName := dc.Name + p, err := db.GetProjectByName(orgId, repo, projectName) + if err != nil { + return fmt.Errorf("error retriving project by name: %v", err) + } + if p == nil { + _, err := db.CreateProject(projectName, repo) + if err != nil { + return fmt.Errorf("could not create project: %v", err) + } + } + } + return nil + }) + + if err != nil { + return fmt.Errorf("error while updating projects from config: %v", err) + } + return nil +} diff --git a/ee/drift/dbmodels/storage.go b/ee/drift/dbmodels/storage.go index d050d0ab5..c80f3168f 100644 --- a/ee/drift/dbmodels/storage.go +++ b/ee/drift/dbmodels/storage.go @@ -92,3 +92,20 @@ func (db *Database) CreateRepo(name string, repoFullName string, repoOrganisatio log.Printf("Repo %s, (id: %v) has been created successfully\n", name, repo.ID) return &repo, nil } + +// GetGithubAppInstallationByIdAndRepo repoFullName should be in the following format: org/repo_name, for example "diggerhq/github-job-scheduler" +func (db *Database) GetRepoByInstllationIdAndRepoFullName(installationId string, repoFullName string) (*model.Repo, error) { + repo := model.Repo{} + result := db.GormDB.Where("github_installation_id = ? AND repo_full_name=?", installationId, repoFullName).Find(&repo) + if result.Error != nil { + if !errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, result.Error + } + } + + // If not found, the values will be default values, which means ID will be 0 + if repo.ID == "" { + return nil, fmt.Errorf("GithubAppInstallation with id=%v doesn't exist", installationId) + } + return &repo, nil +} diff --git a/ee/drift/main.go b/ee/drift/main.go index 2c1133542..445a3de78 100644 --- a/ee/drift/main.go +++ b/ee/drift/main.go @@ -69,6 +69,7 @@ func main() { //authorized := r.Group("/") //authorized.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(dbmodels.CliJobAccessType, dbmodels.AccessPolicyType, models.AdminPolicyType)) + r.POST("github-app-webhook", controller.GithubAppWebHook) r.GET("/github/callback_fe", middleware.WebhookAuth(), controller.GithubAppCallbackPage) port := os.Getenv("DIGGER_PORT") diff --git a/ee/drift/tasks/github.go b/ee/drift/tasks/github.go new file mode 100644 index 000000000..b7b51d1b4 --- /dev/null +++ b/ee/drift/tasks/github.go @@ -0,0 +1,59 @@ +package tasks + +import ( + "fmt" + utils3 "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/ee/drift/dbmodels" + "github.com/diggerhq/digger/ee/drift/utils" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" + utils2 "github.com/diggerhq/digger/next/utils" + "log" + "strconv" + "strings" +) + +func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId string, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) error { + link, err := dbmodels.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + log.Printf("Error getting GetGithubAppInstallationLink: %v", err) + return fmt.Errorf("error getting github app link") + } + + orgId := link.OrganisationID + diggerRepoName := strings.ReplaceAll(repoFullName, "/", "-") + repo, err := dbmodels.DB.GetRepo(orgId, diggerRepoName) + if err != nil { + log.Printf("Error getting Repo: %v", err) + return fmt.Errorf("error getting github app link") + } + if repo == nil { + log.Printf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) + return fmt.Errorf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) + } + + installationId64, err := strconv.ParseInt(installationId, 10, 64) + if err != nil { + log.Printf("failed to convert installation id %v to int64", installationId) + return fmt.Errorf("failed to convert installation id to int64") + } + _, token, err := utils.GetGithubService(gh, installationId64, repoFullName, repoOwner, repoName) + if err != nil { + log.Printf("Error getting github service: %v", err) + return fmt.Errorf("error getting github service") + } + + err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error { + config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) + if err != nil { + log.Printf("ERROR load digger.yml: %v", err) + return fmt.Errorf("error loading digger.yml %v", err) + } + dbmodels.DB.RefreshProjectsFromRepo(link.OrganisationID, *config, repo) + return nil + }) + if err != nil { + return fmt.Errorf("error while cloning repo: %v", err) + } + + return nil +} diff --git a/ee/drift/utils/github.go b/ee/drift/utils/github.go new file mode 100644 index 000000000..048d33114 --- /dev/null +++ b/ee/drift/utils/github.go @@ -0,0 +1,87 @@ +package utils + +import ( + "context" + "encoding/base64" + "fmt" + "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/diggerhq/digger/ee/drift/dbmodels" + github2 "github.com/diggerhq/digger/libs/ci/github" + "github.com/diggerhq/digger/next/utils" + "github.com/google/go-github/v61/github" + "log" + net "net/http" + "os" + "strconv" +) + +func GetGithubClient(gh utils.GithubClientProvider, installationId int64, repoFullName string) (*github.Client, *string, error) { + repo, err := dbmodels.DB.GetRepoByInstllationIdAndRepoFullName(strconv.FormatInt(installationId, 10), repoFullName) + if err != nil { + log.Printf("Error getting repo: %v", err) + return nil, nil, fmt.Errorf("Error getting repo: %v", err) + } + + ghClient, token, err := gh.Get(repo.GithubAppID, installationId) + return ghClient, token, err +} + +func GetGithubService(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string) (*github2.GithubService, *string, error) { + ghClient, token, err := GetGithubClient(gh, installationId, repoFullName) + if err != nil { + log.Printf("Error creating github app client: %v", err) + return nil, nil, fmt.Errorf("Error creating github app client: %v", err) + } + + ghService := github2.GithubService{ + Client: ghClient, + RepoName: repoName, + Owner: repoOwner, + } + + return &ghService, token, nil +} + +type DiggerGithubRealClientProvider struct { +} + +func (gh DiggerGithubRealClientProvider) NewClient(netClient *net.Client) (*github.Client, error) { + ghClient := github.NewClient(netClient) + return ghClient, nil +} + +func (gh DiggerGithubRealClientProvider) Get(githubAppId int64, installationId int64) (*github.Client, *string, error) { + githubAppPrivateKey := "" + githubAppPrivateKeyB64 := os.Getenv("GITHUB_APP_PRIVATE_KEY_BASE64") + if githubAppPrivateKeyB64 != "" { + decodedBytes, err := base64.StdEncoding.DecodeString(githubAppPrivateKeyB64) + if err != nil { + return nil, nil, fmt.Errorf("error initialising github app installation: please set GITHUB_APP_PRIVATE_KEY_BASE64 env variable\n") + } + githubAppPrivateKey = string(decodedBytes) + } else { + githubAppPrivateKey = os.Getenv("GITHUB_APP_PRIVATE_KEY") + if githubAppPrivateKey != "" { + log.Printf("WARNING: GITHUB_APP_PRIVATE_KEY will be deprecated in future releases, " + + "please use GITHUB_APP_PRIVATE_KEY_BASE64 instead") + } else { + return nil, nil, fmt.Errorf("error initialising github app installation: please set GITHUB_APP_PRIVATE_KEY_BASE64 env variable\n") + } + } + + tr := net.DefaultTransport + itr, err := ghinstallation.New(tr, githubAppId, installationId, []byte(githubAppPrivateKey)) + if err != nil { + return nil, nil, fmt.Errorf("error initialising github app installation: %v\n", err) + } + + token, err := itr.Token(context.Background()) + if err != nil { + return nil, nil, fmt.Errorf("error initialising git app token: %v\n", err) + } + ghClient, err := gh.NewClient(&net.Client{Transport: itr}) + if err != nil { + log.Printf("error creating new client: %v", err) + } + return ghClient, &token, nil +}