Render output grouped by modules (#1453)

* Part 1: introduction of group modules

backend/ci_backends/github_actions.go

@@ -3,6 +3,7 @@ package ci_backends
 import (
 	"context"
 	"github.com/diggerhq/digger/backend/models"
+	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
 	"github.com/google/go-github/v61/github"
 	"log"
 	"strconv"
@@ -15,9 +16,14 @@ type GithubActionCi struct {
 func (g GithubActionCi) TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error {
 	client := g.Client
 	log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", repoOwner, repoName, commentId)
+	inputs := orchestrator_scheduler.WorkflowInput{
+		Id:        job.DiggerJobID,
+		JobString: jobString,
+		CommentId: strconv.FormatInt(commentId, 10),
+	}
 	_, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, "digger_workflow.yml", github.CreateWorkflowDispatchEventRequest{
 		Ref:    job.Batch.BranchName,
-		Inputs: map[string]interface{}{"job": jobString, "id": job.DiggerJobID, "comment_id": strconv.FormatInt(commentId, 10)},
+		Inputs: inputs.ToMap(),
 	})
 
 	return err

backend/controllers/github.go

@@ -446,7 +446,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		return nil
 	}
 
-	impactedProjects, _, err := dg_github.ProcessGitHubPullRequestEvent(payload, config, projectsGraph, ghService)
+	impactedProjects, impactedProjectsSourceMapping, _, err := dg_github.ProcessGitHubPullRequestEvent(payload, config, projectsGraph, ghService)
 	if err != nil {
 		log.Printf("Error processing event: %v", err)
 		utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: Error processing event: %v", err))
@@ -500,7 +500,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	}
 
 	batchType := getBatchType(jobsForImpactedProjects)
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(organisationId, impactedJobsMap, impactedProjectsMap, impactedProjectsSourceMapping, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -649,7 +649,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		return fmt.Errorf("error while fetching branch name")
 	}
 
-	impactedProjects, requestedProject, _, err := dg_github.ProcessGitHubIssueCommentEvent(payload, config, projectsGraph, ghService)
+	impactedProjects, impactedProjectsSourceMapping, requestedProject, _, err := dg_github.ProcessGitHubIssueCommentEvent(payload, config, projectsGraph, ghService)
 	if err != nil {
 		log.Printf("Error processing event: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Error processing event: %v", err))
@@ -702,7 +702,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	}
 
 	batchType := getBatchType(jobs)
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(orgId, impactedProjectsJobMap, impactedProjectsMap, impactedProjectsSourceMapping, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr, batchType)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))

backend/controllers/github_after_merge.go

@@ -165,7 +165,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 			return fmt.Errorf("error getting digger config")
 		}
 
-		impactedProjects, requestedProject, _, err := dg_github.ProcessGitHubPushEvent(payload, config, projectsGraph, ghService)
+		impactedProjects, impactedProjectsSourceMapping, requestedProject, _, err := dg_github.ProcessGitHubPushEvent(payload, config, projectsGraph, ghService)
 		if err != nil {
 			log.Printf("Error processing event: %v", err)
 			return fmt.Errorf("error processing event")
@@ -217,7 +217,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 				return fmt.Errorf("error creating job token")
 			}
 
-			planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, orgName, planJobToken.Value, backendHostName, impactedProjects[i]))
+			planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, orgName, planJobToken.Value, backendHostName, impactedProjects[i], impactedProjectsSourceMapping))
 			if err != nil {
 				log.Printf("Error creating jobspec: %v %v", projectName, err)
 				return fmt.Errorf("error creating jobspec")
@@ -230,7 +230,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 				return fmt.Errorf("error creating job token")
 			}
 
-			applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, orgName, applyJobToken.Value, backendHostName, impactedProjects[i]))
+			applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, orgName, applyJobToken.Value, backendHostName, impactedProjects[i], impactedProjectsSourceMapping))
 			if err != nil {
 				log.Printf("Error creating jobs: %v %v", projectName, err)
 				return fmt.Errorf("error creating jobs")

backend/controllers/github_test.go

@@ -730,10 +730,12 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
 	projectMap := make(map[string]configuration.Project)
 	projectMap["dev"] = project1
 
+	impactedProjectsSourceMapping := make(map[string]configuration.ProjectToSourceMapping)
+
 	graph, err := configuration.CreateProjectDependencyGraph(projects)
 	assert.NoError(t, err)
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, impactedProjectsSourceMapping, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -762,7 +764,9 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	impactedProjectsSourceMapping := make(map[string]configuration.ProjectToSourceMapping)
+
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, impactedProjectsSourceMapping, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 
@@ -795,7 +799,9 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	impactedProjectsSourceMapping := make(map[string]configuration.ProjectToSourceMapping)
+
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, impactedProjectsSourceMapping, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -840,7 +846,9 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
 	projectMap["555"] = project5
 	projectMap["666"] = project6
 
-	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
+	impactedProjectsSourceMapping := make(map[string]configuration.ProjectToSourceMapping)
+
+	_, result, err := utils.ConvertJobsToDiggerJobs(1, jobs, projectMap, impactedProjectsSourceMapping, graph, 123, "", 2, "", "", "test", 123, "test", orchestrator_scheduler.BatchTypeApply)
 	assert.NoError(t, err)
 	assert.Equal(t, 6, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)

backend/controllers/projects.go

@@ -1,6 +1,7 @@
 package controllers
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/diggerhq/digger/backend/middleware"
@@ -9,6 +10,7 @@ import (
 	"github.com/diggerhq/digger/backend/utils"
 	"github.com/diggerhq/digger/libs/digger_config"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
+	"github.com/diggerhq/digger/libs/terraform_utils"
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
 	"log"
@@ -308,16 +310,11 @@ func RunHistoryForProject(c *gin.Context) {
 	c.JSON(http.StatusOK, response)
 }
 
-type JobSummary struct {
-	ResourcesCreated uint `json:"resources_created"`
-	ResourcesUpdated uint `json:"resources_updated"`
-	ResourcesDeleted uint `json:"resources_deleted"`
-}
-
 type SetJobStatusRequest struct {
-	Status       string      `json:"status"`
-	Timestamp    time.Time   `json:"timestamp"`
-	JobSummary   *JobSummary `json:"job_summary"`
+	Status     string                                  `json:"status"`
+	Timestamp  time.Time                               `json:"timestamp"`
+	JobSummary *terraform_utils.PlanSummary            `json:"job_summary"`
+	Footprint  *terraform_utils.TerraformPlanFootprint `json:"job_plan_footprint"`
 	PrCommentUrl string      `json:"pr_comment_url"`
 }
 
@@ -352,6 +349,13 @@ func SetJobStatusForProject(c *gin.Context) {
 	switch request.Status {
 	case "started":
 		job.Status = orchestrator_scheduler.DiggerJobStarted
+		err := models.DB.UpdateDiggerJob(job)
+		if err != nil {
+			log.Printf("Error updating job status: %v", err)
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Error updating job status"})
+			return
+		}
+
 		client, _, err := utils.GetGithubClient(&utils.DiggerGithubRealClientProvider{}, job.Batch.GithubInstallationId, job.Batch.RepoFullName)
 		if err != nil {
 			log.Printf("Error Creating github client: %v", err)
@@ -369,10 +373,17 @@ func SetJobStatusForProject(c *gin.Context) {
 		}
 	case "succeeded":
 		job.Status = orchestrator_scheduler.DiggerJobSucceeded
+		if request.Footprint != nil {
+			job.PlanFootprint, err = json.Marshal(request.Footprint)
+			if err != nil {
+				log.Printf("Error marshalling plan footprint: %v", err)
+				c.JSON(http.StatusInternalServerError, gin.H{"error": "Error marshalling plan footprint"})
+			}
+		}
 		job.PRCommentUrl = request.PrCommentUrl
 		err := models.DB.UpdateDiggerJob(job)
 		if err != nil {
-			log.Printf("Unexpected status %v", request.Status)
+			log.Printf("Error updating job status: %v", request.Status)
 			c.JSON(http.StatusInternalServerError, gin.H{"error": "Error saving job"})
 			return
 		}
@@ -468,6 +479,7 @@ func SetJobStatusForProject(c *gin.Context) {
 
 	}
 
+	log.Printf("!!!Batch to json struct: %v", res)
 	c.JSON(http.StatusOK, res)
 }
 

backend/migrations/20240510162721.sql

@@ -0,0 +1,2 @@
+-- Modify "digger_jobs" table
+ALTER TABLE "public"."digger_jobs" ADD COLUMN "plan_footprint" bytea NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:NqQPnp6fRT51TKn9UgHkCyPB2iXeq9puN/eo6Jqan4w=
+h1:twh+xriKbq+sSGNo0DpJTPFSuQiImJoxGVyxq+CgJVA=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -18,4 +18,5 @@ h1:NqQPnp6fRT51TKn9UgHkCyPB2iXeq9puN/eo6Jqan4w=
 20240405150942.sql h1:0JIQlXqQmfgfBcill47gAef3LnnfdwK6ry98eHraUbo=
 20240405160110.sql h1:8bXZtrh8ZFFuCEXWIZ4fSjca0SSk1gsa2BqK7dIZ0To=
 20240409161739.sql h1:x0dZOsILJhmeQ6w8JKkllXZb2oz+QqV/PGLo+8R2pWI=
-20240518182629.sql h1:LbpPAy5jI2fBxoVlkBqxDZAKvZvJttcCMYyDaxN+KJU=
+20240510162721.sql h1:X2iIVRSrJU/Qjk176u9MAYL8kcOy/meS/YwPwrRF/Ok=
+20240518182629.sql h1:TZWESUb/S12o6IoP8JOUnLpA6UF9sPlSFlNSyeA3aX0=

backend/models/scheduler.go

@@ -41,9 +41,11 @@ type DiggerJob struct {
 	DiggerJobSummary   DiggerJobSummary
 	DiggerJobSummaryID uint
 	SerializedJobSpec  []byte
-	WorkflowFile       string
-	WorkflowRunUrl     *string
-	StatusUpdatedAt    time.Time
+	// represents a footprint of terraform plan json for similarity checks
+	PlanFootprint   []byte
+	WorkflowFile    string
+	WorkflowRunUrl  *string
+	StatusUpdatedAt time.Time
 }
 
 type DiggerJobSummary struct {
@@ -90,6 +92,7 @@ func (j *DiggerJob) MapToJsonStruct() (interface{}, error) {
 		DiggerJobId:      j.DiggerJobID,
 		Status:           j.Status,
 		JobString:        j.SerializedJobSpec,
+		PlanFootprint:    j.PlanFootprint,
 		ProjectName:      job.ProjectName,
 		WorkflowRunUrl:   j.WorkflowRunUrl,
 		PRCommentUrl:     j.PRCommentUrl,

backend/utils/github.go

@@ -8,6 +8,7 @@ import (
 	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/libs/orchestrator"
 	github2 "github.com/diggerhq/digger/libs/orchestrator/github"
+	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
@@ -221,9 +222,14 @@ func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string,
 
 func TriggerGithubWorkflow(client *github.Client, repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error {
 	log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", repoOwner, repoName, commentId)
+	inputs := orchestrator_scheduler.WorkflowInput{
+		Id:        job.DiggerJobID,
+		JobString: jobString,
+		CommentId: strconv.FormatInt(commentId, 10),
+	}
 	_, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, job.WorkflowFile, github.CreateWorkflowDispatchEventRequest{
 		Ref:    job.Batch.BranchName,
-		Inputs: map[string]interface{}{"job": jobString, "id": job.DiggerJobID, "comment_id": strconv.FormatInt(commentId, 10)},
+		Inputs: inputs.ToMap(),
 	})
 
 	return err

backend/utils/graphs.go

@@ -15,7 +15,7 @@ import (
 )
 
 // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value
-func ConvertJobsToDiggerJobs(organisationId uint, jobsMap map[string]orchestrator.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commentId int64, diggerConfigStr string, batchType orchestrator_scheduler.DiggerBatchType) (*uuid.UUID, map[string]*models.DiggerJob, error) {
+func ConvertJobsToDiggerJobs(organisationId uint, jobsMap map[string]orchestrator.Job, projectMap map[string]configuration.Project, impactedProjectsSourceMapping map[string]configuration.ProjectToSourceMapping, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commentId int64, diggerConfigStr string, batchType orchestrator_scheduler.DiggerBatchType) (*uuid.UUID, map[string]*models.DiggerJob, error) {
 	result := make(map[string]*models.DiggerJob)
 	organisation, err := models.DB.GetOrganisationById(organisationId)
 	if err != nil {
@@ -35,7 +35,7 @@ func ConvertJobsToDiggerJobs(organisationId uint, jobsMap map[string]orchestrato
 			return nil, nil, fmt.Errorf("error creating job token")
 		}
 
-		marshalled, err := json.Marshal(orchestrator.JobToJson(job, organisationName, jobToken.Value, backendHostName, projectMap[projectName]))
+		marshalled, err := json.Marshal(orchestrator.JobToJson(job, organisationName, jobToken.Value, backendHostName, projectMap[projectName], impactedProjectsSourceMapping))
 		if err != nil {
 			return nil, nil, err
 		}