diff --git a/action.yml b/action.yml index 4748cb94a..ea72f2014 100644 --- a/action.yml +++ b/action.yml @@ -252,6 +252,16 @@ runs: go-version: '^1.21.1' if: ${{ !startsWith(github.action_ref, 'v') }} + - name: Adding required env vars for next step + uses: actions/github-script@v7 + env: + github-token: $GITHUB_TOKEN + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) + - name: build and run digger if: ${{ !startsWith(github.action_ref, 'v') }} @@ -305,23 +315,6 @@ runs: PATH=$PATH:$(pwd) cd $GITHUB_WORKSPACE digger - - name: generate artifact name based on issue number or pr number - id: artifact - shell: bash - run: | - if [[ "${{ github.event_name }}" == "issue_comment" ]]; then - echo "artifact=plans-${{ github.event.issue.number }}" >> $GITHUB_OUTPUT - else - echo "artifact=plans-${{ github.event.number }}" >> $GITHUB_OUTPUT - fi - - name: upload plan - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.artifact.outputs.artifact }} - path: '${{ github.workspace }}/**/*.tfplan' - retention-days: 14 - if: ${{ inputs.upload-plan-destination == 'github' }} - branding: icon: globe color: purple diff --git a/cli/cmd/digger/main.go b/cli/cmd/digger/main.go index 4cbbea432..dc4fb802b 100644 --- a/cli/cmd/digger/main.go +++ b/cli/cmd/digger/main.go @@ -123,6 +123,11 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker, backend err = json.Unmarshal([]byte(inputs.JobString), &job) commentId64, err := strconv.ParseInt(inputs.CommentId, 10, 64) + err = githubPrService.SetOutput(*job.PullRequestNumber, "DIGGER_PR_NUMBER", fmt.Sprintf("%v", *job.PullRequestNumber)) + if err != nil { + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to set job output. Exiting. %s", err), 4) + } + if err != nil { reportErrorAndExit(githubActor, fmt.Sprintf("Failed to parse jobs json. %s", err), 4) } @@ -313,6 +318,11 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker, backend log.Println("GitHub event converted to commands successfully") logCommands(jobs) + err = githubPrService.SetOutput(prNumber, "DIGGER_PR_NUMBER", fmt.Sprintf("%v", prNumber)) + if err != nil { + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to set job output. Exiting. %s", err), 4) + } + planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, githubActor, &prNumber) reporter := &reporting.CiReporter{ diff --git a/cli/pkg/azure/azure.go b/cli/pkg/azure/azure.go index 881f38404..7c143bbe1 100644 --- a/cli/pkg/azure/azure.go +++ b/cli/pkg/azure/azure.go @@ -336,6 +336,11 @@ func (a *AzureReposService) GetBranchName(prNumber int) (string, error) { return "", nil } +func (svc *AzureReposService) SetOutput(prNumber int, key string, value string) error { + //TODO implement me + return nil +} + func (a *AzureReposService) GetComments(prNumber int) ([]orchestrator.Comment, error) { comments, err := a.Client.GetComments(context.Background(), git.GetCommentsArgs{ Project: &a.ProjectName, diff --git a/cli/pkg/bitbucket/bitbucket.go b/cli/pkg/bitbucket/bitbucket.go index e39170479..1e5395a08 100644 --- a/cli/pkg/bitbucket/bitbucket.go +++ b/cli/pkg/bitbucket/bitbucket.go @@ -463,6 +463,11 @@ func (b *BitbucketAPI) GetBranchName(prNumber int) (string, error) { return pullRequest.Source.Branch.Name, nil } +func (svc *BitbucketAPI) SetOutput(prNumber int, key string, value string) error { + //TODO implement me + return nil +} + // Implement the OrgService interface. func (b *BitbucketAPI) GetUserTeams(organisation string, user string) ([]string, error) { diff --git a/cli/pkg/core/execution/execution.go b/cli/pkg/core/execution/execution.go index 7abc048dd..770a3121f 100644 --- a/cli/pkg/core/execution/execution.go +++ b/cli/pkg/core/execution/execution.go @@ -117,6 +117,7 @@ type DiggerExecutorPlanResult struct { type PlanPathProvider interface { LocalPlanFilePath() string StoredPlanFilePath() string + ArtifactName() string PlanFileName() string } @@ -126,8 +127,12 @@ type ProjectPathProvider struct { ProjectName string } +func (d ProjectPathProvider) ArtifactName() string { + return d.ProjectName +} + func (d ProjectPathProvider) PlanFileName() string { - return strings.ReplaceAll(d.ProjectNamespace, "/", "-") + "#" + d.ProjectName + ".tfplan" + return strings.ReplaceAll(d.ProjectNamespace, "/", "-") + "-" + d.ProjectName + ".tfplan" } func (d ProjectPathProvider) LocalPlanFilePath() string { @@ -193,21 +198,17 @@ func (d DiggerExecutor) Plan() (*terraform.PlanSummary, bool, bool, string, stri return nil, false, false, "", "", fmt.Errorf("error executing plan: %v", err) } if d.PlanStorage != nil { - planExists, err := d.PlanStorage.PlanExists(d.PlanPathProvider.StoredPlanFilePath()) - if err != nil { - return nil, false, false, "", "", fmt.Errorf("error checking if plan exists: %v", err) - } - if planExists { - err = d.PlanStorage.DeleteStoredPlan(d.PlanPathProvider.StoredPlanFilePath()) - if err != nil { - return nil, false, false, "", "", fmt.Errorf("error deleting plan: %v", err) - } + fileBytes, err := os.ReadFile(d.PlanPathProvider.LocalPlanFilePath()) + if err != nil { + fmt.Println("Error reading file:", err) + return nil, false, false, "", "", fmt.Errorf("error reading file bytes: %v", err) } - err = d.PlanStorage.StorePlan(d.PlanPathProvider.LocalPlanFilePath(), d.PlanPathProvider.StoredPlanFilePath()) + err = d.PlanStorage.StorePlanFile(fileBytes, d.PlanPathProvider.ArtifactName(), d.PlanPathProvider.PlanFileName()) if err != nil { - return nil, false, false, "", "", fmt.Errorf("error storing plan: %v", err) + fmt.Println("Error storing artifact file:", err) + return nil, false, false, "", "", fmt.Errorf("error storing artifact file: %v", err) } } plan = cleanupTerraformPlan(!isEmptyPlan, err, stdout, stderr) diff --git a/cli/pkg/core/storage/plan_storage.go b/cli/pkg/core/storage/plan_storage.go index 3fc39686d..356304bca 100644 --- a/cli/pkg/core/storage/plan_storage.go +++ b/cli/pkg/core/storage/plan_storage.go @@ -2,6 +2,7 @@ package storage type PlanStorage interface { StorePlan(localPlanFilePath string, storedPlanFilePath string) error + StorePlanFile(fileContents []byte, artifactName string, fileName string) error RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) DeleteStoredPlan(storedPlanFilePath string) error PlanExists(storedPlanFilePath string) (bool, error) diff --git a/cli/pkg/digger/digger.go b/cli/pkg/digger/digger.go index 9bd50d823..d05dc1c37 100644 --- a/cli/pkg/digger/digger.go +++ b/cli/pkg/digger/digger.go @@ -371,7 +371,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org var planPolicyViolations []string if os.Getenv("PLAN_UPLOAD_DESTINATION") != "" { - storedPlanJson, err := retrievePlanBeforeApply(planStorage, planPathProvider, diggerExecutor) + storedPlanJson, err := retrievePlanBeforeApply(planStorage, planPathProvider, diggerExecutor.Executor.(execution.DiggerExecutor)) if err != nil { msg := fmt.Sprintf("Failed to retrieve stored plan. %v", err) log.Printf(msg) @@ -485,14 +485,14 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org return &execution.DiggerExecutorResult{}, "", nil } -func retrievePlanBeforeApply(planStorage storage.PlanStorage, planPathProvider execution.PlanPathProvider, diggerExecutor execution.LockingExecutorWrapper) (string, error) { - storedPlanExists, err := planStorage.PlanExists(planPathProvider.StoredPlanFilePath()) +func retrievePlanBeforeApply(planStorage storage.PlanStorage, planPathProvider execution.PlanPathProvider, diggerExecutor execution.DiggerExecutor) (string, error) { + storedPlanExists, err := planStorage.PlanExists(diggerExecutor.ProjectName) if err != nil { return "", fmt.Errorf("failed to check if stored plan exists. %v", err) } if storedPlanExists { log.Printf("Pre-apply plan retrieval: stored plan exists") - storedPlanPath, err := planStorage.RetrievePlan(planPathProvider.LocalPlanFilePath(), planPathProvider.StoredPlanFilePath()) + storedPlanPath, err := planStorage.RetrievePlan(planPathProvider.LocalPlanFilePath(), planPathProvider.ArtifactName()) if err != nil { return "", fmt.Errorf("failed to retrieve stored plan path. %v", err) } diff --git a/cli/pkg/digger/digger_test.go b/cli/pkg/digger/digger_test.go index 3c6c59803..c0c74610e 100644 --- a/cli/pkg/digger/digger_test.go +++ b/cli/pkg/digger/digger_test.go @@ -1,6 +1,7 @@ package digger import ( + "os" "sort" "strconv" "strings" @@ -137,6 +138,12 @@ func (m *MockPRManager) GetBranchName(prNumber int) (string, error) { return "", nil } +func (m *MockPRManager) SetOutput(prNumber int, key string, value string) error { + m.Commands = append(m.Commands, RunInfo{"SetOutput", strconv.Itoa(prNumber), time.Now()}) + return nil + +} + type MockProjectLock struct { Commands []RunInfo } @@ -179,6 +186,11 @@ func (m *MockPlanStorage) StorePlan(localPlanFilePath string, storedPlanFilePath return nil } +func (m *MockPlanStorage) StorePlanFile(fileContents []byte, artifactName string, fileName string) error { + m.Commands = append(m.Commands, RunInfo{"StorePlanFile", artifactName, time.Now()}) + return nil +} + func (m *MockPlanStorage) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) { m.Commands = append(m.Commands, RunInfo{"RetrievePlan", localPlanFilePath, time.Now()}) return nil, nil @@ -198,6 +210,11 @@ type MockPlanPathProvider struct { Commands []RunInfo } +func (m MockPlanPathProvider) ArtifactName() string { + m.Commands = append(m.Commands, RunInfo{"ArtifactName", "", time.Now()}) + return "plan" +} + func (m MockPlanPathProvider) PlanFileName() string { m.Commands = append(m.Commands, RunInfo{"PlanFileName", "", time.Now()}) return "plan" @@ -344,11 +361,14 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) { PlanPathProvider: planPathProvider, } + os.WriteFile(planPathProvider.LocalPlanFilePath(), []byte{123}, 0644) + defer os.Remove(planPathProvider.LocalPlanFilePath()) + executor.Plan() commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage, planPathProvider) - assert.Equal(t, []string{"Init ", "Plan -out plan -lock-timeout=3m", "Show -no-color -json plan", "PlanExists plan", "StorePlan plan", "Run echo"}, commandStrings) + assert.Equal(t, []string{"Init ", "Plan -out plan -lock-timeout=3m", "Show -no-color -json plan", "StorePlanFile plan", "Run echo"}, commandStrings) } func allCommandsInOrderWithParams(terraformExecutor *MockTerraformExecutor, commandRunner *MockCommandRunner, prManager *MockPRManager, lock *MockProjectLock, planStorage *MockPlanStorage, planPathProvider *MockPlanPathProvider) []string { diff --git a/cli/pkg/github/mocks.go b/cli/pkg/github/mocks.go index ea8ab2790..2eee45121 100644 --- a/cli/pkg/github/mocks.go +++ b/cli/pkg/github/mocks.go @@ -85,3 +85,8 @@ func (t MockCiService) EditComment(prNumber int, commentId interface{}, comment func (t MockCiService) GetBranchName(prNumber int) (string, error) { return "", nil } + +func (svc MockCiService) SetOutput(prNumber int, key string, value string) error { + //TODO implement me + return nil +} diff --git a/cli/pkg/gitlab/gitlab.go b/cli/pkg/gitlab/gitlab.go index ce4cb3b91..f2936fe34 100644 --- a/cli/pkg/gitlab/gitlab.go +++ b/cli/pkg/gitlab/gitlab.go @@ -250,6 +250,11 @@ func (gitlabService GitLabService) GetBranchName(prNumber int) (string, error) { return "", nil } +func (svc *GitLabService) SetOutput(prNumber int, key string, value string) error { + //TODO implement me + return nil +} + func getMergeRequest(gitlabService GitLabService) *go_gitlab.MergeRequest { projectId := *gitlabService.Context.ProjectId mergeRequestIID := *gitlabService.Context.MergeRequestIId diff --git a/cli/pkg/reporting/reporting_test.go b/cli/pkg/reporting/reporting_test.go index d802f91a8..eae006d8a 100644 --- a/cli/pkg/reporting/reporting_test.go +++ b/cli/pkg/reporting/reporting_test.go @@ -237,3 +237,7 @@ func (t MockCiService) EditComment(prNumber int, commentId interface{}, comment func (t MockCiService) GetBranchName(prNumber int) (string, error) { return "", nil } + +func (svc MockCiService) SetOutput(prNumber int, key string, value string) error { + return nil +} diff --git a/cli/pkg/storage/plan_storage.go b/cli/pkg/storage/plan_storage.go index 92aebf985..044bb8738 100644 --- a/cli/pkg/storage/plan_storage.go +++ b/cli/pkg/storage/plan_storage.go @@ -1,17 +1,18 @@ package storage import ( + "bytes" "context" + "encoding/json" "fmt" + "github.com/diggerhq/digger/cli/pkg/utils" "io" "log" + "net/http" "net/url" "os" "os/exec" "path/filepath" - "strconv" - - "github.com/diggerhq/digger/cli/pkg/utils" "cloud.google.com/go/storage" "github.com/google/go-github/v58/github" @@ -65,6 +66,11 @@ func (psg *PlanStorageGcp) StorePlan(localPlanFilePath string, storedPlanFilePat return nil } +func (psg *PlanStorageGcp) StorePlanFile(fileContents []byte, artifactName string, fileName string) error { + // TODO: implement me + return nil +} + func (psg *PlanStorageGcp) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) { obj := psg.Bucket.Object(storedPlanFilePath) rc, err := obj.NewReader(psg.Context) @@ -104,8 +110,89 @@ func (gps *GithubPlanStorage) StorePlan(localPlanFilePath string, storedPlanFile return nil } +func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName string, fileName string) error { + actionsRuntimeToken := os.Getenv("ACTIONS_RUNTIME_TOKEN") + actionsRuntimeURL := os.Getenv("ACTIONS_RUNTIME_URL") + githubRunID := os.Getenv("GITHUB_RUN_ID") + artifactBase := fmt.Sprintf("%s_apis/pipelines/workflows/%s/artifacts?api-version=6.0-preview", actionsRuntimeURL, githubRunID) + + headers := map[string]string{ + "Accept": "application/json;api-version=6.0-preview", + "Authorization": "Bearer " + actionsRuntimeToken, + "Content-Type": "application/json", + } + + // Create Artifact + createArtifactURL := artifactBase + createArtifactData := map[string]string{"type": "actions_storage", "name": artifactName} + createArtifactBody, _ := json.Marshal(createArtifactData) + createArtifactResponse, err := doRequest("POST", createArtifactURL, headers, createArtifactBody) + if createArtifactResponse == nil || err != nil { + return fmt.Errorf("Could not create artifact with github %v", err) + } + defer createArtifactResponse.Body.Close() + + // Extract Resource URL + createArtifactResponseBody, _ := io.ReadAll(createArtifactResponse.Body) + var createArtifactResponseMap map[string]interface{} + json.Unmarshal(createArtifactResponseBody, &createArtifactResponseMap) + resourceURL := createArtifactResponseMap["fileContainerResourceUrl"].(string) + + // Upload Data + uploadURL := fmt.Sprintf("%s?itemPath=%s/%s", resourceURL, artifactName, fileName) + uploadData := fileContents + dataLen := len(uploadData) + headers["Content-Type"] = "application/octet-stream" + headers["Content-Range"] = fmt.Sprintf("bytes 0-%v/%v", dataLen-1, dataLen) + _, err = doRequest("PUT", uploadURL, headers, uploadData) + if err != nil { + return fmt.Errorf("could not upload artifact file %v", err) + } + + // Update Artifact Size + headers = map[string]string{ + "Accept": "application/json;api-version=6.0-preview", + "Authorization": "Bearer " + actionsRuntimeToken, + "Content-Type": "application/json", + } + updateArtifactURL := fmt.Sprintf("%s&artifactName=%s", artifactBase, artifactName) + updateArtifactData := map[string]int{"size": dataLen} + updateArtifactBody, _ := json.Marshal(updateArtifactData) + _, err = doRequest("PATCH", updateArtifactURL, headers, updateArtifactBody) + if err != nil { + return fmt.Errorf("could finalize artefact upload: %v", err) + } + + return nil +} + +func doRequest(method, url string, headers map[string]string, body []byte) (*http.Response, error) { + client := &http.Client{} + req, err := http.NewRequest(method, url, bytes.NewBuffer(body)) + if err != nil { + fmt.Println("Error creating request:", err) + return nil, fmt.Errorf("error creating request: %v", err) + } + for key, value := range headers { + req.Header.Set(key, value) + } + resp, err := client.Do(req) + if err != nil { + fmt.Println("Error making request:", err) + return nil, fmt.Errorf("error creating request: %v", err) + } + if resp.StatusCode >= 400 { + fmt.Printf("url: %v", url) + fmt.Println("Request failed with status code:", resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + fmt.Printf("body: %v", string(body)) + return nil, fmt.Errorf("error creating request: %v", err) + } + return resp, nil +} + func (gps *GithubPlanStorage) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) { - plansFilename, err := gps.DownloadLatestPlans() + plansFilename, err := gps.DownloadLatestPlans(storedPlanFilePath) if err != nil { return nil, fmt.Errorf("error downloading plan: %v", err) @@ -132,7 +219,7 @@ func (gps *GithubPlanStorage) PlanExists(storedPlanFilePath string) (bool, error return false, err } - latestPlans := getLatestArtifactWithName(artifacts.Artifacts, "plans-"+strconv.Itoa(gps.PullRequestNumber)) + latestPlans := getLatestArtifactWithName(artifacts.Artifacts, storedPlanFilePath) if latestPlans == nil { return false, nil @@ -144,7 +231,7 @@ func (gps *GithubPlanStorage) DeleteStoredPlan(storedPlanFilePath string) error return nil } -func (gps *GithubPlanStorage) DownloadLatestPlans() (string, error) { +func (gps *GithubPlanStorage) DownloadLatestPlans(storedPlanFilePath string) (string, error) { artifacts, _, err := gps.Client.Actions.ListArtifacts(context.Background(), gps.Owner, gps.RepoName, &github.ListOptions{ PerPage: 100, }) @@ -153,7 +240,7 @@ func (gps *GithubPlanStorage) DownloadLatestPlans() (string, error) { return "", err } - latestPlans := getLatestArtifactWithName(artifacts.Artifacts, "plans-"+strconv.Itoa(gps.PullRequestNumber)) + latestPlans := getLatestArtifactWithName(artifacts.Artifacts, storedPlanFilePath) if latestPlans == nil { return "", nil @@ -164,7 +251,7 @@ func (gps *GithubPlanStorage) DownloadLatestPlans() (string, error) { if err != nil { return "", err } - filename := "plans-" + strconv.Itoa(gps.PullRequestNumber) + ".zip" + filename := storedPlanFilePath + ".zip" err = downloadArtifactIntoFile(downloadUrl, filename) diff --git a/cli/pkg/utils/mocks.go b/cli/pkg/utils/mocks.go index f12fafda2..bc54cfa3d 100644 --- a/cli/pkg/utils/mocks.go +++ b/cli/pkg/utils/mocks.go @@ -123,6 +123,10 @@ func (t MockPullRequestManager) GetBranchName(prNumber int) (string, error) { return "", nil } +func (t MockPullRequestManager) SetOutput(prNumber int, key string, value string) error { + return nil +} + type MockPlanStorage struct { } @@ -130,6 +134,10 @@ func (t MockPlanStorage) StorePlan(localPlanFilePath string, storedPlanFilePath return nil } +func (t *MockPlanStorage) StorePlanFile(fileContents []byte, artifactName string, fileName string) error { + return nil +} + func (t MockPlanStorage) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) { return nil, nil } diff --git a/cli/pkg/utils/pullrequestmanager_mock.go b/cli/pkg/utils/pullrequestmanager_mock.go index 3577076ed..ddae866c3 100644 --- a/cli/pkg/utils/pullrequestmanager_mock.go +++ b/cli/pkg/utils/pullrequestmanager_mock.go @@ -90,3 +90,8 @@ func (mockGithubPullrequestManager *MockGithubPullrequestManager) GetBranchName( mockGithubPullrequestManager.commands = append(mockGithubPullrequestManager.commands, "GetBranchName") return "", nil } + +func (mockGithubPullrequestManager MockGithubPullrequestManager) SetOutput(prNumber int, key string, value string) error { + mockGithubPullrequestManager.commands = append(mockGithubPullrequestManager.commands, "SetOutput") + return nil +} diff --git a/libs/orchestrator/ci.go b/libs/orchestrator/ci.go index a6937e930..f8c4c8c46 100644 --- a/libs/orchestrator/ci.go +++ b/libs/orchestrator/ci.go @@ -17,6 +17,7 @@ type PullRequestService interface { // IsClosed closed without merging IsClosed(prNumber int) (bool, error) GetBranchName(prNumber int) (string, error) + SetOutput(prNumber int, key string, value string) error } type OrgService interface { diff --git a/libs/orchestrator/github/github.go b/libs/orchestrator/github/github.go index d84ffabf0..0220dd92c 100644 --- a/libs/orchestrator/github/github.go +++ b/libs/orchestrator/github/github.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "os" "strings" "github.com/diggerhq/digger/libs/digger_config" @@ -192,6 +193,23 @@ func (svc GithubService) IsClosed(prNumber int) (bool, error) { return pr.GetState() == "closed", nil } +func (svc GithubService) SetOutput(prNumber int, key string, value string) error { + gout := os.Getenv("GITHUB_ENV") + if gout == "" { + return fmt.Errorf("GITHUB_ENV not set, could not set the output in digger step") + } + f, err := os.OpenFile(gout, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("could not open file for writing during digger step") + } + _, err = f.WriteString(fmt.Sprintf("%v=%v", key, value)) + if err != nil { + return fmt.Errorf("could not write digger file step") + } + f.Close() + return nil +} + func (svc GithubService) GetBranchName(prNumber int) (string, error) { pr, _, err := svc.Client.PullRequests.Get(context.Background(), svc.Owner, svc.RepoName, prNumber) if err != nil {