feat: enable retention policies

.github/workflows/remote-controller.yaml

@@ -51,7 +51,7 @@ jobs:
     - name: Setup correct Go version
       uses: actions/setup-go@v2
       with:
-        go-version: '1.20'
+        go-version: '1.21'
     - name: Install kubebuilder
       run: |
         #kubebuilder

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20-alpine AS builder
+FROM golang:1.21-alpine AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

controllers/v1beta1/build_helpers.go

@@ -212,7 +212,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp
 			return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err)
 		}
 		if lagoonHarbor.UseV2Functions(curVer) {
-			hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name)
+			hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace)
 			if err != nil {
 				return fmt.Errorf("Error creating harbor project: %v", err)
 			}

go.mod

@@ -1,6 +1,6 @@
 module github.com/uselagoon/remote-controller
 
-go 1.20
+go 1.21
 
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1
@@ -15,6 +15,7 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.7
 	github.com/prometheus/client_golang v1.15.1
+	github.com/uselagoon/machinery v0.0.29
 	github.com/vshn/k8up v1.99.99
 	github.com/xhit/go-str2duration/v2 v2.0.0
 	gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
@@ -32,6 +33,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
+	github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -81,15 +83,15 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

internal/harbor/harbor22x.go

@@ -16,7 +16,8 @@ import (
 )
 
 // CreateProjectV2 will create a project if one doesn't exist, but will update as required.
-func (h *Harbor) CreateProjectV2(ctx context.Context, projectName string) (*harborclientv5model.Project, error) {
+func (h *Harbor) CreateProjectV2(ctx context.Context, namespace corev1.Namespace) (*harborclientv5model.Project, error) {
+	projectName := namespace.Labels["lagoon.sh/project"]
 	exists, err := h.ClientV5.ProjectExists(ctx, projectName)
 	if err != nil {
 		h.Log.Info(fmt.Sprintf("Error checking project %s exists, err: %v", projectName, err))
@@ -278,6 +279,14 @@ func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch strin
 			if err != nil {
 				h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name))
 			}
+			h.Log.Info(
+				fmt.Sprintf(
+					"Deleted harbor repository %s in  project %s, environment %s",
+					repo.Name,
+					projectName,
+					environmentName,
+				),
+			)
 		}
 	}
 	if len(listRepositories) > 100 {
@@ -293,6 +302,14 @@ func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch strin
 					if err != nil {
 						h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name))
 					}
+					h.Log.Info(
+						fmt.Sprintf(
+							"Deleted harbor repository %s in  project %s, environment %s",
+							repo.Name,
+							projectName,
+							environmentName,
+						),
+					)
 				}
 			}
 		}
@@ -321,6 +338,14 @@ func (h *Harbor) DeleteRobotAccount(ctx context.Context, projectName, branch str
 				h.Log.Info(fmt.Sprintf("Error deleting project %s robot account %s", projectName, robot.Name))
 				return
 			}
+			h.Log.Info(
+				fmt.Sprintf(
+					"Deleted harbor robot account %s in  project %s, environment %s",
+					robot.Name,
+					projectName,
+					environmentName,
+				),
+			)
 		}
 	}
 }

internal/harbor/harbor_credentialrotation.go

@@ -94,7 +94,7 @@ func (h *Harbor) RotateRobotCredential(ctx context.Context, cl client.Client, ns
 		return false, fmt.Errorf("error checking harbor version: %v", err)
 	}
 	if h.UseV2Functions(curVer) {
-		hProject, err := h.CreateProjectV2(ctx, ns.Labels["lagoon.sh/project"])
+		hProject, err := h.CreateProjectV2(ctx, ns)
 		if err != nil {
 			return false, fmt.Errorf("error getting or creating project: %v", err)
 		}

internal/messenger/consumer.go

@@ -294,26 +294,18 @@ func (m *Messenger) Consumer(targetName string) { //error {
 			jobSpec := &lagoonv1beta1.LagoonTaskSpec{}
 			json.Unmarshal(message.Body(), jobSpec)
 			// check which key has been received
-			namespace := helpers.GenerateNamespaceName(
-				jobSpec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller
-				jobSpec.Environment.Name,
-				jobSpec.Project.Name,
-				m.NamespacePrefix,
-				m.ControllerNamespace,
-				m.RandomNamespacePrefix,
-			)
 			switch jobSpec.Key {
 			case "deploytarget:build:cancel", "kubernetes:build:cancel":
 				opLog.Info(
 					fmt.Sprintf(
 						"Received build cancellation for project %s, environment %s - %s",
 						jobSpec.Project.Name,
 						jobSpec.Environment.Name,
-						namespace,
+						m.genNamespace(jobSpec),
 					),
 				)
 				m.Cache.Add(jobSpec.Misc.Name, jobSpec.Project.Name)
-				err := m.CancelBuild(namespace, jobSpec)
+				err := m.CancelBuild(m.genNamespace(jobSpec), jobSpec)
 				if err != nil {
 					//@TODO: send msg back to lagoon and update task to failed?
 					message.Ack(false) // ack to remove from queue
@@ -325,11 +317,11 @@ func (m *Messenger) Consumer(targetName string) { //error {
 						"Received task cancellation for project %s, environment %s - %s",
 						jobSpec.Project.Name,
 						jobSpec.Environment.Name,
-						namespace,
+						m.genNamespace(jobSpec),
 					),
 				)
 				m.Cache.Add(jobSpec.Task.TaskName, jobSpec.Project.Name)
-				err := m.CancelTask(namespace, jobSpec)
+				err := m.CancelTask(m.genNamespace(jobSpec), jobSpec)
 				if err != nil {
 					//@TODO: send msg back to lagoon and update task to failed?
 					message.Ack(false) // ack to remove from queue
@@ -343,7 +335,7 @@ func (m *Messenger) Consumer(targetName string) { //error {
 						jobSpec.Environment.Name,
 					),
 				)
-				err := m.ResticRestore(namespace, jobSpec)
+				err := m.ResticRestore(m.genNamespace(jobSpec), jobSpec)
 				if err != nil {
 					opLog.Error(err,
 						fmt.Sprintf(
@@ -363,7 +355,7 @@ func (m *Messenger) Consumer(targetName string) { //error {
 						jobSpec.Project.Name,
 					),
 				)
-				err := m.IngressRouteMigration(namespace, jobSpec)
+				err := m.IngressRouteMigration(m.genNamespace(jobSpec), jobSpec)
 				if err != nil {
 					opLog.Error(err,
 						fmt.Sprintf(
@@ -383,7 +375,7 @@ func (m *Messenger) Consumer(targetName string) { //error {
 						jobSpec.Project.Name,
 					),
 				)
-				err := m.AdvancedTask(namespace, jobSpec)
+				err := m.AdvancedTask(m.genNamespace(jobSpec), jobSpec)
 				if err != nil {
 					opLog.Error(err,
 						fmt.Sprintf(
@@ -403,7 +395,7 @@ func (m *Messenger) Consumer(targetName string) { //error {
 						jobSpec.Project.Name,
 					),
 				)
-				err := m.ActiveStandbySwitch(namespace, jobSpec)
+				err := m.ActiveStandbySwitch(m.genNamespace(jobSpec), jobSpec)
 				if err != nil {
 					opLog.Error(err,
 						fmt.Sprintf(
@@ -416,6 +408,18 @@ func (m *Messenger) Consumer(targetName string) { //error {
 					message.Ack(false) // ack to remove from queue
 					return
 				}
+			case "deploytarget:harborpolicy:update":
+				err := m.HarborPolicy(ctx, jobSpec)
+				if err != nil {
+					opLog.Error(err,
+						fmt.Sprintf(
+							"Harbor policy update for project %s failed",
+							jobSpec.Project.Name,
+						),
+					)
+					message.Ack(false) // ack to remove from queue
+					return
+				}
 			default:
 				// if we get something that we don't know about, spit out the entire message
 				opLog.Info(
@@ -433,3 +437,14 @@ func (m *Messenger) Consumer(targetName string) { //error {
 	}
 	<-forever
 }
+
+func (m *Messenger) genNamespace(jobSpec *lagoonv1beta1.LagoonTaskSpec) string {
+	return helpers.GenerateNamespaceName(
+		jobSpec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller
+		jobSpec.Environment.Name,
+		jobSpec.Project.Name,
+		m.NamespacePrefix,
+		m.ControllerNamespace,
+		m.RandomNamespacePrefix,
+	)
+}

internal/messenger/messenger.go

@@ -3,6 +3,7 @@ package messenger
 import (
 	"github.com/cheshir/go-mq/v2"
 	"github.com/hashicorp/golang-lru/v2/expirable"
+	"github.com/uselagoon/remote-controller/internal/harbor"
 	"github.com/uselagoon/remote-controller/internal/utilities/deletions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -37,6 +38,7 @@ type Messenger struct {
 	DeletionHandler                  *deletions.Deletions
 	EnableDebug                      bool
 	SupportK8upV2                    bool
+	Harbor                           harbor.Harbor
 	Cache                            *expirable.LRU[string, string]
 }
 
@@ -53,6 +55,7 @@ func New(config mq.Config,
 	deletionHandler *deletions.Deletions,
 	enableDebug bool,
 	supportK8upV2 bool,
+	harbor harbor.Harbor,
 	cache *expirable.LRU[string, string],
 ) *Messenger {
 	return &Messenger{
@@ -68,6 +71,7 @@ func New(config mq.Config,
 		DeletionHandler:                  deletionHandler,
 		EnableDebug:                      enableDebug,
 		SupportK8upV2:                    supportK8upV2,
+		Harbor:                           harbor,
 		Cache:                            cache,
 	}
 }