chore(debug): kubernetes agent debug in kubernetes

.gitignore

@@ -4,4 +4,6 @@ dist
 /.vscode/
 .idea
 .DS_Store
-go.work.sum
+go.work.sum
+
+.env

Makefile

@@ -51,7 +51,26 @@ clean: ## Remove all build and download artifacts
 	@echo "Clearing the dist directory..."
 	@rm -f dist/*
 
+##@ VSCode debug
+.PHONY: debug/build debug/up debug/cluster/create debug/cluster/delete debug/cluster/recreate
+
+debug/build: ## Build the agent with the correct flags
+	@echo "Building debug Portainer agent..."
+	@CGO_ENABLED=0 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -gcflags "all=-N -l" --installsuffix cgo -o dist/$(agent) cmd/agent/main.go
+
+debug/up: ## Deploy the edge agent with Tilt
+	@tilt up --context kind-vscode-debug -f vscode-debug/Tiltfile; tilt down -f vscode-debug/Tiltfile --delete-namespaces
+
+debug/cluster/create: ## Create a k8s KinD cluster for live debug in VSCode
+	@vscode-debug/cluster.sh create
+
+debug/cluster/delete: ## Delete the k8s KinD cluster
+	@vscode-debug/cluster.sh delete
+
+debug/cluster/recreate: ## Recreate the k8s KinD cluster
+	@vscode-debug/cluster.sh recreate
+
 ##@ Helpers
 
 help:  ## Display this help
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_/\-]+:.*?##/ { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

vscode-debug/.env.template

@@ -0,0 +1,3 @@
+EDGE_ID=""
+EDGE_KEY=""
+EDGE_ASYNC= false | true

vscode-debug/Dockerfile

@@ -0,0 +1,19 @@
+FROM golang:1.21-alpine
+
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+ENV PATH="/app:$PATH"
+WORKDIR /app
+
+COPY dist/agent /app/
+COPY dist/docker /app/
+COPY dist/docker-compose /app/
+COPY dist/docker-credential-portainer /app/
+COPY dist/kubectl /app/
+
+COPY static /app/static
+COPY config $HOME/.docker/
+
+LABEL io.portainer.agent true
+
+ENTRYPOINT /go/bin/dlv --listen=0.0.0.0:50100 --api-version=2 --headless=true --only-same-user=false --accept-multiclient --check-go-version=false exec /app/agent

vscode-debug/README.md

@@ -0,0 +1,39 @@
+# How to debug the agent on kubernetes
+
+## Prerequisites
+
+- Create kind cluster
+
+```sh
+`make debug/cluster/create`
+```
+
+- Download dependencies & build other binaries
+
+```sh
+make all
+```
+
+- Create a task in `.vscode/launch.json`
+
+```json
+{
+  "name": "Debug Remote",
+  "type": "go",
+  "request": "attach",
+  "mode": "remote",
+  "host": "localhost",
+  "port": 50100,
+  "cwd": "${workspaceFolder}",
+  "trace": "verbose"
+}
+```
+
+- Start the debug edge agent (you can follow the state and logs in a browser tab by pressing `Space` once the tilt process has started)
+
+```sh
+make debug/up
+```
+
+- Launch your debug session in VSCode
+  - you can minimize the VSCode debug window, all logs will happen in tilt UI

vscode-debug/Tiltfile

@@ -0,0 +1,125 @@
+# EDGE_ID="e090ce2a-db10-47ff-be66-6209716a2a83"
+# EDGE_KEY="aHR0cHM6Ly8xOTIuMTY4LjEuMjA6OTQ0M3wxOTIuMTY4LjEuMjA6ODAwMHxHSTNOamRoMkxkdm1mK09KQ1VCaHFyTGxacEk1MXp0VnNVRm5HV3RJU0Y4PXwz"
+# "fa599970-c47c-4f56-a6af-9372b5f48d48" "aHR0cHM6Ly8xOTIuMTY4LjEuMjA6OTQ0M3wxOTIuMTY4LjEuMjA6ODAwMHxHSTNOamRoMkxkdm1mK09KQ1VCaHFyTGxacEk1MXp0VnNVRm5HV3RJU0Y4PXw0"
+
+# config.define_string_list("edge_config", args=True)
+# config.define_bool("async")
+# cfg = config.parse()
+# a = cfg.get('edge_config')
+# ASYNC=cfg.get('async')
+
+# Load 
+load('ext://dotenv', 'dotenv')
+dotenv()
+EDGE_ID=os.getenv("EDGE_ID")
+EDGE_KEY=os.getenv("EDGE_KEY")
+EDGE_ASYNC=os.getenv("EDGE_ASYNC")
+if EDGE_ID == "" or EDGE_KEY == "":
+  print("Empty EDGE_ID or EDGE_KEY")
+  exit(1)
+
+if EDGE_ASYNC != "true":
+  EDGE_ASYNC=False
+
+# Load the restart_process extension with the docker_build_with_restart func for live reloading.
+load('ext://restart_process', 'docker_build_with_restart')
+# Load the configmap extension with the secret_from_dict func to create EDGE_ID configmap
+load('ext://configmap', 'configmap_from_dict')
+# Load the secret extension with the secret_from_dict func to create EDGE_KEY secret
+load('ext://secret', 'secret_from_dict')
+
+# Deploy Portainer resources - NS / SA / CRB
+k8s_yaml('portainer-resources-k8s.yaml')
+
+# Create EDGE_ID and EDGE_KEY resources
+# kubectl create configmap portainer-agent-edge-id "--from-literal=edge.id=$1" -n portainer
+k8s_yaml(configmap_from_dict(
+    name='portainer-agent-edge-id',
+    namespace="portainer",
+    inputs={"edge.id": EDGE_ID}
+))
+# kubectl --context $CONTEXT create secret generic portainer-agent-edge-key "--from-literal=edge.key=$2" -n portainer
+k8s_yaml(secret_from_dict(
+    name='portainer-agent-edge-key',
+    namespace="portainer",
+    inputs={"edge.key": EDGE_KEY}
+))
+
+# Building binary locally.
+local_resource('build-portainer-agent',
+    cmd="cd .. && make debug/build",
+    deps=['*.go', 
+    '../agent.go',
+    '../build',
+    '../chisel',
+    '../cmd',
+    '../config',
+    '../constants',
+    '../crypto',
+    '../dist',
+    '../docker',
+    '../edge',
+    '../exec',
+    '../filesystem',
+    '../ghw',
+    '../healthcheck',
+    '../http',
+    '../internals',
+    '../kubernetes',
+    '../net',
+    '../nomad',
+    '../os',
+    '../release.sh',
+    '../serf',
+    '../static'
+  ],
+  ignore=[
+    '../dist', 
+    '/**/*_test.go'
+  ],
+)
+
+# Use custom Dockerfile for Tilt builds, which only takes locally built binary for live reloading.
+dockerfile = '''
+FROM golang:1.21-alpine
+
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+ENV PATH="/app:$PATH"
+WORKDIR /app
+
+COPY dist/agent /app/
+COPY dist/docker /app/
+COPY dist/docker-compose /app/
+COPY dist/docker-credential-portainer /app/
+COPY dist/kubectl /app/
+
+COPY static /app/static
+COPY config $HOME/.docker/
+
+LABEL io.portainer.agent true
+'''
+
+# Wrap a docker_build to restart the given entrypoint after a Live Update.
+docker_build_with_restart(
+    'portainer/agent',
+    context='..',
+    # dockerfile_contents=dockerfile,
+    # entrypoint="dlv version",
+    dockerfile="Dockerfile",
+    entrypoint='dlv --continue --listen=0.0.0.0:50100 --accept-multiclient --headless exec /app/agent',
+    live_update=[
+        # Copy the binary so it gets restarted.
+        sync('dist', '/app'),
+        sync('static', '/app/static'),
+    ],
+)
+
+# Create the deployment from YAML file path.
+k8s_yaml('portainer-agent-edge-k8s.yaml')
+k8s_kind('Environment', image_json_path='{.spec.runtime.image}')
+
+# Configure the resource.
+k8s_resource("portainer-agent",
+  port_forwards = ["50100:50100"] # Set up the K8s port-forward to be able to connect to it locally.
+)

vscode-debug/cluster.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+cluster_name='vscode-debug'
+
+# find script dir, following symlinks if any
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)
+  SOURCE=$(readlink "$SOURCE")
+  [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPT_DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)
+
+create() {
+  kind create cluster --config=$SCRIPT_DIR/kind.yaml --name $cluster_name
+}
+
+delete() {
+  kind delete clusters $cluster_name
+}
+
+recreate() {
+  delete
+  create
+}
+
+help() {
+  cat <<EOF
+
+Usage: ${0} CMD
+
+  with CMD:
+    - create: create the kind cluster
+    - delete: delete the cluster
+    - recreate: recreate the cluster
+    - * (anything else): show this help
+EOF
+}
+
+if [[ $# -ne 1 ]]; then
+  help
+  exit
+fi
+
+cmd=$1
+
+case $cmd in
+create | delete | recreate)
+  $cmd
+  ;;
+*)
+  help
+  ;;
+esac

vscode-debug/kind.yaml

@@ -0,0 +1,4 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane

vscode-debug/portainer-agent-edge-k8s.yaml

@@ -0,0 +1,76 @@
+# <INFO> Optional: can be added to expose the agent port 80 to associate an Edge key.
+# apiVersion: v1
+# kind: Service
+# metadata:
+#   name: portainer-agent
+#   namespace: portainer
+# spec:
+#   type: LoadBalancer
+#   selector:
+#     app: portainer-agent
+#   ports:
+#     - name: http
+#       protocol: TCP
+#       port: 80
+#       targetPort: 80
+# ---
+# <INFO> Regular service
+apiVersion: v1
+kind: Service
+metadata:
+  name: portainer-agent
+  namespace: portainer
+spec:
+  clusterIP: None
+  selector:
+    app: portainer-agent
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: portainer-agent
+  namespace: portainer
+spec:
+  selector:
+    matchLabels:
+      app: portainer-agent
+  template:
+    metadata:
+      labels:
+        app: portainer-agent
+    spec:
+      serviceAccountName: portainer-sa-clusteradmin
+      containers:
+        - name: portainer-agent
+          image: portainer/agent
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: LOG_LEVEL
+              value: DEBUG
+            - name: KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: EDGE
+              value: '1'
+            - name: EDGE_ASYNC
+              value: "1"
+            - name: EDGE_INSECURE_POLL
+              value: '1'
+            - name: AGENT_CLUSTER_ADDR
+              value: 'portainer-agent'
+            - name: EDGE_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: portainer-agent-edge-id
+                  key: edge.id
+            - name: EDGE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: portainer-agent-edge-key
+                  key: edge.key
+          ports:
+            - containerPort: 9001
+              protocol: TCP
+            - containerPort: 80
+              protocol: TCP

vscode-debug/portainer-resources-k8s.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: portainer
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: portainer-sa-clusteradmin
+  namespace: portainer
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: portainer-crb-clusteradmin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: portainer-sa-clusteradmin
+    namespace: portainer