From dc4958b04b0b94ca22d55965324f29ef2cccbf31 Mon Sep 17 00:00:00 2001 From: LP B Date: Wed, 28 Feb 2024 18:21:37 +0100 Subject: [PATCH] chore(debug): kubernetes agent debug in kubernetes --- .gitignore | 4 +- Makefile | 21 +++- vscode-debug/.env.template | 3 + vscode-debug/Dockerfile | 19 ++++ vscode-debug/README.md | 39 +++++++ vscode-debug/Tiltfile | 125 +++++++++++++++++++++ vscode-debug/cluster.sh | 54 +++++++++ vscode-debug/kind.yaml | 4 + vscode-debug/portainer-agent-edge-k8s.yaml | 76 +++++++++++++ vscode-debug/portainer-resources-k8s.yaml | 23 ++++ 10 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 vscode-debug/.env.template create mode 100644 vscode-debug/Dockerfile create mode 100644 vscode-debug/README.md create mode 100644 vscode-debug/Tiltfile create mode 100755 vscode-debug/cluster.sh create mode 100644 vscode-debug/kind.yaml create mode 100644 vscode-debug/portainer-agent-edge-k8s.yaml create mode 100644 vscode-debug/portainer-resources-k8s.yaml diff --git a/.gitignore b/.gitignore index 5fc404e57..6f14cdd10 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ dist /.vscode/ .idea .DS_Store -go.work.sum \ No newline at end of file +go.work.sum + +.env \ No newline at end of file diff --git a/Makefile b/Makefile index e4a3944f0..67965361e 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,26 @@ clean: ## Remove all build and download artifacts @echo "Clearing the dist directory..." @rm -f dist/* +##@ VSCode debug +.PHONY: debug/build debug/up debug/cluster/create debug/cluster/delete debug/cluster/recreate + +debug/build: ## Build the agent with the correct flags + @echo "Building debug Portainer agent..." + @CGO_ENABLED=0 GOOS=$(PLATFORM) GOARCH=$(ARCH) go build -gcflags "all=-N -l" --installsuffix cgo -o dist/$(agent) cmd/agent/main.go + +debug/up: ## Deploy the edge agent with Tilt + @tilt up --context kind-vscode-debug -f vscode-debug/Tiltfile; tilt down -f vscode-debug/Tiltfile --delete-namespaces + +debug/cluster/create: ## Create a k8s KinD cluster for live debug in VSCode + @vscode-debug/cluster.sh create + +debug/cluster/delete: ## Delete the k8s KinD cluster + @vscode-debug/cluster.sh delete + +debug/cluster/recreate: ## Recreate the k8s KinD cluster + @vscode-debug/cluster.sh recreate + ##@ Helpers help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_/\-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/vscode-debug/.env.template b/vscode-debug/.env.template new file mode 100644 index 000000000..0fd9aa0ba --- /dev/null +++ b/vscode-debug/.env.template @@ -0,0 +1,3 @@ +EDGE_ID="" +EDGE_KEY="" +EDGE_ASYNC= false | true \ No newline at end of file diff --git a/vscode-debug/Dockerfile b/vscode-debug/Dockerfile new file mode 100644 index 000000000..61751a596 --- /dev/null +++ b/vscode-debug/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.21-alpine + +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +ENV PATH="/app:$PATH" +WORKDIR /app + +COPY dist/agent /app/ +COPY dist/docker /app/ +COPY dist/docker-compose /app/ +COPY dist/docker-credential-portainer /app/ +COPY dist/kubectl /app/ + +COPY static /app/static +COPY config $HOME/.docker/ + +LABEL io.portainer.agent true + +ENTRYPOINT /go/bin/dlv --listen=0.0.0.0:50100 --api-version=2 --headless=true --only-same-user=false --accept-multiclient --check-go-version=false exec /app/agent diff --git a/vscode-debug/README.md b/vscode-debug/README.md new file mode 100644 index 000000000..812ceaf6b --- /dev/null +++ b/vscode-debug/README.md @@ -0,0 +1,39 @@ +# How to debug the agent on kubernetes + +## Prerequisites + +- Create kind cluster + +```sh +`make debug/cluster/create` +``` + +- Download dependencies & build other binaries + +```sh +make all +``` + +- Create a task in `.vscode/launch.json` + +```json +{ + "name": "Debug Remote", + "type": "go", + "request": "attach", + "mode": "remote", + "host": "localhost", + "port": 50100, + "cwd": "${workspaceFolder}", + "trace": "verbose" +} +``` + +- Start the debug edge agent (you can follow the state and logs in a browser tab by pressing `Space` once the tilt process has started) + +```sh +make debug/up +``` + +- Launch your debug session in VSCode + - you can minimize the VSCode debug window, all logs will happen in tilt UI \ No newline at end of file diff --git a/vscode-debug/Tiltfile b/vscode-debug/Tiltfile new file mode 100644 index 000000000..675cfdc3a --- /dev/null +++ b/vscode-debug/Tiltfile @@ -0,0 +1,125 @@ +# EDGE_ID="e090ce2a-db10-47ff-be66-6209716a2a83" +# EDGE_KEY="aHR0cHM6Ly8xOTIuMTY4LjEuMjA6OTQ0M3wxOTIuMTY4LjEuMjA6ODAwMHxHSTNOamRoMkxkdm1mK09KQ1VCaHFyTGxacEk1MXp0VnNVRm5HV3RJU0Y4PXwz" +# "fa599970-c47c-4f56-a6af-9372b5f48d48" "aHR0cHM6Ly8xOTIuMTY4LjEuMjA6OTQ0M3wxOTIuMTY4LjEuMjA6ODAwMHxHSTNOamRoMkxkdm1mK09KQ1VCaHFyTGxacEk1MXp0VnNVRm5HV3RJU0Y4PXw0" + +# config.define_string_list("edge_config", args=True) +# config.define_bool("async") +# cfg = config.parse() +# a = cfg.get('edge_config') +# ASYNC=cfg.get('async') + +# Load +load('ext://dotenv', 'dotenv') +dotenv() +EDGE_ID=os.getenv("EDGE_ID") +EDGE_KEY=os.getenv("EDGE_KEY") +EDGE_ASYNC=os.getenv("EDGE_ASYNC") +if EDGE_ID == "" or EDGE_KEY == "": + print("Empty EDGE_ID or EDGE_KEY") + exit(1) + +if EDGE_ASYNC != "true": + EDGE_ASYNC=False + +# Load the restart_process extension with the docker_build_with_restart func for live reloading. +load('ext://restart_process', 'docker_build_with_restart') +# Load the configmap extension with the secret_from_dict func to create EDGE_ID configmap +load('ext://configmap', 'configmap_from_dict') +# Load the secret extension with the secret_from_dict func to create EDGE_KEY secret +load('ext://secret', 'secret_from_dict') + +# Deploy Portainer resources - NS / SA / CRB +k8s_yaml('portainer-resources-k8s.yaml') + +# Create EDGE_ID and EDGE_KEY resources +# kubectl create configmap portainer-agent-edge-id "--from-literal=edge.id=$1" -n portainer +k8s_yaml(configmap_from_dict( + name='portainer-agent-edge-id', + namespace="portainer", + inputs={"edge.id": EDGE_ID} +)) +# kubectl --context $CONTEXT create secret generic portainer-agent-edge-key "--from-literal=edge.key=$2" -n portainer +k8s_yaml(secret_from_dict( + name='portainer-agent-edge-key', + namespace="portainer", + inputs={"edge.key": EDGE_KEY} +)) + +# Building binary locally. +local_resource('build-portainer-agent', + cmd="cd .. && make debug/build", + deps=['*.go', + '../agent.go', + '../build', + '../chisel', + '../cmd', + '../config', + '../constants', + '../crypto', + '../dist', + '../docker', + '../edge', + '../exec', + '../filesystem', + '../ghw', + '../healthcheck', + '../http', + '../internals', + '../kubernetes', + '../net', + '../nomad', + '../os', + '../release.sh', + '../serf', + '../static' + ], + ignore=[ + '../dist', + '/**/*_test.go' + ], +) + +# Use custom Dockerfile for Tilt builds, which only takes locally built binary for live reloading. +dockerfile = ''' +FROM golang:1.21-alpine + +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +ENV PATH="/app:$PATH" +WORKDIR /app + +COPY dist/agent /app/ +COPY dist/docker /app/ +COPY dist/docker-compose /app/ +COPY dist/docker-credential-portainer /app/ +COPY dist/kubectl /app/ + +COPY static /app/static +COPY config $HOME/.docker/ + +LABEL io.portainer.agent true +''' + +# Wrap a docker_build to restart the given entrypoint after a Live Update. +docker_build_with_restart( + 'portainer/agent', + context='..', + # dockerfile_contents=dockerfile, + # entrypoint="dlv version", + dockerfile="Dockerfile", + entrypoint='dlv --continue --listen=0.0.0.0:50100 --accept-multiclient --headless exec /app/agent', + live_update=[ + # Copy the binary so it gets restarted. + sync('dist', '/app'), + sync('static', '/app/static'), + ], +) + +# Create the deployment from YAML file path. +k8s_yaml('portainer-agent-edge-k8s.yaml') +k8s_kind('Environment', image_json_path='{.spec.runtime.image}') + +# Configure the resource. +k8s_resource("portainer-agent", + port_forwards = ["50100:50100"] # Set up the K8s port-forward to be able to connect to it locally. +) \ No newline at end of file diff --git a/vscode-debug/cluster.sh b/vscode-debug/cluster.sh new file mode 100755 index 000000000..1a7e606e9 --- /dev/null +++ b/vscode-debug/cluster.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +cluster_name='vscode-debug' + +# find script dir, following symlinks if any +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPT_DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + +create() { + kind create cluster --config=$SCRIPT_DIR/kind.yaml --name $cluster_name +} + +delete() { + kind delete clusters $cluster_name +} + +recreate() { + delete + create +} + +help() { + cat < Optional: can be added to expose the agent port 80 to associate an Edge key. +# apiVersion: v1 +# kind: Service +# metadata: +# name: portainer-agent +# namespace: portainer +# spec: +# type: LoadBalancer +# selector: +# app: portainer-agent +# ports: +# - name: http +# protocol: TCP +# port: 80 +# targetPort: 80 +# --- +# Regular service +apiVersion: v1 +kind: Service +metadata: + name: portainer-agent + namespace: portainer +spec: + clusterIP: None + selector: + app: portainer-agent +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: portainer-agent + namespace: portainer +spec: + selector: + matchLabels: + app: portainer-agent + template: + metadata: + labels: + app: portainer-agent + spec: + serviceAccountName: portainer-sa-clusteradmin + containers: + - name: portainer-agent + image: portainer/agent + imagePullPolicy: IfNotPresent + env: + - name: LOG_LEVEL + value: DEBUG + - name: KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: EDGE + value: '1' + - name: EDGE_ASYNC + value: "1" + - name: EDGE_INSECURE_POLL + value: '1' + - name: AGENT_CLUSTER_ADDR + value: 'portainer-agent' + - name: EDGE_ID + valueFrom: + configMapKeyRef: + name: portainer-agent-edge-id + key: edge.id + - name: EDGE_KEY + valueFrom: + secretKeyRef: + name: portainer-agent-edge-key + key: edge.key + ports: + - containerPort: 9001 + protocol: TCP + - containerPort: 80 + protocol: TCP diff --git a/vscode-debug/portainer-resources-k8s.yaml b/vscode-debug/portainer-resources-k8s.yaml new file mode 100644 index 000000000..e54337681 --- /dev/null +++ b/vscode-debug/portainer-resources-k8s.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: portainer +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: portainer-sa-clusteradmin + namespace: portainer +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: portainer-crb-clusteradmin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: portainer-sa-clusteradmin + namespace: portainer