plugins: add sgx-config plugin.

Add a sample plugin for SGX EPC resource configuration using
pod annotations. Note that this plugin needs a patched kernel
with support for something like
    echo 'sgx_epc 65536' > /sys/fs/cgroup/$CGRP/misc.max
to work.

Signed-off-by: Krisztian Litkey <[email protected]>

Makefile

@@ -68,7 +68,8 @@ PLUGINS ?= \
 	nri-resource-policy-balloons \
 	nri-resource-policy-template \
 	nri-memory-qos \
-	nri-memtierd
+	nri-memtierd \
+        nri-sgx-config
 
 BINARIES ?= \
 	config-manager

cmd/plugins/sgx-config/Dockerfile

@@ -0,0 +1,22 @@
+ARG GO_VERSION=1.20
+
+FROM golang:${GO_VERSION}-bullseye as builder
+
+WORKDIR /go/builder
+
+# Fetch go dependencies in a separate layer for caching
+COPY go.mod go.sum ./
+COPY pkg/topology/ pkg/topology/
+RUN go mod download
+
+# Build nri-resmgr
+COPY . .
+
+RUN make clean
+RUN make PLUGINS=nri-sgx-config build-plugins-static
+
+FROM gcr.io/distroless/static
+
+COPY --from=builder /go/builder/build/bin/nri-sgx-config /bin/nri-sgx-config
+
+ENTRYPOINT ["/bin/nri-sgx-config", "-idx", "40"]

cmd/plugins/sgx-config/nri-sgx-config-deployment.yaml.in

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: nri-sgx-config
+  name: nri-sgx-config
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: nri-sgx-config
+  template:
+    metadata:
+      labels:
+        app: nri-sgx-config
+    spec:
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      containers:
+        - name: nri-sgx-config
+          command:
+            - nri-sgx-config
+            - --idx
+            - "40"
+          image: IMAGE_PLACEHOLDER
+          imagePullPolicy: IfNotPresent
+          resources:
+            requests:
+              cpu: 100m
+              memory: 100Mi
+          volumeMounts:
+          - name: nri-sockets-vol
+            mountPath: /var/run/nri
+      volumes:
+      - name: nri-sockets-vol
+        hostPath:
+          path: /var/run/nri
+          type: Directory

cmd/plugins/sgx-config/sgx-config.go

@@ -0,0 +1,184 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/yaml"
+
+	"github.com/containerd/nri/pkg/api"
+	"github.com/containerd/nri/pkg/stub"
+)
+
+const (
+	// Key used for SGX annotations.
+	sgxEpcKey = "sgx-epc.nri.io"
+)
+
+var (
+	log     *logrus.Logger
+	verbose bool
+)
+
+// our injector plugin
+type plugin struct {
+	stub stub.Stub
+}
+
+// CreateContainer handles container creation requests.
+func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
+	var (
+		ctrName string
+		sgxEpc  uint64
+		err     error
+	)
+
+	ctrName = containerName(pod, container)
+
+	if verbose {
+		dump("CreateContainer", "pod", pod, "container", container)
+	} else {
+		log.Infof("CreateContainer %s", ctrName)
+	}
+
+	adjust := &api.ContainerAdjustment{}
+
+	sgxEpc, err = parseSgxEpc(container.Name, pod.Annotations)
+	if err != nil {
+		log.Errorf("failed to parse SGX EPC annotation: %v", err)
+		return nil, nil, err
+	}
+
+	if sgxEpc > 0 {
+		adjust.AddLinuxUnified("misc.max", "sgx_epc "+strconv.FormatUint(sgxEpc, 10))
+
+		if verbose {
+			dump(ctrName, "ContainerAdjustment", adjust)
+		} else {
+			log.Infof("adjusted SGX EPC to %d", sgxEpc)
+		}
+	} else {
+		log.Infof("no SGX EPC annotations")
+	}
+
+	return adjust, nil, nil
+}
+
+func parseSgxEpc(ctr string, annotations map[string]string) (uint64, error) {
+	// check container-specific or pod-global SGX EPC annotation and parse it
+	for _, key := range []string{
+		sgxEpcKey + "/container." + ctr,
+		sgxEpcKey + "/pod",
+		sgxEpcKey,
+	} {
+		if value, ok := annotations[key]; ok {
+			epc, err := strconv.ParseUint(value, 10, 64)
+			if err != nil {
+				return 0, fmt.Errorf("failed to parse annotation %s: %w", value, err)
+			}
+			return epc, nil
+		}
+	}
+
+	return 0, nil
+}
+
+// Construct a container name for log messages.
+func containerName(pod *api.PodSandbox, container *api.Container) string {
+	if pod != nil {
+		return pod.Name + "/" + container.Name
+	}
+	return container.Name
+}
+
+// Dump one or more objects, with an optional global prefix and per-object tags.
+func dump(args ...interface{}) {
+	var (
+		prefix string
+		idx    int
+	)
+
+	if len(args)&0x1 == 1 {
+		prefix = args[0].(string)
+		idx++
+	}
+
+	for ; idx < len(args)-1; idx += 2 {
+		tag, obj := args[idx], args[idx+1]
+		msg, err := yaml.Marshal(obj)
+		if err != nil {
+			log.Infof("%s: %s: failed to dump object: %v", prefix, tag, err)
+			continue
+		}
+
+		if prefix != "" {
+			log.Infof("%s: %s:", prefix, tag)
+			for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") {
+				log.Infof("%s:    %s", prefix, line)
+			}
+		} else {
+			log.Infof("%s:", tag)
+			for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") {
+				log.Infof("  %s", line)
+			}
+		}
+	}
+}
+
+func main() {
+	var (
+		pluginName string
+		pluginIdx  string
+		opts       []stub.Option
+		err        error
+	)
+
+	log = logrus.StandardLogger()
+	log.SetFormatter(&logrus.TextFormatter{
+		PadLevelText: true,
+	})
+
+	flag.StringVar(&pluginName, "name", "", "plugin name to register to NRI")
+	flag.StringVar(&pluginIdx, "idx", "", "plugin index to register to NRI")
+	flag.BoolVar(&verbose, "verbose", false, "enable (more) verbose logging")
+	flag.Parse()
+
+	if pluginName != "" {
+		opts = append(opts, stub.WithPluginName(pluginName))
+	}
+	if pluginIdx != "" {
+		opts = append(opts, stub.WithPluginIdx(pluginIdx))
+	}
+
+	p := &plugin{}
+	if p.stub, err = stub.New(p, opts...); err != nil {
+		log.Fatalf("failed to create plugin stub: %v", err)
+	}
+
+	err = p.stub.Run(context.Background())
+	if err != nil {
+		log.Errorf("plugin exited with error %v", err)
+		os.Exit(1)
+	}
+}

deployment/helm/sgx-config/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+appVersion: unstable
+description: |
+  The sgx-config NRI plugin allows control over SGX resource usage using the
+  cgroup v2 misc controller and pod annotations.
+name: nri-sgx-config
+sources:
+ - https://github.com/containers/nri-plugins
+home: https://github.com/containers/nri-plugins
+type: application
+version: v0.0.0

deployment/helm/sgx-config/templates/_helpers.tpl

@@ -0,0 +1,16 @@
+{{/*
+Common labels
+*/}}
+{{- define "sgx-config.labels" -}}
+helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{ include "sgx-config.selectorLabels" . }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "sgx-config.selectorLabels" -}}
+app.kubernetes.io/name: nri-sgx-config
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}

deployment/helm/sgx-config/templates/daemonset.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    {{- include "sgx-config.labels" . | nindent 4 }}
+  name: nri-sgx-config
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+    {{- include "sgx-config.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+      {{- include "sgx-config.labels" . | nindent 8 }}
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      {{- if .Values.nri.patchRuntimeConfig }}
+      initContainers:
+      - name: patch-runtime
+        image: {{ .Values.initContainerImage.name }}:{{ .Values.initContainerImage.tag | default .Chart.AppVersion }}
+        imagePullPolicy: {{ .Values.initContainerImage.pullPolicy }}
+        volumeMounts:
+        - name: containerd-config
+          mountPath: /etc/containerd
+        - name: crio-config
+          mountPath: /etc/crio/crio.conf.d
+        - name: dbus-socket
+          mountPath: /var/run/dbus/system_bus_socket
+        securityContext:
+          privileged: true
+      {{- end }}
+      containers:
+        - name: nri-sgx-config
+          command:
+            - nri-sgx-config
+            - --idx
+            - "40"
+          image: {{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          resources:
+            requests:
+              cpu: {{ .Values.resources.cpu }}
+              memory: {{ .Values.resources.memory }}
+          volumeMounts:
+          - name: nrisockets
+            mountPath: /var/run/nri
+      volumes:
+      - name: nrisockets
+        hostPath:
+          path: /var/run/nri
+          type: Directory
+      {{- if .Values.nri.patchRuntimeConfig }}
+      - name: containerd-config
+        hostPath:
+          path: /etc/containerd/
+          type: DirectoryOrCreate
+      - name: crio-config
+        hostPath:
+          path: /etc/crio/crio.conf.d/
+          type: DirectoryOrCreate
+      - name: dbus-socket
+        hostPath:
+          path: /var/run/dbus/system_bus_socket
+          type: Socket
+      {{- end }}

deployment/helm/sgx-config/values.scheme.json

@@ -0,0 +1,44 @@
+{
+    "$schema": "http://json-schema.org/schema#",
+    "required": [
+        "image",
+        "resources"
+    ],
+    "properties": {
+        "image": {
+            "type": "object",
+            "required": [
+                "name",
+                "tag",
+                "pullPolicy"
+            ],
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "tag": {
+                    "type": "string"
+                },
+                "pullPolicy": {
+                    "type": "string",
+                    "enum": ["Never", "Always", "IfNotPresent"]
+                }
+            }
+        },
+        "resources": {
+            "type": "object",
+            "required": [
+                "cpu",
+                "memory"
+            ],
+            "properties": {
+                "cpu": {
+                    "type": "integer"
+                },
+                "memory": {
+                    "type": "integer"
+                }
+            }
+        }
+    }
+ }