Merge pull request #118 from sipsma/master

Forward agent containerd events to shim and monitor task exits via event subscriptions

.buildkite/pipeline.yml

@@ -32,22 +32,26 @@ steps:
           --target firecracker-containerd-unittest-nonroot \
           --tag localhost/firecracker-containerd-unittest-nonroot:${BUILDKITE_BUILD_NUMBER} .
 
+        docker build \
+          --progress=plain \
+          --file tools/docker/Dockerfile \
+          --target firecracker-containerd-e2etest-naive \
+          --tag localhost/firecracker-containerd-e2etest-naive:${BUILDKITE_BUILD_NUMBER} .
+
   - wait
 
-  - label: ":rotating_light: snapshotter root tests"
+  - label: ":rotating_light: :hammer: snapshotter *root* tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
     artifact_paths:
       - "logs/*"
     command:
       >
-        mkdir $(pwd)/logs
-        && docker run --rm \
+        docker run --rm \
           --privileged \
           --volume /dev:/dev \
           --volume /sys:/sys \
           --volume /run/udev/control:/run/udev/control \
-          --volume $(pwd)/logs:/var/log/firecracker-containerd-test \
           --ipc=host \
           localhost/firecracker-containerd-unittest:${BUILDKITE_BUILD_NUMBER} \
           '
@@ -67,27 +71,43 @@ steps:
             DISABLE_ROOT_TESTS='true' make test EXTRAGOARGS="-v -count=1"
           '
 
-  - label: ":running_shirt_with_sash: runtime tests"
+  - label: ":running_shirt_with_sash: runtime unit tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
     artifact_paths:
       - "logs/*"
     command:
       >
-        mkdir $(pwd)/logs
-        && docker run --rm \
+        docker run --rm \
           --privileged \
           --volume /dev:/dev \
-          --volume /sys:/sys \
-          --volume /run/udev/control:/run/udev/control \
-          --volume $(pwd)/logs:/var/log/firecracker-containerd-test \
-          --ipc=host \
           localhost/firecracker-containerd-unittest:${BUILDKITE_BUILD_NUMBER} \
           '
             cd /firecracker-containerd/runtime
             make test EXTRAGOARGS="-v -count=1"
           '
 
+  - label: ":rotating_light: :running_shirt_with_sash: runtime isolated tests"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+    artifact_paths:
+      - "logs/*"
+    env:
+      FCCD_PACKAGE_DIR: "runtime"
+      FCCD_TESTNAME_REGEX: "_Isolated"
+    command:
+      >
+        mkdir $(pwd)/logs
+        && .buildkite/run-tests-by-name-regex.sh \
+          "--privileged \
+          --volume /dev:/dev \
+          --volume /sys:/sys \
+          --volume /run/udev/control:/run/udev/control \
+          --volume $(pwd)/logs:/var/log/firecracker-containerd-test \
+          --ipc=host \
+          --env ENABLE_ISOLATED_TESTS=1" \
+          "localhost/firecracker-containerd-e2etest-naive:${BUILDKITE_BUILD_NUMBER}"
+
   - label: ":fencer: agent tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
@@ -108,3 +128,15 @@ steps:
             cd /firecracker-containerd/agent
             make test EXTRAGOARGS="-v -count=1"
           '
+
+  - label: ":exclamation: event tests"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+    command:
+      >
+        docker run --rm \
+          localhost/firecracker-containerd-unittest-nonroot:${BUILDKITE_BUILD_NUMBER} \
+          '
+            cd /firecracker-containerd/eventbridge
+            make test EXTRAGOARGS="-v -count=1"
+          '

.buildkite/run-tests-by-name-regex.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -e
+
+DOCKER_RUN_ARGS="$1"
+DOCKER_IMAGE="$2"
+
+# test_names returns (via its stdout) a list of test names that match the provided regular expression
+test_names () {
+	TEST_NAME_REGEX="$1"
+	DOCKER_IMAGE="$2"
+
+	docker run --rm \
+		--env FCCD_PACKAGE_DIR="${FCCD_PACKAGE_DIR}" \
+		"${DOCKER_IMAGE}" \
+		'
+			cd /firecracker-containerd/${FCCD_PACKAGE_DIR}
+			go test -list .
+		' | sed '$d' | grep "${TEST_NAME_REGEX}"
+}
+
+# For each test case with a name matching the provided regex, run each one in its own isolated docker container
+for TESTNAME in $(test_names "${FCCD_TESTNAME_REGEX}" "${DOCKER_IMAGE}"); do
+	docker run --rm \
+		--env TESTNAME="${TESTNAME}" \
+		--env FCCD_PACKAGE_DIR="${FCCD_PACKAGE_DIR}" \
+		${DOCKER_RUN_ARGS} \
+		"${DOCKER_IMAGE}" \
+		'
+			echo "FCCD_PACKAGE_DIR = ${FCCD_PACKAGE_DIR}"
+			echo "TESTNAME = ${TESTNAME}"
+			cd /firecracker-containerd/${FCCD_PACKAGE_DIR}
+			go test -v -count=1 -run "^${TESTNAME}$" .
+		'
+done

agent/main.go

@@ -21,8 +21,9 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/events/exchange"
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/runtime/v2/runc"
 	"github.com/containerd/containerd/runtime/v2/shim"
 	shimapi "github.com/containerd/containerd/runtime/v2/task"
@@ -32,26 +33,19 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sys/unix"
+
+	"github.com/firecracker-microvm/firecracker-containerd/eventbridge"
 )
 
 const (
-	defaultPort = 10789
+	defaultPort      = 10789
+	defaultNamespace = namespaces.Default
 
 	// per prctl(2), we must provide a non-zero arg when calling prctl with
 	// PR_SET_CHILD_SUBREAPER in order to enable subreaping (0 disables it)
 	enableSubreaper = 1
 )
 
-// A fake publisher that we use as a bandaid to prevent nil pointer exceptions when the
-// runc shim tries to publish events.
-// TODO replace this with a publisher that forwards events over vsock to the outer shim
-// on the host.
-type noopPublisher struct{}
-
-func (p *noopPublisher) Publish(ctx context.Context, topic string, event events.Event) error {
-	return nil
-}
-
 func main() {
 	var (
 		id    string
@@ -71,7 +65,7 @@ func main() {
 	signals := make(chan os.Signal, 32)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, unix.SIGCHLD)
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(namespaces.WithNamespace(context.Background(), defaultNamespace))
 	defer cancel()
 
 	group, ctx := errgroup.WithContext(ctx)
@@ -93,7 +87,8 @@ func main() {
 
 	log.G(ctx).WithField("id", id).Info("creating runc shim")
 
-	runcTaskService, err := runc.New(ctx, id, &noopPublisher{})
+	eventExchange := exchange.NewExchange()
+	runcTaskService, err := runc.New(ctx, id, eventExchange)
 	if err != nil {
 		log.G(ctx).WithError(err).Fatal("failed to create runc shim")
 	}
@@ -106,6 +101,7 @@ func main() {
 	}
 
 	shimapi.RegisterTaskService(server, taskService)
+	eventbridge.RegisterGetterService(server, eventbridge.NewGetterService(ctx, eventExchange))
 
 	// Run ttrpc over vsock
 

agent/service.go

@@ -37,7 +37,6 @@ import (
 )
 
 const (
-	defaultNamespace = "default"
 	bundleMountPath  = "/container"
 	defaultStdioPath = "/container/fifo"
 )
@@ -87,7 +86,6 @@ func (ts *TaskService) Create(ctx context.Context, req *shimapi.CreateTaskReques
 	ioctx, cancel := context.WithCancel(ctx)
 	ts.cancels = append(ts.cancels, cancel)
 	ts.proxyStdio(ioctx, req.Stdin, req.Stdout, req.Stderr)
-	ctx = namespaces.WithNamespace(ctx, defaultNamespace)
 	// before create call ensure we remove any existing .init.pid file
 	// We can ignore errors since it's valid for the file to not be present
 	os.Remove(".init.pid")

eventbridge/Makefile

@@ -0,0 +1,28 @@
+# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+EXTRAGOARGS:=
+
+GOMOD := $(shell go env GOMOD)
+GOSUM := $(GOMOD:.mod=.sum)
+SOURCES:=$(shell find . -name '*.go')
+
+all:
+
+test: $(SOURCES) $(GOMOD) $(GOSUM)
+	go test ./... $(EXTRAGOARGS)
+
+clean:
+
+install:
+
+.PHONY: clean all install test