feat: expand Nydusify to support encrypted Nydus images

* convert: generate encrypted Ndyus images from OCI images.
* check: check encrypted Nydus images.
* mount: mount encrypted Nydus images.
* build: build Nydus image with encrypted data blobs from a
source directory.

Signed-off-by: taohong <[email protected]>

contrib/nydusify/cmd/nydusify.go

@@ -425,6 +425,13 @@ func main() {
 					Usage:   "File path to save the metrics collected during conversion in JSON format, for example: './output.json'",
 					EnvVars: []string{"OUTPUT_JSON"},
 				},
+				&cli.StringSliceFlag{
+					Name:  "encrypt-recipients",
+					Value: nil,
+					Usage: "Recipients to encrypt the nydus bootstrap layer, like " +
+						"jwe:<public-key-file-path>, provider:<cmd/gprc>, pgp:<email-address>, pkcs7:<x509-file-path>",
+					EnvVars: []string{"ENCRYPT_RECIPIENTS"},
+				},
 			},
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
@@ -507,14 +514,15 @@ func main() {
 					ChunkDictRef:      chunkDictRef,
 					ChunkDictInsecure: c.Bool("chunk-dict-insecure"),
 
-					PrefetchPatterns: prefetchPatterns,
-					MergePlatform:    c.Bool("merge-platform"),
-					Docker2OCI:       docker2OCI,
-					FsVersion:        fsVersion,
-					FsAlignChunk:     c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
-					Compressor:       c.String("compressor"),
-					ChunkSize:        c.String("chunk-size"),
-					BatchSize:        c.String("batch-size"),
+					PrefetchPatterns:  prefetchPatterns,
+					MergePlatform:     c.Bool("merge-platform"),
+					Docker2OCI:        docker2OCI,
+					FsVersion:         fsVersion,
+					FsAlignChunk:      c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
+					Compressor:        c.String("compressor"),
+					ChunkSize:         c.String("chunk-size"),
+					BatchSize:         c.String("batch-size"),
+					EncryptRecipients: c.StringSlice("encrypt-recipients"),
 
 					OCIRef:       c.Bool("oci-ref"),
 					WithReferrer: c.Bool("with-referrer"),
@@ -606,6 +614,12 @@ func main() {
 					Usage:   "Path to the nydusd binary, default to search in PATH",
 					EnvVars: []string{"NYDUSD"},
 				},
+				&cli.StringSliceFlag{
+					Name:    "decrypt-keys",
+					Value:   nil,
+					Usage:   "Keys to decrypt nydus bootstrap layer.",
+					EnvVars: []string{"DECRYPT_KEYS"},
+				},
 			},
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
@@ -632,6 +646,7 @@ func main() {
 					BackendType:    backendType,
 					BackendConfig:  backendConfig,
 					ExpectedArch:   arch,
+					DecryptKeys:    c.StringSlice("decrypt-keys"),
 				})
 				if err != nil {
 					return err
@@ -765,6 +780,12 @@ func main() {
 					Usage:   "The nydusd binary path, if unset, search in PATH environment",
 					EnvVars: []string{"NYDUSD"},
 				},
+				&cli.StringSliceFlag{
+					Name:    "decrypt-keys",
+					Value:   nil,
+					Usage:   "Keys to decrypt nydus bootstrap layer.",
+					EnvVars: []string{"DECRYPT_KEYS"},
+				},
 			},
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
@@ -809,6 +830,7 @@ func main() {
 					BackendType:    backendType,
 					BackendConfig:  backendConfig,
 					ExpectedArch:   arch,
+					DecryptKeys:    c.StringSlice("decrypt-keys"),
 				})
 				if err != nil {
 					return err
@@ -921,6 +943,14 @@ func main() {
 					Usage:   "Path to the nydus-image binary, default to search in PATH",
 					EnvVars: []string{"NYDUS_IMAGE"},
 				},
+
+				&cli.StringSliceFlag{
+					Name:  "encrypt-recipients",
+					Value: nil,
+					Usage: "Recipients to encrypt the nydus bootstrap layer, like " +
+						"jwe:<public-key-file-path>, provider:<cmd/gprc>, pgp:<email-address>, pkcs7:<x509-file-path>",
+					EnvVars: []string{"ENCRYPT_RECIPIENTS"},
+				},
 			},
 			Before: func(ctx *cli.Context) error {
 				sourcePath := ctx.String("source-dir")
@@ -958,10 +988,11 @@ func main() {
 				}
 
 				if p, err = packer.New(packer.Opt{
-					LogLevel:       logrus.GetLevel(),
-					NydusImagePath: c.String("nydus-image"),
-					OutputDir:      c.String("output-dir"),
-					BackendConfig:  backendConfig,
+					LogLevel:          logrus.GetLevel(),
+					NydusImagePath:    c.String("nydus-image"),
+					OutputDir:         c.String("output-dir"),
+					BackendConfig:     backendConfig,
+					EncryptRecipients: c.StringSlice("encrypt-recipients"),
 				}); err != nil {
 					return err
 				}
@@ -978,6 +1009,7 @@ func main() {
 					Parent:            c.String("parent-bootstrap"),
 					TryCompact:        c.Bool("compact"),
 					CompactConfigPath: c.String("compact-config-file"),
+					Encrypt:           len(c.StringSlice("encrypt-recipients")) != 0,
 				}); err != nil {
 					return err
 				}

contrib/nydusify/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
 	github.com/containerd/containerd v1.7.2
+	github.com/containers/ocicrypt v1.1.7
 	github.com/docker/cli v23.0.3+incompatible
 	github.com/docker/distribution v2.8.2+incompatible
 	github.com/goharbor/acceleration-service v0.2.6
@@ -58,7 +59,6 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containerd/ttrpc v1.2.2 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
-	github.com/containers/ocicrypt v1.1.7 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

contrib/nydusify/pkg/build/builder.go

@@ -29,6 +29,7 @@ type BuilderOption struct {
 	Compressor   string
 	ChunkSize    string
 	FsVersion    string
+	Encrypt      bool
 }
 
 type CompactOption struct {
@@ -143,6 +144,10 @@ func (builder *Builder) Run(option BuilderOption) error {
 		args = append(args, "--chunk-size", option.ChunkSize)
 	}
 
+	if option.Encrypt {
+		args = append(args, "--encrypt")
+	}
+
 	args = append(args, option.RootfsPath)
 
 	return builder.run(args, option.PrefetchPatterns)

contrib/nydusify/pkg/checker/checker.go

@@ -34,6 +34,7 @@ type Opt struct {
 	BackendType    string
 	BackendConfig  string
 	ExpectedArch   string
+	DecryptKeys    []string
 }
 
 // Checker validates Nydus image manifest, bootstrap and mounts filesystem
@@ -119,7 +120,7 @@ func (checker *Checker) check(ctx context.Context) error {
 		return errors.Wrap(err, "create work directory")
 	}
 
-	if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir); err != nil {
+	if err := checker.Output(ctx, sourceParsed, targetParsed, checker.WorkDir, checker.Opt); err != nil {
 		return errors.Wrap(err, "output image information")
 	}
 

contrib/nydusify/pkg/checker/output.go

@@ -28,7 +28,7 @@ func prettyDump(obj interface{}, name string) error {
 // Output outputs OCI and Nydus image manifest, index, config to JSON file.
 // Prefer to use source image to output OCI image information.
 func (checker *Checker) Output(
-	ctx context.Context, sourceParsed, targetParsed *parser.Parsed, outputPath string,
+	ctx context.Context, sourceParsed, targetParsed *parser.Parsed, outputPath string, opt Opt,
 ) error {
 	logrus.Infof("Dumping OCI and Nydus manifests to %s", outputPath)
 
@@ -87,6 +87,14 @@ func (checker *Checker) Output(
 		}
 		defer bootstrapReader.Close()
 
+		if len(opt.DecryptKeys) != 0 && utils.IsEncryptedNydusImage(&targetParsed.NydusImage.Manifest) {
+			logrus.Infof("Decrypting Nydus bootstrap layer")
+			bootstrapReader, err = checker.targetParser.DecryptNydusBootstrap(ctx, bootstrapReader, targetParsed.NydusImage, opt.DecryptKeys)
+			if err != nil {
+				return errors.Wrap(err, "decrypt Nydus bootstrap layer")
+			}
+		}
+
 		if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, target); err != nil {
 			return errors.Wrap(err, "unpack Nydus bootstrap layer")
 		}

contrib/nydusify/pkg/converter/config.go

@@ -6,6 +6,7 @@ package converter
 
 import (
 	"strconv"
+	"strings"
 )
 
 func getConfig(opt Opt) map[string]string {
@@ -36,5 +37,7 @@ func getConfig(opt Opt) map[string]string {
 	cfg["cache_version"] = opt.CacheVersion
 	cfg["cache_max_records"] = strconv.FormatUint(uint64(opt.CacheMaxRecords), 10)
 
+	cfg["encrypt_recipients"] = strings.Join(opt.EncryptRecipients, ",")
+
 	return cfg
 }

contrib/nydusify/pkg/converter/converter.go

@@ -36,16 +36,17 @@ type Opt struct {
 	BackendConfig    string
 	BackendForcePush bool
 
-	MergePlatform    bool
-	Docker2OCI       bool
-	FsVersion        string
-	FsAlignChunk     bool
-	Compressor       string
-	ChunkSize        string
-	BatchSize        string
-	PrefetchPatterns string
-	OCIRef           bool
-	WithReferrer     bool
+	MergePlatform     bool
+	Docker2OCI        bool
+	FsVersion         string
+	FsAlignChunk      bool
+	Compressor        string
+	ChunkSize         string
+	BatchSize         string
+	PrefetchPatterns  string
+	OCIRef            bool
+	WithReferrer      bool
+	EncryptRecipients []string
 
 	AllPlatforms bool
 	Platforms    string

contrib/nydusify/pkg/packer/packer.go

@@ -28,10 +28,11 @@ var (
 )
 
 type Opt struct {
-	LogLevel       logrus.Level
-	NydusImagePath string
-	OutputDir      string
-	BackendConfig  BackendConfig
+	LogLevel          logrus.Level
+	NydusImagePath    string
+	OutputDir         string
+	BackendConfig     BackendConfig
+	EncryptRecipients []string
 }
 
 type Builder interface {
@@ -63,6 +64,7 @@ type PackRequest struct {
 	Parent            string
 	TryCompact        bool
 	CompactConfigPath string
+	Encrypt           bool
 }
 
 type PackResult struct {
@@ -253,6 +255,7 @@ func (p *Packer) Pack(_ context.Context, req PackRequest) (PackResult, error) {
 		Compressor:          req.Compressor,
 		ChunkSize:           req.ChunkSize,
 		FsVersion:           req.FsVersion,
+		Encrypt:             req.Encrypt,
 	}); err != nil {
 		return PackResult{}, errors.Wrapf(err, "failed to build image from directory %s", req.SourceDir)
 	}

contrib/nydusify/pkg/parser/parser.go

@@ -9,8 +9,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"strings"
 
+	"github.com/containers/ocicrypt"
+	enchelpers "github.com/containers/ocicrypt/helpers"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
 
@@ -66,7 +69,8 @@ func FindNydusBootstrapDesc(manifest *ocispec.Manifest) *ocispec.Descriptor {
 	if len(layers) != 0 {
 		desc := &layers[len(layers)-1]
 		if (desc.MediaType == ocispec.MediaTypeImageLayerGzip ||
-			desc.MediaType == images.MediaTypeDockerSchema2LayerGzip) &&
+			desc.MediaType == images.MediaTypeDockerSchema2LayerGzip ||
+			desc.MediaType == images.MediaTypeImageLayerGzipEncrypted) &&
 			desc.Annotations[utils.LayerAnnotationNydusBootstrap] == "true" {
 			return desc
 		}
@@ -174,6 +178,21 @@ func (parser *Parser) PullNydusBootstrap(ctx context.Context, image *Image) (io.
 	return reader, nil
 }
 
+// Decrypt Nydus bootstrap layer if decryption key is provided.
+func (parser *Parser) DecryptNydusBootstrap(ctx context.Context, reader io.Reader, image *Image, decryptKeys []string) (io.ReadCloser, error) {
+	bootstrapDesc := FindNydusBootstrapDesc(&image.Manifest)
+	dcc, err := enchelpers.CreateCryptoConfig([]string{}, decryptKeys)
+	if err != nil {
+		return nil, errors.Wrap(err, "Create crypto config failed")
+	}
+
+	resultReader, _, err := ocicrypt.DecryptLayer(dcc.DecryptConfig, reader, *bootstrapDesc, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "Decrypt Nydus bootstrap layer")
+	}
+	return ioutil.NopCloser(resultReader), nil
+}
+
 func (parser *Parser) matchImagePlatform(desc *ocispec.Descriptor) bool {
 	if parser.interestedArch == desc.Platform.Architecture && desc.Platform.OS == "linux" {
 		return true

contrib/nydusify/pkg/utils/constant.go

@@ -17,6 +17,7 @@ const (
 	LayerAnnotationNydusBootstrap     = "containerd.io/snapshot/nydus-bootstrap"
 	LayerAnnotationNydusFsVersion     = "containerd.io/snapshot/nydus-fs-version"
 	LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
+	LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"
 
 	LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"
 

contrib/nydusify/pkg/utils/utils.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/archive/compression"
+	"github.com/containerd/containerd/images"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -224,3 +225,37 @@ func HashFile(path string) ([]byte, error) {
 
 	return hasher.Sum(nil), nil
 }
+
+// IsEncryptedNydusImage checks if the nydus image is encrypted.
+func IsEncryptedNydusImage(manifest *ocispec.Manifest) bool {
+	layers := manifest.Layers
+	if len(layers) != 0 {
+		desc := layers[len(layers)-1]
+		if IsEncryptedNydusBootstrap(desc) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsEncryptedNydusBlob returns true when the specified descriptor is nydus encrypted blob.
+func IsEncryptedNydusBlob(desc ocispec.Descriptor) bool {
+	if desc.Annotations == nil {
+		return false
+	}
+
+	_, hasAnno := desc.Annotations[LayerAnnotationNydusEncryptedBlob]
+	return hasAnno
+}
+
+// IsEncryptedNydusBootstrap returns true when the specified descriptor is nydus encrypted bootstrap.
+func IsEncryptedNydusBootstrap(desc ocispec.Descriptor) bool {
+	if desc.Annotations == nil {
+		return false
+	}
+
+	_, hasAnno := desc.Annotations[LayerAnnotationNydusBootstrap]
+	encrypted := desc.MediaType == images.MediaTypeImageLayerEncrypted ||
+		desc.MediaType == images.MediaTypeImageLayerGzipEncrypted
+	return encrypted && hasAnno
+}