Add support for COPY --exclude and ADD --exclude options

Fixes: #5678

Signed-off-by: Daniel J Walsh <[email protected]>

cmd/buildah/addcopy.go

@@ -37,6 +37,7 @@ type addCopyResults struct {
 	certDir          string
 	retry            int
 	retryDelay       string
+	excludes         []string
 }
 
 func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -78,6 +79,7 @@ func applyFlagVars(flags *pflag.FlagSet, opts *addCopyResults) {
 	if err := flags.MarkHidden("decryption-key"); err != nil {
 		panic(fmt.Sprintf("error marking decryption-key as hidden: %v", err))
 	}
+	flags.StringSliceVar(&opts.excludes, "exclude", nil, "exclude pattern when copying files")
 	flags.StringVar(&opts.ignoreFile, "ignorefile", "", "path to .containerignore file")
 	flags.StringVar(&opts.contextdir, "contextdir", "", "context directory path")
 	flags.IntVar(&opts.retry, "retry", cli.MaxPullPushRetries, "number of times to retry in case of failure when performing pull")
@@ -237,6 +239,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 		PreserveOwnership: preserveOwnership,
 		Checksum:          iopts.checksum,
 		ContextDir:        contextdir,
+		Excludes:          iopts.excludes,
 		IDMappingOptions:  idMappingOptions,
 		// These next two fields are set based on command line flags
 		// with more generic-sounding names.
@@ -251,7 +254,7 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 		if err != nil {
 			return err
 		}
-		options.Excludes = excludes
+		options.Excludes = append(excludes, options.Excludes...)
 	}
 	if iopts.retryDelay != "" {
 		retryDelay, err := time.ParseDuration(iopts.retryDelay)

docs/buildah-add.1.md

@@ -49,6 +49,11 @@ Build context directory. Specifying a context directory causes Buildah to
 chroot into that context directory. This means copying files pointed at
 by symbolic links outside of the chroot will fail.
 
+**--exclude** *pattern*
+
+Exclude coping files mattching the specified pattern. Option can be specified
+multiple times.
+
 **--from** *containerOrImage*
 
 Use the root directory of the specified working container or image as the root

docs/buildah-copy.1.md

@@ -48,6 +48,11 @@ Build context directory. Specifying a context directory causes Buildah to
 chroot into the context directory. This means copying files pointed at
 by symbolic links outside of the chroot will fail.
 
+**--exclude** *pattern*
+
+Exclude coping files mattching the specified pattern. Option can be specified
+multiple times.
+
 **--from** *containerOrImage*
 
 Use the root directory of the specified working container or image as the root
@@ -86,6 +91,8 @@ talking to an insecure registry.
 
 buildah copy containerID '/myapp/app.conf' '/myapp/app.conf'
 
+buildah copy --exclude=**/*.md docs containerID 'docs' '/docs'
+
 buildah copy --chown myuser:mygroup containerID '/myapp/app.conf' '/myapp/app.conf'
 
 buildah copy --chmod 660 containerID '/myapp/app.conf' '/myapp/app.conf'

imagebuildah/stage_executor.go

@@ -373,10 +373,7 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err
 			return errors.New("COPY --parents is not supported")
 		}
 		if len(cp.Excludes) > 0 {
-			if cp.Download {
-				return errors.New("ADD --excludes is not supported")
-			}
-			return errors.New("COPY --excludes is not supported")
+			excludes = append(excludes, cp.Excludes...)
 		}
 	}
 	s.builder.ContentDigester.Restart()
@@ -1325,12 +1322,12 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 		// Also check the chmod and the chown flags for validity.
 		for _, flag := range step.Flags {
 			command := strings.ToUpper(step.Command)
-			// chmod, chown and from flags should have an '=' sign, '--chmod=', '--chown=' or '--from='
-			if command == "COPY" && (flag == "--chmod" || flag == "--chown" || flag == "--from") {
-				return "", nil, false, fmt.Errorf("COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image|stage> flags")
+			// chmod, chown and from flags should have an '=' sign, '--chmod=', '--chown=' or '--from=' or '--exclude='
+			if command == "COPY" && (flag == "--chmod" || flag == "--chown" || flag == "--from" || flag == "--exclude") {
+				return "", nil, false, fmt.Errorf("COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image|stage> --exclude=<pattern> flags")
 			}
-			if command == "ADD" && (flag == "--chmod" || flag == "--chown" || flag == "--checksum") {
-				return "", nil, false, fmt.Errorf("ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags")
+			if command == "ADD" && (flag == "--chmod" || flag == "--chown" || flag == "--checksum" || flag == "--exclude") {
+				return "", nil, false, fmt.Errorf("ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> --exclude=<pattern> flags")
 			}
 			if strings.Contains(flag, "--from") && command == "COPY" {
 				arr := strings.Split(flag, "=")

tests/bud.bats

@@ -3264,7 +3264,7 @@ _EOF
   imgName=alpine-image
   ctrName=alpine-chown
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${imgName} -f $BUDFILES/copy-chown/Dockerfile.bad $BUDFILES/copy-chown
-  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> flags"
+  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> --exclude=<pattern> flags"
 }
 
 @test "bud with chown copy with unknown substitutions in Dockerfile" {
@@ -3292,7 +3292,7 @@ _EOF
   imgName=alpine-image
   ctrName=alpine-chmod
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${imgName} -f $BUDFILES/copy-chmod/Dockerfile.bad $BUDFILES/copy-chmod
-  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> flags"
+  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> --exclude=<pattern> flags"
 }
 
 @test "bud with chmod add" {
@@ -3328,15 +3328,15 @@ _EOF
   imgName=alpine-image
   ctrName=alpine-chown
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${imgName} -f $BUDFILES/add-chown/Dockerfile.bad $BUDFILES/add-chown
-  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags"
+  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> --exclude=<pattern> flags"
 }
 
 @test "bud with chmod add with bad chmod flag in Dockerfile with --layers" {
   _prefetch alpine
   imgName=alpine-image
   ctrName=alpine-chmod
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${imgName} -f $BUDFILES/add-chmod/Dockerfile.bad $BUDFILES/add-chmod
-  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags"
+  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> --exclude=<pattern> flags"
 }
 
 @test "bud with ADD with checksum flag" {
@@ -3359,7 +3359,7 @@ _EOF
   _prefetch alpine
   target=alpine-image
   run_buildah 125 build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/add-checksum/Containerfile.bad $BUDFILES/add-checksum
-  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> flags"
+  expect_output --substring "ADD only supports the --chmod=<permissions>, --chown=<uid:gid>, and --checksum=<checksum> --exclude=<pattern> flags"
 }
 
 @test "bud with ADD file construct" {
@@ -3454,7 +3454,7 @@ _EOF
   imgName=ubuntu-image
   ctrName=ubuntu-copy
   run_buildah 125 build $WITH_POLICY_JSON -f $BUDFILES/copy-multistage-paths/Dockerfile.invalid_from -t ${imgName} $BUDFILES/copy-multistage-paths
-  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> flags"
+  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> --exclude=<pattern> flags"
 }
 
 @test "bud COPY to root succeeds" {
@@ -3603,7 +3603,7 @@ _EOF
   _prefetch busybox
   target=bad-from-flag
   run_buildah 125 build $WITH_POLICY_JSON --layers -t ${target} -f $BUDFILES/copy-from/Dockerfile.bad $BUDFILES/copy-from
-  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> flags"
+  expect_output --substring "COPY only supports the --chmod=<permissions> --chown=<uid:gid> and the --from=<image\|stage> --exclude=<pattern> flags"
 }
 
 @test "bud with copy-from referencing the base image" {
@@ -5939,6 +5939,14 @@ _EOF
   expect_output --substring 'building.*"COPY \*foo /testdir".*no such file or directory'
 }
 
+@test "bud with copy --exclude" {
+  run_buildah build -t test $WITH_POLICY_JSON $BUDFILES/copy-exclude
+  assert "$output" !~ "test1.txt"
+
+  run_buildah build -t test2 -f Containerfile.missing $WITH_POLICY_JSON $BUDFILES/copy-exclude
+  assert "$output" !~ "test2.txt"
+}
+
 @test "bud with containerfile secret" {
   _prefetch alpine
   mytmpdir=${TEST_SCRATCH_DIR}/my-dir1

tests/bud/copy-exclude/Containerfile

@@ -0,0 +1,3 @@
+FROM alpine
+COPY --exclude=*txt . /testdir
+run ls /testdir

tests/bud/copy-exclude/Containerfile.missing

@@ -0,0 +1,3 @@
+FROM alpine
+COPY --exclude=*foo --exclude=test*.txt . /testdir
+run ls /testdir

tests/conformance/conformance_test.go

@@ -2444,6 +2444,77 @@ var internalTestCases = []testCase{
 		compatScratchConfig: types.OptionalBoolTrue,
 	},
 
+	{
+		name: "add--exclude-includes-star",
+		dockerfileContents: strings.Join([]string{
+			"FROM scratch",
+			"ADD --exclude='!**/*-c' * subdir/",
+		}, "\n"),
+		contextDir:          "dockerignore/populated",
+		fsSkip:              []string{"(dir):subdir:mtime"},
+		compatScratchConfig: types.OptionalBoolTrue,
+		dockerUseBuildKit:   true,
+	},
+
+	{
+		name: "add--exclude-includes-dot-slash-star",
+		dockerfileContents: strings.Join([]string{
+			"FROM scratch",
+			"ADD --exclude='!**/*-c' ./* subdir/",
+		}, "\n"),
+		contextDir:          "dockerignore/populated",
+		fsSkip:              []string{"(dir):subdir:mtime"},
+		compatScratchConfig: types.OptionalBoolTrue,
+		dockerUseBuildKit:   true,
+	},
+
+	{
+		name: "add--exclude-includes-slash-star",
+		dockerfileContents: strings.Join([]string{
+			"FROM scratch",
+			"ADD --exclude='!**/*-c' /* subdir/",
+		}, "\n"),
+		contextDir:          "dockerignore/populated",
+		fsSkip:              []string{"(dir):subdir:mtime"},
+		compatScratchConfig: types.OptionalBoolTrue,
+		dockerUseBuildKit:   true,
+	},
+	{
+		name: "copy--exclude-includes-star",
+		dockerfileContents: strings.Join([]string{
+			"FROM scratch",
+			"COPY --exclude='!**/*-c' * subdir/",
+		}, "\n"),
+		contextDir:          "dockerignore/populated",
+		fsSkip:              []string{"(dir):subdir:mtime"},
+		compatScratchConfig: types.OptionalBoolTrue,
+		dockerUseBuildKit:   true,
+	},
+
+	{
+		name: "copy--exclude-includes-dot-slash-star",
+		dockerfileContents: strings.Join([]string{
+			"FROM scratch",
+			"COPY --exclude='!**/*-c' ./* subdir/",
+		}, "\n"),
+		contextDir:          "dockerignore/populated",
+		fsSkip:              []string{"(dir):subdir:mtime"},
+		compatScratchConfig: types.OptionalBoolTrue,
+		dockerUseBuildKit:   true,
+	},
+
+	{
+		name: "copy--exclude-includes-slash-star",
+		dockerfileContents: strings.Join([]string{
+			"FROM scratch",
+			"COPY --exclude='!**/*-c' /* subdir/",
+		}, "\n"),
+		contextDir:          "dockerignore/populated",
+		fsSkip:              []string{"(dir):subdir:mtime"},
+		compatScratchConfig: types.OptionalBoolTrue,
+		dockerUseBuildKit:   true,
+	},
+
 	{
 		name: "dockerignore-includes-star",
 		dockerfileContents: strings.Join([]string{

tests/copy.bats

@@ -359,6 +359,31 @@ stuff/mystuff"
   expect_output --from="$filelist" "$expect" "container file list"
 }
 
+@test "copy --exclude" {
+  mytest=${TEST_SCRATCH_DIR}/mytest
+  mkdir -p ${mytest}
+  touch ${mytest}/mystuff
+  touch ${mytest}/source.go
+  mkdir -p ${mytest}/notmystuff
+  touch ${mytest}/notmystuff/notmystuff
+
+expect="
+stuff
+stuff/mystuff"
+
+  run_buildah from $WITH_POLICY_JSON scratch
+  cid=$output
+
+  run_buildah copy --exclude=**/*.go --exclude=.ignore --exclude=**/notmystuff $cid ${mytest} /stuff
+
+  run_buildah mount $cid
+  mnt=$output
+  run find $mnt -printf "%P\n"
+  filelist=$(LC_ALL=C sort <<<"$output")
+  run_buildah umount $cid
+  expect_output --from="$filelist" "$expect" "container file list"
+}
+
 @test "copy-quiet" {
   createrandom ${TEST_SCRATCH_DIR}/randomfile
   _prefetch alpine