Add paths field to bundle sync configuration

bundle/bundle.go

@@ -39,6 +39,12 @@ type Bundle struct {
 	// Exclusively use this field for filesystem operations.
 	BundleRoot vfs.Path
 
+	// SyncRoot is a virtual filesystem path to the root of the files that
+	// are synchronized to the workspace. It can be an ancestor to [BundleRoot],
+	// but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
+	SyncRoot     vfs.Path
+	SyncRootPath string
+
 	Config config.Root
 
 	// Metadata about the bundle deployment. This is the interface Databricks services

bundle/bundle_read_only.go

@@ -28,6 +28,10 @@ func (r ReadOnlyBundle) BundleRoot() vfs.Path {
 	return r.b.BundleRoot
 }
 
+func (r ReadOnlyBundle) SyncRoot() vfs.Path {
+	return r.b.SyncRoot
+}
+
 func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
 	return r.b.WorkspaceClient()
 }

bundle/config/mutator/configure_wsfs.go

@@ -24,7 +24,7 @@ func (m *configureWSFS) Name() string {
 }
 
 func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	root := b.BundleRoot.Native()
+	root := b.SyncRoot.Native()
 
 	// The bundle root must be located in /Workspace/
 	if !strings.HasPrefix(root, "/Workspace/") {
@@ -45,6 +45,6 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 		return diag.FromErr(err)
 	}
 
-	b.BundleRoot = p
+	b.SyncRoot = p
 	return nil
 }

bundle/config/mutator/rewrite_sync_paths.go

@@ -45,6 +45,10 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc {
 func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) {
+			v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
 			v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
 			if err != nil {
 				return dyn.InvalidValue, err

bundle/config/mutator/rewrite_sync_paths_test.go

@@ -17,6 +17,10 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
 		RootPath: ".",
 		Config: config.Root{
 			Sync: config.Sync{
+				Paths: []string{
+					".",
+					"../common",
+				},
 				Include: []string{
 					"foo",
 					"bar",
@@ -29,6 +33,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
 		},
 	}
 
+	bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml")
+	bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml")
 	bundletest.SetLocation(b, "sync.include[0]", "./file.yml")
 	bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml")
 	bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml")
@@ -37,6 +43,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
 	assert.NoError(t, diags.Error())
 
+	assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
+	assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
 	assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
 	assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
 	assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])
@@ -48,6 +56,10 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
 		RootPath: "/tmp/dir",
 		Config: config.Root{
 			Sync: config.Sync{
+				Paths: []string{
+					".",
+					"../common",
+				},
 				Include: []string{
 					"foo",
 					"bar",
@@ -60,6 +72,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
 		},
 	}
 
+	bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml")
+	bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml")
 	bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml")
 	bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml")
 	bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml")
@@ -68,6 +82,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
 	assert.NoError(t, diags.Error())
 
+	assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
+	assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
 	assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
 	assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
 	assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])

bundle/config/mutator/sync_default_path.go

@@ -0,0 +1,48 @@
+package mutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type syncDefaultPath struct{}
+
+// SyncDefaultPath configures the default sync path to be equal to the bundle root.
+func SyncDefaultPath() bundle.Mutator {
+	return &syncDefaultPath{}
+}
+
+func (m *syncDefaultPath) Name() string {
+	return "SyncDefaultPath"
+}
+
+func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	isset := false
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		pv, _ := dyn.Get(v, "sync.paths")
+
+		// If the sync paths field is already set, do nothing.
+		// We know it is set if its value is either a nil or a sequence (empty or not).
+		switch pv.Kind() {
+		case dyn.KindNil, dyn.KindSequence:
+			isset = true
+		}
+
+		return v, nil
+	})
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// If the sync paths field is already set, do nothing.
+	if isset {
+		return nil
+	}
+
+	// Set the sync paths to the default value.
+	b.Config.Sync.Paths = []string{"."}
+	return nil
+}

bundle/config/mutator/sync_default_path_test.go

@@ -0,0 +1,82 @@
+package mutator_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) {
+	b := &bundle.Bundle{
+		RootPath: "/tmp/some/dir",
+		Config:   config.Root{},
+	}
+
+	ctx := context.Background()
+	diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath())
+	require.NoError(t, diags.Error())
+	assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
+}
+
+func TestSyncDefaultPath_SkipIfSet(t *testing.T) {
+	tcases := []struct {
+		name   string
+		paths  dyn.Value
+		expect []string
+	}{
+		{
+			name:   "nil",
+			paths:  dyn.V(nil),
+			expect: nil,
+		},
+		{
+			name:   "empty sequence",
+			paths:  dyn.V([]dyn.Value{}),
+			expect: []string{},
+		},
+		{
+			name:   "non-empty sequence",
+			paths:  dyn.V([]dyn.Value{dyn.V("something")}),
+			expect: []string{"something"},
+		},
+	}
+
+	for _, tcase := range tcases {
+		t.Run(tcase.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				RootPath: "/tmp/some/dir",
+				Config:   config.Root{},
+			}
+
+			diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+				err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+					v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping()))
+					if err != nil {
+						return dyn.InvalidValue, err
+					}
+					v, err = dyn.Set(v, "sync.paths", tcase.paths)
+					if err != nil {
+						return dyn.InvalidValue, err
+					}
+					return v, nil
+				})
+				return diag.FromErr(err)
+			})
+			require.NoError(t, diags.Error())
+
+			ctx := context.Background()
+			diags = bundle.Apply(ctx, b, mutator.SyncDefaultPath())
+			require.NoError(t, diags.Error())
+
+			// If the sync paths field is already set, do nothing.
+			assert.Equal(t, tcase.expect, b.Config.Sync.Paths)
+		})
+	}
+}

bundle/config/mutator/sync_infer_root.go

@@ -0,0 +1,120 @@
+package mutator
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/vfs"
+)
+
+type syncInferRoot struct{}
+
+// SyncInferRoot is a mutator that infers the root path of all files to synchronize by looking at the
+// paths in the sync configuration. The sync root may be different from the bundle root
+// when the user intends to synchronize files outside the bundle root.
+//
+// The sync root can be equivalent to or an ancestor of the bundle root, but not a descendant.
+// That is, the sync root must contain the bundle root.
+//
+// This mutator requires all sync-related paths and patterns to be relative to the bundle root path.
+// This is done by the [RewriteSyncPaths] mutator, which must run before this mutator.
+func SyncInferRoot() bundle.Mutator {
+	return &syncInferRoot{}
+}
+
+func (m *syncInferRoot) Name() string {
+	return "SyncInferRoot"
+}
+
+// computeRoot finds the innermost path that contains the specified path.
+// It traverses up the root path until it finds the innermost path.
+// If the path does not exist, it returns an empty string.
+//
+// See "sync_infer_root_internal_test.go" for examples.
+func (m *syncInferRoot) computeRoot(path string, root string) string {
+	for !filepath.IsLocal(path) {
+		// Break if we have reached the root of the filesystem.
+		dir := filepath.Dir(root)
+		if dir == root {
+			return ""
+		}
+
+		// Update the sync path as we navigate up the directory tree.
+		path = filepath.Join(filepath.Base(root), path)
+
+		// Move up the directory tree.
+		root = dir
+	}
+
+	return filepath.Clean(root)
+}
+
+func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+
+	// Use the bundle root path as the starting point for inferring the sync root path.
+	bundleRootPath := filepath.Clean(b.RootPath)
+
+	// Infer the sync root path by looking at each one of the sync paths.
+	// Every sync path must be a descendant of the final sync root path.
+	syncRootPath := bundleRootPath
+	for _, path := range b.Config.Sync.Paths {
+		computedPath := m.computeRoot(path, bundleRootPath)
+		if computedPath == "" {
+			continue
+		}
+
+		// Update sync root path if the computed root path is an ancestor of the current sync root path.
+		if len(computedPath) < len(syncRootPath) {
+			syncRootPath = computedPath
+		}
+	}
+
+	// The new sync root path can only be an ancestor of the previous root path.
+	// Compute the relative path from the sync root to the bundle root.
+	rel, err := filepath.Rel(syncRootPath, bundleRootPath)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// If during computation of the sync root path we hit the root of the filesystem,
+	// then one or more of the sync paths are outside the filesystem.
+	// Check if this happened by verifying that none of the paths escape the root
+	// when joined with the sync root path.
+	for i, path := range b.Config.Sync.Paths {
+		if filepath.IsLocal(filepath.Join(rel, path)) {
+			continue
+		}
+
+		diags = append(diags, diag.Diagnostic{
+			Severity:  diag.Error,
+			Summary:   fmt.Sprintf("invalid sync path %q", path),
+			Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)),
+			Paths:     []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))},
+		})
+	}
+
+	if diags.HasError() {
+		return diags
+	}
+
+	// Update all paths in the sync configuration to be relative to the sync root.
+	for i, p := range b.Config.Sync.Paths {
+		b.Config.Sync.Paths[i] = filepath.Join(rel, p)
+	}
+	for i, p := range b.Config.Sync.Include {
+		b.Config.Sync.Include[i] = filepath.Join(rel, p)
+	}
+	for i, p := range b.Config.Sync.Exclude {
+		b.Config.Sync.Exclude[i] = filepath.Join(rel, p)
+	}
+
+	// Configure the sync root path.
+	b.SyncRoot = vfs.MustNew(syncRootPath)
+	b.SyncRootPath = syncRootPath
+	return nil
+}