diff --git a/config/config.go b/config/config.go index 521497cf52..0315c2d3a0 100644 --- a/config/config.go +++ b/config/config.go @@ -118,13 +118,14 @@ type CgroupConfig struct { // Configure how to start and recover nydusd daemons type DaemonConfig struct { - NydusdPath string `toml:"nydusd_path"` - NydusdConfigPath string `toml:"nydusd_config"` - NydusImagePath string `toml:"nydusimage_path"` - RecoverPolicy string `toml:"recover_policy"` - FsDriver string `toml:"fs_driver"` - ThreadsNumber int `toml:"threads_number"` - LogRotationSize int `toml:"log_rotation_size"` + NydusdPath string `toml:"nydusd_path"` + NydusdConfigPath string `toml:"nydusd_config"` + NydusImagePath string `toml:"nydusimage_path"` + RecoverPolicy string `toml:"recover_policy"` + FsDriver string `toml:"fs_driver"` + ThreadsNumber int `toml:"threads_number"` + LogRotationSize int `toml:"log_rotation_size"` + EnableChunkDeduplication bool `toml:"enable_chunk_deduplication"` } type LoggingConfig struct { @@ -326,6 +327,10 @@ func ParseParameters(args *flags.Args, cfg *SnapshotterConfig) error { daemonConfig.FsDriver = args.FsDriver } + if args.EnableChunkDeduplicationCount > 0 { + daemonConfig.EnableChunkDeduplication = args.EnableChunkDeduplication + } + // --- cache manager configuration // empty diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 0ecb22344c..83c8fa81cd 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -38,19 +38,21 @@ type DaemonConfig interface { } // Daemon configurations factory -func NewDaemonConfig(fsDriver, path string) (DaemonConfig, error) { +func NewDaemonConfig(fsDriver, path string, enableDeduplication bool) (DaemonConfig, error) { switch fsDriver { case config.FsDriverFscache: cfg, err := LoadFscacheConfig(path) if err != nil { return nil, err } + cfg.Config.DeduplicationConfig.Enable = enableDeduplication return cfg, nil case config.FsDriverFusedev: cfg, err := LoadFuseConfig(path) if err != nil { return nil, err } + cfg.Device.Deduplication.Enable = enableDeduplication return cfg, nil default: return nil, errors.Errorf("unsupported, fs driver %q", fsDriver) @@ -118,6 +120,10 @@ type DeviceConfig struct { DisableIndexedMap bool `json:"disable_indexed_map"` } `json:"config"` } `json:"cache"` + Deduplication struct { + Enable bool `json:"enable"` + WorkDir string `json:"work_dir"` + } `json:"deduplication"` } // For nydusd as FUSE daemon. Serialize Daemon info and persist to a json file diff --git a/config/daemonconfig/fscache.go b/config/daemonconfig/fscache.go index cc7bcde081..e97ebdfb21 100644 --- a/config/daemonconfig/fscache.go +++ b/config/daemonconfig/fscache.go @@ -45,6 +45,10 @@ type FscacheDaemonConfig struct { CacheConfig struct { WorkDir string `json:"work_dir"` } `json:"cache_config"` + DeduplicationConfig struct { + Enable bool `json:"enable"` + WorkDir string `json:"work_dir"` + } `json:"deduplication_config"` BlobPrefetchConfig BlobPrefetchConfig `json:"prefetch_config"` MetadataPath string `json:"metadata_path"` } `json:"config"` diff --git a/config/global.go b/config/global.go index 56c7ce294f..9b0c10bfe3 100644 --- a/config/global.go +++ b/config/global.go @@ -111,6 +111,10 @@ func GetDaemonProfileCPUDuration() int64 { return globalConfig.origin.SystemControllerConfig.DebugConfig.ProfileDuration } +func GetEnableChunkDeduplication() bool { + return globalConfig.origin.DaemonConfig.EnableChunkDeduplication +} + func ProcessConfigurations(c *SnapshotterConfig) error { if c.LoggingConfig.LogDir == "" { c.LoggingConfig.LogDir = filepath.Join(c.Root, logging.DefaultLogDirName) diff --git a/internal/flags/flags.go b/internal/flags/flags.go index ef28387081..abe31da493 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -13,18 +13,20 @@ import ( ) type Args struct { - Address string - NydusdConfigPath string - SnapshotterConfigPath string - RootDir string - NydusdPath string - NydusImagePath string - DaemonMode string - FsDriver string - LogLevel string - LogToStdout bool - LogToStdoutCount int - PrintVersion bool + Address string + NydusdConfigPath string + SnapshotterConfigPath string + RootDir string + NydusdPath string + NydusImagePath string + DaemonMode string + FsDriver string + LogLevel string + LogToStdout bool + LogToStdoutCount int + PrintVersion bool + EnableChunkDeduplication bool + EnableChunkDeduplicationCount int } type Flags struct { @@ -97,6 +99,13 @@ func buildFlags(args *Args) []cli.Flag { Usage: "print version and build information", Destination: &args.PrintVersion, }, + &cli.BoolFlag{ + Name: "chunk-deduplication", + Value: false, + Usage: "(experiment) whether to enable local cas chunk deduplication", + Destination: &args.EnableChunkDeduplication, + Count: &args.EnableChunkDeduplicationCount, + }, } } diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index fae78616ee..a1defe600c 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -244,12 +244,14 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { return errors.Wrapf(err, "mount instance %s", rafs.SnapshotID) } + configPath := d.ConfigFile(rafs.SnapshotID) + enableDeduplication := config.GetEnableChunkDeduplication() bootstrap, err := rafs.BootstrapFile() if err != nil { return err } - c, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile(rafs.SnapshotID)) + c, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile(rafs.SnapshotID), enableDeduplication) if err != nil { return errors.Wrapf(err, "Failed to reload instance configuration %s", d.ConfigFile(rafs.SnapshotID)) @@ -260,6 +262,15 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { return errors.Wrap(err, "dump instance configuration") } + // exec static deduplication for bootstrap + if enableDeduplication { + log.L.Debugln("Enable chunk decuplication, sharedFusedevMount, cfg = ", cfg) + bootstrap, err = rafs.DeduplicateBootstrap(bootstrap, configPath) + if err != nil { + return errors.Wrapf(err, "failed to dedup rafs bootstrap") + } + } + err = client.Mount(rafs.RelaMountpoint(), bootstrap, cfg) if err != nil { return errors.Wrapf(err, "mount rafs instance") @@ -279,7 +290,9 @@ func (d *Daemon) sharedErofsMount(rafs *Rafs) error { return errors.Wrapf(err, "failed to create fscache work dir %s", rafs.FscacheWorkDir()) } - c, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile(rafs.SnapshotID)) + configPath := d.ConfigFile(rafs.SnapshotID) + enableDeduplication := config.GetEnableChunkDeduplication() + c, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, configPath, enableDeduplication) if err != nil { log.L.Errorf("Failed to reload daemon configuration %s, %s", d.ConfigFile(rafs.SnapshotID), err) return err @@ -309,6 +322,14 @@ func (d *Daemon) sharedErofsMount(rafs *Rafs) error { rafs.AddAnnotation(AnnoFsCacheDomainID, cfg.DomainID) rafs.AddAnnotation(AnnoFsCacheID, fscacheID) + if enableDeduplication { + log.L.Debugln("Enable chunk deduplication, sharedErofsMount, cfg = ", cfg) + bootstrapPath, err = rafs.DeduplicateBootstrap(bootstrapPath, configPath) + if err != nil { + return errors.Wrapf(err, "dedup rafs bootstrap") + } + } + if err := erofs.Mount(bootstrapPath, cfg.DomainID, fscacheID, mountPoint); err != nil { if !errdefs.IsErofsMounted(err) { return errors.Wrapf(err, "mount erofs to %s", mountPoint) diff --git a/pkg/daemon/rafs.go b/pkg/daemon/rafs.go index 5531ae4fec..a177d96017 100644 --- a/pkg/daemon/rafs.go +++ b/pkg/daemon/rafs.go @@ -8,14 +8,17 @@ package daemon import ( "os" + "os/exec" "path" "path/filepath" + "strings" "sync" "github.com/mohae/deepcopy" "github.com/pkg/errors" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" "github.com/containerd/nydus-snapshotter/config" ) @@ -195,3 +198,38 @@ func (r *Rafs) BootstrapFile() (string, error) { return "", errors.Wrapf(errdefs.ErrNotFound, "bootstrap %s", bootstrap) } + +func buildDeduplicationCommand(bootstrapPath, configPath, nydusImagePath string) *exec.Cmd { + args := []string{ + "dedup", + "--bootstrap", bootstrapPath, + "--config", configPath, + } + + log.L.Infof("start bootstrap deduplication: %s %s", nydusImagePath, strings.Join(args, " ")) + + cmd := exec.Command(nydusImagePath, args...) + + return cmd + +} + +func (r *Rafs) DeduplicateBootstrap(bootstrapPath, configPath string) (string, error) { + nydusImagePath, err := exec.LookPath("nydus-image") + if err != nil { + return "", err + } + + cmd := buildDeduplicationCommand(bootstrapPath, configPath, nydusImagePath) + _, err = cmd.CombinedOutput() + if err != nil { + return "", err + } + + _, err = os.Stat(bootstrapPath + ".dedup") + if err != nil { + return "", err + } + bootstrapPath += ".dedup" + return bootstrapPath, err +} diff --git a/pkg/daemon/rafs_test.go b/pkg/daemon/rafs_test.go new file mode 100644 index 0000000000..706345d95a --- /dev/null +++ b/pkg/daemon/rafs_test.go @@ -0,0 +1,26 @@ +package daemon + +import ( + "fmt" + "strings" + "testing" +) + +func TestBuildDeduplicationCommand(t *testing.T) { + bootstrapPath := "/path/to/bootstrap" + configPath := "/path/to/config.json" + nydusImagePath := "/path/to/nydus-image" + + cmd := buildDeduplicationCommand(bootstrapPath, configPath, nydusImagePath) + + expectedArgs := []string{ + "dedup", + "--bootstrap", bootstrapPath, + "--config", configPath, + } + expectedCmd := fmt.Sprintf("%s %s", nydusImagePath, strings.Join(expectedArgs, " ")) + + if expectedCmd != cmd.String() { + t.Errorf("unexpected command string '%s'", cmd.String()) + } +} diff --git a/pkg/manager/daemon_adaptor.go b/pkg/manager/daemon_adaptor.go index 62ce6e2d04..af8e99989b 100644 --- a/pkg/manager/daemon_adaptor.go +++ b/pkg/manager/daemon_adaptor.go @@ -32,6 +32,20 @@ func (m *Manager) StartDaemon(d *daemon.Daemon) error { return errors.Wrapf(err, "create command for daemon %s", d.ID()) } + // not deduplicate bootstrap for shared daemon mode + if config.GetEnableChunkDeduplication() && !d.IsSharedDaemon() { + rafs := d.Instances.Head() + if rafs == nil { + return errors.Wrapf(errdefs.ErrNotFound, "daemon %s no rafs instance associated", d.ID()) + } + configPath := d.ConfigFile("") + bootstrap, _ := rafs.BootstrapFile() + _, err = rafs.DeduplicateBootstrap(bootstrap, configPath) + if err != nil { + return errors.Errorf("fail to deduplicate bootstrap") + } + } + if err := cmd.Start(); err != nil { return err } @@ -118,7 +132,7 @@ func (m *Manager) BuildDaemonCommand(d *daemon.Daemon, bin string, upgrade bool) var cmdOpts []command.Opt nydusdThreadNum := d.NydusdThreadNum() - + enableDeduplication := config.GetEnableChunkDeduplication() if d.States.FsDriver == config.FsDriverFscache { cmdOpts = append(cmdOpts, command.WithMode("singleton"), @@ -145,6 +159,10 @@ func (m *Manager) BuildDaemonCommand(d *daemon.Daemon, bin string, upgrade bool) return nil, errors.Wrapf(err, "locate bootstrap %s", bootstrap) } + if enableDeduplication { + bootstrap += ".dedup" + } + cmdOpts = append(cmdOpts, command.WithConfig(d.ConfigFile("")), command.WithBootstrap(bootstrap), diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 2ace2fb296..a656cb9fa4 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -476,7 +476,7 @@ func (m *Manager) Recover(ctx context.Context, d.States = *s m.daemonStates.RecoverDaemonState(d) - + enableDeduplication := config.GetEnableChunkDeduplication() if m.SupervisorSet != nil { su := m.SupervisorSet.NewSupervisor(d.ID()) if su == nil { @@ -486,7 +486,7 @@ func (m *Manager) Recover(ctx context.Context, } if d.States.FsDriver == config.FsDriverFusedev { - cfg, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile("")) + cfg, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile(""), enableDeduplication) if err != nil { log.L.Errorf("Failed to reload daemon configuration %s, %s", d.ConfigFile(""), err) return err diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 007ec95755..0bb618dc08 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -69,7 +69,7 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho return nil, errors.Wrap(err, "initialize image verifier") } - daemonConfig, err := daemonconfig.NewDaemonConfig(config.GetFsDriver(), cfg.DaemonConfig.NydusdConfigPath) + daemonConfig, err := daemonconfig.NewDaemonConfig(config.GetFsDriver(), cfg.DaemonConfig.NydusdConfigPath, cfg.DaemonConfig.EnableChunkDeduplication) if err != nil { return nil, errors.Wrap(err, "load daemon configuration") } @@ -781,8 +781,9 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage } var c daemonconfig.DaemonConfig + enableDeduplication := config.GetEnableChunkDeduplication() if daemon.IsSharedDaemon() { - c, err = daemonconfig.NewDaemonConfig(daemon.States.FsDriver, daemon.ConfigFile(instance.SnapshotID)) + c, err = daemonconfig.NewDaemonConfig(daemon.States.FsDriver, daemon.ConfigFile(instance.SnapshotID), enableDeduplication) if err != nil { return nil, errors.Wrapf(err, "Failed to load instance configuration %s", daemon.ConfigFile(instance.SnapshotID)) @@ -811,6 +812,13 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage return nil, errors.Wrapf(err, "remoteMounts: failed to detect filesystem version") } + if enableDeduplication { + configPath := daemon.ConfigFile(instance.SnapshotID) + source, err = instance.DeduplicateBootstrap(source, configPath) + if err != nil { + return nil, errors.Wrapf(err, "remoteMounts: failed to dedup rafs bootstrap") + } + } // when enable nydus-overlayfs, return unified mount slice for runc and kata extraOption := &ExtraOption{ Source: source,