From e93401917d8862255d4bcb7a4ef669c0e889273c Mon Sep 17 00:00:00 2001 From: Jeremy Wei Date: Sun, 21 Jul 2024 21:44:49 -0400 Subject: [PATCH] change export-to-file to a flag in export cmd --- server/export.go | 194 ++++++++++++++++------------------------ server/types/app.go | 8 +- server/util.go | 2 - simapp/simd/cmd/root.go | 10 +-- 4 files changed, 81 insertions(+), 133 deletions(-) diff --git a/server/export.go b/server/export.go index 7445143c..9dcb6e5d 100644 --- a/server/export.go +++ b/server/export.go @@ -19,11 +19,22 @@ import ( ) const ( + FlagIsStreaming = "streaming" + FlagStreamingFile = "streaming-file" FlagHeight = "height" FlagForZeroHeight = "for-zero-height" FlagJailAllowedAddrs = "jail-allowed-addrs" ) +type GenesisDocNoAppState struct { + GenesisTime time.Time `json:"genesis_time"` + ChainID string `json:"chain_id"` + InitialHeight int64 `json:"initial_height,string"` + ConsensusParams *tmtypes.ConsensusParams `json:"consensus_params,omitempty"` + Validators []tmtypes.GenesisValidator `json:"validators,omitempty"` + AppHash tmbytes.HexBytes `json:"app_hash"` +} + // ExportCmd dumps app state to JSON. func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Command { cmd := &cobra.Command{ @@ -40,6 +51,20 @@ func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Com return err } + isStreaming, err := cmd.Flags().GetBool(FlagIsStreaming) + if err != nil { + return err + } + + streamingFile, err := cmd.Flags().GetString(FlagStreamingFile) + if err != nil { + return err + } + + if isStreaming && streamingFile == "" { + return fmt.Errorf("file to export stream to not provided") + } + db, err := openDB(config.RootDir) if err != nil { return err @@ -69,7 +94,56 @@ func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Com forZeroHeight, _ := cmd.Flags().GetBool(FlagForZeroHeight) jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(FlagJailAllowedAddrs) - exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper) + if isStreaming { + file, err := os.Create(streamingFile) + if err != nil { + return err + } + exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, file) + if err != nil { + return fmt.Errorf("error exporting state: %v", err) + } + + doc, err := tmtypes.GenesisDocFromFile(serverCtx.Config.GenesisFile()) + if err != nil { + return err + } + + genesisDocNoAppHash := GenesisDocNoAppState{ + GenesisTime: doc.GenesisTime, + ChainID: doc.ChainID, + AppHash: doc.AppHash, + InitialHeight: exported.Height, + ConsensusParams: &tmtypes.ConsensusParams{ + Block: tmtypes.BlockParams{ + MaxBytes: exported.ConsensusParams.Block.MaxBytes, + MaxGas: exported.ConsensusParams.Block.MaxGas, + }, + Evidence: tmtypes.EvidenceParams{ + MaxAgeNumBlocks: exported.ConsensusParams.Evidence.MaxAgeNumBlocks, + MaxAgeDuration: exported.ConsensusParams.Evidence.MaxAgeDuration, + MaxBytes: exported.ConsensusParams.Evidence.MaxBytes, + }, + Validator: tmtypes.ValidatorParams{ + PubKeyTypes: exported.ConsensusParams.Validator.PubKeyTypes, + }, + }, + Validators: exported.Validators, + } + + // NOTE: Tendermint uses a custom JSON decoder for GenesisDoc + // (except for stuff inside AppState). Inside AppState, we're free + // to encode as protobuf or amino. + encoded, err := json.Marshal(genesisDocNoAppHash) + if err != nil { + return err + } + + file.Write([]byte(fmt.Sprintf("%s", string(sdk.MustSortJSON(encoded))))) + return nil + } + + exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, nil) if err != nil { return fmt.Errorf("error exporting state: %v", err) } @@ -110,122 +184,8 @@ func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Com }, } - cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") - cmd.Flags().Int64(FlagHeight, -1, "Export state from a particular height (-1 means latest height)") - cmd.Flags().Bool(FlagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)") - cmd.Flags().StringSlice(FlagJailAllowedAddrs, []string{}, "Comma-separated list of operator addresses of jailed validators to unjail") - cmd.Flags().String(FlagChainID, "", "Chain ID") - - return cmd -} - -type GenesisDocNoAppState struct { - GenesisTime time.Time `json:"genesis_time"` - ChainID string `json:"chain_id"` - InitialHeight int64 `json:"initial_height,string"` - ConsensusParams *tmtypes.ConsensusParams `json:"consensus_params,omitempty"` - Validators []tmtypes.GenesisValidator `json:"validators,omitempty"` - AppHash tmbytes.HexBytes `json:"app_hash"` -} - -// ExportToFileCmd dumps app state to JSON. It appends the app state module by module to the file. -// This is especially useful when the output is too large to fit in memory. -// TODO: change name to ExportStream, also can make it a flag under export -func ExportToFileCmd(appExporterStream types.AppExporterStream, defaultNodeHome string) *cobra.Command { - cmd := &cobra.Command{ - Use: "export-to-file [jsonfile]", - Short: "Export state to JSON file", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - serverCtx := GetServerContextFromCmd(cmd) - config := serverCtx.Config - filePath := args[0] - file, err := os.Create(filePath) - if err != nil { - return err - } - - homeDir, _ := cmd.Flags().GetString(flags.FlagHome) - config.SetRoot(homeDir) - - if _, err := os.Stat(config.GenesisFile()); os.IsNotExist(err) { - return err - } - - db, err := openDB(config.RootDir) - if err != nil { - return err - } - - if appExporterStream == nil { - if _, err := fmt.Fprintln(os.Stderr, "WARNING: App exporter not defined. Returning genesis file."); err != nil { - return err - } - - genesis, err := ioutil.ReadFile(config.GenesisFile()) - if err != nil { - return err - } - - file.Write(genesis) - return nil - } - - traceWriterFile, _ := cmd.Flags().GetString(flagTraceStore) - traceWriter, err := openTraceWriter(traceWriterFile) - if err != nil { - return err - } - - height, _ := cmd.Flags().GetInt64(FlagHeight) - forZeroHeight, _ := cmd.Flags().GetBool(FlagForZeroHeight) - jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(FlagJailAllowedAddrs) - - exported, err := appExporterStream(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, file) - if err != nil { - return fmt.Errorf("error exporting state: %v", err) - } - - doc, err := tmtypes.GenesisDocFromFile(serverCtx.Config.GenesisFile()) - if err != nil { - return err - } - - genesisDocNoAppHash := GenesisDocNoAppState{ - GenesisTime: doc.GenesisTime, - ChainID: doc.ChainID, - AppHash: doc.AppHash, - InitialHeight: exported.Height, - ConsensusParams: &tmtypes.ConsensusParams{ - Block: tmtypes.BlockParams{ - MaxBytes: exported.ConsensusParams.Block.MaxBytes, - MaxGas: exported.ConsensusParams.Block.MaxGas, - }, - Evidence: tmtypes.EvidenceParams{ - MaxAgeNumBlocks: exported.ConsensusParams.Evidence.MaxAgeNumBlocks, - MaxAgeDuration: exported.ConsensusParams.Evidence.MaxAgeDuration, - MaxBytes: exported.ConsensusParams.Evidence.MaxBytes, - }, - Validator: tmtypes.ValidatorParams{ - PubKeyTypes: exported.ConsensusParams.Validator.PubKeyTypes, - }, - }, - Validators: exported.Validators, - } - - // NOTE: Tendermint uses a custom JSON decoder for GenesisDoc - // (except for stuff inside AppState). Inside AppState, we're free - // to encode as protobuf or amino. - encoded, err := json.Marshal(genesisDocNoAppHash) - if err != nil { - return err - } - - file.Write([]byte(fmt.Sprintf("%s", string(sdk.MustSortJSON(encoded))))) - return nil - }, - } - + cmd.Flags().Bool(FlagIsStreaming, false, "Whether to stream the export in chunks. Useful when genesis is extremely large and cannot fit into memory.") + cmd.Flags().String(FlagStreamingFile, "genesis-stream.json", "The file to export the streamed genesis to") cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") cmd.Flags().Int64(FlagHeight, -1, "Export state from a particular height (-1 means latest height)") cmd.Flags().Bool(FlagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)") diff --git a/server/types/app.go b/server/types/app.go index d6ace439..82ac40cb 100644 --- a/server/types/app.go +++ b/server/types/app.go @@ -85,10 +85,6 @@ type ( // AppExporter is a function that dumps all app state to // JSON-serializable structure and returns the current validator set. - AppExporter func(log.Logger, dbm.DB, io.Writer, int64, bool, []string, AppOptions) (ExportedApp, error) - - // AppExporterStream is a function that dumps all app state to - // JSON-serializable structure and writes the result to a file. - // This is useful when the output is too large to fit in memory. - AppExporterStream func(log.Logger, dbm.DB, io.Writer, int64, bool, []string, AppOptions, *os.File) (ExportedApp, error) + // If a file is specified, + AppExporter func(log.Logger, dbm.DB, io.Writer, int64, bool, []string, AppOptions, *os.File) (ExportedApp, error) ) diff --git a/server/util.go b/server/util.go index 7ee740f2..5158f0e3 100644 --- a/server/util.go +++ b/server/util.go @@ -304,7 +304,6 @@ func AddCommands( defaultNodeHome string, appCreator types.AppCreator, appExport types.AppExporter, - appExportToFile types.AppExporterStream, addStartFlags types.ModuleInitFlags, tracerProviderOptions []trace.TracerProviderOption, ) { @@ -349,7 +348,6 @@ func AddCommands( startCmd, tendermintCmd, ExportCmd(appExport, defaultNodeHome), - ExportToFileCmd(appExportToFile, defaultNodeHome), version.NewVersionCommand(), NewRollbackCmd(appCreator, defaultNodeHome), LatestVersionCmd(defaultNodeHome), diff --git a/simapp/simd/cmd/root.go b/simapp/simd/cmd/root.go index 994d2391..40e9e793 100644 --- a/simapp/simd/cmd/root.go +++ b/simapp/simd/cmd/root.go @@ -170,7 +170,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { panic(err) } - server.AddCommands(rootCmd, simapp.DefaultNodeHome, a.newApp, a.appExport, a.appExporterStream, addModuleInitFlags, []trace.TracerProviderOption{ + server.AddCommands(rootCmd, simapp.DefaultNodeHome, a.newApp, a.appExport, addModuleInitFlags, []trace.TracerProviderOption{ trace.WithBatcher(exp), // Record information about this application in a Resource. trace.WithResource(resource.NewWithAttributes( @@ -313,7 +313,7 @@ func (a appCreator) newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, t // and exports state. func (a appCreator) appExport( logger log.Logger, db dbm.DB, traceStore io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string, - appOpts servertypes.AppOptions) (servertypes.ExportedApp, error) { + appOpts servertypes.AppOptions, file *os.File) (servertypes.ExportedApp, error) { var simApp *simapp.SimApp homePath, ok := appOpts.Get(flags.FlagHome).(string) @@ -333,9 +333,3 @@ func (a appCreator) appExport( return simApp.ExportAppStateAndValidators(forZeroHeight, jailAllowedAddrs) } - -func (a appCreator) appExporterStream( - logger log.Logger, db dbm.DB, traceStore io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string, - appOpts servertypes.AppOptions, file *os.File) (servertypes.ExportedApp, error) { - return a.appExport(logger, db, traceStore, height, forZeroHeight, jailAllowedAddrs, appOpts) -}