change export-to-file to a flag in export cmd

sei-protocol · Jul 22, 2024 · e934019 · e934019
1 parent 8da6f31
commit e934019
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 133 deletions.
diff --git a/server/export.go b/server/export.go
@@ -19,11 +19,22 @@ import (
 )
 
 const (
+	FlagIsStreaming      = "streaming"
+	FlagStreamingFile    = "streaming-file"
 	FlagHeight           = "height"
 	FlagForZeroHeight    = "for-zero-height"
 	FlagJailAllowedAddrs = "jail-allowed-addrs"
 )
 
+type GenesisDocNoAppState struct {
+	GenesisTime     time.Time                  `json:"genesis_time"`
+	ChainID         string                     `json:"chain_id"`
+	InitialHeight   int64                      `json:"initial_height,string"`
+	ConsensusParams *tmtypes.ConsensusParams   `json:"consensus_params,omitempty"`
+	Validators      []tmtypes.GenesisValidator `json:"validators,omitempty"`
+	AppHash         tmbytes.HexBytes           `json:"app_hash"`
+}
+
 // ExportCmd dumps app state to JSON.
 func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Command {
 	cmd := &cobra.Command{
@@ -40,6 +51,20 @@ func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Com
 				return err
 			}
 
+			isStreaming, err := cmd.Flags().GetBool(FlagIsStreaming)
+			if err != nil {
+				return err
+			}
+
+			streamingFile, err := cmd.Flags().GetString(FlagStreamingFile)
+			if err != nil {
+				return err
+			}
+
+			if isStreaming && streamingFile == "" {
+				return fmt.Errorf("file to export stream to not provided")
+			}
+
 			db, err := openDB(config.RootDir)
 			if err != nil {
 				return err
@@ -69,7 +94,56 @@ func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Com
 			forZeroHeight, _ := cmd.Flags().GetBool(FlagForZeroHeight)
 			jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(FlagJailAllowedAddrs)
 
-			exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper)
+			if isStreaming {
+				file, err := os.Create(streamingFile)
+				if err != nil {
+					return err
+				}
+				exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, file)
+				if err != nil {
+					return fmt.Errorf("error exporting state: %v", err)
+				}
+
+				doc, err := tmtypes.GenesisDocFromFile(serverCtx.Config.GenesisFile())
+				if err != nil {
+					return err
+				}
+
+				genesisDocNoAppHash := GenesisDocNoAppState{
+					GenesisTime:   doc.GenesisTime,
+					ChainID:       doc.ChainID,
+					AppHash:       doc.AppHash,
+					InitialHeight: exported.Height,
+					ConsensusParams: &tmtypes.ConsensusParams{
+						Block: tmtypes.BlockParams{
+							MaxBytes: exported.ConsensusParams.Block.MaxBytes,
+							MaxGas:   exported.ConsensusParams.Block.MaxGas,
+						},
+						Evidence: tmtypes.EvidenceParams{
+							MaxAgeNumBlocks: exported.ConsensusParams.Evidence.MaxAgeNumBlocks,
+							MaxAgeDuration:  exported.ConsensusParams.Evidence.MaxAgeDuration,
+							MaxBytes:        exported.ConsensusParams.Evidence.MaxBytes,
+						},
+						Validator: tmtypes.ValidatorParams{
+							PubKeyTypes: exported.ConsensusParams.Validator.PubKeyTypes,
+						},
+					},
+					Validators: exported.Validators,
+				}
+
+				// NOTE: Tendermint uses a custom JSON decoder for GenesisDoc
+				// (except for stuff inside AppState). Inside AppState, we're free
+				// to encode as protobuf or amino.
+				encoded, err := json.Marshal(genesisDocNoAppHash)
+				if err != nil {
+					return err
+				}
+
+				file.Write([]byte(fmt.Sprintf("%s", string(sdk.MustSortJSON(encoded)))))
+				return nil
+			}
+
+			exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, nil)
 			if err != nil {
 				return fmt.Errorf("error exporting state: %v", err)
 			}
@@ -110,122 +184,8 @@ func ExportCmd(appExporter types.AppExporter, defaultNodeHome string) *cobra.Com
 		},
 	}
 
-	cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory")
-	cmd.Flags().Int64(FlagHeight, -1, "Export state from a particular height (-1 means latest height)")
-	cmd.Flags().Bool(FlagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)")
-	cmd.Flags().StringSlice(FlagJailAllowedAddrs, []string{}, "Comma-separated list of operator addresses of jailed validators to unjail")
-	cmd.Flags().String(FlagChainID, "", "Chain ID")
-
-	return cmd
-}
-
-type GenesisDocNoAppState struct {
-	GenesisTime     time.Time                  `json:"genesis_time"`
-	ChainID         string                     `json:"chain_id"`
-	InitialHeight   int64                      `json:"initial_height,string"`
-	ConsensusParams *tmtypes.ConsensusParams   `json:"consensus_params,omitempty"`
-	Validators      []tmtypes.GenesisValidator `json:"validators,omitempty"`
-	AppHash         tmbytes.HexBytes           `json:"app_hash"`
-}
-
-// ExportToFileCmd dumps app state to JSON. It appends the app state module by module to the file.
-// This is especially useful when the output is too large to fit in memory.
-// TODO: change name to ExportStream, also can make it a flag under export
-func ExportToFileCmd(appExporterStream types.AppExporterStream, defaultNodeHome string) *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "export-to-file [jsonfile]",
-		Short: "Export state to JSON file",
-		Args:  cobra.ExactArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			serverCtx := GetServerContextFromCmd(cmd)
-			config := serverCtx.Config
-			filePath := args[0]
-			file, err := os.Create(filePath)
-			if err != nil {
-				return err
-			}
-
-			homeDir, _ := cmd.Flags().GetString(flags.FlagHome)
-			config.SetRoot(homeDir)
-
-			if _, err := os.Stat(config.GenesisFile()); os.IsNotExist(err) {
-				return err
-			}
-
-			db, err := openDB(config.RootDir)
-			if err != nil {
-				return err
-			}
-
-			if appExporterStream == nil {
-				if _, err := fmt.Fprintln(os.Stderr, "WARNING: App exporter not defined. Returning genesis file."); err != nil {
-					return err
-				}
-
-				genesis, err := ioutil.ReadFile(config.GenesisFile())
-				if err != nil {
-					return err
-				}
-
-				file.Write(genesis)
-				return nil
-			}
-
-			traceWriterFile, _ := cmd.Flags().GetString(flagTraceStore)
-			traceWriter, err := openTraceWriter(traceWriterFile)
-			if err != nil {
-				return err
-			}
-
-			height, _ := cmd.Flags().GetInt64(FlagHeight)
-			forZeroHeight, _ := cmd.Flags().GetBool(FlagForZeroHeight)
-			jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(FlagJailAllowedAddrs)
-
-			exported, err := appExporterStream(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, file)
-			if err != nil {
-				return fmt.Errorf("error exporting state: %v", err)
-			}
-
-			doc, err := tmtypes.GenesisDocFromFile(serverCtx.Config.GenesisFile())
-			if err != nil {
-				return err
-			}
-
-			genesisDocNoAppHash := GenesisDocNoAppState{
-				GenesisTime:   doc.GenesisTime,
-				ChainID:       doc.ChainID,
-				AppHash:       doc.AppHash,
-				InitialHeight: exported.Height,
-				ConsensusParams: &tmtypes.ConsensusParams{
-					Block: tmtypes.BlockParams{
-						MaxBytes: exported.ConsensusParams.Block.MaxBytes,
-						MaxGas:   exported.ConsensusParams.Block.MaxGas,
-					},
-					Evidence: tmtypes.EvidenceParams{
-						MaxAgeNumBlocks: exported.ConsensusParams.Evidence.MaxAgeNumBlocks,
-						MaxAgeDuration:  exported.ConsensusParams.Evidence.MaxAgeDuration,
-						MaxBytes:        exported.ConsensusParams.Evidence.MaxBytes,
-					},
-					Validator: tmtypes.ValidatorParams{
-						PubKeyTypes: exported.ConsensusParams.Validator.PubKeyTypes,
-					},
-				},
-				Validators: exported.Validators,
-			}
-
-			// NOTE: Tendermint uses a custom JSON decoder for GenesisDoc
-			// (except for stuff inside AppState). Inside AppState, we're free
-			// to encode as protobuf or amino.
-			encoded, err := json.Marshal(genesisDocNoAppHash)
-			if err != nil {
-				return err
-			}
-
-			file.Write([]byte(fmt.Sprintf("%s", string(sdk.MustSortJSON(encoded)))))
-			return nil
-		},
-	}
-
+	cmd.Flags().Bool(FlagIsStreaming, false, "Whether to stream the export in chunks. Useful when genesis is extremely large and cannot fit into memory.")
+	cmd.Flags().String(FlagStreamingFile, "genesis-stream.json", "The file to export the streamed genesis to")
 	cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory")
 	cmd.Flags().Int64(FlagHeight, -1, "Export state from a particular height (-1 means latest height)")
 	cmd.Flags().Bool(FlagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)")

diff --git a/server/types/app.go b/server/types/app.go
@@ -85,10 +85,6 @@ type (
 
 	// AppExporter is a function that dumps all app state to
 	// JSON-serializable structure and returns the current validator set.
-	AppExporter func(log.Logger, dbm.DB, io.Writer, int64, bool, []string, AppOptions) (ExportedApp, error)
-
-	// AppExporterStream is a function that dumps all app state to
-	// JSON-serializable structure and writes the result to a file.
-	// This is useful when the output is too large to fit in memory.
-	AppExporterStream func(log.Logger, dbm.DB, io.Writer, int64, bool, []string, AppOptions, *os.File) (ExportedApp, error)
+	// If a file is specified,
+	AppExporter func(log.Logger, dbm.DB, io.Writer, int64, bool, []string, AppOptions, *os.File) (ExportedApp, error)
 )
diff --git a/server/util.go b/server/util.go
@@ -304,7 +304,6 @@ func AddCommands(
 	defaultNodeHome string,
 	appCreator types.AppCreator,
 	appExport types.AppExporter,
-	appExportToFile types.AppExporterStream,
 	addStartFlags types.ModuleInitFlags,
 	tracerProviderOptions []trace.TracerProviderOption,
 ) {
@@ -349,7 +348,6 @@ func AddCommands(
 		startCmd,
 		tendermintCmd,
 		ExportCmd(appExport, defaultNodeHome),
-		ExportToFileCmd(appExportToFile, defaultNodeHome),
 		version.NewVersionCommand(),
 		NewRollbackCmd(appCreator, defaultNodeHome),
 		LatestVersionCmd(defaultNodeHome),

diff --git a/simapp/simd/cmd/root.go b/simapp/simd/cmd/root.go
@@ -170,7 +170,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
 		panic(err)
 	}
 
-	server.AddCommands(rootCmd, simapp.DefaultNodeHome, a.newApp, a.appExport, a.appExporterStream, addModuleInitFlags, []trace.TracerProviderOption{
+	server.AddCommands(rootCmd, simapp.DefaultNodeHome, a.newApp, a.appExport, addModuleInitFlags, []trace.TracerProviderOption{
 		trace.WithBatcher(exp),
 		// Record information about this application in a Resource.
 		trace.WithResource(resource.NewWithAttributes(
@@ -313,7 +313,7 @@ func (a appCreator) newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, t
 // and exports state.
 func (a appCreator) appExport(
 	logger log.Logger, db dbm.DB, traceStore io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string,
-	appOpts servertypes.AppOptions) (servertypes.ExportedApp, error) {
+	appOpts servertypes.AppOptions, file *os.File) (servertypes.ExportedApp, error) {
 
 	var simApp *simapp.SimApp
 	homePath, ok := appOpts.Get(flags.FlagHome).(string)
@@ -333,9 +333,3 @@ func (a appCreator) appExport(
 
 	return simApp.ExportAppStateAndValidators(forZeroHeight, jailAllowedAddrs)
 }
-
-func (a appCreator) appExporterStream(
-	logger log.Logger, db dbm.DB, traceStore io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string,
-	appOpts servertypes.AppOptions, file *os.File) (servertypes.ExportedApp, error) {
-	return a.appExport(logger, db, traceStore, height, forZeroHeight, jailAllowedAddrs, appOpts)
-}