Metrics for sync phases (#7390)

* add metrics to measure the time for chain and world state sync

Signed-off-by: [email protected] <[email protected]>
Signed-off-by: Stefan Pingel <[email protected]>
Co-authored-by: Simon Dudley <[email protected]>
Co-authored-by: Sally MacFarlane <[email protected]>

ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java

@@ -43,6 +43,7 @@
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValueStorageFactory;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory;
@@ -120,7 +121,8 @@ public void setUpUnchangedState() {
             syncConfig.getWorldStateMaxRequestsWithoutProgress(),
             syncConfig.getWorldStateMinMillisBeforeStalling(),
             Clock.fixed(Instant.ofEpochSecond(1000), ZoneOffset.UTC),
-            metricsSystem);
+            metricsSystem,
+            SyncDurationMetrics.NO_OP_SYNC_DURATION_METRICS);
   }
 
   private Hash createExistingWorldState() {

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java

@@ -38,6 +38,7 @@
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
 import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.data.SyncStatus;
 import org.hyperledger.besu.plugin.services.BesuEvents;
 import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener;
@@ -67,6 +68,7 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
   private final AtomicBoolean running = new AtomicBoolean(false);
   private final Optional<BlockPropagationManager> blockPropagationManager;
   private final Supplier<Optional<FastSyncDownloader<?>>> fastSyncFactory;
+  private final SyncDurationMetrics syncDurationMetrics;
   private Optional<FastSyncDownloader<?>> fastSyncDownloader;
   private final Optional<FullSyncDownloader> fullSyncDownloader;
   private final ProtocolContext protocolContext;
@@ -118,6 +120,8 @@ public DefaultSynchronizer(
                     metricsSystem,
                     blockBroadcaster));
 
+    syncDurationMetrics = new SyncDurationMetrics(metricsSystem);
+
     this.fullSyncDownloader =
         terminationCondition.shouldStopDownload()
             ? Optional.empty()
@@ -129,7 +133,8 @@ public DefaultSynchronizer(
                     ethContext,
                     syncState,
                     metricsSystem,
-                    terminationCondition));
+                    terminationCondition,
+                    syncDurationMetrics));
 
     if (SyncMode.FAST.equals(syncConfig.getSyncMode())) {
       this.fastSyncFactory =
@@ -144,7 +149,8 @@ public DefaultSynchronizer(
                   ethContext,
                   worldStateStorageCoordinator,
                   syncState,
-                  clock);
+                  clock,
+                  syncDurationMetrics);
     } else if (syncConfig.getSyncMode() == SyncMode.CHECKPOINT) {
       this.fastSyncFactory =
           () ->
@@ -159,7 +165,8 @@ public DefaultSynchronizer(
                   ethContext,
                   worldStateStorageCoordinator,
                   syncState,
-                  clock);
+                  clock,
+                  syncDurationMetrics);
     } else {
       this.fastSyncFactory =
           () ->
@@ -174,7 +181,8 @@ public DefaultSynchronizer(
                   ethContext,
                   worldStateStorageCoordinator,
                   syncState,
-                  clock);
+                  clock,
+                  syncDurationMetrics);
     }
 
     // create a non-resync fast sync downloader:
@@ -205,6 +213,9 @@ public TrailingPeerRequirements calculateTrailingPeerRequirements() {
   public CompletableFuture<Void> start() {
     if (running.compareAndSet(false, true)) {
       LOG.info("Starting synchronizer.");
+
+      syncDurationMetrics.startTimer(SyncDurationMetrics.Labels.TOTAL_SYNC_DURATION);
+
       blockPropagationManager.ifPresent(
           manager -> {
             if (!manager.isRunning()) {
@@ -390,6 +401,10 @@ private Void finalizeSync(final Void unused) {
     blockPropagationManager.ifPresent(BlockPropagationManager::stop);
     LOG.info("Stopping the pruner.");
     running.set(false);
+
+    syncDurationMetrics.stopTimer(SyncDurationMetrics.Labels.FLAT_DB_HEAL);
+    syncDurationMetrics.stopTimer(SyncDurationMetrics.Labels.TOTAL_SYNC_DURATION);
+
     return null;
   }
 

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java

@@ -24,6 +24,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.tasks.exceptions.InvalidBlockException;
 import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason;
 import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.metrics.Counter;
 import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
@@ -51,18 +52,21 @@ public class PipelineChainDownloader implements ChainDownloader {
   private final AtomicBoolean cancelled = new AtomicBoolean(false);
   private final Counter pipelineCompleteCounter;
   private final Counter pipelineErrorCounter;
+  private final SyncDurationMetrics syncDurationMetrics;
   private Pipeline<?> currentDownloadPipeline;
 
   public PipelineChainDownloader(
       final SyncState syncState,
       final AbstractSyncTargetManager syncTargetManager,
       final DownloadPipelineFactory downloadPipelineFactory,
       final EthScheduler scheduler,
-      final MetricsSystem metricsSystem) {
+      final MetricsSystem metricsSystem,
+      final SyncDurationMetrics syncDurationMetrics) {
     this.syncState = syncState;
     this.syncTargetManager = syncTargetManager;
     this.downloadPipelineFactory = downloadPipelineFactory;
     this.scheduler = scheduler;
+    this.syncDurationMetrics = syncDurationMetrics;
 
     final LabelledMetric<Counter> labelledCounter =
         metricsSystem.createLabelledCounter(
@@ -79,6 +83,9 @@ public CompletableFuture<Void> start() {
     if (!started.compareAndSet(false, true)) {
       throw new IllegalStateException("Cannot start a chain download twice");
     }
+
+    syncDurationMetrics.startTimer(SyncDurationMetrics.Labels.CHAIN_DOWNLOAD_DURATION);
+
     return performDownload();
   }
 

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java

@@ -36,6 +36,7 @@
 import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.trie.CompactEncoding;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues;
 
@@ -62,7 +63,8 @@ public static Optional<FastSyncDownloader<?>> createCheckpointDownloader(
       final EthContext ethContext,
       final WorldStateStorageCoordinator worldStateStorageCoordinator,
       final SyncState syncState,
-      final Clock clock) {
+      final Clock clock,
+      final SyncDurationMetrics syncDurationMetrics) {
 
     final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
     final FastSyncStateStorage fastSyncStateStorage =
@@ -149,7 +151,8 @@ public static Optional<FastSyncDownloader<?>> createCheckpointDownloader(
             syncConfig.getWorldStateMaxRequestsWithoutProgress(),
             syncConfig.getWorldStateMinMillisBeforeStalling(),
             clock,
-            metricsSystem);
+            metricsSystem,
+            syncDurationMetrics);
     final FastSyncDownloader<SnapDataRequest> fastSyncDownloader =
         new SnapSyncDownloader(
             fastSyncActions,
@@ -158,7 +161,8 @@ public static Optional<FastSyncDownloader<?>> createCheckpointDownloader(
             fastSyncStateStorage,
             snapTaskCollection,
             fastSyncDataDirectory,
-            snapSyncState);
+            snapSyncState,
+            syncDurationMetrics);
     syncState.setWorldStateDownloadStatus(snapWorldStateDownloader);
     return Optional.of(fastSyncDownloader);
   }

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java

@@ -24,6 +24,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 public class CheckpointSyncActions extends FastSyncActions {
@@ -48,7 +49,8 @@ public CheckpointSyncActions(
   }
 
   @Override
-  public ChainDownloader createChainDownloader(final FastSyncState currentState) {
+  public ChainDownloader createChainDownloader(
+      final FastSyncState currentState, final SyncDurationMetrics syncDurationMetrics) {
     return CheckpointSyncChainDownloader.create(
         syncConfig,
         worldStateStorageCoordinator,
@@ -57,6 +59,7 @@ public ChainDownloader createChainDownloader(final FastSyncState currentState) {
         ethContext,
         syncState,
         metricsSystem,
-        currentState);
+        currentState,
+        syncDurationMetrics);
   }
 }

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java

@@ -25,6 +25,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 public class CheckpointSyncChainDownloader extends FastSyncChainDownloader {
@@ -37,7 +38,8 @@ public static ChainDownloader create(
       final EthContext ethContext,
       final SyncState syncState,
       final MetricsSystem metricsSystem,
-      final FastSyncState fastSyncState) {
+      final FastSyncState fastSyncState,
+      final SyncDurationMetrics syncDurationMetrics) {
 
     final SyncTargetManager syncTargetManager =
         new SyncTargetManager(
@@ -55,6 +57,7 @@ public static ChainDownloader create(
         new CheckpointSyncDownloadPipelineFactory(
             config, protocolSchedule, protocolContext, ethContext, fastSyncState, metricsSystem),
         ethContext.getScheduler(),
-        metricsSystem);
+        metricsSystem,
+        syncDurationMetrics);
   }
 }

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java

@@ -28,6 +28,7 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
 import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.metrics.Counter;
 
@@ -155,7 +156,8 @@ private FastSyncState updateStats(final FastSyncState fastSyncState) {
     return fastSyncState;
   }
 
-  public ChainDownloader createChainDownloader(final FastSyncState currentState) {
+  public ChainDownloader createChainDownloader(
+      final FastSyncState currentState, final SyncDurationMetrics syncDurationMetrics) {
     return FastSyncChainDownloader.create(
         syncConfig,
         worldStateStorageCoordinator,
@@ -164,7 +166,8 @@ public ChainDownloader createChainDownloader(final FastSyncState currentState) {
         ethContext,
         syncState,
         metricsSystem,
-        currentState);
+        currentState,
+        syncDurationMetrics);
   }
 
   private CompletableFuture<FastSyncState> downloadPivotBlockHeader(final Hash hash) {

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 public class FastSyncChainDownloader {
@@ -36,7 +37,8 @@ public static ChainDownloader create(
       final EthContext ethContext,
       final SyncState syncState,
       final MetricsSystem metricsSystem,
-      final FastSyncState fastSyncState) {
+      final FastSyncState fastSyncState,
+      final SyncDurationMetrics syncDurationMetrics) {
 
     final SyncTargetManager syncTargetManager =
         new SyncTargetManager(
@@ -53,6 +55,7 @@ public static ChainDownloader create(
         new FastSyncDownloadPipelineFactory(
             config, protocolSchedule, protocolContext, ethContext, fastSyncState, metricsSystem),
         ethContext.getScheduler(),
-        metricsSystem);
+        metricsSystem,
+        syncDurationMetrics);
   }
 }

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java

@@ -23,6 +23,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader;
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.services.tasks.TaskCollection;
 import org.hyperledger.besu.util.ExceptionUtils;
@@ -52,6 +53,7 @@ public class FastSyncDownloader<REQUEST> {
   private final WorldStateDownloader worldStateDownloader;
   private final TaskCollection<REQUEST> taskCollection;
   private final Path fastSyncDataDirectory;
+  private final SyncDurationMetrics syncDurationMetrics;
   private volatile Optional<TrailingPeerRequirements> trailingPeerRequirements = Optional.empty();
   private final AtomicBoolean running = new AtomicBoolean(false);
 
@@ -66,21 +68,24 @@ public FastSyncDownloader(
       final FastSyncStateStorage fastSyncStateStorage,
       final TaskCollection<REQUEST> taskCollection,
       final Path fastSyncDataDirectory,
-      final FastSyncState initialFastSyncState) {
+      final FastSyncState initialFastSyncState,
+      final SyncDurationMetrics syncDurationMetrics) {
     this.fastSyncActions = fastSyncActions;
     this.worldStateStorageCoordinator = worldStateStorageCoordinator;
     this.worldStateDownloader = worldStateDownloader;
     this.fastSyncStateStorage = fastSyncStateStorage;
     this.taskCollection = taskCollection;
     this.fastSyncDataDirectory = fastSyncDataDirectory;
     this.initialFastSyncState = initialFastSyncState;
+    this.syncDurationMetrics = syncDurationMetrics;
   }
 
   public CompletableFuture<FastSyncState> start() {
     if (!running.compareAndSet(false, true)) {
       throw new IllegalStateException("SyncDownloader already running");
     }
     LOG.info("Starting pivot-based sync");
+
     return start(initialFastSyncState);
   }
 
@@ -189,7 +194,8 @@ protected CompletableFuture<FastSyncState> downloadChainAndWorldState(
       }
       final CompletableFuture<Void> worldStateFuture =
           worldStateDownloader.run(fastSyncActions, currentState);
-      final ChainDownloader chainDownloader = fastSyncActions.createChainDownloader(currentState);
+      final ChainDownloader chainDownloader =
+          fastSyncActions.createChainDownloader(currentState, syncDurationMetrics);
       final CompletableFuture<Void> chainFuture = chainDownloader.start();
 
       // If either download fails, cancel the other one.

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java

@@ -30,6 +30,7 @@
 import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
 import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.SyncDurationMetrics;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues;
 
@@ -60,7 +61,8 @@ public static Optional<FastSyncDownloader<?>> create(
       final EthContext ethContext,
       final WorldStateStorageCoordinator worldStateStorageCoordinator,
       final SyncState syncState,
-      final Clock clock) {
+      final Clock clock,
+      final SyncDurationMetrics syncDurationMetrics) {
 
     final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
     final FastSyncStateStorage fastSyncStateStorage =
@@ -114,7 +116,8 @@ public static Optional<FastSyncDownloader<?>> create(
             syncConfig.getWorldStateMaxRequestsWithoutProgress(),
             syncConfig.getWorldStateMinMillisBeforeStalling(),
             clock,
-            metricsSystem);
+            metricsSystem,
+            syncDurationMetrics);
     final FastSyncDownloader<NodeDataRequest> fastSyncDownloader =
         new FastSyncDownloader<>(
             new FastSyncActions(
@@ -131,7 +134,8 @@ public static Optional<FastSyncDownloader<?>> create(
             fastSyncStateStorage,
             taskCollection,
             fastSyncDataDirectory,
-            fastSyncState);
+            fastSyncState,
+            syncDurationMetrics);
     syncState.setWorldStateDownloadStatus(worldStateDownloader);
     return Optional.of(fastSyncDownloader);
   }