From 97c9ee34a0e169be1df883f35735eaa1be452fc1 Mon Sep 17 00:00:00 2001 From: marta-lokhova Date: Tue, 15 Oct 2024 09:48:46 -0700 Subject: [PATCH] Allow ledger close path to run async during externalize --- src/bucket/test/BucketTestUtils.cpp | 4 + src/catchup/ApplyLedgerWork.cpp | 3 +- src/catchup/CatchupManager.h | 7 +- src/catchup/CatchupManagerImpl.cpp | 111 ++++++++---- src/catchup/CatchupManagerImpl.h | 9 +- src/catchup/DownloadApplyTxsWork.cpp | 6 +- src/catchup/DownloadApplyTxsWork.h | 2 +- src/catchup/ReplayDebugMetaWork.cpp | 3 +- src/herder/Herder.h | 3 +- src/herder/HerderImpl.cpp | 21 +-- src/herder/HerderImpl.h | 3 +- src/herder/test/HerderTests.cpp | 22 +-- src/history/test/HistoryTests.cpp | 14 +- src/history/test/HistoryTestsUtils.cpp | 40 ++--- src/history/test/HistoryTestsUtils.h | 4 - src/ledger/LedgerManager.h | 13 +- src/ledger/LedgerManagerImpl.cpp | 197 ++++++++++------------ src/ledger/LedgerManagerImpl.h | 22 ++- src/main/Config.cpp | 4 +- src/main/Config.h | 2 + src/test/TestAccount.cpp | 1 + src/test/TestUtils.cpp | 24 +++ src/test/TestUtils.h | 5 + src/test/TxTests.cpp | 16 +- src/transactions/test/TxEnvelopeTests.cpp | 1 + 25 files changed, 318 insertions(+), 219 deletions(-) diff --git a/src/bucket/test/BucketTestUtils.cpp b/src/bucket/test/BucketTestUtils.cpp index 0200e49442..ee4b959857 100644 --- a/src/bucket/test/BucketTestUtils.cpp +++ b/src/bucket/test/BucketTestUtils.cpp @@ -78,6 +78,10 @@ closeLedger(Application& app, std::optional skToSignValue, app.getHerder().externalizeValue(TxSetXDRFrame::makeEmpty(lcl), ledgerNum, lcl.header.scpValue.closeTime, upgrades, skToSignValue); + testutil::crankUntil( + app, + [&lm, ledgerNum]() { return lm.getLastClosedLedgerNum() == ledgerNum; }, + std::chrono::seconds(10)); return lm.getLastClosedLedgerHeader().hash; } diff --git a/src/catchup/ApplyLedgerWork.cpp b/src/catchup/ApplyLedgerWork.cpp index 5d910f8bf5..bba96df816 100644 --- a/src/catchup/ApplyLedgerWork.cpp +++ b/src/catchup/ApplyLedgerWork.cpp @@ -23,7 +23,8 @@ BasicWork::State ApplyLedgerWork::onRun() { ZoneScoped; - mApp.getLedgerManager().closeLedger(mLedgerCloseData); + mApp.getLedgerManager().closeLedger(mLedgerCloseData, + /* externalize */ false); return BasicWork::State::WORK_SUCCESS; } diff --git a/src/catchup/CatchupManager.h b/src/catchup/CatchupManager.h index 61c9b5821f..d79077a658 100644 --- a/src/catchup/CatchupManager.h +++ b/src/catchup/CatchupManager.h @@ -52,7 +52,12 @@ class CatchupManager // Process ledgers that could not be applied, and determine if catchup // should run - virtual void processLedger(LedgerCloseData const& ledgerData) = 0; + + // Return true is latest ledger was applied, and there are no syncing + // ledgers, Return false if ledgers are buffered with gaps, and we need to + // start catchup + virtual bool processLedger(LedgerCloseData const& ledgerData, + bool isLatestSlot) = 0; // Forcibly switch the application into catchup mode, treating `toLedger` // as the destination ledger number and count as the number of past ledgers diff --git a/src/catchup/CatchupManagerImpl.cpp b/src/catchup/CatchupManagerImpl.cpp index b1eca69dd7..a7467c8bb8 100644 --- a/src/catchup/CatchupManagerImpl.cpp +++ b/src/catchup/CatchupManagerImpl.cpp @@ -103,9 +103,12 @@ CatchupManagerImpl::getCatchupCount() : mApp.getConfig().CATCHUP_RECENT; } -void -CatchupManagerImpl::processLedger(LedgerCloseData const& ledgerData) +bool +CatchupManagerImpl::processLedger(LedgerCloseData const& ledgerData, + bool isLatestSlot) { + maybeUpdateLastQueuedToApply(); + ZoneScoped; if (catchupWorkIsDone()) { @@ -119,26 +122,32 @@ CatchupManagerImpl::processLedger(LedgerCloseData const& ledgerData) logAndUpdateCatchupStatus(true); } + // Always skip old ledgers uint32_t lastReceivedLedgerSeq = ledgerData.getLedgerSeq(); + if (lastReceivedLedgerSeq <= *mLastQueuedToApply) + { + // If LCL is already at-or-ahead of the ledger we just received from the + // network, we're up to date. Return early, nothing to do. + CLOG_INFO( + Ledger, + "Skipping close ledger: local state is {}, more recent than {}", + *mLastQueuedToApply, ledgerData.getLedgerSeq()); + return true; + } + + // Always add a newer ledger, maybe apply + mSyncingLedgers.emplace(lastReceivedLedgerSeq, ledgerData); mLargestLedgerSeqHeard = std::max(mLargestLedgerSeqHeard, lastReceivedLedgerSeq); // 1. CatchupWork is not running yet - // 2. CatchupManager received ledger that was immediately applied by + // 2. CatchupManager received ledger that should be immediately applied by // LedgerManager: check if we have any sequential ledgers. // If so, attempt to apply mSyncingLedgers and possibly get back in sync - if (!mCatchupWork && lastReceivedLedgerSeq == - mApp.getLedgerManager().getLastClosedLedgerNum()) + if (!mCatchupWork && lastReceivedLedgerSeq == *mLastQueuedToApply + 1) { tryApplySyncingLedgers(); - return; - } - else if (lastReceivedLedgerSeq <= - mApp.getLedgerManager().getLastClosedLedgerNum()) - { - // If LCL is already at-or-ahead of the ledger we just received from the - // network, we're up to date. Return early, nothing to do. - return; + return true; } // For the rest of this method: we know LCL has fallen behind the network @@ -151,6 +160,9 @@ CatchupManagerImpl::processLedger(LedgerCloseData const& ledgerData) // to history and commence catchup, running the (checkpoint-driven) catchup // state machine to ledger L-1 (the end of the checkpoint covering K) and // then replay buffered ledgers from L onwards. + CLOG_INFO(Ledger, + "Close of ledger {} buffered. mSyncingLedgers has {} ledgers", + ledgerData.getLedgerSeq(), mSyncingLedgers.size()); // First: if CatchupWork has started, just buffer and return early. if (mCatchupWork) @@ -160,17 +172,17 @@ CatchupManagerImpl::processLedger(LedgerCloseData const& ledgerData) auto const& config = mCatchupWork->getCatchupConfiguration(); if (ledgerData.getLedgerSeq() <= config.toLedger()) { - return; + return false; } - addAndTrimSyncingLedgers(ledgerData); + trimSyncingLedgers(); logAndUpdateCatchupStatus(true); - return; + return false; } // Next, we buffer every out of sync ledger to allow us to get back in sync // in case the ledgers we're missing are received. - addAndTrimSyncingLedgers(ledgerData); + trimSyncingLedgers(); // Finally we wait some number of ledgers beyond the smallest buffered // checkpoint ledger before we trigger the CatchupWork. This could be any @@ -233,6 +245,7 @@ CatchupManagerImpl::processLedger(LedgerCloseData const& ledgerData) } } logAndUpdateCatchupStatus(true, message); + return false; } void @@ -241,7 +254,9 @@ CatchupManagerImpl::startCatchup( std::set> bucketsToRetain) { ZoneScoped; - auto lastClosedLedger = mApp.getLedgerManager().getLastClosedLedgerNum(); + maybeUpdateLastQueuedToApply(); + + auto lastClosedLedger = *mLastQueuedToApply; if ((configuration.toLedger() != CatchupConfiguration::CURRENT) && (configuration.toLedger() <= lastClosedLedger)) { @@ -329,10 +344,13 @@ CatchupManagerImpl::logAndUpdateCatchupStatus(bool contiguous) std::optional CatchupManagerImpl::maybeGetNextBufferedLedgerToApply() { + releaseAssert(threadIsMain()); + // Since we just applied a ledger, refresh mLastQueuedToApply + maybeUpdateLastQueuedToApply(); + trimSyncingLedgers(); if (!mSyncingLedgers.empty() && - mSyncingLedgers.begin()->first == - mApp.getLedgerManager().getLastClosedLedgerNum() + 1) + mSyncingLedgers.begin()->first == *mLastQueuedToApply + 1) { return std::make_optional( mSyncingLedgers.begin()->second); @@ -370,14 +388,18 @@ CatchupManagerImpl::syncMetrics() } void -CatchupManagerImpl::addAndTrimSyncingLedgers(LedgerCloseData const& ledgerData) +CatchupManagerImpl::maybeUpdateLastQueuedToApply() { - mSyncingLedgers.emplace(ledgerData.getLedgerSeq(), ledgerData); - trimSyncingLedgers(); - - CLOG_INFO(Ledger, - "Close of ledger {} buffered. mSyncingLedgers has {} ledgers", - ledgerData.getLedgerSeq(), mSyncingLedgers.size()); + if (mLastQueuedToApply) + { + mLastQueuedToApply = mApp.getLedgerManager().getLastClosedLedgerNum(); + } + else + { + mLastQueuedToApply = + std::max(*mLastQueuedToApply, + mApp.getLedgerManager().getLastClosedLedgerNum()); + } } void @@ -408,7 +430,7 @@ CatchupManagerImpl::trimSyncingLedgers() // This erases [begin, it). mSyncingLedgers.erase(mSyncingLedgers.begin(), it); }; - removeLedgersLessThan(mApp.getLedgerManager().getLastClosedLedgerNum() + 1); + removeLedgersLessThan(*mLastQueuedToApply + 1); auto& hm = mApp.getHistoryManager(); if (!mSyncingLedgers.empty()) { @@ -439,8 +461,7 @@ void CatchupManagerImpl::tryApplySyncingLedgers() { ZoneScoped; - auto const& ledgerHeader = - mApp.getLedgerManager().getLastClosedLedgerHeader(); + uint32_t nextToClose = *mLastQueuedToApply + 1; // We can apply multiple ledgers here, which might be slow. This is a rare // occurrence so we should be fine. @@ -450,16 +471,38 @@ CatchupManagerImpl::tryApplySyncingLedgers() auto const& lcd = it->second; // we still have a missing ledger - if (ledgerHeader.header.ledgerSeq + 1 != lcd.getLedgerSeq()) + if (nextToClose != lcd.getLedgerSeq()) { break; } - mApp.getLedgerManager().closeLedger(lcd); - CLOG_INFO(History, "Closed buffered ledger: {}", - LedgerManager::ledgerAbbrev(ledgerHeader)); + if (mApp.getConfig() + .ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING.count() > 0) + { + // Close a ledger asynchronously, with an added delay + // Usefult to test async extrnalize flow + mApp.postOnMainThread( + [&app = mApp, lcd]() { + if (app.isStopping()) + { + return; + } + std::this_thread::sleep_for( + app.getConfig() + .ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING); + app.getLedgerManager().closeLedger(lcd, + /* externalize */ true); + }, + "closeLedger"); + } + else + { + mApp.getLedgerManager().closeLedger(lcd, /* externalize */ true); + } + mLastQueuedToApply = lcd.getLedgerSeq(); ++it; + ++nextToClose; } mSyncingLedgers.erase(mSyncingLedgers.cbegin(), it); diff --git a/src/catchup/CatchupManagerImpl.h b/src/catchup/CatchupManagerImpl.h index 8c04a344aa..d474d9473a 100644 --- a/src/catchup/CatchupManagerImpl.h +++ b/src/catchup/CatchupManagerImpl.h @@ -43,8 +43,12 @@ class CatchupManagerImpl : public CatchupManager // maintain the invariants above. std::map mSyncingLedgers; medida::Counter& mSyncingLedgersSize; + // Most recent ledger that was queued to be applied by CatchupManager. + // Once applued, and before a new ledger is scheduled, this is equivalent to + // LCL. + std::optional mLastQueuedToApply; - void addAndTrimSyncingLedgers(LedgerCloseData const& ledgerData); + void maybeUpdateLastQueuedToApply(); void startOnlineCatchup(); void trimSyncingLedgers(); void tryApplySyncingLedgers(); @@ -61,7 +65,8 @@ class CatchupManagerImpl : public CatchupManager CatchupManagerImpl(Application& app); ~CatchupManagerImpl() override; - void processLedger(LedgerCloseData const& ledgerData) override; + bool processLedger(LedgerCloseData const& ledgerData, + bool isLatestSlot) override; void startCatchup(CatchupConfiguration configuration, std::shared_ptr archive, diff --git a/src/catchup/DownloadApplyTxsWork.cpp b/src/catchup/DownloadApplyTxsWork.cpp index 1746060d69..29b118bec9 100644 --- a/src/catchup/DownloadApplyTxsWork.cpp +++ b/src/catchup/DownloadApplyTxsWork.cpp @@ -27,7 +27,7 @@ DownloadApplyTxsWork::DownloadApplyTxsWork( : BatchWork(app, "download-apply-ledgers") , mRange(range) , mDownloadDir(downloadDir) - , mLastApplied(lastApplied) + , mLastQueuedToApply(lastApplied) , mCheckpointToQueue( app.getHistoryManager().checkpointContainingLedger(range.mFirst)) , mWaitForPublish(waitForPublish) @@ -171,7 +171,7 @@ DownloadApplyTxsWork::resetIter() mCheckpointToQueue = mApp.getHistoryManager().checkpointContainingLedger(mRange.mFirst); mLastYieldedWork.reset(); - mLastApplied = mApp.getLedgerManager().getLastClosedLedgerHeader(); + mLastQueuedToApply = mApp.getLedgerManager().getLastClosedLedgerHeader(); } bool @@ -189,7 +189,7 @@ DownloadApplyTxsWork::hasNext() const void DownloadApplyTxsWork::onSuccess() { - mLastApplied = mApp.getLedgerManager().getLastClosedLedgerHeader(); + mLastQueuedToApply = mApp.getLedgerManager().getLastClosedLedgerHeader(); } std::string diff --git a/src/catchup/DownloadApplyTxsWork.h b/src/catchup/DownloadApplyTxsWork.h index e5a81d9240..3259fdacfe 100644 --- a/src/catchup/DownloadApplyTxsWork.h +++ b/src/catchup/DownloadApplyTxsWork.h @@ -25,7 +25,7 @@ class DownloadApplyTxsWork : public BatchWork { LedgerRange const mRange; TmpDir const& mDownloadDir; - LedgerHeaderHistoryEntry& mLastApplied; + LedgerHeaderHistoryEntry& mLastQueuedToApply; uint32_t mCheckpointToQueue; std::shared_ptr mLastYieldedWork; bool const mWaitForPublish; diff --git a/src/catchup/ReplayDebugMetaWork.cpp b/src/catchup/ReplayDebugMetaWork.cpp index 2d2dcd7fde..6fcbefc13e 100644 --- a/src/catchup/ReplayDebugMetaWork.cpp +++ b/src/catchup/ReplayDebugMetaWork.cpp @@ -164,7 +164,8 @@ ReplayDebugMetaWork::applyLastLedger() if (lcl + 1 == debugTxSet.ledgerSeq) { mApp.getLedgerManager().closeLedger( - LedgerCloseData::toLedgerCloseData(debugTxSet)); + LedgerCloseData::toLedgerCloseData(debugTxSet), + /* externalize */ false); } else { diff --git a/src/herder/Herder.h b/src/herder/Herder.h index 5ed657b5f3..bb34067a2e 100644 --- a/src/herder/Herder.h +++ b/src/herder/Herder.h @@ -114,7 +114,8 @@ class Herder // restores Herder's state from disk virtual void start() = 0; - virtual void lastClosedLedgerIncreased(bool latest) = 0; + virtual void lastClosedLedgerIncreased(bool latest, + TxSetXDRFrameConstPtr txSet) = 0; // Setup Herder's state to fully participate in consensus virtual void setTrackingSCPState(uint64_t index, StellarValue const& value, diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp index e3e1eeb601..33584f1722 100644 --- a/src/herder/HerderImpl.cpp +++ b/src/herder/HerderImpl.cpp @@ -3,6 +3,9 @@ // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 #include "herder/HerderImpl.h" +#include "bucket/BucketListSnapshot.h" +#include "bucket/BucketManager.h" +#include "bucket/BucketSnapshotManager.h" #include "crypto/Hex.h" #include "crypto/KeyUtils.h" #include "crypto/SHA.h" @@ -14,9 +17,6 @@ #include "herder/TxSetFrame.h" #include "herder/TxSetUtils.h" #include "ledger/LedgerManager.h" -#include "ledger/LedgerTxn.h" -#include "ledger/LedgerTxnEntry.h" -#include "ledger/LedgerTxnHeader.h" #include "lib/json/json.h" #include "main/Application.h" #include "main/Config.h" @@ -249,10 +249,6 @@ HerderImpl::newSlotExternalized(bool synchronous, StellarValue const& value) // start timing next externalize from this point mLastExternalize = mApp.getClock().now(); - // In order to update the transaction queue we need to get the - // applied transactions. - updateTransactionQueue(mPendingEnvelopes.getTxSet(value.txSetHash)); - // perform cleanups // Evict slots that are outside of our ledger validity bracket auto minSlotToRemember = getMinLedgerSeqToRemember(); @@ -359,7 +355,7 @@ HerderImpl::processExternalized(uint64 slotIndex, StellarValue const& value, writeDebugTxSet(ledgerData); } - mLedgerManager.valueExternalized(ledgerData); + mLedgerManager.valueExternalized(ledgerData, isLatestSlot); } void @@ -1136,14 +1132,19 @@ HerderImpl::safelyProcessSCPQueue(bool synchronous) } void -HerderImpl::lastClosedLedgerIncreased(bool latest) +HerderImpl::lastClosedLedgerIncreased(bool latest, TxSetXDRFrameConstPtr txSet) { + releaseAssert(threadIsMain()); maybeSetupSorobanQueue( mLedgerManager.getLastClosedLedgerHeader().header.ledgerVersion); // Ensure potential upgrades are handled in overlay maybeHandleUpgrade(); + // In order to update the transaction queue we need to get the + // applied transactions. + updateTransactionQueue(txSet); + if (latest) { releaseAssert(isTracking()); @@ -1531,7 +1532,7 @@ HerderImpl::getUpgradesJson() void HerderImpl::forceSCPStateIntoSyncWithLastClosedLedger() { - auto const& header = mLedgerManager.getLastClosedLedgerHeader().header; + auto header = mLedgerManager.getLastClosedLedgerHeader().header; setTrackingSCPState(header.ledgerSeq, header.scpValue, /* isTrackingNetwork */ true); } diff --git a/src/herder/HerderImpl.h b/src/herder/HerderImpl.h index be1d3d8e12..dc3975ff53 100644 --- a/src/herder/HerderImpl.h +++ b/src/herder/HerderImpl.h @@ -75,7 +75,8 @@ class HerderImpl : public Herder void start() override; - void lastClosedLedgerIncreased(bool latest) override; + void lastClosedLedgerIncreased(bool latest, + TxSetXDRFrameConstPtr txSet) override; SCP& getSCP(); HerderSCPDriver& diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index 9366adb7c3..5149898bb6 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -97,22 +97,9 @@ TEST_CASE_VERSIONS("standalone", "[herder][acceptance]") }; auto waitForExternalize = [&]() { - bool stop = false; auto prev = app->getLedgerManager().getLastClosedLedgerNum(); - VirtualTimer checkTimer(*app); - - auto check = [&](asio::error_code const& error) { - REQUIRE(!error); - REQUIRE(app->getLedgerManager().getLastClosedLedgerNum() > - prev); - stop = true; - }; - - checkTimer.expires_from_now( - Herder::EXP_LEDGER_TIMESPAN_SECONDS + - std::chrono::seconds(1)); - checkTimer.async_wait(check); - while (!stop) + while (app->getLedgerManager().getLastClosedLedgerNum() <= + prev + 1) { app->getClock().crank(true); } @@ -4443,6 +4430,7 @@ TEST_CASE("do not flood invalid transactions", "[herder]") VirtualClock clock; auto cfg = getTestConfig(); cfg.FLOOD_TX_PERIOD_MS = 1; // flood as fast as possible + cfg.ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING = std::chrono::seconds(0); auto app = createTestApplication(clock, cfg); auto& lm = app->getLedgerManager(); @@ -4496,6 +4484,8 @@ TEST_CASE("do not flood too many soroban transactions", cfg.FLOOD_OP_RATE_PER_LEDGER = 2.0; cfg.FLOOD_SOROBAN_TX_PERIOD_MS = 50; cfg.FLOOD_SOROBAN_RATE_PER_LEDGER = 2.0; + cfg.ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING = + std::chrono::seconds(0); return cfg; }); @@ -4673,6 +4663,8 @@ TEST_CASE("do not flood too many transactions", "[herder][transactionqueue]") cfg.FORCE_SCP = false; cfg.FLOOD_TX_PERIOD_MS = 100; cfg.FLOOD_OP_RATE_PER_LEDGER = 2.0; + cfg.ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING = + std::chrono::seconds(0); return cfg; }); diff --git a/src/history/test/HistoryTests.cpp b/src/history/test/HistoryTests.cpp index 2b882a9099..5c98b793c0 100644 --- a/src/history/test/HistoryTests.cpp +++ b/src/history/test/HistoryTests.cpp @@ -808,7 +808,7 @@ TEST_CASE("Retriggering catchups after trimming mSyncingLedgers", CatchupConfiguration::Mode::ONLINE}, *app); - catchupSimulation.crankUntil( + testutil::crankUntil( app, [&]() { return app->getCatchupManager().catchupWorkIsDone(); }, std::chrono::seconds{ std::max(expectedCatchupWork.mTxSetsApplied + 15, 60)}); @@ -873,6 +873,14 @@ TEST_CASE("Retriggering catchups after trimming mSyncingLedgers", catchupSimulation.externalizeLedger(herder, triggerLedger + bufferLedgers + 1); + testutil::crankUntil( + app, + [&]() { + return lm.getLastClosedLedgerNum() == + triggerLedger + bufferLedgers + 1; + }, + std::chrono::seconds{60}); + REQUIRE(lm.isSynced()); REQUIRE(lm.getLastClosedLedgerNum() == @@ -1540,18 +1548,21 @@ TEST_CASE("Introduce and fix gap without starting catchup", catchupSimulation.externalizeLedger(herder, nextLedger + 2); catchupSimulation.externalizeLedger(herder, nextLedger + 3); catchupSimulation.externalizeLedger(herder, nextLedger + 5); + testutil::crankFor(app->getClock(), std::chrono::seconds(5)); REQUIRE(!lm.isSynced()); REQUIRE(cm.getLargestLedgerSeqHeard() > lm.getLastClosedLedgerNum()); // Fill in the first gap. There will still be buffered ledgers left because // of the second gap catchupSimulation.externalizeLedger(herder, nextLedger + 1); + testutil::crankFor(app->getClock(), std::chrono::seconds(5)); REQUIRE(!lm.isSynced()); REQUIRE(cm.getLargestLedgerSeqHeard() > lm.getLastClosedLedgerNum()); // Fill in the second gap. All buffered ledgers should be applied, but we // wait for another ledger to close to get in sync catchupSimulation.externalizeLedger(herder, nextLedger + 4); + testutil::crankFor(app->getClock(), std::chrono::seconds(5)); REQUIRE(lm.isSynced()); REQUIRE(cm.getLargestLedgerSeqHeard() == lm.getLastClosedLedgerNum()); REQUIRE(!cm.isCatchupInitialized()); @@ -1584,6 +1595,7 @@ TEST_CASE("Receive trigger and checkpoint ledger out of order", catchupSimulation.externalizeLedger(herder, checkpointLedger + 1); catchupSimulation.externalizeLedger(herder, checkpointLedger); catchupSimulation.externalizeLedger(herder, checkpointLedger + 2); + testutil::crankFor(app->getClock(), std::chrono::seconds(10)); REQUIRE(lm.isSynced()); REQUIRE(cm.getLargestLedgerSeqHeard() == lm.getLastClosedLedgerNum()); diff --git a/src/history/test/HistoryTestsUtils.cpp b/src/history/test/HistoryTestsUtils.cpp index 5119d372a4..74a5bd0365 100644 --- a/src/history/test/HistoryTestsUtils.cpp +++ b/src/history/test/HistoryTestsUtils.cpp @@ -689,24 +689,6 @@ CatchupSimulation::getLastPublishedCheckpoint() const return pair; } -void -CatchupSimulation::crankUntil(Application::pointer app, - std::function const& predicate, - VirtualClock::duration timeout) -{ - auto start = std::chrono::system_clock::now(); - while (!predicate()) - { - app->getClock().crank(false); - auto current = std::chrono::system_clock::now(); - auto diff = current - start; - if (diff > timeout) - { - break; - } - } -} - Application::pointer CatchupSimulation::createCatchupApplication( uint32_t count, Config::TestDbMode dbMode, std::string const& appName, @@ -754,9 +736,9 @@ CatchupSimulation::catchupOffline(Application::pointer app, uint32_t toLedger, auto expectedCatchupWork = computeCatchupPerformedWork(lastLedger, catchupConfiguration, *app); - crankUntil(app, finished, - std::chrono::seconds{std::max( - expectedCatchupWork.mTxSetsApplied + 15, 60)}); + testutil::crankUntil(app, finished, + std::chrono::seconds{std::max( + expectedCatchupWork.mTxSetsApplied + 15, 60)}); // Finished successfully auto success = cm.isCatchupInitialized() && @@ -869,9 +851,9 @@ CatchupSimulation::catchupOnline(Application::pointer app, uint32_t initLedger, auto expectedCatchupWork = computeCatchupPerformedWork(lastLedger, catchupConfiguration, *app); - crankUntil(app, catchupIsDone, - std::chrono::seconds{std::max( - expectedCatchupWork.mTxSetsApplied + 15, 60)}); + testutil::crankUntil(app, catchupIsDone, + std::chrono::seconds{std::max( + expectedCatchupWork.mTxSetsApplied + 15, 60)}); if (lm.getLastClosedLedgerNum() == triggerLedger + bufferLedgers) { @@ -879,6 +861,14 @@ CatchupSimulation::catchupOnline(Application::pointer app, uint32_t initLedger, externalize(triggerLedger + bufferLedgers + 1); } + testutil::crankUntil( + app, + [&]() { + return lm.getLastClosedLedgerNum() == + triggerLedger + bufferLedgers + 1; + }, + std::chrono::seconds{60}); + auto result = caughtUp(); if (result) { @@ -917,6 +907,8 @@ CatchupSimulation::externalizeLedger(HerderImpl& herder, uint32_t ledger) lcd.getLedgerSeq(), lcd.getTxSet()); herder.getHerderSCPDriver().valueExternalized( lcd.getLedgerSeq(), xdr::xdr_to_opaque(lcd.getValue())); + + // TODO: crank the clock } void diff --git a/src/history/test/HistoryTestsUtils.h b/src/history/test/HistoryTestsUtils.h index eace1f3a6e..0de12845c4 100644 --- a/src/history/test/HistoryTestsUtils.h +++ b/src/history/test/HistoryTestsUtils.h @@ -271,10 +271,6 @@ class CatchupSimulation // this method externalizes through herder void externalizeLedger(HerderImpl& herder, uint32_t ledger); - void crankUntil(Application::pointer app, - std::function const& predicate, - VirtualClock::duration duration); - void setUpgradeLedger(uint32_t ledger, ProtocolVersion upgradeVersion); }; } diff --git a/src/ledger/LedgerManager.h b/src/ledger/LedgerManager.h index 88d0ca8bb8..59894d7f35 100644 --- a/src/ledger/LedgerManager.h +++ b/src/ledger/LedgerManager.h @@ -90,7 +90,8 @@ class LedgerManager // close event. This is the most common cause of LedgerManager advancing // from one ledger to the next: the network reached consensus on // `ledgerData`. - virtual void valueExternalized(LedgerCloseData const& ledgerData) = 0; + virtual void valueExternalized(LedgerCloseData const& ledgerData, + bool isLatestSlot) = 0; // Return the LCL header and (complete, immutable) hash. virtual LedgerHeaderHistoryEntry const& @@ -178,7 +179,15 @@ class LedgerManager // changes. This is normally done automatically as part of // `valueExternalized()`; this method is present in the public interface to // permit testing. - virtual void closeLedger(LedgerCloseData const& ledgerData) = 0; + virtual void closeLedger(LedgerCloseData const& ledgerData, + bool externalize) = 0; +#ifdef BUILD_TESTS + void + closeLedger(LedgerCloseData const& ledgerData) + { + closeLedger(ledgerData, /* externalize */ false); + } +#endif // deletes old entries stored in the database virtual void deleteOldEntries(Database& db, uint32_t ledgerSeq, diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 2b7f328671..f9979807c4 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -43,6 +43,7 @@ #include "util/XDROperators.h" #include "util/XDRStream.h" #include "work/WorkScheduler.h" +#include "xdrpp/printer.h" #include @@ -548,17 +549,10 @@ LedgerManagerImpl::getLastClosedLedgerNum() const return mLastClosedLedger.header.ledgerSeq; } -SorobanNetworkConfig& -LedgerManagerImpl::getSorobanNetworkConfigInternal() -{ - releaseAssert(mSorobanNetworkConfig); - return *mSorobanNetworkConfig; -} - SorobanNetworkConfig const& LedgerManagerImpl::getSorobanNetworkConfig() { - return getSorobanNetworkConfigInternal(); + return *mSorobanNetworkConfig; } bool @@ -571,7 +565,7 @@ LedgerManagerImpl::hasSorobanNetworkConfig() const SorobanNetworkConfig& LedgerManagerImpl::getMutableSorobanNetworkConfig() { - return getSorobanNetworkConfigInternal(); + return *mSorobanNetworkConfig; } std::vector const& LedgerManagerImpl::getLastClosedLedgerTxMeta() @@ -638,7 +632,8 @@ LedgerManagerImpl::publishSorobanMetrics() // called by txherder void -LedgerManagerImpl::valueExternalized(LedgerCloseData const& ledgerData) +LedgerManagerImpl::valueExternalized(LedgerCloseData const& ledgerData, + bool isLatestSlot) { ZoneScoped; @@ -661,60 +656,21 @@ LedgerManagerImpl::valueExternalized(LedgerCloseData const& ledgerData) releaseAssert(false); } - closeLedgerIf(ledgerData); - + // if catchup work is running, we don't want ledger manager to close + // this ledger and potentially cause issues. auto& cm = mApp.getCatchupManager(); - - cm.processLedger(ledgerData); - - // We set the state to synced - // if we have closed the latest ledger we have heard of. - bool appliedLatest = false; - if (cm.getLargestLedgerSeqHeard() == getLastClosedLedgerNum()) + bool catchupIsRunning = + cm.isCatchupInitialized() && !cm.catchupWorkIsDone(); + if (lcl + 1 == ledgerData.getLedgerSeq() && catchupIsRunning) { - setState(LM_SYNCED_STATE); - appliedLatest = true; - } - - // New ledger(s) got closed, notify Herder - if (getLastClosedLedgerNum() > lcl) - { - CLOG_DEBUG(Ledger, - "LedgerManager::valueExternalized LCL advanced {} -> {}", - lcl, getLastClosedLedgerNum()); - mApp.getHerder().lastClosedLedgerIncreased(appliedLatest); + CLOG_INFO(Ledger, + "Can't close ledger: {} in LM because catchup is running", + ledgerAbbrev(mLastClosedLedger)); + return; } -} - -void -LedgerManagerImpl::closeLedgerIf(LedgerCloseData const& ledgerData) -{ - ZoneScoped; - if (mLastClosedLedger.header.ledgerSeq + 1 == ledgerData.getLedgerSeq()) - { - auto& cm = mApp.getCatchupManager(); - // if catchup work is running, we don't want ledger manager to close - // this ledger and potentially cause issues. - if (cm.isCatchupInitialized() && !cm.catchupWorkIsDone()) - { - CLOG_INFO( - Ledger, - "Can't close ledger: {} in LM because catchup is running", - ledgerAbbrev(mLastClosedLedger)); - return; - } - closeLedger(ledgerData); - CLOG_INFO(Ledger, "Closed ledger: {}", ledgerAbbrev(mLastClosedLedger)); - } - else if (ledgerData.getLedgerSeq() <= mLastClosedLedger.header.ledgerSeq) - { - CLOG_INFO( - Ledger, - "Skipping close ledger: local state is {}, more recent than {}", - mLastClosedLedger.header.ledgerSeq, ledgerData.getLedgerSeq()); - } - else + bool synced = cm.processLedger(ledgerData, isLatestSlot); + if (!synced) { if (mState != LM_CATCHING_UP_STATE) { @@ -787,6 +743,35 @@ LedgerManagerImpl::emitNextMeta() mNextMetaToEmit.reset(); } +void +maybeSimulateSleep(Config const& cfg, size_t opSize, + LogSlowExecution& closeTime) +{ + if (!cfg.OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.empty()) + { + // Sleep for a parameterized amount of time in simulation mode + std::discrete_distribution distribution( + cfg.OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.begin(), + cfg.OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.end()); + std::chrono::microseconds sleepFor{0}; + for (size_t i = 0; i < opSize; i++) + { + sleepFor += + cfg.OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING[distribution( + gRandomEngine)]; + } + std::chrono::microseconds applicationTime = + closeTime.checkElapsedTime(); + if (applicationTime < sleepFor) + { + sleepFor -= applicationTime; + CLOG_DEBUG(Perf, "Simulate application: sleep for {} microseconds", + sleepFor.count()); + std::this_thread::sleep_for(sleepFor); + } + } +} + /* This is the main method that closes the current ledger based on the close context that was computed by SCP or by the historical module @@ -794,7 +779,8 @@ during replays. */ void -LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData) +LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, + bool externalize) { #ifdef BUILD_TESTS mLastLedgerTxMeta.clear(); @@ -907,7 +893,7 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData) applicableTxSet->getTxsInApplyOrder(); // first, prefetch source accounts for txset, then charge fees - prefetchTxSourceIds(txs); + prefetchTxSourceIds(mApp.getLedgerTxnRoot(), txs, mApp.getConfig()); auto const mutableTxResults = processFeesSeqNums(txs, ltx, *applicableTxSet, ledgerCloseMeta); @@ -1061,37 +1047,40 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData) // step 5 mApp.getBucketManager().forgetUnreferencedBuckets(); - if (!mApp.getConfig().OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.empty()) - { - // Sleep for a parameterized amount of time in simulation mode - std::discrete_distribution distribution( - mApp.getConfig().OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.begin(), - mApp.getConfig().OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.end()); - std::chrono::microseconds sleepFor{0}; - auto txSetSizeOp = applicableTxSet->sizeOpTotal(); - for (size_t i = 0; i < txSetSizeOp; i++) - { - sleepFor += - mApp.getConfig() - .OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING[distribution( - gRandomEngine)]; - } - std::chrono::microseconds applicationTime = - closeLedgerTime.checkElapsedTime(); - if (applicationTime < sleepFor) - { - sleepFor -= applicationTime; - CLOG_DEBUG(Perf, "Simulate application: sleep for {} microseconds", - sleepFor.count()); - std::this_thread::sleep_for(sleepFor); - } - } + maybeSimulateSleep(mApp.getConfig(), txs.size(), closeLedgerTime); std::chrono::duration ledgerTimeSeconds = ledgerTime.Stop(); CLOG_DEBUG(Perf, "Applied ledger in {} seconds", ledgerTimeSeconds.count()); + + ledgerCloseComplete(mApp, ledgerSeq, externalize, ledgerData); + FrameMark; } +void +ledgerCloseComplete(Application& app, uint32_t lcl, bool externalize, + LedgerCloseData const& ledgerData) +{ + releaseAssert(threadIsMain()); + + // Continue execution on main + // We set the state to synced + // if we have closed the latest ledger we have heard of. + bool appliedLatest = false; + + if (app.getCatchupManager().getLargestLedgerSeqHeard() <= lcl) + { + app.getLedgerManager().moveToSynced(); + appliedLatest = true; + } + if (externalize) + { + // New ledger(s) got closed, notify Herder + app.getHerder().lastClosedLedgerIncreased(appliedLatest, + ledgerData.getTxSet()); + } +} + void LedgerManagerImpl::deleteOldEntries(Database& db, uint32_t ledgerSeq, uint32_t count) @@ -1322,12 +1311,7 @@ LedgerManagerImpl::updateNetworkConfig(AbstractLedgerTxn& rootLtx) { ZoneScoped; - uint32_t ledgerVersion{}; - { - LedgerTxn ltx(rootLtx, false, - TransactionMode::READ_ONLY_WITHOUT_SQL_TXN); - ledgerVersion = ltx.loadHeader().current().ledgerVersion; - } + uint32_t ledgerVersion = rootLtx.loadHeader().current().ledgerVersion; if (protocolVersionStartsFrom(ledgerVersion, SOROBAN_PROTOCOL_VERSION)) { @@ -1457,26 +1441,28 @@ LedgerManagerImpl::processFeesSeqNums( void LedgerManagerImpl::prefetchTxSourceIds( - std::vector const& txs) + AbstractLedgerTxnParent& rootLtx, + std::vector const& txs, Config const& config) { ZoneScoped; - if (mApp.getConfig().PREFETCH_BATCH_SIZE > 0) + if (config.PREFETCH_BATCH_SIZE > 0) { UnorderedSet keys; for (auto const& tx : txs) { tx->insertKeysForFeeProcessing(keys); } - mApp.getLedgerTxnRoot().prefetchClassic(keys); + rootLtx.prefetchClassic(keys); } } void LedgerManagerImpl::prefetchTransactionData( - std::vector const& txs) + AbstractLedgerTxnParent& rootLtx, + std::vector const& txs, Config const& config) { ZoneScoped; - if (mApp.getConfig().PREFETCH_BATCH_SIZE > 0) + if (config.PREFETCH_BATCH_SIZE > 0) { UnorderedSet sorobanKeys; auto lkMeter = make_unique(); @@ -1485,7 +1471,7 @@ LedgerManagerImpl::prefetchTransactionData( { if (tx->isSoroban()) { - if (mApp.getConfig().isUsingBucketListDB()) + if (config.isUsingBucketListDB()) { tx->insertKeysForTxApply(sorobanKeys, lkMeter.get()); } @@ -1497,15 +1483,14 @@ LedgerManagerImpl::prefetchTransactionData( } // Prefetch classic and soroban keys separately for greater visibility // into the performance of each mode. - if (mApp.getConfig().isUsingBucketListDB()) + if (config.isUsingBucketListDB()) { if (!sorobanKeys.empty()) { - mApp.getLedgerTxnRoot().prefetchSoroban(sorobanKeys, - lkMeter.get()); + rootLtx.prefetchSoroban(sorobanKeys, lkMeter.get()); } } - mApp.getLedgerTxnRoot().prefetchClassic(classicKeys); + rootLtx.prefetchClassic(classicKeys); } } @@ -1535,7 +1520,7 @@ LedgerManagerImpl::applyTransactions( ltx.loadHeader().current().ledgerSeq, txSet.summary()); } - prefetchTransactionData(txs); + prefetchTransactionData(mApp.getLedgerTxnRoot(), txs, mApp.getConfig()); Hash sorobanBasePrngSeed = txSet.getContentsHash(); uint64_t txNum{0}; @@ -1718,8 +1703,8 @@ LedgerManagerImpl::transferLedgerEntriesToBucketList( ltxEvictions.commit(); } - getSorobanNetworkConfigInternal().maybeSnapshotBucketListSize( - lh.ledgerSeq, ltx, mApp); + mSorobanNetworkConfig->maybeSnapshotBucketListSize(lh.ledgerSeq, ltx, + mApp); } ltx.getAllEntries(initEntries, liveEntries, deadEntries); diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h index a5b1ae860a..f3b8d81007 100644 --- a/src/ledger/LedgerManagerImpl.h +++ b/src/ledger/LedgerManagerImpl.h @@ -74,7 +74,7 @@ class LedgerManagerImpl : public LedgerManager std::unique_ptr mNextMetaToEmit; - std::vector processFeesSeqNums( + static std::vector processFeesSeqNums( std::vector const& txs, AbstractLedgerTxn& ltxOuter, ApplicableTxSetFrame const& txSet, std::unique_ptr const& ledgerCloseMeta); @@ -95,10 +95,14 @@ class LedgerManagerImpl : public LedgerManager uint32_t initialLedgerVers); void storeCurrentLedger(LedgerHeader const& header, bool storeHeader); - void - prefetchTransactionData(std::vector const& txs); - void prefetchTxSourceIds(std::vector const& txs); - void closeLedgerIf(LedgerCloseData const& ledgerData); + static void + prefetchTransactionData(AbstractLedgerTxnParent& rootLtx, + std::vector const& txs, + Config const& config); + static void + prefetchTxSourceIds(AbstractLedgerTxnParent& rootLtx, + std::vector const& txs, + Config const& config); State mState; @@ -110,8 +114,6 @@ class LedgerManagerImpl : public LedgerManager void emitNextMeta(); - SorobanNetworkConfig& getSorobanNetworkConfigInternal(); - // Publishes soroban metrics, including select network config limits as well // as the actual ledger usage. void publishSorobanMetrics(); @@ -146,7 +148,8 @@ class LedgerManagerImpl : public LedgerManager State getState() const override; std::string getStateHuman() const override; - void valueExternalized(LedgerCloseData const& ledgerData) override; + void valueExternalized(LedgerCloseData const& ledgerData, + bool isLatestSlot) override; uint32_t getLastMaxTxSetSize() const override; uint32_t getLastMaxTxSetSizeOps() const override; @@ -186,7 +189,8 @@ class LedgerManagerImpl : public LedgerManager std::shared_ptr archive, std::set> bucketsToRetain) override; - void closeLedger(LedgerCloseData const& ledgerData) override; + void closeLedger(LedgerCloseData const& ledgerData, + bool externalize) override; void deleteOldEntries(Database& db, uint32_t ledgerSeq, uint32_t count) override; diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 58c1eb8dc8..52b43fdd2b 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -67,7 +67,8 @@ static const std::unordered_set TESTING_ONLY_OPTIONS = { "ARTIFICIALLY_SET_SURVEY_PHASE_DURATION_FOR_TESTING", "ARTIFICIALLY_DELAY_BUCKET_APPLICATION_FOR_TESTING", "ARTIFICIALLY_SLEEP_MAIN_THREAD_FOR_TESTING", - "ARTIFICIALLY_SKIP_CONNECTION_ADJUSTMENT_FOR_TESTING"}; + "ARTIFICIALLY_SKIP_CONNECTION_ADJUSTMENT_FOR_TESTING", + "ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING"}; // Options that should only be used for testing static const std::unordered_set TESTING_SUGGESTED_OPTIONS = { @@ -190,6 +191,7 @@ Config::Config() : NODE_SEED(SecretKey::random()) ARTIFICIALLY_REPLAY_WITH_NEWEST_BUCKET_LOGIC_FOR_TESTING = false; ARTIFICIALLY_DELAY_BUCKET_APPLICATION_FOR_TESTING = std::chrono::seconds::zero(); + ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING = std::chrono::milliseconds(0); ALLOW_LOCALHOST_FOR_TESTING = false; USE_CONFIG_FOR_GENESIS = false; FAILURE_SAFETY = -1; diff --git a/src/main/Config.h b/src/main/Config.h index 600a349259..1ea2e79a0f 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -262,6 +262,8 @@ class Config : public std::enable_shared_from_this // This config should only be enabled when testing. std::chrono::microseconds ARTIFICIALLY_SLEEP_MAIN_THREAD_FOR_TESTING; + std::chrono::milliseconds ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING; + // Timeout before publishing externalized values to archive std::chrono::seconds PUBLISH_TO_ARCHIVE_DELAY; diff --git a/src/test/TestAccount.cpp b/src/test/TestAccount.cpp index 89b58e169b..0b01ab16ef 100644 --- a/src/test/TestAccount.cpp +++ b/src/test/TestAccount.cpp @@ -74,6 +74,7 @@ TestAccount::getTrustlineBalance(PoolID const& poolID) const int64_t TestAccount::getBalance() const { + REQUIRE(doesAccountExist(mApp, getPublicKey())); LedgerSnapshot lsg(mApp); auto const entry = lsg.getAccount(getPublicKey()); return entry.current().data.account().balance; diff --git a/src/test/TestUtils.cpp b/src/test/TestUtils.cpp index 7750fe345a..9f7ceca9ae 100644 --- a/src/test/TestUtils.cpp +++ b/src/test/TestUtils.cpp @@ -35,6 +35,30 @@ crankFor(VirtualClock& clock, VirtualClock::duration duration) ; } +void +crankUntil(Application::pointer app, std::function const& predicate, + VirtualClock::duration timeout) +{ + crankUntil(*app, predicate, timeout); +} + +void +crankUntil(Application& app, std::function const& predicate, + VirtualClock::duration timeout) +{ + auto start = std::chrono::system_clock::now(); + while (!predicate()) + { + app.getClock().crank(false); + auto current = std::chrono::system_clock::now(); + auto diff = current - start; + if (diff > timeout) + { + break; + } + } +} + void shutdownWorkScheduler(Application& app) { diff --git a/src/test/TestUtils.h b/src/test/TestUtils.h index 83f3e6d4f9..ec1a239902 100644 --- a/src/test/TestUtils.h +++ b/src/test/TestUtils.h @@ -22,6 +22,11 @@ namespace testutil { void crankSome(VirtualClock& clock); void crankFor(VirtualClock& clock, VirtualClock::duration duration); +void crankUntil(Application& app, std::function const& predicate, + VirtualClock::duration duration); +void crankUntil(Application::pointer app, + std::function const& predicate, + VirtualClock::duration duration); void injectSendPeersAndReschedule(VirtualClock::time_point& end, VirtualClock& clock, VirtualTimer& timer, LoopbackPeerConnection& connection); diff --git a/src/test/TxTests.cpp b/src/test/TxTests.cpp index b306e737c1..d7f086b468 100644 --- a/src/test/TxTests.cpp +++ b/src/test/TxTests.cpp @@ -561,6 +561,10 @@ closeLedgerOn(Application& app, uint32 ledgerSeq, TimePoint closeTime, } app.getHerder().externalizeValue(txSet.first, ledgerSeq, closeTime, emptyUpgradeSteps); + while (app.getLedgerManager().getLastClosedLedgerNum() != ledgerSeq) + { + app.getClock().crank(true); + } REQUIRE(app.getLedgerManager().getLastClosedLedgerNum() == ledgerSeq); return getTransactionHistoryResults(app.getDatabase(), ledgerSeq); } @@ -581,7 +585,10 @@ closeLedgerOn(Application& app, uint32 ledgerSeq, time_t closeTime, { app.getHerder().externalizeValue(txSet, ledgerSeq, closeTime, emptyUpgradeSteps); - + while (app.getLedgerManager().getLastClosedLedgerNum() != ledgerSeq) + { + app.getClock().crank(true); + } auto z1 = getTransactionHistoryResults(app.getDatabase(), ledgerSeq); REQUIRE(app.getLedgerManager().getLastClosedLedgerNum() == ledgerSeq); @@ -1771,11 +1778,16 @@ executeUpgrades(Application& app, xdr::xvector const& upgrades, auto& lm = app.getLedgerManager(); auto currLh = app.getLedgerManager().getLastClosedLedgerHeader().header; - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); auto lastCloseTime = lcl.header.scpValue.closeTime; app.getHerder().externalizeValue(txSet, lcl.header.ledgerSeq + 1, lastCloseTime, upgrades); + while (app.getLedgerManager().getLastClosedLedgerNum() != + lcl.header.ledgerSeq + 1) + { + app.getClock().crank(true); + } if (upgradesIgnored) { auto const& newHeader = lm.getLastClosedLedgerHeader().header; diff --git a/src/transactions/test/TxEnvelopeTests.cpp b/src/transactions/test/TxEnvelopeTests.cpp index 99dc560533..6003a1e1a6 100644 --- a/src/transactions/test/TxEnvelopeTests.cpp +++ b/src/transactions/test/TxEnvelopeTests.cpp @@ -1883,6 +1883,7 @@ TEST_CASE_VERSIONS("txenvelope", "[tx][envelope]") .header.scpValue.closeTime; app->getHerder().externalizeValue(txSet, 3, lastCloseTime, emptyUpgradeSteps); + testutil::crankFor(app->getClock(), std::chrono::seconds(1)); REQUIRE(app->getLedgerManager().getLastClosedLedgerNum() == 3); };