From f6308564ed728ac594202c09248938c06bb95091 Mon Sep 17 00:00:00 2001 From: marta-lokhova Date: Fri, 11 Oct 2024 21:02:30 -0700 Subject: [PATCH] Some fixes --- src/history/CheckpointBuilder.cpp | 14 +++++++++----- src/history/CheckpointBuilder.h | 9 ++++++--- src/history/HistoryManagerImpl.cpp | 22 +++++++++++++++------- src/ledger/LedgerHeaderUtils.cpp | 3 ++- src/transactions/TransactionSQL.cpp | 8 ++++---- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/history/CheckpointBuilder.cpp b/src/history/CheckpointBuilder.cpp index 1bb2493e04..e12faf02dc 100644 --- a/src/history/CheckpointBuilder.cpp +++ b/src/history/CheckpointBuilder.cpp @@ -92,7 +92,8 @@ CheckpointBuilder::CheckpointBuilder(Application& app) : mApp(app) void CheckpointBuilder::appendTransactionSet(uint32_t ledgerSeq, TxSetXDRFrameConstPtr const& txSet, - TransactionResultSet const& resultSet) + TransactionResultSet const& resultSet, + bool skipStartupCheck) { ZoneScoped; TransactionHistoryEntry txs; @@ -113,11 +114,12 @@ CheckpointBuilder::appendTransactionSet(uint32_t ledgerSeq, void CheckpointBuilder::appendTransactionSet(uint32_t ledgerSeq, TransactionHistoryEntry const& txSet, - TransactionResultSet const& resultSet) + TransactionResultSet const& resultSet, + bool skipStartupCheck) { ZoneScoped; if (!mStartupValidationComplete && - ledgerSeq != LedgerManager::GENESIS_LEDGER_SEQ) + ledgerSeq != LedgerManager::GENESIS_LEDGER_SEQ && !skipStartupCheck) { throw std::runtime_error("Startup validation not performed"); } @@ -134,11 +136,13 @@ CheckpointBuilder::appendTransactionSet(uint32_t ledgerSeq, } void -CheckpointBuilder::appendLedgerHeader(LedgerHeader const& header) +CheckpointBuilder::appendLedgerHeader(LedgerHeader const& header, + bool skipStartupCheck) { ZoneScoped; if (!mStartupValidationComplete && - header.ledgerSeq != LedgerManager::GENESIS_LEDGER_SEQ) + header.ledgerSeq != LedgerManager::GENESIS_LEDGER_SEQ && + !skipStartupCheck) { throw std::runtime_error("Startup validation not performed"); } diff --git a/src/history/CheckpointBuilder.h b/src/history/CheckpointBuilder.h index 1165465f5e..4698b0ae63 100644 --- a/src/history/CheckpointBuilder.h +++ b/src/history/CheckpointBuilder.h @@ -30,11 +30,14 @@ class CheckpointBuilder CheckpointBuilder(Application& app); void appendTransactionSet(uint32_t ledgerSeq, TxSetXDRFrameConstPtr const& txSet, - TransactionResultSet const& resultSet); + TransactionResultSet const& resultSet, + bool skipStartupCheck = false); void appendTransactionSet(uint32_t ledgerSeq, TransactionHistoryEntry const& txSet, - TransactionResultSet const& resultSet); - void appendLedgerHeader(LedgerHeader const& header); + TransactionResultSet const& resultSet, + bool skipStartupCheck = false); + void appendLedgerHeader(LedgerHeader const& header, + bool skipStartupCheck = false); // Cleanup publish files according to the latest LCL. // Publish files might contain dirty data if a crash occurred after append diff --git a/src/history/HistoryManagerImpl.cpp b/src/history/HistoryManagerImpl.cpp index a959277e74..0ccdccb4eb 100644 --- a/src/history/HistoryManagerImpl.cpp +++ b/src/history/HistoryManagerImpl.cpp @@ -104,6 +104,17 @@ HistoryManagerImpl::dropSQLBasedPublish() // function is atomic releaseAssert(threadIsMain()); + // In case previous schema migration rolled back, cleanup files + fs::deltree(publishQueuePath(mApp.getConfig())); + fs::deltree(getPublishHistoryDir(FileType::HISTORY_FILE_TYPE_LEDGER, + mApp.getConfig()) + .parent_path()); + createPublishDir(FileType::HISTORY_FILE_TYPE_LEDGER, mApp.getConfig()); + createPublishDir(FileType::HISTORY_FILE_TYPE_TRANSACTIONS, + mApp.getConfig()); + createPublishDir(FileType::HISTORY_FILE_TYPE_RESULTS, mApp.getConfig()); + HistoryManager::createPublishQueueDir(mApp.getConfig()); + std::set checkpointLedgers; // Migrate all the existing queued checkpoints to the new format { @@ -126,8 +137,6 @@ HistoryManagerImpl::dropSQLBasedPublish() } auto freq = getCheckpointFrequency(); - uint32_t lclCheckpoint = checkpointContainingLedger( - mApp.getLedgerManager().getLastClosedLedgerNum()); uint32_t lastQueued = 0; for (auto const& checkpoint : checkpointLedgers) { @@ -142,14 +151,13 @@ HistoryManagerImpl::dropSQLBasedPublish() lastQueued = std::max(lastQueued, checkpoint); } - if (lclCheckpoint < lastQueued) + if (lastQueued != 0) { + releaseAssert(isFirstLedgerInCheckpoint(lastQueued + 1)); // Then, reconstruct any partial checkpoints that haven't yet been // queued - populateCheckpointFilesFromDB( - mApp, mApp.getDatabase().getSession(), - firstLedgerInCheckpointContaining(lclCheckpoint), freq, - *mCheckpoint); + populateCheckpointFilesFromDB(mApp, mApp.getDatabase().getSession(), + lastQueued + 1, freq, *mCheckpoint); } // Now it's safe to drop obsolete SQL tables diff --git a/src/ledger/LedgerHeaderUtils.cpp b/src/ledger/LedgerHeaderUtils.cpp index e21eafc7cf..6d06989242 100644 --- a/src/ledger/LedgerHeaderUtils.cpp +++ b/src/ledger/LedgerHeaderUtils.cpp @@ -200,7 +200,8 @@ copyToStream(Database& db, soci::session& sess, uint32_t ledgerSeq, lhe.header = decodeFromData(headerEncoded); lhe.hash = xdrSha256(lhe.header); CLOG_DEBUG(Ledger, "Streaming ledger-header {}", lhe.header.ledgerSeq); - checkpointBuilder.appendLedgerHeader(lhe.header); + checkpointBuilder.appendLedgerHeader(lhe.header, + /* skipStartupCheck */ true); ++n; st.fetch(); } diff --git a/src/transactions/TransactionSQL.cpp b/src/transactions/TransactionSQL.cpp index a36efdf4f1..c35a725f36 100644 --- a/src/transactions/TransactionSQL.cpp +++ b/src/transactions/TransactionSQL.cpp @@ -169,8 +169,8 @@ writeNonGeneralizedTxSetToStream( TransactionHistoryEntry hist; hist.ledgerSeq = ledgerSeq; txSet->toXDR(hist.txSet); - checkpointBuilder.appendTransactionSet(ledgerSeq, hist, - results.txResultSet); + checkpointBuilder.appendTransactionSet(ledgerSeq, hist, results.txResultSet, + /* skipStartupCheck */ true); } void @@ -205,8 +205,8 @@ writeGeneralizedTxSetToStream(uint32 ledgerSeq, hist.ext.v(1); xdr_argpack_archive(unpacker, hist.ext.generalizedTxSet()); - checkpointBuilder.appendTransactionSet(ledgerSeq, hist, - results.txResultSet); + checkpointBuilder.appendTransactionSet(ledgerSeq, hist, results.txResultSet, + /* skipStartupCheck */ true); } void