Add tests

stellar · Sep 12, 2024 · ccf6b27 · ccf6b27
1 parent eace482
commit ccf6b27
Show file tree

Hide file tree

Showing 4 changed files with 281 additions and 60 deletions.
diff --git a/src/history/test/HistoryTests.cpp b/src/history/test/HistoryTests.cpp
@@ -6,9 +6,10 @@
 #include "bucket/test/BucketTestUtils.h"
 #include "catchup/CatchupManagerImpl.h"
 #include "catchup/test/CatchupWorkTests.h"
+#include "history/CheckpointBuilder.h"
 #include "history/FileTransferInfo.h"
 #include "history/HistoryArchiveManager.h"
-#include "history/HistoryManager.h"
+#include "history/HistoryManagerImpl.h"
 #include "history/test/HistoryTestsUtils.h"
 #include "historywork/GetHistoryArchiveStateWork.h"
 #include "historywork/GunzipFileWork.h"
@@ -472,6 +473,59 @@ TEST_CASE("History publish", "[history][publish]")
     catchupSimulation.ensureOfflineCatchupPossible(checkpointLedger);
 }
 
+TEST_CASE("History publish with restart", "[history][publish]")
+{
+    auto catchupSimulation =
+        CatchupSimulation(VirtualClock::VIRTUAL_TIME,
+                          std::make_shared<TmpDirHistoryConfigurator>(), true,
+                          Config::TESTDB_ON_DISK_SQLITE);
+    auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(2);
+
+    // Restart at various points in the checkpoint, core should continue
+    // properly writing checkpoint files
+    auto ledgerNums = std::vector<uint32_t>{
+        LedgerManager::GENESIS_LEDGER_SEQ,
+        10,
+        catchupSimulation.getLastCheckpointLedger(1),
+        catchupSimulation.getApp()
+            .getHistoryManager()
+            .firstLedgerInCheckpointContaining(checkpointLedger),
+        checkpointLedger - 1,
+        checkpointLedger};
+    for (auto ledgerNum : ledgerNums)
+    {
+        SECTION("Restart at ledger " + std::to_string(ledgerNum))
+        {
+            SECTION("graceful")
+            {
+                catchupSimulation.ensureOfflineCatchupPossible(checkpointLedger,
+                                                               ledgerNum);
+            }
+            SECTION("crash leaves dirty data")
+            {
+                auto& hm = static_cast<HistoryManagerImpl&>(
+                    catchupSimulation.getApp().getHistoryManager());
+                hm.mThrowOnLastAppend = true;
+                REQUIRE_THROWS_AS(
+                    catchupSimulation.ensureOfflineCatchupPossible(
+                        checkpointLedger),
+                    std::runtime_error);
+                // Restart app, truncate dirty data in checkpoints, proceed to
+                // publish
+                catchupSimulation.restartApp();
+                catchupSimulation.ensureOfflineCatchupPossible(
+                    checkpointLedger);
+            }
+
+            // Now catchup to ensure published checkpoints are valid
+            auto app = catchupSimulation.createCatchupApplication(
+                std::numeric_limits<uint32_t>::max(),
+                Config::TESTDB_ON_DISK_SQLITE, "app");
+            REQUIRE(catchupSimulation.catchupOffline(app, checkpointLedger));
+        }
+    }
+}
+
 TEST_CASE("History publish to multiple archives", "[history]")
 {
     Config cfg(getTestConfig());
@@ -1630,3 +1684,117 @@ TEST_CASE("Externalize gap while catchup work is running", "[history][catchup]")
     REQUIRE(catchupSimulation.catchupOnline(app, lcl + 2, 0, 0, 0,
                                             {128, 129, 127}));
 }
+
+TEST_CASE("CheckpointBuilder", "[history][publish]")
+{
+    VirtualClock clock;
+    auto cfg = getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE);
+    TmpDirHistoryConfigurator().configure(cfg, true);
+
+    auto app = createTestApplication(clock, cfg);
+    releaseAssert(app->getLedgerManager().getLastClosedLedgerNum() ==
+                  LedgerManager::GENESIS_LEDGER_SEQ);
+    auto& hm = static_cast<HistoryManagerImpl&>(app->getHistoryManager());
+    auto& cb = hm.getCheckpointBuilder();
+    auto lcl = app->getLedgerManager().getLastClosedLedgerNum();
+
+    auto generate = [&](uint32_t count, bool appendHeaders = true) {
+        for (int i = lcl; i < lcl + count; ++i)
+        {
+            LedgerHeaderHistoryEntry lh;
+            lh.header.ledgerSeq = i;
+            cb.appendTransactionSet(i, TxSetXDRFrame::makeEmpty(lh),
+                                    TransactionResultSet{});
+            // Do not append last ledger in a checkpoint if `appendHeaders` is
+            // false
+            if (!appendHeaders && i == count)
+            {
+                continue;
+            }
+            cb.appendLedgerHeader(lh.header);
+        }
+    };
+
+    auto validateHdr = [&](std::string path, uint32_t ledger) {
+        XDRInputFileStream hdrIn;
+        hdrIn.open(path);
+        LedgerHeaderHistoryEntry entry;
+        while (hdrIn && hdrIn.readOne(entry))
+        {
+            REQUIRE(entry.header.ledgerSeq <= ledger);
+        }
+        REQUIRE(entry.header.ledgerSeq == ledger);
+    };
+
+    auto tmpCheckpointCheck = [&](uint32_t ledger, bool isFinalized) {
+        auto checkpoint =
+            app->getHistoryManager().checkpointContainingLedger(ledger);
+        FileTransferInfo res(FileType::HISTORY_FILE_TYPE_RESULTS, checkpoint,
+                             cfg);
+        FileTransferInfo txs(FileType::HISTORY_FILE_TYPE_TRANSACTIONS,
+                             checkpoint, cfg);
+        FileTransferInfo headers(FileType::HISTORY_FILE_TYPE_LEDGER, checkpoint,
+                                 cfg);
+        if (isFinalized)
+        {
+            REQUIRE(fs::exists(res.localPath_nogz()));
+            REQUIRE(fs::exists(txs.localPath_nogz()));
+            REQUIRE(!fs::exists(res.localPath_nogz_dirty()));
+            REQUIRE(!fs::exists(txs.localPath_nogz_dirty()));
+            REQUIRE(!fs::exists(headers.localPath_nogz_dirty()));
+            validateHdr(headers.localPath_nogz(), ledger);
+        }
+        else
+        {
+            REQUIRE(!fs::exists(res.localPath_nogz()));
+            REQUIRE(!fs::exists(txs.localPath_nogz()));
+            REQUIRE(!fs::exists(headers.localPath_nogz()));
+            REQUIRE(fs::exists(res.localPath_nogz_dirty()));
+            REQUIRE(fs::exists(txs.localPath_nogz_dirty()));
+            validateHdr(headers.localPath_nogz_dirty(), ledger);
+        }
+    };
+
+    SECTION("truncate")
+    {
+        SECTION("truncate transactions, but not headers")
+        {
+            generate(10, false);
+            tmpCheckpointCheck(9, false);
+        }
+        SECTION("truncate both")
+        {
+            generate(10);
+            tmpCheckpointCheck(10, false);
+        }
+        SECTION("truncate due to partial write")
+        {
+            generate(10);
+            tmpCheckpointCheck(10, false);
+            FileTransferInfo headers(
+                FileType::HISTORY_FILE_TYPE_LEDGER,
+                app->getHistoryManager().checkpointContainingLedger(10),
+                app->getConfig());
+            auto sz =
+                std::filesystem::file_size(headers.localPath_nogz_dirty());
+            std::filesystem::resize_file(headers.localPath_nogz_dirty(),
+                                         sz - 1);
+        }
+        CheckpointBuilder cb2{*app};
+        cb2.cleanup(9);
+        tmpCheckpointCheck(9, false);
+    }
+    SECTION("checkpoint complete")
+    {
+        auto ledgerSeq = hm.checkpointContainingLedger(1);
+        // Checkpoint not finalized
+        generate(ledgerSeq);
+        tmpCheckpointCheck(ledgerSeq, false);
+        cb.checkpointComplete(ledgerSeq);
+        tmpCheckpointCheck(ledgerSeq, true);
+        REQUIRE(!cb.mOpen);
+        // any subssequent call to checkpointComplete is a no-op
+        cb.checkpointComplete(ledgerSeq);
+        tmpCheckpointCheck(ledgerSeq, true);
+    }
+}