Merge pull request #1523 from CounterpartyXCP/testbook

Restore Test Book

.github/workflows/test_book.yml

@@ -0,0 +1,40 @@
+name: Testnet Test Book
+
+on:
+  push:
+    branches: ["testbook"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Rust toolchain
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        default: true
+    - name: Install dependencies
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y libgirepository1.0-dev libleveldb-dev
+        python -m pip install --upgrade pip
+        pip install pylint maturin pytest
+        cd counterparty-rs && pip install -e . && cd ..
+        cd counterparty-lib && pip install -e . && cd ..
+        cd counterparty-cli && pip install -e . && cd ..
+    - name: Bootstrap testnet database
+      run: |
+        counterparty-server --testnet bootstrap
+    - name: Run tests
+      run: |
+        cd counterparty-lib
+        pytest counterpartylib/test/book_test.py --testbook -s

counterparty-cli/counterpartycli/server.py

@@ -32,7 +32,7 @@
     [('--backend-connect',), {'default': 'localhost', 'help': 'the hostname or IP of the backend server'}],
     [('--backend-port',), {'type': int, 'help': 'the backend port to connect to'}],
     [('--backend-user',), {'default': 'bitcoinrpc', 'help': 'the username used to communicate with backend'}],
-    [('--backend-password',), {'help': 'the password used to communicate with backend'}],
+    [('--backend-password',), {'default': 'rpc', 'help': 'the password used to communicate with backend'}],
     [('--backend-ssl',), {'action': 'store_true', 'default': False, 'help': 'use SSL to connect to backend (default: false)'}],
     [('--backend-ssl-no-verify',), {'action': 'store_true', 'default': False, 'help': 'verify SSL certificate of backend; disallow use of self‐signed certificates (default: true)'}],
     [('--backend-poll-interval',), {'type': float, 'default': 0.5, 'help': 'poll interval, in seconds (default: 0.5)'}],

counterparty-lib/counterpartylib/lib/backend/addrindexrs.py

@@ -639,9 +639,11 @@ def get_oldest_tx(address, block_index=None):
     current_block_index = block_index or ledger.CURRENT_BLOCK_INDEX
     hardcoded_key = f"{current_block_index}-{address}"
     if hardcoded_key in GET_OLDEST_TX_HARDCODED:
-        return GET_OLDEST_TX_HARDCODED[hardcoded_key]
+        result = GET_OLDEST_TX_HARDCODED[hardcoded_key]
+    else:
+        global ADDRINDEXRS_CLIENT
+        if ADDRINDEXRS_CLIENT is None:
+            ADDRINDEXRS_CLIENT = AddrindexrsSocket()
+        result = ADDRINDEXRS_CLIENT.get_oldest_tx(address, block_index=current_block_index)
 
-    global ADDRINDEXRS_CLIENT
-    if ADDRINDEXRS_CLIENT is None:
-        ADDRINDEXRS_CLIENT = AddrindexrsSocket()
-    return ADDRINDEXRS_CLIENT.get_oldest_tx(address, block_index=current_block_index)
+    return result

counterparty-lib/counterpartylib/lib/check.py

@@ -209,6 +209,7 @@
     2400000: {'ledger_hash': '5c0606e2729d9b2a2181388231fd816ce3279c4183137bf62e9c699dbdc2f140', 'txlist_hash': '36bbd14c69e5fc17cb1e69303affcb808909757395d28e3e3da83394260bf0dd'},
     2500000: {'ledger_hash': '76262b272c47b5a17f19ffa0ba72256617bd18e51fad4c3c5b3c776cb3a1037b', 'txlist_hash': '26567e5c45a59426b2bcdeb177167a92c5fc21e8fd2000ae9a24eb09e3945f70'},
     2540000: {'ledger_hash': '6d3e77f1c059b062f4eb131cc8eb1d8355598de756905d83803df0009a514f48', 'txlist_hash': '45e134cb4196bc5cfb4ef9b356e02025f752a9bc0ae635bc9ced2c471ecfcb6c'},
+    2580000: {'ledger_hash': 'f84878f8e1293cd5365fc66ec7acbc24b4201b79ab3125d7e60e26624f49037a', 'txlist_hash': 'dc1f037beb8a632c5277fa0e6fd52866f0370d1f93de024420a7c31ce956c812'},
 }
 
 CONSENSUS_HASH_VERSION_REGTEST = 1

counterparty-lib/counterpartylib/test/book_test.py

@@ -6,10 +6,8 @@
 from counterpartylib.test import util_test
 
 
-def test_book(book):
-    if book == 'testnet':
-        util_test.reparse(testnet=True)
-    elif book == 'mainnet':
-        util_test.reparse(testnet=False)
-    else:
+def test_book(skip):
+    if skip:
         pytest.skip("Skipping test book")
+    else:
+        util_test.reparse(testnet=True)

counterparty-lib/counterpartylib/test/conftest.py

@@ -116,7 +116,7 @@ def pytest_generate_tests(metafunc):
                 args.append((scenario_name, INTEGRATION_SCENARIOS[scenario_name][1], INTEGRATION_SCENARIOS[scenario_name][0], metafunc.config))
         metafunc.parametrize('scenario_name, base_scenario_name, transactions, pytest_config', args)
     elif metafunc.function.__name__ == 'test_book':
-        metafunc.parametrize('book', [(metafunc.config.getoption("testbook"),)])
+        metafunc.parametrize('skip', [not metafunc.config.getoption("testbook")])
 
 
 def pytest_addoption(parser):
@@ -126,7 +126,7 @@ def pytest_addoption(parser):
     parser.addoption("--gentxhex", action='store_true', default=False, help="Generate and print unsigned hex for *.compose() tests")
     parser.addoption("--savescenarios", action='store_true', default=False, help="Generate sql dump and log in .new files")
     parser.addoption("--alternative", action='store_true', default=False)
-    parser.addoption("--testbook", default='no', help="Include test book (use with one of the following values: `testnet` or `mainnet`)")
+    parser.addoption("--testbook", action='store_true', default=False, help="Include testnet test book")
 
 
 @pytest.fixture(scope="function")

counterparty-lib/counterpartylib/test/util_test.py

@@ -742,12 +742,26 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         for k in self._before_empty:
             del self.mock_protocol_changes[k]
 
-def reparse(testnet=True, checkpoint_count=10):
+
+def connect_to_addrindexrs_mock():
+    return True
+
+def get_oldest_tx_mock(address, block_index=None):
+    if address == "mrHFGUKSiNMeErqByjX97qPKfumdZxe6mC" and block_index==99999999999:
+        return {"block_index": 2576100, "tx_hash": "d6751cc77da77d9270e16046a19954bfaffc005269f0c6a9248e680be6d1f468"}
+    return {}
+
+def reparse(testnet=True, checkpoint_count=5):
     """
     Reparse all transaction from the database.
      - Create a new in-memory DB, copy the DB that is on-disk
      - Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB
     """
+
+    # mock the backend
+    server.connect_to_addrindexrs = connect_to_addrindexrs_mock
+    backend.get_oldest_tx = get_oldest_tx_mock
+
     # create a new in-memory DB
     options = dict(COUNTERPARTYD_OPTIONS)
     server.initialise(database_file=':memory:', testnet=testnet, **options)
@@ -775,6 +789,7 @@ def reparse(testnet=True, checkpoint_count=10):
 
     # we start one block after the checkpoint before the first one we want to check
     block_index = sorted(list(CHECKPOINTS.keys()))[-checkpoint_count - 1]
+    print(f"Checking from block {block_index}")
 
     # Initialise missing tables
     blocks.initialise(memory_db)

counterparty-lib/tools/compareledger.py

@@ -109,7 +109,7 @@ def check_hashes(database_file_1, database_file_2, hash_name="ledger_hash"):
 
 
 def get_checkpoints(database_file):
-    checkpoints = [1600000, 1700000, 1800000, 1900000, 2000000, 2200000, 2400000, 2500000, 2540000]
+    checkpoints = [2580000]
     db = apsw.Connection(database_file_1, flags=apsw.SQLITE_OPEN_READONLY)
     cursor = db.cursor()
     for checkpoint in checkpoints:
@@ -138,6 +138,6 @@ def get_last_block(database_file_1, database_file_2):
 
 LAST_BLOCK = 290000
 #compare_ledger(database_file_1, database_file_2)
-check_hashes(database_file_1, database_file_2, "txlist_hash")
-#get_checkpoints(database_file_1)
+#check_hashes(database_file_1, database_file_2, "txlist_hash")
+get_checkpoints(database_file_1)
 #get_last_block(database_file_1, database_file_2)