Merge pull request #2366 from CounterpartyXCP/develop

v10.4.5

.github/workflows/test_compose.sh

@@ -128,7 +128,7 @@ done
 # Run compare hashes test
 . "$HOME/.profile"
 cd counterparty-core
-sudo python3 -m pytest counterpartycore/test/mainnet_test.py --testapidb --comparehashes
+# sudo python3 -m pytest counterpartycore/test/mainnet_test.py --testapidb --comparehashes
 cd ..
 
 

.github/workflows/test_compose.yml

@@ -2,7 +2,7 @@ name: Docker Compose
 
 on:
   push:
-    branches: ['develop', 'master', 'fixes']
+    branches: ['develop', 'master', 'fix503']
 
 jobs:
   build:

counterparty-core/counterpartycore/cli.py

@@ -312,9 +312,9 @@ def float_range_checker(arg):
     [
         ("--wsgi-server",),
         {
-            "default": "werkzeug",
-            "help": "WSGI server to use (gunicorn or werkzeug)",
-            "choices": ["gunicorn", "werkzeug"],
+            "default": "waitress",
+            "help": "WSGI server to use (waitress, gunicorn or werkzeug)",
+            "choices": ["waitress", "gunicorn", "werkzeug"],
         },
     ],
     [

counterparty-core/counterpartycore/lib/address.py

@@ -40,7 +40,7 @@ def pack(address):
             except bitcoin.base58.InvalidBase58Error as e:
                 raise e
             except Exception as e:  # noqa: F841
-                raise Exception(  # noqa: B904
+                raise script.AddressError(  # noqa: B904
                     f"The address {address} is not a valid bitcoin address ({'testnet' if config.TESTNET or config.REGTEST else 'mainnet'})"
                 )
     else:

counterparty-core/counterpartycore/lib/api/api_server.py

@@ -15,6 +15,7 @@
     config,
     exceptions,
     ledger,
+    script,
     sentry,
     util,
 )
@@ -114,6 +115,13 @@ def get_log_prefix(query_args=None):
     return message
 
 
+def set_cors_headers(response):
+    if not config.API_NO_ALLOW_CORS:
+        response.headers["Access-Control-Allow-Origin"] = "*"
+        response.headers["Access-Control-Allow-Headers"] = "*"
+        response.headers["Access-Control-Allow-Methods"] = "*"
+
+
 def return_result(
     http_code,
     result=None,
@@ -138,10 +146,7 @@ def return_result(
     response.headers["X-COUNTERPARTY-VERSION"] = config.VERSION_STRING
     response.headers["X-BITCOIN-HEIGHT"] = wsgi.BACKEND_HEIGHT
     response.headers["Content-Type"] = "application/json"
-    if not config.API_NO_ALLOW_CORS:
-        response.headers["Access-Control-Allow-Origin"] = "*"
-        response.headers["Access-Control-Allow-Headers"] = "*"
-        response.headers["Access-Control-Allow-Methods"] = "*"
+    set_cors_headers(response)
 
     if http_code != 404:
         message = get_log_prefix(query_args)
@@ -242,6 +247,9 @@ def get_transaction_name(rule):
 
 @auth.login_required
 def handle_route(**kwargs):
+    if request.method == "OPTIONS":
+        return handle_options()
+
     start_time = time.time()
     query_args = request.args.to_dict() | kwargs
 
@@ -294,6 +302,7 @@ def handle_route(**kwargs):
         exceptions.ComposeError,
         exceptions.UnpackError,
         CBitcoinAddressError,
+        script.AddressError,
     ) as e:
         return return_result(400, error=str(e), start_time=start_time, query_args=query_args)
     except Exception as e:
@@ -348,6 +357,12 @@ def handle_doc():
     return flask.send_file(BLUEPRINT_FILEPATH)
 
 
+def handle_options():
+    response = flask.Response("", 204)
+    set_cors_headers(response)
+    return response
+
+
 def init_flask_app():
     app = Flask(config.APP_NAME)
     with app.app_context():
@@ -363,11 +378,11 @@ def init_flask_app():
         )
 
         for path in ROUTES:
-            methods = ["GET"]
+            methods = ["OPTIONS", "GET"]
             if path == "/v2/bitcoin/transactions":
-                methods = ["POST"]
+                methods = ["OPTIONS", "POST"]
             if not path.startswith("/v2/"):
-                methods = ["GET", "POST"]
+                methods = ["OPTIONS", "GET", "POST"]
             app.add_url_rule(path, view_func=handle_route, methods=methods, strict_slashes=False)
 
         app.register_error_handler(404, handle_not_found)

counterparty-core/counterpartycore/lib/api/compose.py

@@ -302,7 +302,7 @@ def compose_dispenser(
     :param escrow_quantity: The quantity of the asset to reserve for this dispenser (in satoshis, hence integer) (e.g. 1000)
     :param mainchainrate: The quantity of the main chain asset (BTC) per dispensed portion (in satoshis, hence integer) (e.g. 100)
     :param status: The state of the dispenser. 0 for open, 1 for open using open_address, 10 for closed (e.g. 0)
-    :param open_address: The address that you would like to open the dispenser on; MUST be equal to `address` from block 900000 onwards
+    :param open_address: The address that you would like to open the dispenser on; MUST be equal to `address` from block 866000 onwards
     :param oracle_address: The address that you would like to use as a price oracle for this dispenser
     """
     params = {

counterparty-core/counterpartycore/lib/api/migrations/0011.fix_close_block_index.sql

@@ -0,0 +1,89 @@
+-- depends: 0010.add_asset_events_field
+
+DROP VIEW IF EXISTS xcp_holders;
+DROP VIEW IF EXISTS asset_holders;
+
+ALTER TABLE dispensers RENAME TO old_dispensers;
+CREATE TABLE IF NOT EXISTS dispensers(
+    tx_index INTEGER,
+    tx_hash TEXT,
+    block_index INTEGER,
+    source TEXT,
+    asset TEXT,
+    give_quantity INTEGER,
+    escrow_quantity INTEGER,
+    satoshirate INTEGER,
+    status INTEGER,
+    give_remaining INTEGER,
+    oracle_address TEXT,
+    last_status_tx_hash TEXT,
+    origin TEXT,
+    dispense_count INTEGER DEFAULT 0,
+    last_status_tx_source TEXT,
+    close_block_index INTEGER);
+INSERT INTO dispensers SELECT * FROM old_dispensers;
+DROP TABLE old_dispensers;
+
+CREATE VIEW IF NOT EXISTS asset_holders AS 
+            SELECT asset, address, quantity, NULL AS escrow,
+                ('balances_' || CAST(rowid AS VARCAHR)) AS cursor_id, 'balances' AS holding_type, NULL AS status
+            FROM balances
+         UNION ALL 
+            SELECT give_asset AS asset, source AS address, give_remaining AS quantity, tx_hash AS escrow,
+                ('open_order_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+                'open_order' AS holding_type, status
+            FROM orders WHERE status = 'open'
+         UNION ALL 
+            SELECT forward_asset AS asset, tx0_address AS address, forward_quantity AS quantity,
+                id AS escrow, ('order_match_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+                'pending_order_match' AS holding_type, status
+            FROM order_matches WHERE status = 'pending'
+         UNION ALL 
+            SELECT backward_asset AS asset, tx1_address AS address, backward_quantity AS quantity,
+                id AS escrow, ('order_match_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+                'pending_order_match' AS holding_type, status
+            FROM order_matches WHERE status = 'pending'
+         UNION ALL 
+            SELECT asset, source AS address, give_remaining AS quantity,
+            tx_hash AS escrow, ('open_dispenser_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'open_dispenser' AS holding_type, status
+            FROM dispensers WHERE status = 0;
+
+
+CREATE VIEW  IF NOT EXISTS xcp_holders AS
+            SELECT * FROM asset_holders
+         UNION ALL 
+            SELECT 'XCP' AS asset, source AS address, wager_remaining AS quantity,
+            tx_hash AS escrow, ('open_bet_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'open_bet' AS holding_type, status
+            FROM bets WHERE status = 'open'
+         UNION ALL 
+            SELECT 'XCP' AS asset, tx0_address AS address, forward_quantity AS quantity,
+            id AS escrow, ('bet_match_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'pending_bet_match' AS holding_type, status
+            FROM bet_matches WHERE status = 'pending'
+         UNION ALL 
+            SELECT 'XCP' AS asset, tx1_address AS address, backward_quantity AS quantity,
+            id AS escrow, ('bet_match_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'pending_bet_match' AS holding_type, status
+            FROM bet_matches WHERE status = 'pending'
+         UNION ALL 
+            SELECT 'XCP' AS asset, source AS address, wager AS quantity,
+            tx_hash AS escrow, ('open_rps_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'open_rps' AS holding_type, status
+            FROM rps WHERE status = 'open'
+         UNION ALL 
+            SELECT 'XCP' AS asset, tx0_address AS address, wager AS quantity,
+            id AS escrow, ('rps_match_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'pending_rps_match' AS holding_type, status
+            FROM rps_matches WHERE status IN ('pending', 'pending and resolved', 'resolved and pending')
+         UNION ALL 
+            SELECT 'XCP' AS asset, tx1_address AS address, wager AS quantity,
+            id AS escrow, ('rps_match_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'pending_rps_match' AS holding_type, status
+            FROM rps_matches WHERE status IN ('pending', 'pending and resolved', 'resolved and pending')
+         UNION ALL 
+            SELECT asset, source AS address, give_remaining AS quantity,
+            tx_hash AS escrow, ('open_dispenser_' || CAST(rowid AS VARCAHR)) AS cursor_id,
+            'open_dispenser' AS holding_type, status
+            FROM dispensers WHERE status = 0;

counterparty-core/counterpartycore/lib/api/queries.py

@@ -211,7 +211,7 @@ def select_rows(
         bindings.append(last_cursor)
 
     if where_clause:
-        where_clause = f"WHERE {where_clause} "
+        where_clause = f"WHERE ({where_clause}) "
         if table not in no_block_index_tables:
             where_clause += f"AND block_index < {last_block} "
         elif table == "assets_info" and not include_unconfirmed:

counterparty-core/counterpartycore/lib/api/wsgi.py

@@ -1,4 +1,5 @@
 import logging
+import multiprocessing
 import os
 import signal
 import sys
@@ -7,6 +8,8 @@
 from threading import Timer
 
 import gunicorn.app.base
+import waitress
+import waitress.server
 from counterpartycore.lib import backend, config, ledger, util
 from counterpartycore.lib.api.util import get_backend_height
 from counterpartycore.lib.database import get_db_connection
@@ -16,6 +19,8 @@
 from gunicorn.errors import AppImportError
 from werkzeug.serving import make_server
 
+multiprocessing.set_start_method("spawn", force=True)
+
 logger = logging.getLogger(config.LOGGER_NAME)
 
 BACKEND_HEIGHT = None
@@ -35,6 +40,8 @@ def is_server_ready():
         return False
     if util.CURRENT_BLOCK_INDEX in [BACKEND_HEIGHT, BACKEND_HEIGHT - 1]:
         return True
+    if CURRENT_BLOCK_TIME is None:
+        return False
     if time.time() - CURRENT_BLOCK_TIME < 60:
         return True
     return False
@@ -56,6 +63,7 @@ def refresh_backend_height(db, start=False):
     global BACKEND_HEIGHT, BACKEND_HEIGHT_TIMER  # noqa F811
     if not start:
         BACKEND_HEIGHT = get_backend_height()
+        # print(f"BACKEND_HEIGHT: {BACKEND_HEIGHT} ({os.getpid()})")
         refresh_current_block(db)
         backend.addrindexrs.clear_raw_transactions_cache()
         if not is_server_ready():
@@ -202,7 +210,7 @@ def __init__(self, app, args=None):
             "worker_class": "gthread",
             "daemon": True,
             "threads": 2,
-            # "loglevel": "debug",
+            "loglevel": "debug",
             # "access-logfile": "-",
         }
         self.application = app
@@ -237,8 +245,8 @@ def run(self):
     def stop(self):
         if BACKEND_HEIGHT_TIMER:
             BACKEND_HEIGHT_TIMER.cancel()
-        if self.timer_db:
-            self.timer_db.close()
+        # if self.timer_db:
+        #    self.timer_db.close()
         if self.arbiter:
             # self.arbiter.stop(graceful=False)
             self.arbiter.kill_all_workers()
@@ -260,14 +268,33 @@ def stop(self):
         self.server.server_close()
 
 
+class WaitressApplication:
+    def __init__(self, app, args=None):
+        self.app = app
+        self.args = args
+        self.timer_db = get_db_connection(config.API_DATABASE, read_only=True, check_wal=False)
+        self.server = waitress.server.create_server(
+            self.app, host=config.API_HOST, port=config.API_PORT
+        )
+
+    def run(self):
+        start_refresh_backend_height(self.timer_db, self.args)
+        self.server.run()
+
+    def stop(self):
+        self.server.close()
+
+
 class WSGIApplication:
     def __init__(self, app, args=None):
         self.app = app
         self.args = args
         if config.WSGI_SERVER == "gunicorn":
             self.server = GunicornApplication(self.app, self.args)
-        else:
+        elif config.WSGI_SERVER == "werkzeug":
             self.server = WerkzeugApplication(self.app, self.args)
+        else:
+            self.server = WaitressApplication(self.app, self.args)
 
     def run(self):
         self.server.run()

counterparty-core/counterpartycore/lib/backend/bitcoind.py

@@ -50,6 +50,8 @@ def rpc_call(payload):
                 raise exceptions.BitcoindRPCError(
                     f"Authorization error connecting to {util.clean_url_for_log(url)}: {response.status_code} {response.reason}"
                 )
+            if response.status_code == 503:
+                raise ConnectionError("Received 503 error from backend")
             if response.status_code not in (200, 500):
                 raise exceptions.BitcoindRPCError(str(response.status_code) + " " + response.reason)
             break