implement protocol 2.0; change db schema

contrib/query.py

@@ -72,7 +72,7 @@ async def query(args):
         if not hashX:
             continue
         n = None
-        history = await db.limited_history(hashX, limit=limit)
+        history = await db.limited_history(hashX=hashX, limit=limit)
         for n, (tx_hash, height) in enumerate(history, start=1):
             print(f'History #{n:,d}: height {height:,d} '
                   f'tx_hash {hash_to_hex_str(tx_hash)}')

docs/HOWTO.rst

@@ -29,17 +29,20 @@ functions. For example, `x11_hash`_ is required for DASH. Scrypt coins
 require a Python interpreter compiled and/or linked with OpenSSL 1.1.0
 or higher.
 
-You **must** be running a non-pruning bitcoin daemon with::
+You **must** be running a non-pruning bitcoin daemon.
+It is also recommended to have::
 
   txindex=1
 
-set in its configuration file.  If you have an existing installation
-of bitcoind and have not previously set this you will need to reindex
-the blockchain with::
+set in its configuration file, for better performance when serving many sessions.
+If you have an existing installation of bitcoind and have not previously set this
+but you do now, you will need to reindex the blockchain with::
 
   bitcoind -reindex
 
 which can take some time.
+If you intend to use a bitcoind without txindex enabled, you need to configure the
+`DAEMON_HAS_TXINDEX` :ref:`environment variable <environment>`.
 
 While not a requirement for running ElectrumX, it is intended to be
 run with supervisor software such as Daniel Bernstein's

docs/environment.rst

@@ -281,13 +281,14 @@ These environment variables are optional:
   version string. For example to drop versions from 1.0 to 1.2 use
   the regex ``1\.[0-2]\.\d+``.
 
-.. envvar:: DROP_CLIENT_UNKNOWN
+.. envvar:: DAEMON_HAS_TXINDEX
 
-  Set to anything non-empty to deny serving clients which do not
-  identify themselves first by issuing the server.version method
-  call with a non-empty client identifier. The connection is dropped 
-  on first actual method call. This might help to filter out simple 
-  robots. This behavior is off by default.
+  Set this environment variable to empty if the connected bitcoind
+  does not have txindex enabled. Defaults to True (has txindex).
+  We rely on bitcoind to lookup arbitrary txs, regardless of this setting.
+  Without txindex, tx lookup works as in `bitcoind PR #10275`_.
+  Note that performance when serving many sessions is better with txindex
+  (but initial sync should be unaffected).
 
 
 Resource Usage Limits
@@ -506,3 +507,4 @@ your available physical RAM:
 
 .. _lib/coins.py: https://github.com/spesmilo/electrumx/blob/master/electrumx/lib/coins.py
 .. _uvloop: https://pypi.python.org/pypi/uvloop
+.. _bitcoind PR #10275: https://github.com/bitcoin/bitcoin/pull/10275

electrumx/lib/tx.py

@@ -29,7 +29,7 @@
 
 from dataclasses import dataclass
 from hashlib import blake2s
-from typing import Sequence
+from typing import Sequence, Optional
 
 from electrumx.lib.hash import sha256, double_sha256, hash_to_hex_str
 from electrumx.lib.script import OpCodes
@@ -105,6 +105,13 @@ def serialize(self):
         ))
 
 
+@dataclass
+class TXOSpendStatus:
+    prev_height: Optional[int]  # block height TXO is mined at. None if the outpoint never existed
+    spender_txhash: bytes = None
+    spender_height: int = None
+
+
 class Deserializer:
     '''Deserializes blocks into transactions.
 

electrumx/lib/util.py

@@ -34,6 +34,7 @@
 import sys
 from collections.abc import Container, Mapping
 from struct import Struct
+from typing import Optional
 
 
 # Use system-compiled JSON lib if available, fallback to stdlib
@@ -162,7 +163,7 @@ def chunks(items, size):
         yield items[i: i + size]
 
 
-def resolve_limit(limit):
+def resolve_limit(limit: Optional[int]) -> int:
     if limit is None or limit < 0:
         return -1
     assert isinstance(limit, int)
@@ -316,6 +317,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 struct_le_Q = Struct('<Q')
 struct_be_H = Struct('>H')
 struct_be_I = Struct('>I')
+struct_be_Q = Struct('>Q')
 structB = Struct('B')
 
 unpack_le_int32_from = struct_le_i.unpack_from
@@ -329,6 +331,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 unpack_le_uint32 = struct_le_I.unpack
 unpack_le_uint64 = struct_le_Q.unpack
 unpack_be_uint32 = struct_be_I.unpack
+unpack_be_uint64 = struct_be_Q.unpack
 
 pack_le_int32 = struct_le_i.pack
 pack_le_int64 = struct_le_q.pack
@@ -337,6 +340,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 pack_le_uint64 = struct_le_Q.pack
 pack_be_uint16 = struct_be_H.pack
 pack_be_uint32 = struct_be_I.pack
+pack_be_uint64 = struct_be_Q.pack
 pack_byte = structB.pack
 
 hex_to_bytes = bytes.fromhex