diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index 7092a5646..536d3defe 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -105,11 +105,6 @@ impl ElectrsClient { Ok(ret) } - pub async fn is_tx_output_spent(&self, txid: &Txid, vout: u32) -> Result { - let spending_value: SpendingValue = self.get_and_decode(&format!("/tx/{txid}/outspend/{vout}")).await?; - Ok(spending_value.spent) - } - pub async fn get_blocks_tip_height(&self) -> Result { Ok(self.get("/blocks/tip/height").await?.parse()?) } diff --git a/bitcoin/src/electrs/types.rs b/bitcoin/src/electrs/types.rs index bb72552d7..5320a3c61 100644 --- a/bitcoin/src/electrs/types.rs +++ b/bitcoin/src/electrs/types.rs @@ -15,7 +15,7 @@ pub struct TransactionStatus { } // https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L167-L189 -#[derive(Deserialize, Clone)] +#[derive(Deserialize)] pub struct TxInValue { pub txid: Txid, pub vout: u32, @@ -33,7 +33,7 @@ pub struct TxInValue { } // https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L239-L270 -#[derive(Deserialize, Clone)] +#[derive(Deserialize)] pub struct TxOutValue { pub scriptpubkey: ScriptBuf, pub scriptpubkey_asm: String, @@ -66,15 +66,3 @@ pub struct UtxoValue { pub status: TransactionStatus, pub value: u64, } - -// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L448-L457 -#[derive(Deserialize)] -pub struct SpendingValue { - pub spent: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub txid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub vin: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub status: Option, -} diff --git a/bitcoin/src/iter.rs b/bitcoin/src/iter.rs index 94994c153..e5d9ed130 100644 --- a/bitcoin/src/iter.rs +++ b/bitcoin/src/iter.rs @@ -207,18 +207,15 @@ mod tests { async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; fn list_transactions(&self, max_count: Option) -> Result, Error>; - fn list_addresses(&self) -> Result, Error>; async fn get_block_count(&self) -> Result; async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result, Error>; async fn get_transaction(&self, txid: &Txid, block_hash: Option) -> Result; async fn get_proof(&self, txid: Txid, block_hash: &BlockHash) -> Result, Error>; async fn get_block_hash(&self, height: u32) -> Result; async fn get_new_address(&self) -> Result; - async fn get_new_sweep_address(&self) -> Result; - async fn get_last_sweep_height(&self) -> Result, Error>; async fn get_new_public_key(&self) -> Result; - fn dump_private_key(&self, address: &Address) -> Result; - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), Error>; + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result; + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), Error>; async fn add_new_deposit_key( &self, public_key: PublicKey, @@ -253,7 +250,6 @@ mod tests { fee_rate: SatPerVbyte, num_confirmations: u32, ) -> Result; - async fn sweep_funds(&self, address: Address) -> Result; async fn create_or_load_wallet(&self) -> Result<(), Error>; async fn rescan_blockchain(&self, start_height: usize, end_height: usize) -> Result<(), Error>; async fn rescan_electrs_for_addresses( diff --git a/bitcoin/src/lib.rs b/bitcoin/src/lib.rs index 05da3fd28..a47dbf8d6 100644 --- a/bitcoin/src/lib.rs +++ b/bitcoin/src/lib.rs @@ -53,7 +53,6 @@ pub use sp_core::H256; use std::{ convert::TryInto, future::Future, - str::FromStr, sync::Arc, time::{Duration, Instant}, }; @@ -101,8 +100,6 @@ const RANDOMIZATION_FACTOR: f64 = 0.25; const DERIVATION_KEY_LABEL: &str = "derivation-key"; const DEPOSIT_LABEL: &str = "deposit"; -const SWEEP_ADDRESS: &str = "sweep-address"; - fn get_exponential_backoff() -> ExponentialBackoff { ExponentialBackoff { current_interval: INITIAL_INTERVAL, @@ -150,8 +147,6 @@ pub trait BitcoinCoreApi { fn list_transactions(&self, max_count: Option) -> Result, Error>; - fn list_addresses(&self) -> Result, Error>; - async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result, Error>; async fn get_transaction(&self, txid: &Txid, block_hash: Option) -> Result; @@ -162,15 +157,11 @@ pub trait BitcoinCoreApi { async fn get_new_address(&self) -> Result; - async fn get_new_sweep_address(&self) -> Result; - - async fn get_last_sweep_height(&self) -> Result, Error>; - async fn get_new_public_key(&self) -> Result; - fn dump_private_key(&self, address: &Address) -> Result; + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result; - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), Error>; + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), Error>; async fn add_new_deposit_key(&self, public_key: PublicKey, secret_key: Vec) -> Result<(), Error>; @@ -213,8 +204,6 @@ pub trait BitcoinCoreApi { num_confirmations: u32, ) -> Result; - async fn sweep_funds(&self, address: Address) -> Result; - async fn create_or_load_wallet(&self) -> Result<(), Error>; async fn rescan_blockchain(&self, start_height: usize, end_height: usize) -> Result<(), Error>; @@ -370,7 +359,7 @@ impl BitcoinCoreBuilder { #[derive(Clone)] pub struct BitcoinCore { - pub rpc: Arc, + rpc: Arc, wallet_name: Option, network: Network, transaction_creation_lock: Arc>, @@ -482,6 +471,7 @@ impl BitcoinCore { signed_funded_raw_tx.complete, signed_funded_raw_tx.errors ); + return Err(Error::TransactionSigningError); } @@ -724,27 +714,6 @@ impl BitcoinCoreApi for BitcoinCore { .list_transactions(None, max_count.or(Some(DEFAULT_MAX_TX_COUNT)), None, None)?) } - // TODO: remove this once the wallet migration has completed - fn list_addresses(&self) -> Result, Error> { - // Lists groups of addresses which have had their common ownership - // made public by common use as inputs or as the resulting change - // in past transactions - let groupings: Vec>> = self.rpc.call("listaddressgroupings", &[])?; - let addresses = groupings - .into_iter() - .flatten() - .filter_map(|group| { - group - .get(0) - .and_then(|v| v.as_str()) - .map(Address::from_str)? - .and_then(|x| x.require_network(self.network)) - .ok() - }) - .collect::>(); - Ok(addresses) - } - /// Get the raw transaction identified by `Txid` and stored /// in the specified block. /// @@ -799,26 +768,6 @@ impl BitcoinCoreApi for BitcoinCore { .require_network(self.network)?) } - async fn get_new_sweep_address(&self) -> Result { - Ok(self - .rpc - .get_new_address(Some(SWEEP_ADDRESS), Some(AddressType::Bech32))? - .require_network(self.network)?) - } - - async fn get_last_sweep_height(&self) -> Result, Error> { - Ok(self - .rpc - .list_transactions(Some(SWEEP_ADDRESS), Some(DEFAULT_MAX_TX_COUNT), None, None)? - .into_iter() - // we want to return None if there is no sweep tx for full nodes or new - // pruned nodes and we should return an error if any tx is still in the mempool - .map(|tx| tx.info.blockheight.ok_or(Error::ConfirmationError)) - .collect::, _>>()? - .into_iter() - .min()) - } - /// Gets a new public key for an address in the wallet async fn get_new_public_key(&self) -> Result { let address = self @@ -830,16 +779,15 @@ impl BitcoinCoreApi for BitcoinCore { Ok(public_key) } - fn dump_private_key(&self, address: &Address) -> Result { - Ok(self.rpc.dump_private_key(address)?) + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result { + let address = Address::p2wpkh(public_key, self.network).map_err(ConversionError::from)?; + Ok(self.rpc.dump_private_key(&address)?) } - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), Error> { - Ok(self.rpc.import_private_key( - private_key, - is_derivation_key.then_some(DERIVATION_KEY_LABEL), - Some(false), - )?) + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), Error> { + Ok(self + .rpc + .import_private_key(private_key, Some(DERIVATION_KEY_LABEL), Some(false))?) } /// Derive and import the private key for the master public key and public secret @@ -1063,66 +1011,6 @@ impl BitcoinCoreApi for BitcoinCore { .await?) } - async fn sweep_funds(&self, address: Address) -> Result { - let unspent = self.rpc.list_unspent(Some(0), None, None, None, None)?; - - let mut amount = Amount::ZERO; - let mut utxos = Vec::::new(); - - for entry in unspent { - if self.electrs_client.is_tx_output_spent(&entry.txid, entry.vout).await? { - log::info!("{}:{} already spent", entry.txid, entry.vout); - // skip if already spent - continue; - } - amount += entry.amount; - utxos.push(json::CreateRawTransactionInput { - txid: entry.txid, - vout: entry.vout, - sequence: None, - }) - } - - log::info!("Sweeping {} from {} utxos", amount, utxos.len()); - let mut outputs = serde_json::Map::::new(); - outputs.insert(address.to_string(), serde_json::Value::from(amount.to_btc())); - - let args = [ - serde_json::to_value::<&[json::CreateRawTransactionInput]>(&utxos)?, - serde_json::to_value(outputs)?, - serde_json::to_value(0i64)?, /* locktime - default 0: see https://developer.bitcoin.org/reference/rpc/createrawtransaction.html */ - serde_json::to_value(true)?, // BIP125-replaceable, aka Replace By Fee (RBF) - ]; - let raw_tx: String = self.rpc.call("createrawtransaction", &args)?; - - let funding_opts = FundRawTransactionOptions { - fee_rate: None, - add_inputs: Some(false), - subtract_fee_from_outputs: Some(vec![0]), - ..Default::default() - }; - let funded_raw_tx = self.rpc.fund_raw_transaction(raw_tx, Some(&funding_opts), None)?; - - let signed_funded_raw_tx = - self.rpc - .sign_raw_transaction_with_wallet(&funded_raw_tx.transaction()?, None, None)?; - - if signed_funded_raw_tx.errors.is_some() { - log::warn!( - "Received bitcoin funding errors (complete={}): {:?}", - signed_funded_raw_tx.complete, - signed_funded_raw_tx.errors - ); - return Err(Error::TransactionSigningError); - } - - let transaction = signed_funded_raw_tx.transaction()?; - let txid = self.rpc.send_raw_transaction(&transaction)?; - log::info!("Sent sweep tx: {txid}"); - - Ok(txid) - } - /// Create or load a wallet on Bitcoin Core. async fn create_or_load_wallet(&self) -> Result<(), Error> { let wallet_name = if let Some(ref wallet_name) = self.wallet_name { @@ -1185,7 +1073,7 @@ impl BitcoinCoreApi for BitcoinCore { // filter to only import // a) payments in the blockchain (not in mempool), and // b) payments TO the address (as bitcoin core will already know about transactions spending FROM it) - let confirmed_payments_to = all_transactions.iter().filter(|tx| { + let confirmed_payments_to = all_transactions.into_iter().filter(|tx| { if let Some(status) = &tx.status { if !status.confirmed { return false; diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index bb5324f85..1c6ba6cf7 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -137,12 +137,6 @@ impl BitcoinCoreApi for BitcoinLight { Ok(Default::default()) } - // TODO: remove this later - fn list_addresses(&self) -> Result, BitcoinError> { - // don't need to migrate keys - Ok(Default::default()) - } - async fn get_raw_tx(&self, txid: &Txid, _block_hash: &BlockHash) -> Result, BitcoinError> { Ok(self.electrs.get_raw_tx(txid).await?) } @@ -168,29 +162,15 @@ impl BitcoinCoreApi for BitcoinLight { Ok(self.get_change_address()?) } - async fn get_new_sweep_address(&self) -> Result { - Ok(self.get_change_address()?) - } - - async fn get_last_sweep_height(&self) -> Result, BitcoinError> { - Ok(None) - } - async fn get_new_public_key(&self) -> Result { Ok(self.private_key.public_key(&self.secp_ctx)) } - fn dump_private_key(&self, address: &Address) -> Result { - self.wallet - .key_store - .read() - .map_err(Into::::into)? - .get(address) - .ok_or(Error::NoPrivateKey.into()) - .cloned() + fn dump_derivation_key(&self, _public_key: &PublicKey) -> Result { + Ok(self.private_key) } - fn import_private_key(&self, _private_key: &PrivateKey, _is_derivation_key: bool) -> Result<(), BitcoinError> { + fn import_derivation_key(&self, _private_key: &PrivateKey) -> Result<(), BitcoinError> { // nothing to do Ok(()) } @@ -339,10 +319,6 @@ impl BitcoinCoreApi for BitcoinLight { .await?) } - async fn sweep_funds(&self, _address: Address) -> Result { - Ok(Txid::all_zeros()) - } - async fn create_or_load_wallet(&self) -> Result<(), BitcoinError> { // nothing to do Ok(()) diff --git a/bitcoin/tests/integration_test.rs b/bitcoin/tests/integration_test.rs index 6a36f2355..865a1c734 100644 --- a/bitcoin/tests/integration_test.rs +++ b/bitcoin/tests/integration_test.rs @@ -1,10 +1,6 @@ #![cfg(feature = "uses-bitcoind")] -use bitcoin::{ - Amount, Auth, BitcoinCore, BitcoinCoreApi, BitcoinCoreBuilder, Error, Network, PrivateKey, PublicKey, RpcApi, -}; -use bitcoincore_rpc::json; -use rand::{distributions::Alphanumeric, Rng}; +use bitcoin::{Auth, BitcoinCore, BitcoinCoreApi, BitcoinCoreBuilder, Error, Network, PrivateKey, PublicKey}; use regex::Regex; use std::env::var; @@ -30,22 +26,6 @@ async fn should_get_new_address() -> Result<(), Error> { Ok(()) } -#[tokio::test] -async fn should_list_addresses() -> Result<(), Error> { - let btc_rpc = new_bitcoin_core(Some("Alice".to_string()))?; - btc_rpc.create_or_load_wallet().await?; - - let address = btc_rpc.get_new_address().await?; - btc_rpc.mine_blocks(101, Some(address.clone())); - - assert!( - btc_rpc.list_addresses()?.contains(&address), - "Address not found in groupings" - ); - - Ok(()) -} - #[tokio::test] async fn should_get_new_public_key() -> Result<(), Error> { let btc_rpc = new_bitcoin_core(Some("Bob".to_string()))?; @@ -93,52 +73,3 @@ async fn should_add_new_deposit_key() -> Result<(), Error> { Ok(()) } - -fn rand_wallet_name() -> String { - rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(7) - .map(char::from) - .collect() -} - -#[tokio::test] -async fn should_sweep_funds() -> Result<(), Error> { - let btc_rpc1 = new_bitcoin_core(Some(rand_wallet_name()))?; - btc_rpc1.create_or_load_wallet().await?; - - let btc_rpc2 = new_bitcoin_core(Some(rand_wallet_name()))?; - btc_rpc2.create_or_load_wallet().await?; - - let btc_rpc3 = new_bitcoin_core(Some(rand_wallet_name()))?; - btc_rpc3.create_or_load_wallet().await?; - - // blocks must have 100 confirmations for reward to be spent - let address1 = btc_rpc1.get_new_address().await?; - btc_rpc1.mine_blocks(101, Some(address1)); - - let address2 = btc_rpc2.get_new_address().await?; - let txid = btc_rpc1.rpc.send_to_address( - &address2, - Amount::from_sat(100000), - None, - None, - None, - None, - None, - Some(json::EstimateMode::Economical), - )?; - btc_rpc1.mine_blocks(1, None); - - assert_eq!(btc_rpc2.get_balance(None)?.to_sat(), 100000); - - let address3 = btc_rpc3.get_new_address().await?; - let _txid = btc_rpc2.sweep_funds(address3).await?; - btc_rpc1.mine_blocks(1, None); - - assert_eq!(btc_rpc2.get_balance(None)?.to_sat(), 0); - // balance before minus fees - assert_eq!(btc_rpc3.get_balance(None)?.to_sat(), 97800); - - Ok(()) -} diff --git a/runtime/src/integration/bitcoin_simulator.rs b/runtime/src/integration/bitcoin_simulator.rs index 36891a013..8a81a2859 100644 --- a/runtime/src/integration/bitcoin_simulator.rs +++ b/runtime/src/integration/bitcoin_simulator.rs @@ -341,9 +341,6 @@ impl BitcoinCoreApi for MockBitcoinCore { fn list_transactions(&self, max_count: Option) -> Result, BitcoinError> { Ok(vec![]) } - fn list_addresses(&self) -> Result, BitcoinError> { - Ok(vec![]) - } async fn get_block_count(&self) -> Result { Ok((self.blocks.read().await.len() - 1).try_into().unwrap()) } @@ -405,15 +402,6 @@ impl BitcoinCoreApi for MockBitcoinCore { let address = BtcAddress::P2PKH(H160::from(bytes)); Ok(address.to_address(Network::Regtest)?) } - - async fn get_new_sweep_address(&self) -> Result { - self.get_new_address().await - } - - async fn get_last_sweep_height(&self) -> Result, BitcoinError> { - Ok(None) - } - async fn get_new_public_key(&self) -> Result { let secp = Secp256k1::new(); let raw_secret_key: [u8; SECRET_KEY_SIZE] = thread_rng().gen(); @@ -421,10 +409,10 @@ impl BitcoinCoreApi for MockBitcoinCore { let public_key = secp256k1::PublicKey::from_secret_key(&secp, &secret_key); Ok(PublicKey::new(public_key)) } - fn dump_private_key(&self, address: &Address) -> Result { + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result { todo!() } - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), BitcoinError> { + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), BitcoinError> { todo!() } async fn add_new_deposit_key(&self, _public_key: PublicKey, _secret_key: Vec) -> Result<(), BitcoinError> { @@ -523,9 +511,6 @@ impl BitcoinCoreApi for MockBitcoinCore { .unwrap(); Ok(metadata) } - async fn sweep_funds(&self, _address: Address) -> Result { - Ok(Txid::all_zeros()) - } async fn create_or_load_wallet(&self) -> Result<(), BitcoinError> { Ok(()) } @@ -536,7 +521,6 @@ impl BitcoinCoreApi for MockBitcoinCore { async fn rescan_electrs_for_addresses(&self, addresses: Vec
) -> Result<(), BitcoinError> { Ok(()) } - fn get_utxo_count(&self) -> Result { Ok(0) } diff --git a/vault/src/connection_manager.rs b/vault/src/connection_manager.rs index 10fb679ce..63eeffb66 100644 --- a/vault/src/connection_manager.rs +++ b/vault/src/connection_manager.rs @@ -26,9 +26,7 @@ pub trait Service { fn new_service( btc_parachain: BtcParachain, - bitcoin_core_master: DynBitcoinCoreApi, - bitcoin_core_shared: DynBitcoinCoreApi, - bitcoin_core_shared_v2: DynBitcoinCoreApi, + bitcoin_core: DynBitcoinCoreApi, config: Config, monitoring_config: MonitoringConfig, shutdown: ShutdownSender, @@ -85,7 +83,7 @@ impl ConnectionManager { let shutdown_tx = ShutdownSender::new(); let prefix = self.wallet_name.clone().unwrap_or_else(|| "vault".to_string()); - let bitcoin_core_master = self.bitcoin_config.new_client(Some(format!("{prefix}-master"))).await?; + let bitcoin_core = self.bitcoin_config.new_client(Some(format!("{prefix}-master"))).await?; // only open connection to parachain after bitcoind sync to prevent timeout let signer = self.signer.clone(); @@ -100,18 +98,7 @@ impl ConnectionManager { .await?; let config_copy = self.bitcoin_config.clone(); - let network_copy = bitcoin_core_master.network(); - - // use a separate wallet for all bitcoin transactions - // to make exporting the private key easier from the - // master wallet if we switch to descriptor wallets - let bitcoin_core_shared = - config_copy.new_client_with_network(Some(format!("{prefix}-shared")), network_copy)?; - bitcoin_core_shared.create_or_load_wallet().await?; - let bitcoin_core_shared_v2 = - config_copy.new_client_with_network(Some(format!("{prefix}-shared-v2")), network_copy)?; - bitcoin_core_shared_v2.create_or_load_wallet().await?; - + let network_copy = bitcoin_core.network(); let constructor = move |vault_id: VaultId| { let collateral_currency: CurrencyId = vault_id.collateral_currency(); let wrapped_currency: CurrencyId = vault_id.wrapped_currency(); @@ -130,9 +117,7 @@ impl ConnectionManager { let service = S::new_service( btc_parachain, - bitcoin_core_master, - bitcoin_core_shared, - bitcoin_core_shared_v2, + bitcoin_core, config, self.monitoring_config.clone(), shutdown_tx.clone(), diff --git a/vault/src/execution.rs b/vault/src/execution.rs index 26bdb2128..7f42d81d6 100644 --- a/vault/src/execution.rs +++ b/vault/src/execution.rs @@ -708,7 +708,6 @@ mod tests { async fn get_foreign_asset_metadata(&self, id: u32) -> Result; async fn get_lend_tokens(&self) -> Result, RuntimeError>; } - #[async_trait] pub trait VaultRegistryPallet { async fn get_vault(&self, vault_id: &VaultId) -> Result; @@ -801,7 +800,6 @@ mod tests { async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; fn list_transactions(&self, max_count: Option) -> Result, BitcoinError>; - fn list_addresses(&self) -> Result, BitcoinError>; async fn get_block_count(&self) -> Result; async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result, BitcoinError>; async fn get_transaction(&self, txid: &Txid, block_hash: Option) -> Result; @@ -809,11 +807,9 @@ mod tests { async fn get_block_hash(&self, height: u32) -> Result; async fn get_pruned_height(&self) -> Result; async fn get_new_address(&self) -> Result; - async fn get_new_sweep_address(&self) -> Result; - async fn get_last_sweep_height(&self) -> Result, BitcoinError>; async fn get_new_public_key(&self) -> Result; - fn dump_private_key(&self, address: &Address) -> Result; - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), BitcoinError>; + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result; + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), BitcoinError>; async fn add_new_deposit_key(&self, public_key: PublicKey, secret_key: Vec) -> Result<(), BitcoinError>; async fn get_best_block_hash(&self) -> Result; async fn get_block(&self, hash: &BlockHash) -> Result; @@ -822,7 +818,6 @@ mod tests { async fn wait_for_transaction_metadata(&self, txid: Txid, num_confirmations: u32, block_hash: Option, is_wallet: bool) -> Result; async fn create_and_send_transaction(&self, address: Address, sat: u64, fee_rate: SatPerVbyte, request_id: Option) -> Result; async fn send_to_address(&self, address: Address, sat: u64, request_id: Option, fee_rate: SatPerVbyte, num_confirmations: u32) -> Result; - async fn sweep_funds(&self, address: Address) -> Result; async fn create_or_load_wallet(&self) -> Result<(), BitcoinError>; async fn rescan_blockchain(&self, start_height: usize, end_height: usize) -> Result<(), BitcoinError>; async fn rescan_electrs_for_addresses(&self, addresses: Vec
) -> Result<(), BitcoinError>; diff --git a/vault/src/issue.rs b/vault/src/issue.rs index 51e6fa764..9586f5294 100644 --- a/vault/src/issue.rs +++ b/vault/src/issue.rs @@ -1,13 +1,13 @@ use crate::{ - delay::RandomDelay, metrics::publish_expected_bitcoin_balance, service::DynBitcoinCoreApi, system::DatabaseConfig, - Error, Event, IssueRequests, VaultIdManager, + delay::RandomDelay, metrics::publish_expected_bitcoin_balance, service::DynBitcoinCoreApi, Error, Event, + IssueRequests, VaultIdManager, }; use bitcoin::{BlockHash, Error as BitcoinError, Hash, PublicKey, Transaction, TransactionExt}; use futures::{channel::mpsc::Sender, future, SinkExt, StreamExt, TryFutureExt}; use runtime::{ - AccountId, BtcAddress, BtcPublicKey, BtcRelayPallet, CancelIssueEvent, ExecuteIssueEvent, H256Le, - InterBtcIssueRequest, InterBtcParachain, IssuePallet, IssueRequestStatus, PartialAddress, PrettyPrint, - RequestIssueEvent, UtilFuncs, H256, + BtcAddress, BtcPublicKey, BtcRelayPallet, CancelIssueEvent, ExecuteIssueEvent, H256Le, InterBtcIssueRequest, + InterBtcParachain, IssuePallet, IssueRequestStatus, PartialAddress, PrettyPrint, RequestIssueEvent, UtilFuncs, + VaultId, H256, }; use sha2::{Digest, Sha256}; use std::{ @@ -82,11 +82,10 @@ struct RescanStatus { newest_issue_height: u32, queued_rescan_range: Option<(usize, usize)>, // start, end(including) } - impl RescanStatus { // there was a bug pre-v2 that set rescanning status to an invalid range. // by changing the keyname we effectively force a reset - const KEY: &str = "rescan-status-v4"; + const KEY: &str = "rescan-status-v2"; fn update(&mut self, mut issues: Vec, current_bitcoin_height: usize) { // Only look at issues that haven't been processed yet issues.retain(|issue| issue.opentime > self.newest_issue_height); @@ -134,26 +133,30 @@ impl RescanStatus { Some((start, chunk_end)) } - fn get(account_id: &AccountId, db: &DatabaseConfig) -> Result { - Ok(db.get(account_id, Self::KEY)?.unwrap_or_default()) + fn get(vault_id: &VaultId, db: &crate::system::DatabaseConfig) -> Result { + Ok(db.get(vault_id, Self::KEY)?.unwrap_or_default()) } - - fn store(&self, account_id: &AccountId, db: &DatabaseConfig) -> Result<(), Error> { - db.put(account_id, Self::KEY, self)?; + fn store(&self, vault_id: &VaultId, db: &crate::system::DatabaseConfig) -> Result<(), Error> { + db.put(vault_id, Self::KEY, self)?; Ok(()) } } -pub async fn add_keys_from_past_issue_request_old( +pub async fn add_keys_from_past_issue_request( bitcoin_core: &DynBitcoinCoreApi, btc_parachain: &InterBtcParachain, - db: &DatabaseConfig, + vault_id: &VaultId, + db: &crate::system::DatabaseConfig, ) -> Result<(), Error> { - let account_id = btc_parachain.get_account_id(); - let mut scanning_status = RescanStatus::get(&account_id, db)?; - tracing::info!("Scanning: {scanning_status:?}"); + let mut scanning_status = RescanStatus::get(vault_id, db)?; + tracing::info!("initial status: = {scanning_status:?}"); - let issue_requests = btc_parachain.get_vault_issue_requests(account_id.clone()).await?; + let issue_requests: Vec<_> = btc_parachain + .get_vault_issue_requests(btc_parachain.get_account_id().clone()) + .await? + .into_iter() + .filter(|(_, issue)| &issue.vault == vault_id) + .collect(); for (issue_id, request) in issue_requests.clone().into_iter() { if let Err(e) = add_new_deposit_key(bitcoin_core, issue_id, request.btc_public_key).await { @@ -161,7 +164,7 @@ pub async fn add_keys_from_past_issue_request_old( } } - // read height only _after_ the last add_new_deposit_key. If a new block arrives + // read height only _after_ the last add_new_deposit_key.If a new block arrives // while we rescan, bitcoin core will correctly recognize addressed associated with the // privkey let btc_end_height = bitcoin_core.get_block_count().await? as usize; @@ -182,7 +185,6 @@ pub async fn add_keys_from_past_issue_request_old( issue_requests .into_iter() .filter_map(|(_, request)| { - // only import if BEFORE current pruning height if (request.btc_height as usize) < btc_pruned_start_height { Some(request.btc_address.to_address(bitcoin_core.network()).ok()?) } else { @@ -195,87 +197,7 @@ pub async fn add_keys_from_past_issue_request_old( } // save progress s.t. we don't rescan pruned range again if we crash now - scanning_status.store(account_id, db)?; - - let mut chunk_size = 1; - // rescan the blockchain in chunks, so that we can save progress. The code below - // aims to have each chunk take about 10 seconds (arbitrarily chosen value). - while let Some((chunk_start, chunk_end)) = scanning_status.process_blocks(chunk_size) { - tracing::info!("Rescanning bitcoin chain from {} to {}...", chunk_start, chunk_end); - - let start_time = Instant::now(); - - bitcoin_core.rescan_blockchain(chunk_start, chunk_end).await?; - - // with the code below the rescan time should remain between 5 and 20 seconds - // after the first couple of rounds. - if start_time.elapsed() < Duration::from_secs(10) { - chunk_size = chunk_size.saturating_mul(2); - } else { - chunk_size = (chunk_size.checked_div(2).ok_or(Error::ArithmeticUnderflow)?).max(1); - } - - scanning_status.store(account_id, db)?; - } - - Ok(()) -} - -pub async fn add_keys_from_past_issue_request_new( - bitcoin_core: &DynBitcoinCoreApi, - btc_parachain: &InterBtcParachain, - db: &DatabaseConfig, -) -> Result<(), Error> { - let account_id = btc_parachain.get_account_id(); - let mut scanning_status = RescanStatus::get(&account_id, db)?; - tracing::info!("Scanning: {scanning_status:?}"); - - let issue_requests = btc_parachain.get_vault_issue_requests(account_id.clone()).await?; - - for (issue_id, request) in issue_requests.clone().into_iter() { - if let Err(e) = add_new_deposit_key(bitcoin_core, issue_id, request.btc_public_key).await { - tracing::error!("Failed to add deposit key #{}: {}", issue_id, e.to_string()); - } - } - - // read height only _after_ the last add_new_deposit_key. If a new block arrives - // while we rescan, bitcoin core will correctly recognize addressed associated with the - // privkey - let btc_end_height = bitcoin_core.get_block_count().await? as usize; - let btc_pruned_start_height = bitcoin_core.get_pruned_height().await? as usize; - let btc_last_sweep_height = bitcoin_core.get_last_sweep_height().await?; - - let issues = issue_requests.clone().into_iter().map(|(_key, issue)| issue).collect(); - scanning_status.update(issues, btc_end_height); - - // use electrs to scan the portion that is not scannable by bitcoin core - if let Some((start, end)) = scanning_status.prune(btc_pruned_start_height) { - tracing::info!( - "Also checking electrs for issue requests between {} and {}...", - start, - end - ); - bitcoin_core - .rescan_electrs_for_addresses( - issue_requests - .into_iter() - .filter_map(|(_, request)| { - // only import if address is AFTER last sweep height and BEFORE current pruning height - if btc_last_sweep_height.map_or(true, |sweep_height| request.btc_height > sweep_height) - && (request.btc_height as usize) < btc_pruned_start_height - { - Some(request.btc_address.to_address(bitcoin_core.network()).ok()?) - } else { - None - } - }) - .collect(), - ) - .await?; - } - - // save progress s.t. we don't rescan pruned range again if we crash now - scanning_status.store(account_id, db)?; + scanning_status.store(vault_id, db)?; let mut chunk_size = 1; // rescan the blockchain in chunks, so that we can save progress. The code below @@ -295,7 +217,7 @@ pub async fn add_keys_from_past_issue_request_new( chunk_size = (chunk_size.checked_div(2).ok_or(Error::ArithmeticUnderflow)?).max(1); } - scanning_status.store(account_id, db)?; + scanning_status.store(vault_id, db)?; } Ok(()) @@ -543,9 +465,9 @@ mod tests { use super::*; use runtime::{ subxt::utils::Static, + AccountId, CurrencyId::Token, TokenSymbol::{DOT, IBTC, INTR}, - VaultId, }; fn dummy_issues(heights: Vec<(u32, usize)>) -> Vec { diff --git a/vault/src/metrics.rs b/vault/src/metrics.rs index 9a37b7003..f0bccce45 100644 --- a/vault/src/metrics.rs +++ b/vault/src/metrics.rs @@ -2,10 +2,7 @@ use std::{collections::HashMap, convert::TryInto, sync::Arc}; use crate::{ execution::parachain_blocks_to_bitcoin_blocks_rounded_up, - service::{ - warp::{Rejection, Reply}, - DynBitcoinCoreApi, - }, + service::warp::{Rejection, Reply}, system::{VaultData, VaultIdManager}, Error, }; @@ -31,7 +28,7 @@ const SLEEP_DURATION: Duration = Duration::from_secs(5 * 60); const SECONDS_PER_HOUR: f64 = 3600.0; const CURRENCY_LABEL: &str = "currency"; -const EXPECTED_BTC_BALANCE_TYPE_LABEL: &str = "type"; +const BTC_BALANCE_TYPE_LABEL: &str = "type"; const REQUEST_STATUS_LABEL: &str = "status"; const TASK_NAME: &str = "task"; const TOKIO_POLLING_INTERVAL_MS: u64 = 10000; @@ -45,10 +42,9 @@ lazy_static! { &[CURRENCY_LABEL] ) .expect("Failed to create prometheus metric"); - pub static ref AVERAGE_BTC_FEE: StatefulGauge = StatefulGauge { - gauge: Gauge::new("avg_btc_fee", "Average Bitcoin Fee").expect("Failed to create prometheus metric"), - data: Arc::new(RwLock::new(AverageTracker { total: 0, count: 0 })), - }; + pub static ref AVERAGE_BTC_FEE: GaugeVec = + GaugeVec::new(Opts::new("avg_btc_fee", "Average Bitcoin Fee"), &[CURRENCY_LABEL]) + .expect("Failed to create prometheus metric"); pub static ref LOCKED_COLLATERAL: GaugeVec = GaugeVec::new(Opts::new("locked_collateral", "Locked Collateral"), &[CURRENCY_LABEL]) .expect("Failed to create prometheus metric"); @@ -71,13 +67,14 @@ lazy_static! { &[TASK_NAME] ) .expect("Failed to create prometheus metric"); - pub static ref UTXO_COUNT: IntGauge = - IntGauge::new("utxo_count", "Number of Unspent Bitcoin Outputs",).expect("Failed to create prometheus metric"); - pub static ref ACTUAL_BTC_BALANCE: Gauge = - Gauge::new("actual_btc_balance", "Actual Bitcoin Balance",).expect("Failed to create prometheus metric"); - pub static ref EXPECTED_BTC_BALANCE: GaugeVec = GaugeVec::new( - Opts::new("expected_btc_balance", "Expected Bitcoin Balance"), - &[CURRENCY_LABEL, EXPECTED_BTC_BALANCE_TYPE_LABEL] + pub static ref UTXO_COUNT: IntGaugeVec = IntGaugeVec::new( + Opts::new("utxo_count", "Number of Unspent Bitcoin Outputs"), + &[CURRENCY_LABEL] + ) + .expect("Failed to create prometheus metric"); + pub static ref BTC_BALANCE: GaugeVec = GaugeVec::new( + Opts::new("btc_balance", "Bitcoin Balance"), + &[CURRENCY_LABEL, BTC_BALANCE_TYPE_LABEL] ) .expect("Failed to create prometheus metric"); pub static ref ISSUES: GaugeVec = GaugeVec::new( @@ -92,22 +89,21 @@ lazy_static! { .expect("Failed to create prometheus metric"); pub static ref NATIVE_CURRENCY_BALANCE: Gauge = Gauge::new("native_currency_balance", "Native Currency Balance").expect("Failed to create prometheus metric"); - pub static ref FEE_BUDGET_SURPLUS: StatefulGauge = StatefulGauge { - gauge: Gauge::new("fee_budget_surplus", "Fee Budget Surplus").expect("Failed to create prometheus metric"), - data: Arc::new(RwLock::new(0)), - }; + pub static ref FEE_BUDGET_SURPLUS: GaugeVec = + GaugeVec::new(Opts::new("fee_budget_surplus", "Fee Budget Surplus"), &[CURRENCY_LABEL]) + .expect("Failed to create prometheus metric"); pub static ref RESTART_COUNT: IntCounter = IntCounter::new("restart_count", "Number of service restarts").expect("Failed to create prometheus metric"); } #[derive(Clone, Debug)] -pub struct AverageTracker { +struct AverageTracker { total: u64, count: u64, } #[derive(Clone, Debug)] -pub struct StatefulGauge { +struct StatefulGauge { gauge: Gauge, data: Arc>, } @@ -116,6 +112,7 @@ pub struct StatefulGauge { struct BtcBalance { upperbound: Gauge, lowerbound: Gauge, + actual: Gauge, } #[derive(Clone, Debug)] @@ -134,6 +131,9 @@ pub struct PerCurrencyMetrics { btc_balance: BtcBalance, issues: RequestCounter, redeems: RequestCounter, + average_btc_fee: StatefulGauge, + fee_budget_surplus: StatefulGauge, + utxo_count: IntGauge, } #[async_trait] @@ -169,10 +169,9 @@ impl PerCurrencyMetrics { fn new_with_label(label: &str) -> Self { let labels = HashMap::from([(CURRENCY_LABEL, label)]); - let expected_btc_balance_gauge = |balance_type: &'static str| { - let labels = - HashMap::<&str, &str>::from([(CURRENCY_LABEL, label), (EXPECTED_BTC_BALANCE_TYPE_LABEL, balance_type)]); - EXPECTED_BTC_BALANCE.with(&labels) + let btc_balance_gauge = |balance_type: &'static str| { + let labels = HashMap::<&str, &str>::from([(CURRENCY_LABEL, label), (BTC_BALANCE_TYPE_LABEL, balance_type)]); + BTC_BALANCE.with(&labels) }; let request_type_label = |balance_type: &'static str| { HashMap::<&str, &str>::from([(CURRENCY_LABEL, label), (REQUEST_STATUS_LABEL, balance_type)]) @@ -183,9 +182,19 @@ impl PerCurrencyMetrics { collateralization: COLLATERALIZATION.with(&labels), required_collateral: REQUIRED_COLLATERAL.with(&labels), remaining_time_to_redeem_hours: REMAINING_TIME_TO_REDEEM_HOURS.with(&labels), + utxo_count: UTXO_COUNT.with(&labels), + fee_budget_surplus: StatefulGauge { + gauge: FEE_BUDGET_SURPLUS.with(&labels), + data: Arc::new(RwLock::new(0)), + }, + average_btc_fee: StatefulGauge { + gauge: AVERAGE_BTC_FEE.with(&labels), + data: Arc::new(RwLock::new(AverageTracker { total: 0, count: 0 })), + }, btc_balance: BtcBalance { - upperbound: expected_btc_balance_gauge("required_upperbound"), - lowerbound: expected_btc_balance_gauge("required_lowerbound"), + upperbound: btc_balance_gauge("required_upperbound"), + lowerbound: btc_balance_gauge("required_lowerbound"), + actual: btc_balance_gauge("actual"), }, issues: RequestCounter { open_count: ISSUES.with(&request_type_label("open")), @@ -236,7 +245,7 @@ impl PerCurrencyMetrics { .fold(0i64, |acc, x| async move { acc.saturating_add(x) }) .await; - *FEE_BUDGET_SURPLUS.data.write().await = fee_budget_surplus; + *vault.metrics.fee_budget_surplus.data.write().await = fee_budget_surplus; publish_fee_budget_surplus(vault).await?; } Ok(()) @@ -256,30 +265,29 @@ impl PerCurrencyMetrics { .iter() .filter_map(|tx| tx.detail.fee.map(|amount| amount.to_sat().unsigned_abs())) .fold((0, 0), |(total, count), x| (total + x, count + 1)); - *AVERAGE_BTC_FEE.data.write().await = AverageTracker { total, count }; + *vault.metrics.average_btc_fee.data.write().await = AverageTracker { total, count }; - publish_utxo_count(&vault.btc_rpc); - publish_bitcoin_balance(&vault.btc_rpc); + publish_utxo_count(vault); + publish_bitcoin_balance(vault); let _ = tokio::join!( Self::initialize_fee_budget_surplus(vault, parachain_rpc.clone(), bitcoin_transactions), - publish_average_bitcoin_fee(), + publish_average_bitcoin_fee(vault), publish_expected_bitcoin_balance(vault, parachain_rpc.clone()), - publish_locked_collateral(vault, ¶chain_rpc), - publish_required_collateral(vault, ¶chain_rpc), - publish_collateralization(vault, ¶chain_rpc), + publish_locked_collateral(vault, parachain_rpc.clone()), + publish_required_collateral(vault, parachain_rpc.clone()), + publish_collateralization(vault, parachain_rpc.clone()), ); } } pub fn register_custom_metrics() -> Result<(), RuntimeError> { - REGISTRY.register(Box::new(AVERAGE_BTC_FEE.gauge.clone()))?; + REGISTRY.register(Box::new(AVERAGE_BTC_FEE.clone()))?; REGISTRY.register(Box::new(LOCKED_COLLATERAL.clone()))?; REGISTRY.register(Box::new(COLLATERALIZATION.clone()))?; REGISTRY.register(Box::new(REQUIRED_COLLATERAL.clone()))?; - REGISTRY.register(Box::new(FEE_BUDGET_SURPLUS.gauge.clone()))?; - REGISTRY.register(Box::new(ACTUAL_BTC_BALANCE.clone()))?; - REGISTRY.register(Box::new(EXPECTED_BTC_BALANCE.clone()))?; + REGISTRY.register(Box::new(FEE_BUDGET_SURPLUS.clone()))?; + REGISTRY.register(Box::new(BTC_BALANCE.clone()))?; REGISTRY.register(Box::new(NATIVE_CURRENCY_BALANCE.clone()))?; REGISTRY.register(Box::new(ISSUES.clone()))?; REGISTRY.register(Box::new(REDEEMS.clone()))?; @@ -324,7 +332,7 @@ fn raw_value_as_currency(value: u128, currency: CurrencyId) -> Result( vault: &VaultData, - parachain_rpc: &P, + parachain_rpc: P, ) -> Result<(), Error> { if let Ok(actual_collateral) = parachain_rpc.get_vault_total_collateral(vault.vault_id.clone()).await { let actual_collateral = raw_value_as_currency(actual_collateral, vault.vault_id.collateral_currency())?; @@ -335,7 +343,7 @@ pub async fn publish_locked_collateral( pub async fn publish_required_collateral( vault: &VaultData, - parachain_rpc: &P, + parachain_rpc: P, ) -> Result<(), Error> { if let Ok(required_collateral) = parachain_rpc .get_required_collateral_for_vault(vault.vault_id.clone()) @@ -347,8 +355,8 @@ pub async fn publish_required_collateral( Ok(()) } -pub async fn publish_collateralization(vault: &VaultData, parachain_rpc: &P) { - // if the collateralization is infinite, return 0 rather than logging an error so +pub async fn publish_collateralization(vault: &VaultData, parachain_rpc: P) { + // if the collateralization is infinite, return 0 rather than logging an error, so // the metrics do change in case of a replacement let collateralization = parachain_rpc .get_collateralization_from_vault(vault.vault_id.clone(), false) @@ -366,45 +374,47 @@ pub async fn update_bitcoin_metrics( // update the average fee if let Some(amount) = new_fee_entry { { - let mut tmp = AVERAGE_BTC_FEE.data.write().await; + let mut tmp = vault.metrics.average_btc_fee.data.write().await; *tmp = AverageTracker { total: tmp.total.saturating_add(amount.to_sat().unsigned_abs()), count: tmp.count.saturating_add(1), }; } - publish_average_bitcoin_fee().await; + publish_average_bitcoin_fee(vault).await; if let Ok(budget) = TryInto::::try_into(fee_budget.unwrap_or(0)) { let surplus = budget.saturating_sub(amount.to_sat().abs()); - let mut tmp = FEE_BUDGET_SURPLUS.data.write().await; + let mut tmp = vault.metrics.fee_budget_surplus.data.write().await; *tmp = tmp.saturating_add(surplus); } publish_fee_budget_surplus(vault).await?; } - publish_bitcoin_balance(&vault.btc_rpc); + publish_bitcoin_balance(vault); Ok(()) } async fn publish_fee_budget_surplus(vault: &VaultData) -> Result<(), Error> { - let surplus = *FEE_BUDGET_SURPLUS.data.read().await; - FEE_BUDGET_SURPLUS + let surplus = *vault.metrics.fee_budget_surplus.data.read().await; + vault + .metrics + .fee_budget_surplus .gauge .set(surplus as f64 / vault.vault_id.wrapped_currency().inner()?.one() as f64); Ok(()) } -async fn publish_average_bitcoin_fee() { - let average = match AVERAGE_BTC_FEE.data.read().await { +async fn publish_average_bitcoin_fee(vault: &VaultData) { + let average = match vault.metrics.average_btc_fee.data.read().await { x if x.count > 0 => x.total as f64 / x.count as f64, _ => Default::default(), }; - AVERAGE_BTC_FEE.gauge.set(average); + vault.metrics.average_btc_fee.gauge.set(average); } -fn publish_bitcoin_balance(btc_rpc: &DynBitcoinCoreApi) { - match btc_rpc.get_balance(None) { - Ok(bitcoin_balance) => ACTUAL_BTC_BALANCE.set(bitcoin_balance.to_btc()), +fn publish_bitcoin_balance(vault: &VaultData) { + match vault.btc_rpc.get_balance(None) { + Ok(bitcoin_balance) => vault.metrics.btc_balance.actual.set(bitcoin_balance.to_btc()), Err(e) => { // unexpected error, but not critical so just continue tracing::warn!("Failed to get Bitcoin balance: {}", e); @@ -423,10 +433,10 @@ async fn publish_native_currency_balance Result; fn get_balance(&self, min_confirmations: Option) -> Result; fn list_transactions(&self, max_count: Option) -> Result, BitcoinError>; - fn list_addresses(&self) -> Result, BitcoinError>; async fn get_block_count(&self) -> Result; async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result, BitcoinError>; async fn get_transaction(&self, txid: &Txid, block_hash: Option) -> Result; @@ -782,11 +792,9 @@ mod tests { async fn get_block_hash(&self, height: u32) -> Result; async fn get_pruned_height(&self) -> Result; async fn get_new_address(&self) -> Result; - async fn get_new_sweep_address(&self) -> Result; - async fn get_last_sweep_height(&self) -> Result, BitcoinError>; async fn get_new_public_key(&self) -> Result; - fn dump_private_key(&self, address: &Address) -> Result; - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), BitcoinError>; + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result; + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), BitcoinError>; async fn add_new_deposit_key(&self, public_key: PublicKey, secret_key: Vec) -> Result<(), BitcoinError>; async fn get_best_block_hash(&self) -> Result; async fn get_block(&self, hash: &BlockHash) -> Result; @@ -795,7 +803,6 @@ mod tests { async fn wait_for_transaction_metadata(&self, txid: Txid, num_confirmations: u32, block_hash: Option, is_wallet: bool) -> Result; async fn create_and_send_transaction(&self, address: Address, sat: u64, fee_rate: SatPerVbyte, request_id: Option) -> Result; async fn send_to_address(&self, address: Address, sat: u64, request_id: Option, fee_rate: SatPerVbyte, num_confirmations: u32) -> Result; - async fn sweep_funds(&self, address: Address) -> Result; async fn create_or_load_wallet(&self) -> Result<(), BitcoinError>; async fn rescan_blockchain(&self, start_height: usize, end_height: usize) -> Result<(), BitcoinError>; async fn rescan_electrs_for_addresses(&self, addresses: Vec
) -> Result<(), BitcoinError>; @@ -1021,9 +1028,9 @@ mod tests { update_bitcoin_metrics(&vault_data, Some(SignedAmount::from_sat(125)), Some(122)) .await .unwrap(); - let average_btc_fee = AVERAGE_BTC_FEE.gauge.get(); - let fee_budget_surplus = FEE_BUDGET_SURPLUS.gauge.get(); - let bitcoin_balance = ACTUAL_BTC_BALANCE.get(); + let average_btc_fee = vault_data.metrics.average_btc_fee.gauge.get(); + let fee_budget_surplus = vault_data.metrics.fee_budget_surplus.gauge.get(); + let bitcoin_balance = vault_data.metrics.btc_balance.actual.get(); assert_eq!(average_btc_fee, 125.0); assert_eq!(fee_budget_surplus, -0.00000003); @@ -1035,8 +1042,15 @@ mod tests { let mut mock_bitcoin = MockBitcoin::default(); mock_bitcoin.expect_get_utxo_count().returning(move || Ok(102)); let btc_rpc: DynBitcoinCoreApi = Arc::new(mock_bitcoin); - publish_utxo_count(&btc_rpc); - let utxo_count = UTXO_COUNT.get(); + + let vault_data = VaultData { + vault_id: dummy_vault_id(), + btc_rpc, + metrics: PerCurrencyMetrics::dummy(), + }; + publish_utxo_count(&vault_data); + + let utxo_count = vault_data.metrics.utxo_count.get(); assert_eq!(utxo_count, 102); } @@ -1059,7 +1073,7 @@ mod tests { metrics: PerCurrencyMetrics::dummy(), }; - publish_locked_collateral(&vault_data, ¶chain_rpc).await.unwrap(); + publish_locked_collateral(&vault_data, parachain_rpc).await.unwrap(); let total_collateral = vault_data.metrics.locked_collateral.get(); assert_eq!(total_collateral, 0.0000000075); @@ -1112,7 +1126,7 @@ mod tests { metrics: PerCurrencyMetrics::dummy(), }; - publish_collateralization(&vault_data, ¶chain_rpc).await; + publish_collateralization(&vault_data, parachain_rpc).await; let collateralization_metrics = vault_data.metrics.collateralization.get(); assert_eq!( @@ -1141,7 +1155,7 @@ mod tests { metrics: PerCurrencyMetrics::dummy(), }; - publish_required_collateral(&vault_data, ¶chain_rpc).await.unwrap(); + publish_required_collateral(&vault_data, parachain_rpc).await.unwrap(); let required_collateral = vault_data.metrics.required_collateral.get(); assert_eq!(required_collateral, 0.000000005); diff --git a/vault/src/replace.rs b/vault/src/replace.rs index d7e413dd8..6f72bb44d 100644 --- a/vault/src/replace.rs +++ b/vault/src/replace.rs @@ -241,7 +241,6 @@ mod tests { async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; fn list_transactions(&self, max_count: Option) -> Result, BitcoinError>; - fn list_addresses(&self) -> Result, BitcoinError>; async fn get_block_count(&self) -> Result; async fn get_raw_tx(&self, txid: &Txid, block_hash: &BlockHash) -> Result, BitcoinError>; async fn get_transaction(&self, txid: &Txid, block_hash: Option) -> Result; @@ -249,11 +248,9 @@ mod tests { async fn get_block_hash(&self, height: u32) -> Result; async fn get_pruned_height(&self) -> Result; async fn get_new_address(&self) -> Result; - async fn get_new_sweep_address(&self) -> Result; - async fn get_last_sweep_height(&self) -> Result, BitcoinError>; async fn get_new_public_key(&self) -> Result; - fn dump_private_key(&self, address: &Address) -> Result; - fn import_private_key(&self, private_key: &PrivateKey, is_derivation_key: bool) -> Result<(), BitcoinError>; + fn dump_derivation_key(&self, public_key: &PublicKey) -> Result; + fn import_derivation_key(&self, private_key: &PrivateKey) -> Result<(), BitcoinError>; async fn add_new_deposit_key( &self, public_key: PublicKey, @@ -287,7 +284,6 @@ mod tests { fee_rate: SatPerVbyte, num_confirmations: u32, ) -> Result; - async fn sweep_funds(&self, address: Address) -> Result; async fn create_or_load_wallet(&self) -> Result<(), BitcoinError>; async fn rescan_blockchain(&self, start_height: usize, end_height: usize) -> Result<(), BitcoinError>; async fn rescan_electrs_for_addresses(&self, addresses: Vec
) -> Result<(), BitcoinError>; diff --git a/vault/src/system.rs b/vault/src/system.rs index b61ed0146..24f79745a 100644 --- a/vault/src/system.rs +++ b/vault/src/system.rs @@ -9,7 +9,7 @@ use crate::{ }; use async_trait::async_trait; use backoff::Error as BackoffError; -use bitcoin::{Address, ConversionError, Error as BitcoinError, Network, PublicKey}; +use bitcoin::{Error as BitcoinError, Network, PublicKey}; use clap::Parser; use futures::{ channel::{mpsc, mpsc::Sender}, @@ -19,9 +19,9 @@ use futures::{ use git_version::git_version; use runtime::{ cli::{parse_duration_minutes, parse_duration_ms}, - AccountId, BtcRelayPallet, CollateralBalancesPallet, CurrencyId, Error as RuntimeError, InterBtcParachain, - PrettyPrint, RegisterVaultEvent, StoreMainChainHeaderEvent, TryFromSymbol, UpdateActiveBlockEvent, UtilFuncs, - VaultCurrencyPair, VaultId, VaultRegistryPallet, + BtcRelayPallet, CollateralBalancesPallet, CurrencyId, Error as RuntimeError, InterBtcParachain, PrettyPrint, + RegisterVaultEvent, RuntimeCurrencyInfo, StoreMainChainHeaderEvent, TryFromSymbol, UpdateActiveBlockEvent, + UtilFuncs, VaultCurrencyPair, VaultId, VaultRegistryPallet, }; use std::{collections::HashMap, pin::Pin, sync::Arc, time::Duration}; use tokio::{sync::RwLock, time::sleep}; @@ -125,10 +125,6 @@ pub struct VaultServiceConfig { /// the path is generated from the --keyname argument #[clap(long)] pub db_path: Option, - - /// run the wallet migration, but don't start regular vault services - #[clap(long)] - pub only_migrate: bool, } async fn active_block_listener( @@ -182,25 +178,35 @@ pub struct DatabaseConfig { } impl DatabaseConfig { - fn prefixed_key(account_id: &AccountId, key: &str) -> Result { + fn prefixed_key(vault_id: &VaultId, key: &str) -> Result { Ok(format!( - "{}-{}", - account_id.pretty_print(), /* technically not needed since each client should have their own - * db, but doesn't hurt to be safe */ + "{}-{}-{}-{}", + vault_id.account_id.pretty_print(), /* technically not needed since each client should have their own + * db, but doesn't hurt to be safe */ + vault_id + .currencies + .collateral + .symbol() + .map_err(|_| BitcoinError::FailedToConstructWalletName)?, + vault_id + .currencies + .wrapped + .symbol() + .map_err(|_| BitcoinError::FailedToConstructWalletName)?, key )) } - pub fn put(&self, account_id: &AccountId, key: &str, value: &V) -> Result<(), Error> { + pub fn put(&self, vault_id: &VaultId, key: &str, value: &V) -> Result<(), Error> { let db = rocksdb::DB::open_default(self.path.clone())?; - let key = Self::prefixed_key(account_id, key)?; + let key = Self::prefixed_key(vault_id, key)?; db.put(key, serde_json::to_vec(value)?)?; Ok(()) } - pub fn get(&self, account_id: &AccountId, key: &str) -> Result, Error> { + pub fn get(&self, vault_id: &VaultId, key: &str) -> Result, Error> { let db = rocksdb::DB::open_default(self.path.clone())?; - let key = Self::prefixed_key(account_id, key)?; + let key = Self::prefixed_key(vault_id, key)?; let value = match db.get(key)? { None => return Ok(None), @@ -218,9 +224,7 @@ pub struct VaultIdManager { vault_data: Arc>>, btc_parachain: InterBtcParachain, btc_rpc_master_wallet: DynBitcoinCoreApi, - pub(crate) btc_rpc_shared_wallet: DynBitcoinCoreApi, - pub(crate) btc_rpc_shared_wallet_v2: DynBitcoinCoreApi, - // TODO: remove this + // TODO: refactor this #[allow(clippy::type_complexity)] constructor: Arc Result + Send + Sync>>, db: DatabaseConfig, @@ -230,8 +234,6 @@ impl VaultIdManager { pub fn new( btc_parachain: InterBtcParachain, btc_rpc_master_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet_v2: DynBitcoinCoreApi, constructor: impl Fn(VaultId) -> Result + Send + Sync + 'static, db_path: String, ) -> Self { @@ -239,8 +241,6 @@ impl VaultIdManager { vault_data: Arc::new(RwLock::new(HashMap::new())), constructor: Arc::new(Box::new(constructor)), btc_rpc_master_wallet, - btc_rpc_shared_wallet, - btc_rpc_shared_wallet_v2, btc_parachain, db: DatabaseConfig { path: db_path }, } @@ -250,8 +250,6 @@ impl VaultIdManager { pub fn from_map( btc_parachain: InterBtcParachain, btc_rpc_master_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet_v2: DynBitcoinCoreApi, map: HashMap, db_path: &str, ) -> Self { @@ -272,8 +270,6 @@ impl VaultIdManager { vault_data: Arc::new(RwLock::new(vault_data)), constructor: Arc::new(Box::new(|_| unimplemented!())), btc_rpc_master_wallet, - btc_rpc_shared_wallet, - btc_rpc_shared_wallet_v2, btc_parachain, db: DatabaseConfig { path: db_path.to_string(), @@ -290,26 +286,25 @@ impl VaultIdManager { .await .map_err(Error::WalletInitializationFailure)?; - let btc_rpc_master = &self.btc_rpc_master_wallet; - let btc_rpc_shared = self.btc_rpc_shared_wallet.clone(); - tracing::info!("Adding derivation key..."); let derivation_key = self .btc_parachain .get_public_key() .await? .ok_or(BitcoinError::MissingPublicKey)?; + + // migration to the new shared public key setup: copy the public key from the + // currency-specific wallet to the master wallet. This can be removed once all + // vaults have migrated let public_key = PublicKey::from_slice(&derivation_key.0).map_err(BitcoinError::KeyError)?; - let address = Address::p2wpkh(&public_key, btc_rpc_master.network()) - .map_err(ConversionError::from) - .map_err(BitcoinError::ConversionError)?; + if let Ok(private_key) = btc_rpc.dump_derivation_key(&public_key) { + self.btc_rpc_master_wallet.import_derivation_key(&private_key)?; + } // Copy the derivation key from the master wallet to use currency-specific wallet - match btc_rpc_master.dump_private_key(&address) { + match self.btc_rpc_master_wallet.dump_derivation_key(&public_key) { Ok(private_key) => { - // TODO: remove this after the migration is complete - btc_rpc_shared.import_private_key(&private_key, true)?; - self.btc_rpc_shared_wallet_v2.import_private_key(&private_key, true)?; + btc_rpc.import_derivation_key(&private_key)?; } Err(err) => { tracing::error!("Could not find the derivation key in the bitcoin wallet"); @@ -317,33 +312,15 @@ impl VaultIdManager { } } - tracing::info!("Merging wallet for {:?}", vault_id); - // issue keys should be imported separately but we need to iterate - // through currency specific wallets to get change addresses - for address in btc_rpc.list_addresses()? { - tracing::info!("Found {:?}", address); - // get private key from currency specific wallet - let private_key = btc_rpc.dump_private_key(&address)?; - // import key into main wallet - btc_rpc_shared.import_private_key(&private_key, false)?; - } + tracing::info!("Adding keys from past issues..."); - // only sweep if using pruned node and there is no sweep tx yet to shared-v2 - if btc_rpc_shared.get_pruned_height().await? != 0 - && self.btc_rpc_shared_wallet_v2.get_last_sweep_height().await?.is_none() - { - // sweep to old shared wallet which will then sweep again to the v2 wallet - let shared_wallet_address = btc_rpc_shared.get_new_address().await?; - if let Err(err) = btc_rpc.sweep_funds(shared_wallet_address).await { - tracing::error!("Could not sweep funds: {err}"); - } - } + issue::add_keys_from_past_issue_request(&btc_rpc, &self.btc_parachain, &vault_id, &self.db).await?; tracing::info!("Initializing metrics..."); let metrics = PerCurrencyMetrics::new(&vault_id); let data = VaultData { vault_id: vault_id.clone(), - btc_rpc: self.btc_rpc_shared_wallet_v2.clone(), + btc_rpc: btc_rpc.clone(), metrics: metrics.clone(), }; PerCurrencyMetrics::initialize_values(self.btc_parachain.clone(), &data).await; @@ -353,24 +330,23 @@ impl VaultIdManager { Ok(()) } - pub async fn fetch_vault_ids(&self, only_migrate: bool) -> Result<(), Error> { + pub async fn fetch_vault_ids(&self) -> Result<(), Error> { for vault_id in self .btc_parachain .get_vaults_by_account_id(self.btc_parachain.get_account_id()) .await? { - match (only_migrate, is_vault_registered(&self.btc_parachain, &vault_id).await) { - // TODO: import keys for liquidated vaults? - (false, Err(Error::RuntimeError(RuntimeError::VaultLiquidated))) => { + match is_vault_registered(&self.btc_parachain, &vault_id).await { + Err(Error::RuntimeError(RuntimeError::VaultLiquidated)) => { tracing::error!( "[{}] Vault is liquidated -- not going to process events for this vault.", vault_id.pretty_print() ); } - (_, Ok(_)) | (true, Err(Error::RuntimeError(RuntimeError::VaultLiquidated))) => { + Ok(_) => { self.add_vault_id(vault_id.clone()).await?; } - (_, Err(x)) => { + Err(x) => { return Err(x); } } @@ -378,31 +354,6 @@ impl VaultIdManager { Ok(()) } - // only run AFTER the separate currency wallet sweeps - async fn sweep_shared_wallet(&self) -> Result<(), Error> { - if self.btc_rpc_shared_wallet.get_pruned_height().await? == 0 - || self.btc_rpc_shared_wallet_v2.get_last_sweep_height().await?.is_some() - { - // no need to sweep, full node can rescan or already has sweep tx - return Ok(()); - } - - // sweep funds from shared wallet to shared-v2 - let shared_v2_wallet_address = self.btc_rpc_shared_wallet_v2.get_new_sweep_address().await?; - match self.btc_rpc_shared_wallet.sweep_funds(shared_v2_wallet_address).await { - Ok(txid) => { - self.btc_rpc_shared_wallet - .wait_for_transaction_metadata(txid, 1, None, true) - .await?; - } - Err(err) => { - tracing::error!("Could not sweep funds: {err}"); - } - } - - Ok(()) - } - pub async fn listen_for_vault_id_registrations(self) -> Result<(), Error> { Ok(self .btc_parachain @@ -419,7 +370,6 @@ impl VaultIdManager { .await?) } - // TODO: we can refactor this since we only use one wallet pub async fn get_bitcoin_rpc(&self, vault_id: &VaultId) -> Option { self.vault_data.read().await.get(vault_id).map(|x| x.btc_rpc.clone()) } @@ -459,8 +409,6 @@ impl VaultIdManager { pub struct VaultService { btc_parachain: InterBtcParachain, btc_rpc_master_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet_v2: DynBitcoinCoreApi, config: VaultServiceConfig, monitoring_config: MonitoringConfig, shutdown: ShutdownSender, @@ -475,8 +423,6 @@ impl Service for VaultService { fn new_service( btc_parachain: InterBtcParachain, btc_rpc_master_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet_v2: DynBitcoinCoreApi, config: VaultServiceConfig, monitoring_config: MonitoringConfig, shutdown: ShutdownSender, @@ -486,8 +432,6 @@ impl Service for VaultService { VaultService::new( btc_parachain, btc_rpc_master_wallet, - btc_rpc_shared_wallet, - btc_rpc_shared_wallet_v2, config, monitoring_config, shutdown, @@ -561,8 +505,6 @@ impl VaultService { fn new( btc_parachain: InterBtcParachain, btc_rpc_master_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet: DynBitcoinCoreApi, - btc_rpc_shared_wallet_v2: DynBitcoinCoreApi, config: VaultServiceConfig, monitoring_config: MonitoringConfig, shutdown: ShutdownSender, @@ -572,19 +514,10 @@ impl VaultService { Self { btc_parachain: btc_parachain.clone(), btc_rpc_master_wallet: btc_rpc_master_wallet.clone(), - btc_rpc_shared_wallet: btc_rpc_shared_wallet.clone(), - btc_rpc_shared_wallet_v2: btc_rpc_shared_wallet_v2.clone(), config, monitoring_config, shutdown, - vault_id_manager: VaultIdManager::new( - btc_parachain, - btc_rpc_master_wallet, - btc_rpc_shared_wallet, - btc_rpc_shared_wallet_v2, - constructor, - db_path, - ), + vault_id_manager: VaultIdManager::new(btc_parachain, btc_rpc_master_wallet, constructor, db_path), } } @@ -694,28 +627,7 @@ impl VaultService { }?; // purposefully _after_ maybe_register_vault and _before_ other calls - self.vault_id_manager.fetch_vault_ids(self.config.only_migrate).await?; - - tracing::info!("Adding keys from past issues..."); - issue::add_keys_from_past_issue_request_old( - &self.btc_rpc_shared_wallet, - &self.btc_parachain, - &self.vault_id_manager.db, - ) - .await?; - - self.vault_id_manager.sweep_shared_wallet().await?; - issue::add_keys_from_past_issue_request_new( - &self.btc_rpc_shared_wallet_v2, - &self.btc_parachain, - &self.vault_id_manager.db, - ) - .await?; - - if self.config.only_migrate { - tracing::info!("Only migrating - quitting now."); - return Err(BackoffError::Permanent(Error::ClientShutdown)); - } + self.vault_id_manager.fetch_vault_ids().await?; let startup_height = self.await_parachain_block().await?; diff --git a/vault/tests/vault_integration_tests.rs b/vault/tests/vault_integration_tests.rs index fd195a48e..bfb420fff 100644 --- a/vault/tests/vault_integration_tests.rs +++ b/vault/tests/vault_integration_tests.rs @@ -95,8 +95,6 @@ async fn test_redeem_succeeds() { let btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( vault_provider.clone(), - btc_rpc_master_wallet.clone(), - btc_rpc_master_wallet.clone(), btc_rpc_master_wallet, btc_rpcs, "test_redeem_succeeds", @@ -166,8 +164,6 @@ async fn test_replace_succeeds() { let new_btc_rpc_master_wallet = btc_rpc.clone(); let _vault_id_manager = VaultIdManager::from_map( new_vault_provider.clone(), - new_btc_rpc_master_wallet.clone(), - new_btc_rpc_master_wallet.clone(), new_btc_rpc_master_wallet, btc_rpcs, "test_replace_succeeds1", @@ -181,8 +177,6 @@ async fn test_replace_succeeds() { let old_btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( old_vault_provider.clone(), - old_btc_rpc_master_wallet.clone(), - old_btc_rpc_master_wallet.clone(), old_btc_rpc_master_wallet, btc_rpcs, "test_replace_succeeds2", @@ -357,8 +351,6 @@ async fn test_cancellation_succeeds() { let new_btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( new_vault_provider.clone(), - new_btc_rpc_master_wallet.clone(), - new_btc_rpc_master_wallet.clone(), new_btc_rpc_master_wallet, btc_rpcs, "test_cancellation_succeeds", @@ -627,8 +619,6 @@ async fn test_automatic_issue_execution_succeeds() { let btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( vault2_provider.clone(), - btc_rpc_master_wallet.clone(), - btc_rpc_master_wallet.clone(), btc_rpc_master_wallet, btc_rpcs, "test_automatic_issue_execution_succeeds", @@ -728,8 +718,6 @@ async fn test_automatic_issue_execution_succeeds_with_big_transaction() { let btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( vault2_provider.clone(), - btc_rpc_master_wallet.clone(), - btc_rpc_master_wallet.clone(), btc_rpc_master_wallet, btc_rpcs, "test_automatic_issue_execution_succeeds_with_big_transaction", @@ -818,8 +806,6 @@ async fn test_execute_open_requests_succeeds() { let btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( vault_provider.clone(), - btc_rpc_master_wallet.clone(), - btc_rpc_master_wallet.clone(), btc_rpc_master_wallet, btc_rpcs, "test_execute_open_requests_succeeds", @@ -1220,8 +1206,6 @@ mod test_with_bitcoind { let btc_rpc_master_wallet = btc_rpc.clone(); let vault_id_manager = VaultIdManager::from_map( vault_provider.clone(), - btc_rpc_master_wallet.clone(), - btc_rpc_master_wallet.clone(), btc_rpc_master_wallet, btc_rpcs, "test_automatic_rbf_succeeds",