From c91b34ef87a520fea626366b6cf1986df85d8a2b Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 22 Aug 2024 12:03:54 -0600 Subject: [PATCH 1/5] Support older `sqlite` versions. The `FALSE` constant was introduced in sqlite version 3.23.0, but Android does not support this version of sqlite until API level 30; we support back to Android API 27 so we have to use `0` as the constant for `FALSE` instead. --- zcash_client_sqlite/CHANGELOG.md | 5 + zcash_client_sqlite/src/wallet/db.rs | 2 +- .../src/wallet/init/migrations.rs | 4 + .../init/migrations/support_legacy_sqlite.rs | 91 +++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index e577920ec..6f1bb2f4a 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,6 +7,11 @@ and this library adheres to Rust's notion of ## [Unreleased] +### Changed +- The `v_tx_outputs` view was modified slightly to support older versions of + `sqlite`. + + ## [0.11.1] - 2024-08-21 ### Fixed diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 22d08b3f3..3fe97c66c 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -883,7 +883,7 @@ SELECT transactions.txid AS txid, NULL AS to_account_id, sent_notes.to_address AS to_address, sent_notes.value AS value, - FALSE AS is_change, + 0 AS is_change, sent_notes.memo AS memo FROM sent_notes JOIN transactions diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 10c97b790..ed29f7f48 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -15,6 +15,7 @@ mod sapling_memo_consistency; mod sent_notes_to_internal; mod shardtree_support; mod spend_key_available; +mod support_legacy_sqlite; mod tx_retrieval_queue; mod ufvk_support; mod utxos_table; @@ -74,6 +75,8 @@ pub(super) fn all_migrations( // \ \ ephemeral_addresses / / // \ \ | / / // ------------------------------ tx_retrieval_queue ---------------------------- + // | + // support_legacy_sqlite vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -131,6 +134,7 @@ pub(super) fn all_migrations( Box::new(tx_retrieval_queue::Migration { params: params.clone(), }), + Box::new(support_legacy_sqlite::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs b/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs new file mode 100644 index 000000000..54bd7f1fb --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs @@ -0,0 +1,91 @@ +//! A migration that removes the use of `FALSE` in sqlite view definitions. +//! This is necessary to support older +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::{migrations::tx_retrieval_queue, WalletMigrationError}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xc9ed1fb5_b2c3_467f_89dc_2591dcca5562); + +const DEPENDENCIES: &[Uuid] = &[tx_retrieval_queue::MIGRATION_ID]; + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + DEPENDENCIES.iter().copied().collect() + } + + fn description(&self) -> &'static str { + "Removes the FALSE keyword from the v_tx_outputs view definition" + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch( + r#" + DROP VIEW v_tx_outputs; + CREATE VIEW v_tx_outputs AS + -- select all outputs received by the wallet + SELECT transactions.txid AS txid, + ro.pool AS output_pool, + ro.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + ro.account_id AS to_account_id, + NULL AS to_address, + ro.value AS value, + ro.is_change AS is_change, + ro.memo AS memo + FROM v_received_outputs ro + JOIN transactions + ON transactions.id_tx = ro.transaction_id + -- join to the sent_notes table to obtain `from_account_id` + LEFT JOIN sent_notes ON sent_notes.id = ro.sent_note_id + UNION + -- select all outputs sent from the wallet to external recipients + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + NULL AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id + -- exclude any sent notes for which a row exists in the v_received_outputs view + WHERE ro.account_id IS NULL + "#, + )?; + + Ok(()) + } + + fn down(&self, _: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} + +#[cfg(test)] +mod tests { + use crate::wallet::init::migrations::tests::test_migrate; + + #[test] + fn migrate() { + test_migrate(&[super::MIGRATION_ID]); + } +} From 61584f7cc0b6770b81d49049b6a1af560ad4de68 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 27 Aug 2024 15:03:03 -0600 Subject: [PATCH 2/5] zcash_client_sqlite: Verify sqlite version compatibility on wallet init. --- zcash_client_sqlite/CHANGELOG.md | 18 ++++--- zcash_client_sqlite/Cargo.toml | 2 +- zcash_client_sqlite/src/wallet/init.rs | 51 +++++++++++++++++++ .../init/migrations/support_legacy_sqlite.rs | 7 ++- 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 6f1bb2f4a..0878b3fd4 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -9,8 +9,14 @@ and this library adheres to Rust's notion of ### Changed - The `v_tx_outputs` view was modified slightly to support older versions of - `sqlite`. - + `sqlite`. Queries to the exposed `v_tx_outputs` and `v_transactions` views + are supported for SQLite versions back to `3.19.x`. +- `zcash_client_sqlite::wallet::init::WalletMigrationError` has an additional + variant, `DatabaseNotSupported`. The `init_wallet_db` function now checks + that the sqlite version in use is compatible with the features required by + the wallet and returns this error if not. SQLite version `3.35` or higher + is required for use with `zcash_client_sqlite`. + ## [0.11.1] - 2024-08-21 @@ -23,7 +29,7 @@ and this library adheres to Rust's notion of `zcash_client_sqlite` now provides capabilities for the management of ephemeral transparent addresses in support of the creation of ZIP 320 transaction pairs. -In addition, `zcash_client_sqlite` now provides improved tracking of transparent +In addition, `zcash_client_sqlite` now provides improved tracking of transparent wallet history in support of the API changes in `zcash_client_backend 0.13`, and the `v_transactions` view has been modified to provide additional metadata about the relationship of each transaction to the wallet, in particular whether @@ -70,11 +76,11 @@ or not the transaction represents a wallet-internal shielding operation. ## [0.10.1] - 2024-03-25 ### Fixed -- The `sent_notes` table's `received_note` constraint was excessively restrictive - after zcash/librustzcash#1306. Any databases that have migrations from +- The `sent_notes` table's `received_note` constraint was excessively restrictive + after zcash/librustzcash#1306. Any databases that have migrations from zcash_client_sqlite 0.10.0 applied should be wiped and restored from seed. In order to ensure that the incorrect migration is not used, the migration - id for the `full_account_ids` migration has been changed from + id for the `full_account_ids` migration has been changed from `0x1b104345_f27e_42da_a9e3_1de22694da43` to `0x6d02ec76_8720_4cc6_b646_c4e2ce69221c` ## [0.10.0] - 2024-03-25 diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 069a495b6..201000624 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -71,6 +71,7 @@ schemer.workspace = true schemer-rusqlite.workspace = true time.workspace = true uuid.workspace = true +regex = "1.4" # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) @@ -87,7 +88,6 @@ orchard = { workspace = true, features = ["test-dependencies"] } proptest.workspace = true rand_chacha.workspace = true rand_core.workspace = true -regex = "1.4" tempfile = "3.5.0" zcash_keys = { workspace = true, features = ["test-dependencies"] } zcash_note_encryption.workspace = true diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 9b0994e2f..b69c625e7 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -3,6 +3,7 @@ use std::fmt; use std::rc::Rc; +use regex::Regex; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; @@ -20,8 +21,15 @@ use crate::{error::SqliteClientError, WalletDb}; mod migrations; +const SQLITE_MAJOR_VERSION: u32 = 3; +const MIN_SQLITE_MINOR_VERSION: u32 = 35; + #[derive(Debug)] pub enum WalletMigrationError { + /// A feature required by the wallet database is not supported by the version of + /// SQLite that the migration is running against. + DatabaseNotSupported(String), + /// The seed is required for the migration. SeedRequired, @@ -100,6 +108,13 @@ impl From for WalletMigrationError { impl fmt::Display for WalletMigrationError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { + WalletMigrationError::DatabaseNotSupported(version) => { + write!( + f, + "The installed SQLite version {} does not support operations required by the wallet.", + version + ) + } WalletMigrationError::SeedRequired => { write!( f, @@ -305,6 +320,8 @@ fn init_wallet_db_internal( ) -> Result<(), MigratorError> { let seed = seed.map(Rc::new); + verify_sqlite_version_compatibility(&wdb.conn).map_err(MigratorError::Adapter)?; + // Turn off foreign key enforcement, to ensure that table replacement does not break foreign // key references in table definitions. // @@ -357,6 +374,40 @@ fn init_wallet_db_internal( Ok(()) } +/// Verify that the sqlite version in use supports the features required by this library. +/// Note that the version of sqlite available to the database backend may be different +/// from what is used to query the views that are part of the public API. +fn verify_sqlite_version_compatibility( + conn: &rusqlite::Connection, +) -> Result<(), WalletMigrationError> { + let sqlite_version = + conn.query_row("SELECT sqlite_version()", [], |row| row.get::<_, String>(0))?; + + let version_re = Regex::new(r"^(?[0-9]+)\.(?[0-9]+).+$").unwrap(); + let captures = + version_re + .captures(&sqlite_version) + .ok_or(WalletMigrationError::DatabaseNotSupported( + "Unknown".to_owned(), + ))?; + let parse_int = |value: &str| { + value.parse::().map_err(|_| { + WalletMigrationError::CorruptedData(format!( + "Cannot decode SQLite major version {}", + &captures["major"] + )) + }) + }; + let major = parse_int(&captures["major"])?; + let minor = parse_int(&captures["minor"])?; + + if major != SQLITE_MAJOR_VERSION || minor < MIN_SQLITE_MINOR_VERSION { + Err(WalletMigrationError::DatabaseNotSupported(sqlite_version)) + } else { + Ok(()) + } +} + #[cfg(test)] #[allow(deprecated)] mod tests { diff --git a/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs b/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs index 54bd7f1fb..29385a30e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/support_legacy_sqlite.rs @@ -1,5 +1,4 @@ -//! A migration that removes the use of `FALSE` in sqlite view definitions. -//! This is necessary to support older +//! Modifies definitions to avoid keywords that may not be available in older SQLite versions. use std::collections::HashSet; use rusqlite; @@ -9,7 +8,7 @@ use uuid::Uuid; use crate::wallet::init::{migrations::tx_retrieval_queue, WalletMigrationError}; -pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xc9ed1fb5_b2c3_467f_89dc_2591dcca5562); +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x156d8c8f_2173_4b59_89b6_75697d5a2103); const DEPENDENCIES: &[Uuid] = &[tx_retrieval_queue::MIGRATION_ID]; @@ -25,7 +24,7 @@ impl schemer::Migration for Migration { } fn description(&self) -> &'static str { - "Removes the FALSE keyword from the v_tx_outputs view definition" + "Modifies definitions to avoid keywords that may not be available in older SQLite versions." } } From 1b8334104dae260272e34b8f624612bcd3b1037e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 3 Sep 2024 12:12:33 -0600 Subject: [PATCH 3/5] zcash_client_sqlite: Accept 2-part `major.minor` SQLite versions. --- zcash_client_sqlite/src/wallet/init.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index b69c625e7..ca6c4b52e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -383,23 +383,23 @@ fn verify_sqlite_version_compatibility( let sqlite_version = conn.query_row("SELECT sqlite_version()", [], |row| row.get::<_, String>(0))?; - let version_re = Regex::new(r"^(?[0-9]+)\.(?[0-9]+).+$").unwrap(); + let version_re = Regex::new(r"^(?[0-9]+)\.(?[0-9]+).*$").unwrap(); let captures = version_re .captures(&sqlite_version) .ok_or(WalletMigrationError::DatabaseNotSupported( "Unknown".to_owned(), ))?; - let parse_int = |value: &str| { - value.parse::().map_err(|_| { + let parse_version_part = |part: &str| { + captures[part].parse::().map_err(|_| { WalletMigrationError::CorruptedData(format!( - "Cannot decode SQLite major version {}", - &captures["major"] + "Cannot decode SQLite {} version component {}", + part, &captures[part] )) }) }; - let major = parse_int(&captures["major"])?; - let minor = parse_int(&captures["minor"])?; + let major = parse_version_part("major")?; + let minor = parse_version_part("minor")?; if major != SQLITE_MAJOR_VERSION || minor < MIN_SQLITE_MINOR_VERSION { Err(WalletMigrationError::DatabaseNotSupported(sqlite_version)) From 5e6a8f02508893bce0410c9e3a941ea151d1d92f Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 22 Aug 2024 12:09:33 -0600 Subject: [PATCH 4/5] Release zcash_client_sqlite version 0.11.2 --- Cargo.lock | 2 +- zcash_client_sqlite/CHANGELOG.md | 2 ++ zcash_client_sqlite/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4f3541b8..dc8cb7e20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5877,7 +5877,7 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.11.1" +version = "0.11.2" dependencies = [ "assert_matches", "bip32", diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 0878b3fd4..0b5e9865c 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -7,6 +7,8 @@ and this library adheres to Rust's notion of ## [Unreleased] +## [0.11.2] - 2024-08-21 + ### Changed - The `v_tx_outputs` view was modified slightly to support older versions of `sqlite`. Queries to the exposed `v_tx_outputs` and `v_transactions` views diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 201000624..25f2288c5 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.11.1" +version = "0.11.2" authors = [ "Jack Grigg ", "Kris Nuttycombe " From 01576f10aee25c97530458955df1989a54c9d34c Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 3 Sep 2024 12:26:56 -0600 Subject: [PATCH 5/5] Update audit metadata for zcash_client_sqlite 0.11.2 release. --- supply-chain/imports.lock | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 0145b6d3b..c464e8c34 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -238,15 +238,8 @@ user-login = "nuttycom" user-name = "Kris Nuttycombe" [[publisher.zcash_client_sqlite]] -version = "0.11.0" -when = "2024-08-20" -user-id = 169181 -user-login = "nuttycom" -user-name = "Kris Nuttycombe" - -[[publisher.zcash_client_sqlite]] -version = "0.11.1" -when = "2024-08-21" +version = "0.11.2" +when = "2024-09-03" user-id = 169181 user-login = "nuttycom" user-name = "Kris Nuttycombe"