Use sqlx inplace of rusqlite

teos-common/Cargo.toml

@@ -10,7 +10,6 @@ edition = "2018"
 # General
 hex = { version = "0.4.3", features = [ "serde" ] }
 prost = "0.9"
-rusqlite = { version = "0.26.0", features = [ "bundled", "limits" ] }
 serde = "1.0.130"
 serde_json = "1.0"
 tonic = "0.6"

teos-common/src/lib.rs

@@ -11,7 +11,6 @@ pub mod protos {
 pub mod appointment;
 pub mod constants;
 pub mod cryptography;
-pub mod dbm;
 pub mod errors;
 pub mod net;
 pub mod receipts;

teos/Cargo.toml

@@ -14,14 +14,20 @@ path = "src/cli.rs"
 name = "teosd"
 path = "src/main.rs"
 
+[features]
+# By default, enable both SQLite snd PostgreSQL in the output binary.
+default = ["sqlite", "postgres"]
+sqlite = ["sqlx/sqlite"]
+postgres = ["sqlx/postgres"]
+
 [dependencies]
 # General
 hex = { version = "0.4.3", features = [ "serde" ] }
 home = "0.5.3"
 log = "0.4"
 prost = "0.9"
 rcgen = { version = "0.8", features = ["pem", "x509-parser"] }
-rusqlite = { version = "0.26.0", features = [ "bundled", "limits" ] }
+sqlx = { version = "0.7", features = ["runtime-tokio", "tls-native-tls", "migrate", "any"] }
 serde = "1.0.130"
 serde_json = "1.0"
 simple_logger = "2.1.0"

teos/build.rs

@@ -1,4 +1,6 @@
-fn main() -> Result<(), Box<dyn std::error::Error>> {
+fn main() {
+    // trigger recompilation when a new migration is added without a change in the source code.
+    println!("cargo:rerun-if-changed=migrations");
     tonic_build::configure()
         .extern_path(".common.teos.v2", "::teos-common::protos")
         .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]")
@@ -23,7 +25,6 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 "proto/teos/v2/user.proto",
             ],
             &["proto/teos/v2", "../teos-common/proto/"],
-        )?;
-
-    Ok(())
+        )
+        .unwrap();
 }

teos/migrations/README.md

@@ -0,0 +1,10 @@
+# Structure:
+- `postgres`: Contains `.sql` migrations for postgres databases.
+- `sqlite`: Contains `.sql` migrations for sqlite databases.
+
+
+# Migrations Extra Documentation (`migrations/*/*.md`):
+
+Migrations cannot be edited once applied to the database. Thus, writing/editing any comments or explanations in the `.sql` files would break the tower for users who have applied those migrations.
+
+Any additional comments that we need to add after a migration has been applied should be in `MID_MNAME.md` instead.

teos/migrations/postgres/000_init.sql

@@ -0,0 +1,51 @@
+-- INT is 4 bytes signed integer (i32) in PostgreSQL.
+-- Many fields in here map to (u32)s in Rust, which has double the capacity of (i32)s on the positive side.
+-- Some database calls might break because of this.
+-- A solution for this could be either one of:
+--   1- Find a one to one mapping between the Rust (u32)s and PostgreSQL's (i32)s since they are essentially the same size.
+--   2- Use PostgreSQL's BIGINT which is equivalent to an i64.
+
+CREATE TABLE IF NOT EXISTS users (
+    user_id BYTEA PRIMARY KEY,
+    available_slots BIGINT NOT NULL,
+    subscription_start BIGINT NOT NULL,
+    subscription_expiry BIGINT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS appointments (
+    UUID BYTEA PRIMARY KEY,
+    locator BYTEA NOT NULL,
+    encrypted_blob BYTEA NOT NULL,
+    to_self_delay BIGINT NOT NULL,
+    user_signature TEXT NOT NULL,
+    start_block BIGINT NOT NULL,
+    user_id BYTEA NOT NULL,
+    FOREIGN KEY(user_id)
+        REFERENCES users(user_id)
+        ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS trackers (
+    UUID BYTEA PRIMARY KEY,
+    dispute_tx BYTEA NOT NULL,
+    penalty_tx BYTEA NOT NULL,
+    height BIGINT NOT NULL,
+    confirmed BIGINT NOT NULL,
+    FOREIGN KEY(UUID)
+        REFERENCES appointments(UUID)
+        ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS last_known_block (
+    id INT PRIMARY KEY,
+    block_hash BYTEA NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS keys (
+    id SERIAL PRIMARY KEY,
+    secret_key TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS locators_index ON appointments (
+    locator
+);

teos/migrations/sqlite/000_init.sql

@@ -0,0 +1,40 @@
+CREATE TABLE IF NOT EXISTS users (
+    user_id INT PRIMARY KEY,
+    available_slots INT NOT NULL,
+    subscription_start INT NOT NULL,
+    subscription_expiry INT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS appointments (
+    UUID INT PRIMARY KEY,
+    locator INT NOT NULL,
+    encrypted_blob BLOB NOT NULL,
+    to_self_delay INT NOT NULL,
+    user_signature BLOB NOT NULL,
+    start_block INT NOT NULL,
+    user_id INT NOT NULL,
+    FOREIGN KEY(user_id)
+        REFERENCES users(user_id)
+        ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS trackers (
+    UUID INT PRIMARY KEY,
+    dispute_tx BLOB NOT NULL,
+    penalty_tx BLOB NOT NULL,
+    height INT NOT NULL,
+    confirmed BOOL NOT NULL,
+    FOREIGN KEY(UUID)
+        REFERENCES appointments(UUID)
+        ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS last_known_block (
+    id INT PRIMARY KEY,
+    block_hash INT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS keys (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    key INT NOT NULL
+);

teos/migrations/sqlite/001_datatypes_correction.sql

@@ -0,0 +1,51 @@
+-- Change `user_id` from INT to BLOB.
+CREATE TABLE tmp_users (
+    user_id BLOB PRIMARY KEY,
+    available_slots INT NOT NULL,
+    subscription_start INT NOT NULL,
+    subscription_expiry INT NOT NULL
+);
+INSERT INTO tmp_users SELECT * FROM users;
+-- We couldn't drop `users` before copying `appointments`, as the former will cascade delete the latter.
+-- Same for `trackers` and `appointments`.
+-- DROP TABLE users;
+
+-- Change `UUID` & `locator` & `user_id` from INT to BLOB.
+-- Change `user_signature` from BLOB to TEXT.
+CREATE TABLE tmp_appointments (
+    UUID BLOB PRIMARY KEY,
+    locator BLOB NOT NULL,
+    encrypted_blob BLOB NOT NULL,
+    to_self_delay INT NOT NULL,
+    user_signature TEXT NOT NULL,
+    start_block INT NOT NULL,
+    user_id BLOB NOT NULL,
+    FOREIGN KEY(user_id)
+        REFERENCES tmp_users(user_id)
+        ON DELETE CASCADE
+);
+INSERT INTO tmp_appointments SELECT * FROM appointments;
+
+-- Change `UUID` from INT to BLOB.
+-- Change `confirmed` from BOOL to INT (due to https://github.com/launchbadge/sqlx/issues/2657).
+CREATE TABLE tmp_trackers (
+    UUID BLOB PRIMARY KEY,
+    dispute_tx BLOB NOT NULL,
+    penalty_tx BLOB NOT NULL,
+    height INT NOT NULL,
+    confirmed INT NOT NULL,
+    FOREIGN KEY(UUID)
+        REFERENCES tmp_appointments(UUID)
+        ON DELETE CASCADE
+);
+INSERT INTO tmp_trackers SELECT * FROM trackers;
+
+-- We can drop these now after all the data has been copied.
+DROP TABLE users;
+DROP TABLE appointments;
+DROP TABLE trackers;
+
+-- Foreign key references are automatically adjusted (tmp_* -> *).
+ALTER TABLE tmp_users RENAME TO users;
+ALTER TABLE tmp_appointments RENAME TO appointments;
+ALTER TABLE tmp_trackers RENAME TO trackers;

teos/migrations/sqlite/002_more_datatypes_corrections.sql

@@ -0,0 +1,17 @@
+-- Change `block_hash` from INT to BLOB.
+CREATE TABLE tmp_last_known_block (
+    id INT PRIMARY KEY,
+    block_hash BLOB NOT NULL
+);
+INSERT INTO tmp_last_known_block SELECT * FROM last_known_block;
+DROP TABLE last_known_block;
+ALTER TABLE tmp_last_known_block RENAME TO last_known_block;
+
+-- Change `key` from INT to TEXT.
+CREATE TABLE tmp_keys (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    key TEXT NOT NULL
+);
+INSERT INTO tmp_keys SELECT * FROM keys;
+DROP TABLE keys;
+ALTER TABLE tmp_keys RENAME TO keys;

teos/migrations/sqlite/003_keys_rename.sql

@@ -0,0 +1,2 @@
+-- Rename `key` to `secret_key` as the word `key` is reserved in some databases.
+ALTER TABLE keys RENAME key TO secret_key;

teos/migrations/sqlite/004_locator_index.sql

@@ -0,0 +1,5 @@
+-- This index greatly enhances the performance of locator based selection queries:
+-- "SELECT ... FROM appointments WHERE locator = ..."
+CREATE INDEX IF NOT EXISTS locators_index ON appointments (
+    locator
+);

teos/src/api/http.rs

@@ -842,7 +842,8 @@ mod tests_methods {
         );
         internal_api
             .get_watcher()
-            .add_dummy_tracker_to_responder(&tracker);
+            .add_dummy_tracker_to_responder(&tracker)
+            .await;
 
         // Try to add it via the http API
         let appointment = generate_dummy_appointment(Some(&dispute_tx.txid())).inner;