From b50c6146faa371d073d111c45590e1132fba3ea9 Mon Sep 17 00:00:00 2001 From: technillogue Date: Sat, 29 Jul 2023 16:45:38 -0400 Subject: [PATCH 1/2] use rustls feature in addition, 5min keepalive, connection pool 20 Signed-off-by: technillogue --- Cargo.lock | 106 ++++++++++++++++++++++++++++++ storage/Cargo.toml | 2 +- storage/src/backend/connection.rs | 5 +- 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4413c54deb8..280e23008aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -779,6 +779,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1584,6 +1597,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -1593,19 +1607,38 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rlimit" version = "0.9.1" @@ -1668,6 +1701,27 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.10" @@ -1701,6 +1755,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -1814,6 +1878,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "strsim" version = "0.10.0" @@ -1956,6 +2026,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-uring" version = "0.4.0" @@ -2070,6 +2151,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -2261,6 +2348,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index a47ff12d547..36e297fdb48 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -23,7 +23,7 @@ leaky-bucket = { version = "0.12.1", optional = true } libc = "0.2" log = "0.4.8" nix = "0.24" -reqwest = { version = "0.11.14", features = ["blocking", "json"], optional = true } +reqwest = { version = "0.11.14", features = ["blocking", "json", "rustls", "rustls-tls"], optional = true } serde = { version = "1.0.110", features = ["serde_derive", "rc"] } serde_json = "1.0.53" sha1 = { version = "0.10.5", optional = true } diff --git a/storage/src/backend/connection.rs b/storage/src/backend/connection.rs index 6b6b2e69e43..b96829e22d5 100644 --- a/storage/src/backend/connection.rs +++ b/storage/src/backend/connection.rs @@ -611,7 +611,10 @@ impl Connection { let mut cb = Client::builder() .timeout(timeout) .connect_timeout(connect_timeout) - .redirect(Policy::none()); + .redirect(Policy::none()) + .use_rustls_tls() + .tcp_keepalive(Some(Duration::from_secs(5 * 60))) + .pool_max_idle_per_host(20); if config.skip_verify { cb = cb.danger_accept_invalid_certs(true); From dfa8491f6670507b86990f0b872583a2a9bf3838 Mon Sep 17 00:00:00 2001 From: technillogue Date: Sat, 29 Jul 2023 16:46:04 -0400 Subject: [PATCH 2/2] make connection pool size configurable and add throughput logging Signed-off-by: technillogue --- storage/src/backend/connection.rs | 9 +++++++-- storage/src/cache/mod.rs | 15 +++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/storage/src/backend/connection.rs b/storage/src/backend/connection.rs index b96829e22d5..edb45ffa24d 100644 --- a/storage/src/backend/connection.rs +++ b/storage/src/backend/connection.rs @@ -10,7 +10,7 @@ use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicI16, AtomicU8, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use std::{fmt, thread}; +use std::{env, fmt, thread}; use log::{max_level, Level}; @@ -607,6 +607,11 @@ impl Connection { } else { None }; + // get pool size from envvar + let pool_max_idle_per_host = match env::var("REGISTRY_CLIENT_POOL_MAX_IDLE_PER_HOST") { + Ok(val) => val.parse::().unwrap_or(20), + Err(_) => 20, + }; let mut cb = Client::builder() .timeout(timeout) @@ -614,7 +619,7 @@ impl Connection { .redirect(Policy::none()) .use_rustls_tls() .tcp_keepalive(Some(Duration::from_secs(5 * 60))) - .pool_max_idle_per_host(20); + .pool_max_idle_per_host(pool_max_idle_per_host); if config.skip_verify { cb = cb.danger_accept_invalid_certs(true); diff --git a/storage/src/cache/mod.rs b/storage/src/cache/mod.rs index cc65f842919..d9b5eaae5ea 100644 --- a/storage/src/cache/mod.rs +++ b/storage/src/cache/mod.rs @@ -270,13 +270,17 @@ pub trait BlobCache: Send + Sync { ))); } let duration = Instant::now().duration_since(start).as_millis(); + let duration_s = duration as f64 / 1000.0; + let throughput_mbps = blob_size as f64 / duration_s / 1_000_000.0; + debug!( - "read_chunks_from_backend: {} {} {} bytes at {}, duration {}ms", + "read_chunks_from_backend: {} {} {} bytes at {}, duration {}ms, throughput {:.4}Mbps", std::thread::current().name().unwrap_or_default(), if prefetch { "prefetch" } else { "fetch" }, blob_size, blob_offset, - duration + duration, + throughput_mbps ); let chunks = chunks.iter().map(|v| v.as_ref()).collect(); @@ -328,12 +332,15 @@ pub trait BlobCache: Send + Sync { } let duration = Instant::now().duration_since(start).as_millis(); + let duration_s = duration as f64 / 1000.0; + let throughput_mbps = chunk.compressed_size() as f64 / duration_s / 1_000_000.0; debug!( - "read_chunk_from_backend: {} {} bytes at {}, duration {}ms", + "read_chunk_from_backend: {} {} bytes at {}, duration {}ms, throughput {:.4}Mbps", std::thread::current().name().unwrap_or_default(), chunk.compressed_size(), chunk.compressed_offset(), - duration + duration, + throughput_mbps ); self.validate_chunk_data(chunk, buffer, false) .map_err(|e| {