From 1585c975b276b8a81685484e707e0ba2a11df014 Mon Sep 17 00:00:00 2001 From: Troy Benson Date: Sat, 12 Oct 2024 18:13:16 +0000 Subject: [PATCH] feat(foundations): allow for jinja2 templates in configs --- Cargo.lock | 12 +++++ dev/docker-compose.yml | 26 +++++----- foundations/Cargo.toml | 2 + foundations/src/settings/cli.rs | 68 +++++++++++++++++-------- image-processor/proto/build.rs | 2 +- image-processor/src/config.rs | 8 ++- image-processor/src/event_queue/nats.rs | 14 ++++- 7 files changed, 94 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8d503ad..fcf8247f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2515,6 +2515,17 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1028b628753a7e1a88fc59c9ba4b02ecc3bc0bd3c7af23df667bc28df9b3310e" +dependencies = [ + "aho-corasick", + "serde", + "serde_json", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3755,6 +3766,7 @@ dependencies = [ "hyper-util", "itertools 0.13.0", "matchers", + "minijinja", "num_cpus", "once_cell", "opentelemetry", diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index a5d72dfd..c1866c6f 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -11,18 +11,18 @@ services: volumes: - mongo:/data/db - nats: - image: ghcr.io/scuffletv/ci/nats:latest - pull_policy: "always" - ports: - - "127.0.0.1:4222:4222" - - "127.0.0.1:8222:8222" - - "127.0.0.1:6222:6222" - volumes: - - nats:/data - command: - - --jetstream - - --store_dir=/data + # nats: + # image: ghcr.io/scuffletv/ci/nats:latest + # pull_policy: "always" + # ports: + # - "127.0.0.1:4222:4222" + # - "127.0.0.1:8222:8222" + # - "127.0.0.1:6222:6222" + # volumes: + # - nats:/data + # command: + # - --jetstream + # - --store_dir=/data minio: image: ghcr.io/scuffletv/ci/minio:latest @@ -52,6 +52,6 @@ services: " volumes: - nats: + # nats: minio: mongo: diff --git a/foundations/Cargo.toml b/foundations/Cargo.toml index afc744d8..bf7fd0ec 100644 --- a/foundations/Cargo.toml +++ b/foundations/Cargo.toml @@ -38,6 +38,7 @@ scc = { version = "2", optional = true } serde = { version = "1", optional = true, features = ["derive", "rc"] } toml = { version = "0.8", optional = true } +minijinja = { version = "2.3.1", optional = true, features = ["json", "custom_syntax"] } clap = { version = "4", optional = true } const-str = { version = "0.5", optional = true } @@ -163,6 +164,7 @@ settings = [ "toml", "macros", "humantime-serde", + "minijinja", ] cli = [ diff --git a/foundations/src/settings/cli.rs b/foundations/src/settings/cli.rs index 425588f0..e024b0ac 100644 --- a/foundations/src/settings/cli.rs +++ b/foundations/src/settings/cli.rs @@ -1,10 +1,12 @@ use anyhow::Context; use clap::ArgAction; +use minijinja::syntax::SyntaxConfig; use super::{Settings, SettingsParser}; const GENERATE_ARG_ID: &str = "generate"; const CONFIG_ARG_ID: &str = "config"; +const ALLOW_TEMPLATE: &str = "jinja"; pub use clap; @@ -33,6 +35,14 @@ fn default_cmd() -> clap::Command { .num_args(0..=1) .default_missing_value("./config.toml"), ) + .arg( + clap::Arg::new(ALLOW_TEMPLATE) + .long("jinja") + .help("Allows for the expansion of templates in the configuration file using Jinja syntax") + .action(ArgAction::Set) + .num_args(0..=1) + .default_missing_value("true"), + ) } impl Default for Cli { @@ -71,23 +81,6 @@ impl Cli { self } - fn load_file(file: &str, optional: bool) -> anyhow::Result> { - let contents = match std::fs::read_to_string(file) { - Ok(contents) => contents, - Err(err) => { - if optional { - return Ok(None); - } - - return Err(err).with_context(|| format!("Error reading configuration file: {file}")); - } - }; - - let incoming = toml::from_str(&contents).with_context(|| format!("Error parsing configuration file: {file}"))?; - - Ok(Some(incoming)) - } - pub fn parse(mut self) -> anyhow::Result> { let args = self.app.get_matches(); @@ -103,6 +96,22 @@ impl Cli { std::process::exit(0); } + let mut allow_template = args.get_one::(ALLOW_TEMPLATE).copied().unwrap_or(true).then(|| { + let mut env = minijinja::Environment::new(); + + env.add_global("env", std::env::vars().collect::>()); + env.set_syntax( + SyntaxConfig::builder() + .block_delimiters("{%", "%}") + .variable_delimiters("${{", "}}") + .comment_delimiters("{#", "#}") + .build() + .unwrap(), + ); + + env + }); + let mut files = if let Some(files) = args.get_many::(CONFIG_ARG_ID) { files.cloned().map(|file| (file, false)).collect::>() } else { @@ -114,9 +123,28 @@ impl Cli { } for (file, optional) in files { - if let Some(value) = Self::load_file(&file, optional)? { - self.settings.merge(value); - } + let content = match std::fs::read_to_string(file) { + Ok(content) => content, + Err(err) => { + if optional && err.kind() == std::io::ErrorKind::NotFound { + continue; + } + + return Err(err).context("read"); + } + }; + + let content = if let Some(env) = &mut allow_template { + env.template_from_str(&content) + .context("template")? + .render(()) + .context("render")? + } else { + content + }; + + let incoming = toml::from_str(&content).context("parse")?; + self.settings.merge(incoming); } Ok(Matches { diff --git a/image-processor/proto/build.rs b/image-processor/proto/build.rs index c5ba555b..d3962480 100644 --- a/image-processor/proto/build.rs +++ b/image-processor/proto/build.rs @@ -13,7 +13,7 @@ fn main() -> Result<(), Box> { #[cfg(feature = "serde")] let config = config.file_descriptor_set_path(&descriptor_path); - config.compile( + config.compile_protos( &[ "scuffle/image_processor/service.proto", "scuffle/image_processor/types.proto", diff --git a/image-processor/src/config.rs b/image-processor/src/config.rs index d177d1c8..2e80131c 100644 --- a/image-processor/src/config.rs +++ b/image-processor/src/config.rs @@ -264,8 +264,12 @@ pub struct NatsEventQueueConfig { /// The name of the event queue pub name: String, /// The Nats URL - /// For example: nats://localhost:4222 - pub url: String, + /// For example: localhost:4222 + pub servers: Vec, + #[serde(default)] + pub username: Option, + #[serde(default)] + pub password: Option, /// The message encoding for the event queue #[serde(default)] pub message_encoding: MessageEncoding, diff --git a/image-processor/src/event_queue/nats.rs b/image-processor/src/event_queue/nats.rs index 1ce6f9c4..93b19d53 100644 --- a/image-processor/src/event_queue/nats.rs +++ b/image-processor/src/event_queue/nats.rs @@ -1,3 +1,4 @@ +use async_nats::ConnectOptions; use prost::Message; use scuffle_image_processor_proto::EventCallback; @@ -24,8 +25,17 @@ pub enum NatsEventQueueError { impl NatsEventQueue { #[tracing::instrument(skip(config), name = "NatsEventQueue::new", fields(name = %config.name), err)] pub async fn new(config: &NatsEventQueueConfig) -> Result { - tracing::debug!("setting up nats event queue"); - let nats = async_nats::connect(&config.url).await.map_err(NatsEventQueueError::from)?; + let nats = async_nats::connect_with_options(&config.servers, { + let options = ConnectOptions::default(); + + if let Some(username) = &config.username { + options.user_and_password(username.clone(), config.password.clone().unwrap_or_default()) + } else { + options + } + }) + .await + .map_err(NatsEventQueueError::from)?; Ok(Self { name: config.name.clone(),