feat(foundations): allow for jinja2 templates in configs

dev/docker-compose.yml

@@ -11,18 +11,18 @@ services:
     volumes:
       - mongo:/data/db
 
-  nats:
-    image: ghcr.io/scuffletv/ci/nats:latest
-    pull_policy: "always"
-    ports:
-      - "127.0.0.1:4222:4222"
-      - "127.0.0.1:8222:8222"
-      - "127.0.0.1:6222:6222"
-    volumes:
-      - nats:/data
-    command:
-      - --jetstream
-      - --store_dir=/data
+  # nats:
+  #   image: ghcr.io/scuffletv/ci/nats:latest
+  #   pull_policy: "always"
+  #   ports:
+  #     - "127.0.0.1:4222:4222"
+  #     - "127.0.0.1:8222:8222"
+  #     - "127.0.0.1:6222:6222"
+  #   volumes:
+  #     - nats:/data
+  #   command:
+  #     - --jetstream
+  #     - --store_dir=/data
 
   minio:
     image: ghcr.io/scuffletv/ci/minio:latest
@@ -52,6 +52,6 @@ services:
       "
 
 volumes:
-  nats:
+  # nats:
   minio:
   mongo:

foundations/Cargo.toml

@@ -38,6 +38,7 @@ scc = { version = "2", optional = true }
 
 serde = { version = "1", optional = true, features = ["derive", "rc"] }
 toml = { version = "0.8", optional = true }
+minijinja = { version = "2.3.1", optional = true, features = ["json", "custom_syntax"] }
 
 clap = { version = "4", optional = true }
 const-str = { version = "0.5", optional = true }
@@ -163,6 +164,7 @@ settings = [
     "toml",
     "macros",
     "humantime-serde",
+    "minijinja",
 ]
 
 cli = [

foundations/src/settings/cli.rs

@@ -1,10 +1,12 @@
 use anyhow::Context;
 use clap::ArgAction;
+use minijinja::syntax::SyntaxConfig;
 
 use super::{Settings, SettingsParser};
 
 const GENERATE_ARG_ID: &str = "generate";
 const CONFIG_ARG_ID: &str = "config";
+const ALLOW_TEMPLATE: &str = "jinja";
 
 pub use clap;
 
@@ -33,6 +35,14 @@ fn default_cmd() -> clap::Command {
 				.num_args(0..=1)
 				.default_missing_value("./config.toml"),
 		)
+		.arg(
+			clap::Arg::new(ALLOW_TEMPLATE)
+				.long("jinja")
+				.help("Allows for the expansion of templates in the configuration file using Jinja syntax")
+				.action(ArgAction::Set)
+				.num_args(0..=1)
+				.default_missing_value("true"),
+		)
 }
 
 impl<S: Settings + serde::de::DeserializeOwned + serde::Serialize + Default> Default for Cli<S> {
@@ -71,23 +81,6 @@ impl<S: Settings + serde::de::DeserializeOwned + serde::Serialize> Cli<S> {
 		self
 	}
 
-	fn load_file(file: &str, optional: bool) -> anyhow::Result<Option<toml::Value>> {
-		let contents = match std::fs::read_to_string(file) {
-			Ok(contents) => contents,
-			Err(err) => {
-				if optional {
-					return Ok(None);
-				}
-
-				return Err(err).with_context(|| format!("Error reading configuration file: {file}"));
-			}
-		};
-
-		let incoming = toml::from_str(&contents).with_context(|| format!("Error parsing configuration file: {file}"))?;
-
-		Ok(Some(incoming))
-	}
-
 	pub fn parse(mut self) -> anyhow::Result<Matches<S>> {
 		let args = self.app.get_matches();
 
@@ -103,6 +96,22 @@ impl<S: Settings + serde::de::DeserializeOwned + serde::Serialize> Cli<S> {
 			std::process::exit(0);
 		}
 
+		let mut allow_template = args.get_one::<bool>(ALLOW_TEMPLATE).copied().unwrap_or(true).then(|| {
+			let mut env = minijinja::Environment::new();
+
+			env.add_global("env", std::env::vars().collect::<std::collections::HashMap<_, _>>());
+			env.set_syntax(
+				SyntaxConfig::builder()
+					.block_delimiters("{%", "%}")
+					.variable_delimiters("${{", "}}")
+					.comment_delimiters("{#", "#}")
+					.build()
+					.unwrap(),
+			);
+
+			env
+		});
+
 		let mut files = if let Some(files) = args.get_many::<String>(CONFIG_ARG_ID) {
 			files.cloned().map(|file| (file, false)).collect::<Vec<_>>()
 		} else {
@@ -114,9 +123,28 @@ impl<S: Settings + serde::de::DeserializeOwned + serde::Serialize> Cli<S> {
 		}
 
 		for (file, optional) in files {
-			if let Some(value) = Self::load_file(&file, optional)? {
-				self.settings.merge(value);
-			}
+			let content = match std::fs::read_to_string(file) {
+				Ok(content) => content,
+				Err(err) => {
+					if optional && err.kind() == std::io::ErrorKind::NotFound {
+						continue;
+					}
+
+					return Err(err).context("read");
+				}
+			};
+
+			let content = if let Some(env) = &mut allow_template {
+				env.template_from_str(&content)
+					.context("template")?
+					.render(())
+					.context("render")?
+			} else {
+				content
+			};
+
+			let incoming = toml::from_str(&content).context("parse")?;
+			self.settings.merge(incoming);
 		}
 
 		Ok(Matches {

image-processor/proto/build.rs

@@ -13,7 +13,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 	#[cfg(feature = "serde")]
 	let config = config.file_descriptor_set_path(&descriptor_path);
 
-	config.compile(
+	config.compile_protos(
 		&[
 			"scuffle/image_processor/service.proto",
 			"scuffle/image_processor/types.proto",

image-processor/src/config.rs

@@ -264,8 +264,12 @@ pub struct NatsEventQueueConfig {
 	/// The name of the event queue
 	pub name: String,
 	/// The Nats URL
-	/// For example: nats://localhost:4222
-	pub url: String,
+	/// For example: localhost:4222
+	pub servers: Vec<String>,
+	#[serde(default)]
+	pub username: Option<String>,
+	#[serde(default)]
+	pub password: Option<String>,
 	/// The message encoding for the event queue
 	#[serde(default)]
 	pub message_encoding: MessageEncoding,

image-processor/src/event_queue/nats.rs

@@ -1,3 +1,4 @@
+use async_nats::ConnectOptions;
 use prost::Message;
 use scuffle_image_processor_proto::EventCallback;
 
@@ -24,8 +25,17 @@ pub enum NatsEventQueueError {
 impl NatsEventQueue {
 	#[tracing::instrument(skip(config), name = "NatsEventQueue::new", fields(name = %config.name), err)]
 	pub async fn new(config: &NatsEventQueueConfig) -> Result<Self, EventQueueError> {
-		tracing::debug!("setting up nats event queue");
-		let nats = async_nats::connect(&config.url).await.map_err(NatsEventQueueError::from)?;
+		let nats = async_nats::connect_with_options(&config.servers, {
+			let options = ConnectOptions::default();
+
+			if let Some(username) = &config.username {
+				options.user_and_password(username.clone(), config.password.clone().unwrap_or_default())
+			} else {
+				options
+			}
+		})
+		.await
+		.map_err(NatsEventQueueError::from)?;
 
 		Ok(Self {
 			name: config.name.clone(),