Merge pull request #127 from GreenmaskIO/feat/static_parameters_value…

…_template
GreenmaskIO · May 17, 2024 · 0eb7f8f · 0eb7f8f
2 parents 871c992 + 723b7b3
commit 0eb7f8f
Show file tree

Hide file tree

Showing 17 changed files with 153 additions and 33 deletions.
diff --git a/docs/built_in_transformers/parameters_templating.md b/docs/built_in_transformers/parameters_templating.md
@@ -0,0 +1,77 @@
+# Parameters templating
+
+## Description
+
+It is allowed to generate parameter values from templates. It is useful when you don't want to write values manually,
+but instead want to generate and initialize them dynamically.
+
+Here you can find the list of template functions that can be used in the
+template [Custom functions](advanced_transformers/custom_functions/index.md).
+
+You can encode and decode objects using the driver function bellow.
+
+### Template functions
+
+| Function               | Description                                                                                                                                                                                                                                               | Signature                                                           |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|
+| `.GetColumnType`       | Returns a string with the column type.                                                                                                                                                                                                                    | `.GetColumnType(name string) (typeName string, err error)`          |
+| `.EncodeValueByColumn` | Encodes a value of any type into its raw string representation using the specified column name. Encoding is performed through the PostgreSQL driver. Throws an error if types are incompatible.                                                           | `.EncodeValueByColumn(name string, value any) (res any, err error)` |
+| `.DecodeValueByColumn` | Decodes a value from its raw string representation to a Golang type using the specified column name. Decoding is performed through the PostgreSQL driver. Throws an error if types are incompatible.                                                      | `.DecodeValueByColumn(name string, value any) (res any, err error)` |
+| `.EncodeValueByType`   | Encodes a value of any type into its string representation using the specified type name. Encoding is performed through the PostgreSQL driver. Throws an error if types are incompatible.                                                                 | `.EncodeValueByType(name string, value any) (res any, err error)`   |
+| `.DecodeValueByType`   | Decodes a value from its raw string representation to a Golang type using the specified type name. Decoding is performed through the PostgreSQL driver. Throws an error if types are incompatible.                                                        | `.DecodeValueByType(name string, value any) (res any, err error)`   |
+| `.DecodeValue`         | Decodes a value from its raw string representation to a Golang type using the data type assigned to the table column specified in the `column` parameter. Decoding is performed through the PostgreSQL driver. Throws an error if types are incompatible. | `.DecodeValueByColumn(value any) (res any, err error)`              |
+| `.EncodeValue`         | Encodes a value of any type into its string representation using the type assigned to the table column specified in the `column` parameter. Encoding is performed through the PostgreSQL driver. Throws an error if types are incompatible.               | `.EncodeValue(value any) (res any, err error)`                      |
+
+!!! warning
+
+    If column parameter is not linked to column parameter, then functions `.DecodeValue` and `.EncodeValue` will return 
+    an error. You can use `.DecodeValueByType` and `.EncodeValueByType` or `.DecodeValueByColumn` and 
+    `.EncodeValueByColumn` instead.
+
+### Example
+
+In the example below, the min and max values for the `birth_date` column are generated dynamically using the `now`
+template function. The value returns the current date and time. The `tsModify` function is then used to subtract 30
+(and 18) years. But because the parameter type is mapped on `column` parameter type, the `EncodeValue` function is used
+to encode the value into the column type.
+
+For example, if we have the now date as `2021-01-01`, the dynamically calculated `min` value will be `1994-01-01` and
+the `max` value will be `2006-01-01`.
+
+```sql
+CREATE TABLE account
+(
+    id         SERIAL PRIMARY KEY,
+    gender     VARCHAR(1) NOT NULL,
+    email      TEXT       NOT NULL NOT NULL UNIQUE,
+    first_name TEXT       NOT NULL,
+    last_name  TEXT       NOT NULL,
+    birth_date DATE,
+    created_at TIMESTAMP  NOT NULL DEFAULT NOW()
+);
+
+INSERT INTO account (first_name, gender, last_name, birth_date, email)
+VALUES ('John', 'M', 'Smith', '1980-01-01', '[email protected]');
+```
+
+```yaml
+- schema: "public"
+  name: "account"
+  transformers:
+    - name: "RandomDate"
+      params:
+        column: "birth_date"
+        min: '{{ now | tsModify "-30 years" | .EncodeValue }}' # 1994
+        max: '{{ now | tsModify "-18 years" | .EncodeValue }}' # 2006
+```
+
+Result
+
+<table>
+<tr>
+<th>Column</th><th>OriginalValue</th><th>TransformedValue</th>
+</tr>
+<tr>
+<td>birth_date</td><td><span style="color:green">1980-01-01</span></td><td><span style="color:red">1995-09-06</span></td>
+</tr>
+</table>
diff --git a/internal/db/postgres/transformers/dict.go b/internal/db/postgres/transformers/dict.go
@@ -54,6 +54,7 @@ var DictTransformerDefinition = utils.NewTransformerDefinition(
 		`fail if value is not matched with dict otherwise keep value`,
 	).SetRequired(false).
 		SetDefaultValue(toolkit.ParamsValue("true")),
+
 	toolkit.MustNewParameterDefinition(
 		"validate",
 		`perform encode-decode procedure using column type, ensuring that value has correct type`,

diff --git a/internal/db/postgres/transformers/noise_date.go b/internal/db/postgres/transformers/noise_date.go
@@ -48,18 +48,21 @@ var NoiseDateTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"min_ratio",
 		"min random duration for noise. Dy default 5% of the max_ratio",
-	).SetCastDbType("interval"),
+	).SetCastDbType("interval").
+		SetSupportTemplate(true),
 
 	toolkit.MustNewParameterDefinition(
 		"max_ratio",
 		"max random duration for noise",
 	).SetRequired(true).
-		SetCastDbType("interval"),
+		SetCastDbType("interval").
+		SetSupportTemplate(true),
 
 	toolkit.MustNewParameterDefinition(
 		"min",
 		"min threshold date (and/or time) of value",
-	).SetLinkParameter("column").
+	).SetSupportTemplate(true).
+		SetLinkParameter("column").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes("date", "timestamp", "timestamptz"),
@@ -68,7 +71,8 @@ var NoiseDateTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"max",
 		"max threshold date (and/or time) of value",
-	).SetLinkParameter("column").
+	).SetSupportTemplate(true).
+		SetLinkParameter("column").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes("date", "timestamp", "timestamptz"),

diff --git a/internal/db/postgres/transformers/noise_float.go b/internal/db/postgres/transformers/noise_float.go
@@ -43,12 +43,14 @@ var NoiseFloatTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"decimal",
 		"Numbers of decimal",
-	).SetDefaultValue(toolkit.ParamsValue("4")),
+	).SetSupportTemplate(true).
+		SetDefaultValue(toolkit.ParamsValue("4")),
 
 	toolkit.MustNewParameterDefinition(
 		"min",
 		"min float value threshold",
 	).SetLinkParameter("column").
+		SetSupportTemplate(true).
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes("float4", "float8", "int2", "int4", "int8"),
@@ -58,6 +60,7 @@ var NoiseFloatTransformerDefinition = utils.NewTransformerDefinition(
 		"max",
 		"max float value threshold",
 	).SetLinkParameter("column").
+		SetSupportTemplate(true).
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes("float4", "float8", "int2", "int4", "int8"),

diff --git a/internal/db/postgres/transformers/noise_int.go b/internal/db/postgres/transformers/noise_int.go
@@ -43,18 +43,20 @@ var NoiseIntTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"min",
 		"min value threshold limiter",
-	).SetDynamicMode(
-		toolkit.NewDynamicModeProperties().
-			SetCompatibleTypes("int2", "int4", "int8"),
-	),
+	).SetSupportTemplate(true).
+		SetDynamicMode(
+			toolkit.NewDynamicModeProperties().
+				SetCompatibleTypes("int2", "int4", "int8"),
+		),
 
 	toolkit.MustNewParameterDefinition(
 		"max",
 		"max value threshold limiter",
-	).SetDynamicMode(
-		toolkit.NewDynamicModeProperties().
-			SetCompatibleTypes("int2", "int4", "int8"),
-	),
+	).SetSupportTemplate(true).
+		SetDynamicMode(
+			toolkit.NewDynamicModeProperties().
+				SetCompatibleTypes("int2", "int4", "int8"),
+		),
 
 	minRatioParameterDefinition,
 

diff --git a/internal/db/postgres/transformers/noise_numeric.go b/internal/db/postgres/transformers/noise_numeric.go
@@ -43,12 +43,14 @@ var NoiseNumericTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"decimal",
 		"Numbers of decimal",
-	).SetDefaultValue(toolkit.ParamsValue("4")),
+	).SetSupportTemplate(true).
+		SetDefaultValue(toolkit.ParamsValue("4")),
 
 	toolkit.MustNewParameterDefinition(
 		"min",
 		"min float value threshold",
-	).SetLinkParameter("column").
+	).SetSupportTemplate(true).
+		SetLinkParameter("column").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes(
@@ -65,7 +67,8 @@ var NoiseNumericTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"max",
 		"max float value threshold",
-	).SetLinkParameter("column").
+	).SetSupportTemplate(true).
+		SetLinkParameter("column").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes(

diff --git a/internal/db/postgres/transformers/random_date.go b/internal/db/postgres/transformers/random_date.go
@@ -51,6 +51,7 @@ var timestampTransformerDefinition = utils.NewTransformerDefinition(
 		"min",
 		"min threshold date (and/or time) of value",
 	).SetRequired(true).
+		SetSupportTemplate(true).
 		SetLinkParameter("column").
 		SetSupportTemplate(true).
 		SetDynamicMode(
@@ -62,6 +63,7 @@ var timestampTransformerDefinition = utils.NewTransformerDefinition(
 		"max",
 		"max threshold date (and/or time) of value",
 	).SetRequired(true).
+		SetSupportTemplate(true).
 		SetLinkParameter("column").
 		SetSupportTemplate(true).
 		SetDynamicMode(

diff --git a/internal/db/postgres/transformers/random_float.go b/internal/db/postgres/transformers/random_float.go
@@ -50,6 +50,7 @@ var floatTransformerDefinition = utils.NewTransformerDefinition(
 		"min",
 		"min float value threshold",
 	).SetRequired(true).
+		SetSupportTemplate(true).
 		SetLinkParameter("column").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
@@ -60,6 +61,7 @@ var floatTransformerDefinition = utils.NewTransformerDefinition(
 		"max",
 		"max float value threshold",
 	).SetRequired(true).
+		SetSupportTemplate(true).
 		SetLinkParameter("column").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
@@ -69,7 +71,8 @@ var floatTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"decimal",
 		"Numbers of decimal",
-	).SetDefaultValue(toolkit.ParamsValue("4")),
+	).SetSupportTemplate(true).
+		SetDefaultValue(toolkit.ParamsValue("4")),
 
 	keepNullParameterDefinition,
 

diff --git a/internal/db/postgres/transformers/random_int.go b/internal/db/postgres/transformers/random_int.go
@@ -51,6 +51,7 @@ var integerTransformerDefinition = utils.NewTransformerDefinition(
 		"min",
 		"min int value threshold",
 	).SetLinkParameter("column").
+		SetSupportTemplate(true).
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes("int2", "int4", "int8"),
@@ -60,6 +61,7 @@ var integerTransformerDefinition = utils.NewTransformerDefinition(
 		"max",
 		"max int value threshold",
 	).SetLinkParameter("column").
+		SetSupportTemplate(true).
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
 				SetCompatibleTypes("int2", "int4", "int8"),

diff --git a/internal/db/postgres/transformers/random_ip.go b/internal/db/postgres/transformers/random_ip.go
@@ -41,6 +41,7 @@ var RandomIpDefinition = utils.NewTransformerDefinition(
 		"subnet",
 		"Subnet for generating random ip in V4 or V6 format",
 	).SetRequired(true).
+		SetSupportTemplate(true).
 		SetCastDbType("cidr").
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().

diff --git a/internal/db/postgres/transformers/random_mac.go b/internal/db/postgres/transformers/random_mac.go
@@ -53,16 +53,20 @@ var RandomMacAddressDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"keep_original_vendor",
 		"Keep original vendor. Default false",
-	).SetRequired(false).SetDefaultValue(toolkit.ParamsValue("false")),
+	).SetRequired(false).
+		SetDefaultValue(toolkit.ParamsValue("false")),
 
 	toolkit.MustNewParameterDefinition(
 		"cast_type",
 		"Cast type, supported types are: individual, group, any.",
-	).SetRequired(false).SetAllowedValues(
-		toolkit.ParamsValue(castTypeNameIndividual),
-		toolkit.ParamsValue(castTypeNameGroup),
-		toolkit.ParamsValue(castTypeNameAny),
-	).SetDefaultValue(toolkit.ParamsValue(castTypeNameAny)).SetUnmarshaler(scanCastType),
+	).SetRequired(false).
+		SetAllowedValues(
+			toolkit.ParamsValue(castTypeNameIndividual),
+			toolkit.ParamsValue(castTypeNameGroup),
+			toolkit.ParamsValue(castTypeNameAny),
+		).
+		SetDefaultValue(toolkit.ParamsValue(castTypeNameAny)).
+		SetUnmarshaler(scanCastType),
 
 	toolkit.MustNewParameterDefinition(
 		"management_type",
@@ -71,7 +75,8 @@ var RandomMacAddressDefinition = utils.NewTransformerDefinition(
 		toolkit.ParamsValue(managementTypeNameUniversal),
 		toolkit.ParamsValue(managementTypeNameLocal),
 		toolkit.ParamsValue(managementTypeNameAny),
-	).SetDefaultValue(toolkit.ParamsValue(managementTypeNameAny)).SetUnmarshaler(scanManagementType),
+	).SetDefaultValue(toolkit.ParamsValue(managementTypeNameAny)).
+		SetUnmarshaler(scanManagementType),
 
 	engineParameterDefinition,
 )

diff --git a/internal/db/postgres/transformers/random_numeric.go b/internal/db/postgres/transformers/random_numeric.go
@@ -33,12 +33,14 @@ var numericTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"decimal",
 		"the value decimal",
-	).SetDefaultValue([]byte("0")),
+	).SetSupportTemplate(true).
+		SetDefaultValue([]byte("0")),
 
 	toolkit.MustNewParameterDefinition(
 		"min",
 		"min int value threshold",
 	).SetLinkParameter("column").
+		SetSupportTemplate(true).
 		SetRequired(true).
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().
@@ -58,6 +60,7 @@ var numericTransformerDefinition = utils.NewTransformerDefinition(
 		"max",
 		"max int value threshold",
 	).SetLinkParameter("column").
+		SetSupportTemplate(true).
 		SetRequired(true).
 		SetDynamicMode(
 			toolkit.NewDynamicModeProperties().

diff --git a/internal/db/postgres/transformers/random_person.go b/internal/db/postgres/transformers/random_person.go
@@ -48,7 +48,8 @@ var randomPersonTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"fallback_gender",
 		"Specify fallback gender if not mapped when using dynamic mode in \"gender\" parameter",
-	).SetDefaultValue(toolkit.ParamsValue("Any")),
+	).SetSupportTemplate(true).
+		SetDefaultValue(toolkit.ParamsValue("Any")),
 
 	// TODO: Allow user to override the default names, surnames and genders with kind of dictionary
 

diff --git a/internal/db/postgres/transformers/random_string.go b/internal/db/postgres/transformers/random_string.go
@@ -42,12 +42,14 @@ var stringTransformerDefinition = utils.NewTransformerDefinition(
 	toolkit.MustNewParameterDefinition(
 		"min_length",
 		"min length of string",
-	).SetRequired(true),
+	).SetSupportTemplate(true).
+		SetRequired(true),
 
 	toolkit.MustNewParameterDefinition(
 		"max_length",
 		"max length of string",
-	).SetRequired(true),
+	).SetSupportTemplate(true).
+		SetRequired(true),
 
 	toolkit.MustNewParameterDefinition(
 		"symbols",