APP-4245 - [Inputs] Added support for secure Redis (SSL and User/Pass)

release_notes.md

@@ -1,5 +1,9 @@
 # Release Notes
 
+## 4.0.3
+
+-   APP-4245 - [Inputs] Added support for secure Redis (SSL and User/Pass)
+
 ## 4.0.2
 
 -   APP-4155 - [API] Added Mitre Attack module for lookup by id or name

tcex/__metadata__.py

@@ -1,3 +1,3 @@
 """TcEx Framework Module"""
 __license__ = 'Apache-2.0'
-__version__ = '4.0.2'
+__version__ = '4.0.3'

tcex/app/app.py

@@ -62,8 +62,14 @@ def key_value_store(self) -> KeyValueStore:
             self.model.tc_kvstore_host,
             self.model.tc_kvstore_port,
             self.model.tc_kvstore_type,
+            self.model.tc_playbook_kvstore_id,
             self.model.tc_kvstore_pass,
             self.model.tc_kvstore_user,
+            self.model.tc_kvstore_tls_enabled,
+            self.model.tc_kvstore_tls_port,
+            self.model.tc_svc_broker_cacert_file,
+            self.model.tc_svc_broker_cert_file,
+            self.model.tc_svc_broker_key_file,
         )
 
     @cached_property

tcex/input/input.py

@@ -10,11 +10,9 @@
 # third-party
 from pydantic import ValidationError  # TYPE-CHECKING
 from pydantic import BaseModel, Extra
-from redis import Redis
 
 # first-party
 from tcex.app.config.install_json import InstallJson
-from tcex.app.key_value_store import RedisClient
 from tcex.input.field_type import Sensitive
 from tcex.input.model.advanced_request_model import AdvancedRequestModel
 from tcex.input.model.app_external_model import AppExternalModel
@@ -75,74 +73,6 @@ def __init__(self, config: dict | None = None, config_file: str | None = None):
         self.log = _logger
         self.util = Util()
 
-    @staticmethod
-    def _get_redis_client(host: str, port: int, db: int) -> Redis:
-        """Return RedisClient client"""
-        return RedisClient(host=host, port=port, db=db).client
-
-    def _load_aot_params(
-        self,
-        tc_aot_enabled: bool,
-        tc_kvstore_type: str,
-        tc_kvstore_host: str,
-        tc_kvstore_port: int,
-        tc_action_channel: str,
-        tc_terminate_seconds: int,
-    ) -> dict[str, dict | list | str]:
-        """Subscribe to AOT action channel."""
-        params = {}
-        if tc_aot_enabled is not True:
-            return params
-
-        if not all(
-            [
-                tc_kvstore_type,
-                tc_kvstore_host,
-                tc_kvstore_port,
-                tc_action_channel,
-                tc_terminate_seconds,
-            ]
-        ):
-            return params
-
-        if tc_kvstore_type == 'Redis':
-            # get an instance of redis client
-            redis_client = self._get_redis_client(
-                host=tc_kvstore_host,
-                port=tc_kvstore_port,
-                db=0,
-            )
-
-            try:
-                self.log.info('feature=inputs, event=blocking-for-aot')
-                msg_data = redis_client.blpop(
-                    keys=tc_action_channel,
-                    timeout=int(tc_terminate_seconds),
-                )
-
-                if msg_data is None:  # pragma: no cover
-                    # send exit to tcex.exit method
-                    registry.exit.exit_aot_terminate(
-                        code=1, msg='AOT subscription timeout reached.'
-                    )
-                else:
-                    msg_data = json.loads(msg_data[1])
-                    msg_type = msg_data.get('type', 'terminate')
-                    if msg_type == 'execute':
-                        params = msg_data.get('params', {})
-                    elif msg_type == 'terminate':
-                        # send exit to tcex.exit method
-                        registry.exit.exit_aot_terminate(
-                            code=0, msg='Received AOT terminate message.'
-                        )
-            except Exception as e:  # pragma: no cover
-                # send exit to tcex.exit method
-                registry.exit.exit_aot_terminate(
-                    code=1, msg=f'Exception during AOT subscription ({e}).'
-                )
-
-        return params
-
     def _load_config_file(self):
         """Load config file params provided passed to inputs."""
         # default file contents
@@ -242,31 +172,6 @@ def contents(self) -> dict:
         # file params
         _contents.update(self._load_file_params())
 
-        # aot params - must be loaded last so that it has the kv store channels
-        tc_aot_enabled = self.util.to_bool(_contents.get('tc_aot_enabled', False))
-        if tc_aot_enabled is True:
-            tc_kvstore_type = _contents.get('tc_kvstore_type')
-            tc_kvstore_host = _contents.get('tc_kvstore_host')
-            tc_kvstore_port = _contents.get('tc_kvstore_port')
-            tc_action_channel = _contents.get('tc_action_channel')
-            tc_terminate_seconds = _contents.get('tc_terminate_seconds')
-
-            if (
-                tc_kvstore_type
-                and tc_kvstore_host
-                and tc_kvstore_port
-                and tc_action_channel
-                and tc_terminate_seconds
-            ):
-                param_data = self._load_aot_params(
-                    tc_aot_enabled=tc_aot_enabled,
-                    tc_kvstore_type=tc_kvstore_type,
-                    tc_kvstore_host=tc_kvstore_host,
-                    tc_kvstore_port=tc_kvstore_port,
-                    tc_action_channel=tc_action_channel,
-                    tc_terminate_seconds=tc_terminate_seconds,
-                )
-                _contents.update(param_data)
         return _contents
 
     @cached_property

tcex/input/model/cert_model.py

@@ -0,0 +1,43 @@
+"""TcEx Framework Module"""
+# third-party
+from pydantic import BaseModel, Field
+
+
+class CertModel(BaseModel):
+    """Input Service Model
+
+    Supported runtimeLevel:
+    * ApiService
+    * Playbook
+    * Organization
+    * TriggerService
+    * WebhookTriggerService
+    """
+
+    # @bcs - for service App these should be required, for any other App type making them
+    # required would force a min server version of 7.4
+    tc_svc_broker_cacert_file: str | None = Field(
+        None,
+        description='The Broker SSL CA (full chain) certificate.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_cert_file: str | None = Field(
+        None,
+        description='The Broker SSL Server certificate.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_jks_file: str | None = Field(
+        'Unused',
+        description='Input for Java Apps.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_jks_pwd: str | None = Field(
+        'Unused',
+        description='Input for Java Apps.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_key_file: str | None = Field(
+        None,
+        description='The Broker SSL Key (full chain) certificate.',
+        inclusion_reason='runtimeLevel',
+    )

tcex/input/model/common_model.py

@@ -3,10 +3,11 @@
 # first-party
 from tcex.input.model.api_model import ApiModel
 from tcex.input.model.batch_model import BatchModel
+from tcex.input.model.cert_model import CertModel
 from tcex.input.model.logging_model import LoggingModel
 from tcex.input.model.path_model import PathModel
 from tcex.input.model.proxy_model import ProxyModel
 
 
-class CommonModel(ApiModel, BatchModel, LoggingModel, PathModel, ProxyModel):
+class CommonModel(ApiModel, BatchModel, CertModel, LoggingModel, PathModel, ProxyModel):
     """Model Definition"""

tcex/input/model/module_app_model.py

@@ -5,6 +5,7 @@
 
 # first-party
 from tcex.input.model.api_model import ApiModel
+from tcex.input.model.cert_model import CertModel
 from tcex.input.model.path_model import PathModel
 from tcex.input.model.playbook_common_model import PlaybookCommonModel
 from tcex.input.model.playbook_model import PlaybookModel
@@ -13,7 +14,7 @@
 
 
 class ModuleAppModel(
-    ApiModel, PathModel, PlaybookCommonModel, PlaybookModel, ProxyModel, ServiceModel
+    ApiModel, CertModel, PathModel, PlaybookCommonModel, PlaybookModel, ProxyModel, ServiceModel
 ):
     """Model Definition
 

tcex/input/model/playbook_common_model.py

@@ -36,6 +36,16 @@ class PlaybookCommonModel(BaseModel):
         description='The KV Store port number.',
         inclusion_reason='runtimeLevel',
     )
+    tc_kvstore_tls_enabled: bool = Field(
+        False,
+        description='If true, KV Store requires SSL connection.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_kvstore_tls_port: int = Field(
+        6379,
+        description='The KV Store TLS port number.',
+        inclusion_reason='runtimeLevel',
+    )
     tc_kvstore_type: str = Field(
         'Redis',
         description='The KV Store type (Redis or TCKeyValueAPI).',

tcex/input/model/service_model.py

@@ -15,16 +15,6 @@ class ServiceModel(BaseModel):
     * WebhookTriggerService
     """
 
-    tc_svc_broker_cacert_file: str = Field(
-        None,
-        description='The Broker SSL CA (full chain) certificate.',
-        inclusion_reason='runtimeLevel',
-    )
-    tc_svc_broker_cert_file: str = Field(
-        None,
-        description='The Broker SSL Server certificate.',
-        inclusion_reason='runtimeLevel',
-    )
     tc_svc_broker_conn_timeout: int = Field(
         60,
         description='The broker connection startup timeout in seconds.',
@@ -36,16 +26,6 @@ class ServiceModel(BaseModel):
         description='The Broker service hostname.',
         inclusion_reason='runtimeLevel',
     )
-    tc_svc_broker_jks_file: str | None = Field(
-        'Unused',
-        description='Input for Java Apps.',
-        inclusion_reason='runtimeLevel',
-    )
-    tc_svc_broker_jks_pwd: str | None = Field(
-        'Unused',
-        description='Input for Java Apps.',
-        inclusion_reason='runtimeLevel',
-    )
     tc_svc_broker_port: int = Field(
         None,
         description='The Broker service port number.',

tests/api/tc/v3/v3_helpers.py

@@ -743,6 +743,19 @@ def obj_api_options(self):
                 # fix discrepancy between <endpoint>/fields and <endpoint>
                 names = ['createdBy']
 
+            if self.v3_helper.v3_object in ['groups', 'indicators'] and 'externalDates' in names:
+                names = [
+                    'externalDateAdded',
+                    'externalDateExpires',
+                    'externalLastModified',
+                ]
+
+            if self.v3_helper.v3_object in ['groups', 'indicators'] and 'sightings' in names:
+                names = [
+                    'firstSeen',
+                    'lastSeen',
+                ]
+
             if self.v3_helper.v3_object == 'indicators':
                 if 'genericCustomIndicatorValues' in names:
                     # fix discrepancy between <endpoint>/fields and <endpoint>