From e05fd64a65ee9ef353c22cfb4603b025008021b6 Mon Sep 17 00:00:00 2001 From: Bracey Summers Date: Wed, 18 Oct 2023 17:17:12 -0500 Subject: [PATCH] APP-4245 - [Inputs] Added support for secure Redis (SSL and User/Pass) --- release_notes.md | 4 + tcex/__metadata__.py | 2 +- tcex/app/app.py | 6 ++ tcex/app/key_value_store | 2 +- tcex/app/playbook | 2 +- tcex/input/input.py | 95 ----------------------- tcex/input/model/cert_model.py | 43 ++++++++++ tcex/input/model/common_model.py | 3 +- tcex/input/model/module_app_model.py | 3 +- tcex/input/model/playbook_common_model.py | 10 +++ tcex/input/model/service_model.py | 20 ----- tests/api/tc/v3/v3_helpers.py | 13 ++++ 12 files changed, 83 insertions(+), 120 deletions(-) create mode 100644 tcex/input/model/cert_model.py diff --git a/release_notes.md b/release_notes.md index dfa44169f..4e27f6f45 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,9 @@ # Release Notes +## 4.0.3 + +- APP-4245 - [Inputs] Added support for secure Redis (SSL and User/Pass) + ## 4.0.2 - APP-4155 - [API] Added Mitre Attack module for lookup by id or name diff --git a/tcex/__metadata__.py b/tcex/__metadata__.py index 1a4990914..9a9d1ac20 100644 --- a/tcex/__metadata__.py +++ b/tcex/__metadata__.py @@ -1,3 +1,3 @@ """TcEx Framework Module""" __license__ = 'Apache-2.0' -__version__ = '4.0.2' +__version__ = '4.0.3' diff --git a/tcex/app/app.py b/tcex/app/app.py index 5c3016f79..0c412f1f1 100644 --- a/tcex/app/app.py +++ b/tcex/app/app.py @@ -62,8 +62,14 @@ def key_value_store(self) -> KeyValueStore: self.model.tc_kvstore_host, self.model.tc_kvstore_port, self.model.tc_kvstore_type, + self.model.tc_playbook_kvstore_id, self.model.tc_kvstore_pass, self.model.tc_kvstore_user, + self.model.tc_kvstore_tls_enabled, + self.model.tc_kvstore_tls_port, + self.model.tc_svc_broker_cacert_file, + self.model.tc_svc_broker_cert_file, + self.model.tc_svc_broker_key_file, ) @cached_property diff --git a/tcex/app/key_value_store b/tcex/app/key_value_store index cab3d2870..2c8e730f1 160000 --- a/tcex/app/key_value_store +++ b/tcex/app/key_value_store @@ -1 +1 @@ -Subproject commit cab3d28704444e40b3a5efeb8080d56ebe773f79 +Subproject commit 2c8e730f1ae90adfe2ad6a8642adc1b2bcbef83b diff --git a/tcex/app/playbook b/tcex/app/playbook index 04ba65c86..5033012c4 160000 --- a/tcex/app/playbook +++ b/tcex/app/playbook @@ -1 +1 @@ -Subproject commit 04ba65c8609024b05dea7ee1b41924fefbe65584 +Subproject commit 5033012c4bed258d53f029fcdd22136a868ca82c diff --git a/tcex/input/input.py b/tcex/input/input.py index cc918a9a7..fc212eeb9 100644 --- a/tcex/input/input.py +++ b/tcex/input/input.py @@ -10,11 +10,9 @@ # third-party from pydantic import ValidationError # TYPE-CHECKING from pydantic import BaseModel, Extra -from redis import Redis # first-party from tcex.app.config.install_json import InstallJson -from tcex.app.key_value_store import RedisClient from tcex.input.field_type import Sensitive from tcex.input.model.advanced_request_model import AdvancedRequestModel from tcex.input.model.app_external_model import AppExternalModel @@ -75,74 +73,6 @@ def __init__(self, config: dict | None = None, config_file: str | None = None): self.log = _logger self.util = Util() - @staticmethod - def _get_redis_client(host: str, port: int, db: int) -> Redis: - """Return RedisClient client""" - return RedisClient(host=host, port=port, db=db).client - - def _load_aot_params( - self, - tc_aot_enabled: bool, - tc_kvstore_type: str, - tc_kvstore_host: str, - tc_kvstore_port: int, - tc_action_channel: str, - tc_terminate_seconds: int, - ) -> dict[str, dict | list | str]: - """Subscribe to AOT action channel.""" - params = {} - if tc_aot_enabled is not True: - return params - - if not all( - [ - tc_kvstore_type, - tc_kvstore_host, - tc_kvstore_port, - tc_action_channel, - tc_terminate_seconds, - ] - ): - return params - - if tc_kvstore_type == 'Redis': - # get an instance of redis client - redis_client = self._get_redis_client( - host=tc_kvstore_host, - port=tc_kvstore_port, - db=0, - ) - - try: - self.log.info('feature=inputs, event=blocking-for-aot') - msg_data = redis_client.blpop( - keys=tc_action_channel, - timeout=int(tc_terminate_seconds), - ) - - if msg_data is None: # pragma: no cover - # send exit to tcex.exit method - registry.exit.exit_aot_terminate( - code=1, msg='AOT subscription timeout reached.' - ) - else: - msg_data = json.loads(msg_data[1]) - msg_type = msg_data.get('type', 'terminate') - if msg_type == 'execute': - params = msg_data.get('params', {}) - elif msg_type == 'terminate': - # send exit to tcex.exit method - registry.exit.exit_aot_terminate( - code=0, msg='Received AOT terminate message.' - ) - except Exception as e: # pragma: no cover - # send exit to tcex.exit method - registry.exit.exit_aot_terminate( - code=1, msg=f'Exception during AOT subscription ({e}).' - ) - - return params - def _load_config_file(self): """Load config file params provided passed to inputs.""" # default file contents @@ -242,31 +172,6 @@ def contents(self) -> dict: # file params _contents.update(self._load_file_params()) - # aot params - must be loaded last so that it has the kv store channels - tc_aot_enabled = self.util.to_bool(_contents.get('tc_aot_enabled', False)) - if tc_aot_enabled is True: - tc_kvstore_type = _contents.get('tc_kvstore_type') - tc_kvstore_host = _contents.get('tc_kvstore_host') - tc_kvstore_port = _contents.get('tc_kvstore_port') - tc_action_channel = _contents.get('tc_action_channel') - tc_terminate_seconds = _contents.get('tc_terminate_seconds') - - if ( - tc_kvstore_type - and tc_kvstore_host - and tc_kvstore_port - and tc_action_channel - and tc_terminate_seconds - ): - param_data = self._load_aot_params( - tc_aot_enabled=tc_aot_enabled, - tc_kvstore_type=tc_kvstore_type, - tc_kvstore_host=tc_kvstore_host, - tc_kvstore_port=tc_kvstore_port, - tc_action_channel=tc_action_channel, - tc_terminate_seconds=tc_terminate_seconds, - ) - _contents.update(param_data) return _contents @cached_property diff --git a/tcex/input/model/cert_model.py b/tcex/input/model/cert_model.py new file mode 100644 index 000000000..841b10fef --- /dev/null +++ b/tcex/input/model/cert_model.py @@ -0,0 +1,43 @@ +"""TcEx Framework Module""" +# third-party +from pydantic import BaseModel, Field + + +class CertModel(BaseModel): + """Input Service Model + + Supported runtimeLevel: + * ApiService + * Playbook + * Organization + * TriggerService + * WebhookTriggerService + """ + + # @bcs - for service App these should be required, for any other App type making them + # required would force a min server version of 7.4 + tc_svc_broker_cacert_file: str | None = Field( + None, + description='The Broker SSL CA (full chain) certificate.', + inclusion_reason='runtimeLevel', + ) + tc_svc_broker_cert_file: str | None = Field( + None, + description='The Broker SSL Server certificate.', + inclusion_reason='runtimeLevel', + ) + tc_svc_broker_jks_file: str | None = Field( + 'Unused', + description='Input for Java Apps.', + inclusion_reason='runtimeLevel', + ) + tc_svc_broker_jks_pwd: str | None = Field( + 'Unused', + description='Input for Java Apps.', + inclusion_reason='runtimeLevel', + ) + tc_svc_broker_key_file: str | None = Field( + None, + description='The Broker SSL Key (full chain) certificate.', + inclusion_reason='runtimeLevel', + ) diff --git a/tcex/input/model/common_model.py b/tcex/input/model/common_model.py index 1cf73ff9d..b9564f7ef 100644 --- a/tcex/input/model/common_model.py +++ b/tcex/input/model/common_model.py @@ -3,10 +3,11 @@ # first-party from tcex.input.model.api_model import ApiModel from tcex.input.model.batch_model import BatchModel +from tcex.input.model.cert_model import CertModel from tcex.input.model.logging_model import LoggingModel from tcex.input.model.path_model import PathModel from tcex.input.model.proxy_model import ProxyModel -class CommonModel(ApiModel, BatchModel, LoggingModel, PathModel, ProxyModel): +class CommonModel(ApiModel, BatchModel, CertModel, LoggingModel, PathModel, ProxyModel): """Model Definition""" diff --git a/tcex/input/model/module_app_model.py b/tcex/input/model/module_app_model.py index 139d2204b..b8811b712 100644 --- a/tcex/input/model/module_app_model.py +++ b/tcex/input/model/module_app_model.py @@ -5,6 +5,7 @@ # first-party from tcex.input.model.api_model import ApiModel +from tcex.input.model.cert_model import CertModel from tcex.input.model.path_model import PathModel from tcex.input.model.playbook_common_model import PlaybookCommonModel from tcex.input.model.playbook_model import PlaybookModel @@ -13,7 +14,7 @@ class ModuleAppModel( - ApiModel, PathModel, PlaybookCommonModel, PlaybookModel, ProxyModel, ServiceModel + ApiModel, CertModel, PathModel, PlaybookCommonModel, PlaybookModel, ProxyModel, ServiceModel ): """Model Definition diff --git a/tcex/input/model/playbook_common_model.py b/tcex/input/model/playbook_common_model.py index 535670f44..7dc36d71a 100644 --- a/tcex/input/model/playbook_common_model.py +++ b/tcex/input/model/playbook_common_model.py @@ -36,6 +36,16 @@ class PlaybookCommonModel(BaseModel): description='The KV Store port number.', inclusion_reason='runtimeLevel', ) + tc_kvstore_tls_enabled: bool = Field( + False, + description='If true, KV Store requires SSL connection.', + inclusion_reason='runtimeLevel', + ) + tc_kvstore_tls_port: int = Field( + 6379, + description='The KV Store TLS port number.', + inclusion_reason='runtimeLevel', + ) tc_kvstore_type: str = Field( 'Redis', description='The KV Store type (Redis or TCKeyValueAPI).', diff --git a/tcex/input/model/service_model.py b/tcex/input/model/service_model.py index 585f7a19d..89943970e 100644 --- a/tcex/input/model/service_model.py +++ b/tcex/input/model/service_model.py @@ -15,16 +15,6 @@ class ServiceModel(BaseModel): * WebhookTriggerService """ - tc_svc_broker_cacert_file: str = Field( - None, - description='The Broker SSL CA (full chain) certificate.', - inclusion_reason='runtimeLevel', - ) - tc_svc_broker_cert_file: str = Field( - None, - description='The Broker SSL Server certificate.', - inclusion_reason='runtimeLevel', - ) tc_svc_broker_conn_timeout: int = Field( 60, description='The broker connection startup timeout in seconds.', @@ -36,16 +26,6 @@ class ServiceModel(BaseModel): description='The Broker service hostname.', inclusion_reason='runtimeLevel', ) - tc_svc_broker_jks_file: str | None = Field( - 'Unused', - description='Input for Java Apps.', - inclusion_reason='runtimeLevel', - ) - tc_svc_broker_jks_pwd: str | None = Field( - 'Unused', - description='Input for Java Apps.', - inclusion_reason='runtimeLevel', - ) tc_svc_broker_port: int = Field( None, description='The Broker service port number.', diff --git a/tests/api/tc/v3/v3_helpers.py b/tests/api/tc/v3/v3_helpers.py index d6d950596..aa11c6438 100644 --- a/tests/api/tc/v3/v3_helpers.py +++ b/tests/api/tc/v3/v3_helpers.py @@ -743,6 +743,19 @@ def obj_api_options(self): # fix discrepancy between /fields and names = ['createdBy'] + if self.v3_helper.v3_object in ['groups', 'indicators'] and 'externalDates' in names: + names = [ + 'externalDateAdded', + 'externalDateExpires', + 'externalLastModified', + ] + + if self.v3_helper.v3_object in ['groups', 'indicators'] and 'sightings' in names: + names = [ + 'firstSeen', + 'lastSeen', + ] + if self.v3_helper.v3_object == 'indicators': if 'genericCustomIndicatorValues' in names: # fix discrepancy between /fields and