Add ids_flag and timestamp to Misp processing, search using controlle…

…r attributes

common/cuckoo/common/misp.py

@@ -149,11 +149,30 @@ def __init__(self, misp_url, api_key, verify_tls=True, timeout=5):
                 f"Failed to create MISP client. Error: {e}"
             ).with_traceback(e.__traceback__)
 
-    def find_events(self, value, type_attribute=None, limit=1):
+    def find_event(self, eventid):
         try:
             events = self._client.search(
+                eventid=eventid, controller='events'
+            )
+        except (pymisp.PyMISPError, requests.exceptions.RequestException) as e:
+            raise MispError(
+                f"Event query failed for value '{eventid}'. "
+                f"Type: '{type_attribute}'. Error: {e}"
+            ).with_traceback(e.__traceback__)
+        try:
+            return events[0]
+        except (ValueError, TypeError) as e:
+            raise MispError(
+                f"Failure while reading MISP response JSON. Error: {e}"
+            )
+
+    def find_events(self, value, type_attribute=None, limit=1, to_ids=1, publish_timestamp="365d"):
+        try:
+            attributes = self._client.search(
                 value=value, type_attribute=type_attribute, limit=limit,
-                metadata=True, return_format="json", object_name=None
+                metadata=True, return_format="json", object_name=None,
+                to_ids=to_ids, publish_timestamp=publish_timestamp,
+                controller='attributes'
             )
         except (pymisp.PyMISPError, requests.exceptions.RequestException) as e:
             raise MispError(
@@ -163,47 +182,52 @@ def find_events(self, value, type_attribute=None, limit=1):
 
         try:
             return [
-                ExistingMispEvent(event_dict["Event"], value, self._misp_url)
-                for event_dict in events
+                ExistingMispEvent(self.find_event(attribute['event_id'])["Event"], value, self._misp_url)
+                for attribute in attributes['Attribute']
             ]
         except (ValueError, TypeError) as e:
-            raise MispError(
-                f"Failure while reading MISP response JSON. Error: {e}"
-            )
+            return []
 
-    def find_file_md5(self, md5, limit=1):
+    def find_file_md5(self, md5, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=md5, type_attribute="md5", limit=limit
+            value=md5, type_attribute="md5", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
-    def find_file_sha1(self, sha1, limit=1):
+    def find_file_sha1(self, sha1, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=sha1, type_attribute="sha1", limit=limit
+            value=sha1, type_attribute="sha1", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
-    def find_file_sha256(self, sha256, limit=1):
+    def find_file_sha256(self, sha256, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=sha256, type_attribute="sha256", limit=limit
+            value=sha256, type_attribute="sha256", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
-    def find_file_sha512(self, sha512, limit=1):
+    def find_file_sha512(self, sha512, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=sha512, type_attribute="sha512", limit=limit
+            value=sha512, type_attribute="sha512", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
-    def find_url(self, url, limit=1):
+    def find_url(self, url, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=url, type_attribute="url", limit=limit
+            value=url, type_attribute="url", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
-    def find_ip_dst(self, ip, limit=1):
+    def find_ip_dst(self, ip, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=ip, type_attribute="ip-dst", limit=limit
+            value=ip, type_attribute="ip-dst", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
-    def find_domain(self, domain, limit=1):
+    def find_domain(self, domain, limit=1, to_ids=1, publish_timestamp="365d"):
         return self.find_events(
-            value=domain, type_attribute="domain", limit=limit
+            value=domain, type_attribute="domain", limit=limit,
+            to_ids=to_ids, publish_timestamp=publish_timestamp
         )
 
     def create_event(self, new_misp_event):

processing/cuckoo/processing/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (C) 2019-2023 Estonian Information System Authority.
 # See the file 'LICENSE' for copying permission.
 
-__version__ = "0.1.4"
+__version__ = "0.1.5"

processing/cuckoo/processing/config.py

@@ -92,6 +92,8 @@ def constraints(self, value):
             "timeout": config.Int(default_val=5, min_value=0),
             "pre": {
                 "event_limit": config.Int(default_val=1, min_value=1),
+                "query_ids_flag": config.Int(default_val=1, min_value=0, max_value=1),
+                "publish_timestamp": config.String(default_val="365d"),
                 "file": {
                     "hashes": config.List(
                         config.String, default_val=["sha256"]
@@ -112,7 +114,21 @@ def constraints(self, value):
                         "domain": 1,
                         "url": 1
                     }
-                )
+                ),
+                "query_ids_flags": config.Dict(
+                    config.Int, default_val={
+                        "dst_ip": 1,
+                        "domain": 1,
+                        "url": 1
+                    }
+                ),
+                "publish_timestamps": config.Dict(
+                    config.String, default_val={
+                        "dst_ip": "365d",
+                        "domain": "365d",
+                        "url": "365d"
+                    }
+                ),
             }
         },
         "reporting": {

processing/cuckoo/processing/data/conftemplates/misp.yaml.jinja2

@@ -21,6 +21,10 @@ processing:
   pre:
     # The maximum amount of events that should be asked from MISP per query.
     event_limit: {{ processing.pre.event_limit }}
+    #To restrict the returned data set to to_ids only attributes set this parameter to 1
+    query_ids_flag: {{ processing.pre.query_ids_flag }}
+    # Filter on event’s publish timestamp.
+    publish_timestamp: {{ processing.pre.publish_timestamp }}
 
     file:
       # What target file hashes to use when searching for MISP events.
@@ -46,6 +50,18 @@ processing:
       {{ ioctype }}: {{ limit }}
     {% endfor %}
 
+    # To restrict the returned data set to to_ids only attributes
+    query_ids_flags:
+    {% for ioctype, limit in processing.post.query_ids_flags.items() %}
+      {{ ioctype }}: {{ limit }}
+    {% endfor %}
+
+    # To restrict the returned data set by publish_timestamp
+    publish_timestamps:
+    {% for ioctype, limit in processing.post.publish_timestamps.items() %}
+      {{ ioctype }}: {{ limit }}
+    {% endfor %}
+
 reporting:
   # Enable the reporting of discovered IOCs as a new event.
   enabled: {{ reporting.enabled }}

processing/cuckoo/processing/post/misp.py

@@ -28,6 +28,12 @@ def init_once(cls):
         cls.event_limits = cfg(
             "misp", "processing", "post", "event_limits", subpkg="processing"
         )
+        cls.query_ids_flags = cfg(
+            "misp", "processing", "post", "query_ids_flags", subpkg="processing"
+        )
+        cls.publish_timestamps = cfg(
+            "misp", "processing", "post", "publish_timestamps", subpkg="processing"
+        )
 
     def init(self):
         try:
@@ -40,7 +46,7 @@ def init(self):
                 f"Failed to connect to MISP server. Error: {e}"
             )
 
-    def _search_dst_ips(self, query_limit=None, event_limit=1):
+    def _search_dst_ips(self, query_limit=None, event_limit=1, to_ids=1, publish_timestamp="365d"):
         network = self.ctx.result.get("network", {})
 
         queries = 0
@@ -50,11 +56,13 @@ def _search_dst_ips(self, query_limit=None, event_limit=1):
                 break
 
             queries += 1
-            events.extend(self.misp_client.find_ip_dst(ip, limit=event_limit))
+            events.extend(self.misp_client.find_ip_dst(ip, limit=event_limit,
+                to_ids=to_ids, publish_timestamp=publish_timestamp)
+            )
 
         return events
 
-    def _search_domains(self, query_limit=None, event_limit=1):
+    def _search_domains(self, query_limit=None, event_limit=1, to_ids=1, publish_timestamp="365d"):
         network = self.ctx.result.get("network", {})
 
         queries = 0
@@ -65,12 +73,13 @@ def _search_domains(self, query_limit=None, event_limit=1):
 
             queries += 1
             events.extend(
-                self.misp_client.find_domain(domain, limit=event_limit)
+                self.misp_client.find_domain(domain, limit=event_limit,
+                    to_ids=to_ids, publish_timestamp=publish_timestamp)
             )
 
         return events
 
-    def _search_urls(self, query_limit=None, event_limit=1):
+    def _search_urls(self, query_limit=None, event_limit=1, to_ids=1, publish_timestamp="365d"):
         network = self.ctx.result.get("network", {})
 
         queries = 0
@@ -84,7 +93,8 @@ def _search_urls(self, query_limit=None, event_limit=1):
                 continue
 
             queries += 1
-            events.extend(self.misp_client.find_url(url, limit=event_limit))
+            events.extend(self.misp_client.find_url(url, limit=event_limit,
+                to_ids=to_ids, publish_timestamp=publish_timestamp))
 
         return events
 
@@ -98,19 +108,26 @@ def start(self):
             events.extend(
                 self._search_dst_ips(
                     query_limit=self.query_limits.get("dst_ip"),
-                    event_limit=self.event_limits.get("dst_ip", 1)
+                    event_limit=self.event_limits.get("dst_ip", 1),
+                    to_ids=self.query_ids_flags.get("dst_ip", 1),
+                    publish_timestamp=self.publish_timestamps.get("dst_ip", 1)
                 )
             )
             events.extend(
                 self._search_urls(
                     query_limit=self.query_limits.get("url"),
-                    event_limit=self.event_limits.get("url", 1)
+                    event_limit=self.event_limits.get("url", 1),
+                    to_ids=self.query_ids_flags.get("url", 1),
+                    publish_timestamp=self.publish_timestamps.get("url", 1)
+
                 )
             )
             events.extend(
                 self._search_domains(
                     query_limit=self.query_limits.get("domain"),
-                    event_limit=self.event_limits.get("domain", 1)
+                    event_limit=self.event_limits.get("domain", 1),
+                    to_ids=self.query_ids_flags.get("domain", 1),
+                    publish_timestamp=self.publish_timestamps.get("domain", 1)
                 )
             )
         except MispError as e:

processing/cuckoo/processing/pre/misp.py

@@ -30,6 +30,12 @@ def init_once(cls):
         cls.event_limit = cfg(
             "misp", "processing", "pre", "event_limit", subpkg="processing"
         )
+        cls.to_ids = cfg(
+            "misp", "processing", "pre", "query_ids_flag", subpkg="processing"
+        )
+        cls.publish_timestamp = cfg(
+            "misp", "processing", "pre", "publish_timestamp", subpkg="processing"
+        )
 
     def init(self):
         try:
@@ -57,7 +63,8 @@ def _search_events_hashes(self, target):
                 continue
 
             events.extend(
-                lookup_handler(target[hashalgo], limit=self.event_limit)
+                lookup_handler(target[hashalgo], limit=self.event_limit,
+                to_ids=self.to_ids, publish_timestamp=self.publish_timestamp)
             )
 
         return events
@@ -67,7 +74,8 @@ def start(self):
         events = []
         try:
             if self.ctx.analysis.category == "url":
-                events = self.misp_client.find_url(target.target)
+                events = self.misp_client.find_url(target.target, limit=self.event_limit,
+                to_ids=self.to_ids, publish_timestamp=self.publish_timestamp)
             elif self.ctx.analysis.category == "file":
                 events = self._search_events_hashes(target)
         except MispError as e: