From eb8433b8eac58ec4d33d3565501af27fc7adf428 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Wed, 2 Oct 2024 17:18:22 -0700 Subject: [PATCH 1/2] Add null check while adding fetched iocs into per-indicator-type map (#1335) * add null check while adding fetched iocs into per-indicator-type map Signed-off-by: Surya Sashank Nistala * adds tests verifying monitor configured on multiple indicator types Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala --- .../iocscan/service/IoCScanService.java | 38 ++-- .../ThreatIntelMonitorRestApiIT.java | 178 +++++++++++++++++- 2 files changed, 195 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java index 861880da9..ad591e42f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -128,28 +128,28 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) { String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase(); String concreteIndex = getIndexName(datum); - if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex) - && false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty() - ) { + if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) { // if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc. String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0); - List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); - for (String field : fields) { - List vals = getValuesAsStringList(datum, field); - String id = getId(datum); - String docId = id + ":" + index; - Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); - iocs.addAll(vals); - docIdToIocsMap.put(docId, iocs); - for (String ioc : vals) { - Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); - docIds.add(docId); - iocValueToDocIdMap.put(ioc, docIds); - } - if (false == vals.isEmpty()) { - iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + List fieldsConfiguredInMonitorForCurrentIndex = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + if(fieldsConfiguredInMonitorForCurrentIndex != null && false == fieldsConfiguredInMonitorForCurrentIndex.isEmpty()) { + for (String field : fieldsConfiguredInMonitorForCurrentIndex) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String docId = id + ":" + index; + Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); iocs.addAll(vals); - iocsPerIocTypeMap.put(iocType, iocs); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } } } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 6af6c4275..ad7b3f969 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -48,7 +48,6 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; -import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH; public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -81,6 +80,71 @@ public String indexSourceConfigsAndIocs(List iocVals) throws IOException return indexTifSourceConfig(testIocDtos); } + public String indexSourceConfigsAndIocs(List ipVals, List hashVals, List domainVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < ipVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.IPV4_TYPE), + ipVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < hashVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.HASHES_TYPE), + hashVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < domainVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.DOMAIN_NAME_TYPE), + domainVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + return indexTifSourceConfig(testIocDtos); + } + private String indexTifSourceConfig(List testIocDtos) throws IOException { SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( "configId", @@ -100,7 +164,7 @@ private String indexTifSourceConfig(List testIocDtos) throws IOExce null, null, false, - List.of(IOCType.IPV4_TYPE), + List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE, IOCType.DOMAIN_NAME_TYPE), true ); @@ -251,6 +315,96 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } + + + public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List ipVals = List.of("ip1", "ip2"); + List hashVals = List.of("h1", "h2"); + List domainVals = List.of("d1", "d2"); + String createdId = indexSourceConfigsAndIocs(ipVals, hashVals, domainVals); + + String ipIndex = "ipAlias"; + createTestAlias(ipIndex, 1, true); + String hashIndex = "hashAlias"; + createTestAlias(hashIndex, 1, true); + String domainIndex = "domainAlias"; + createTestAlias(domainIndex, 1, true); + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(ipIndex, hashIndex, domainIndex); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : ipVals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(ipIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : hashVals) { + String doc = String.format("{\"hash\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(hashIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : domainVals) { + String doc = String.format("{\"domain\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(domainIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON, false)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON, false)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(6, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(6, searchHits.size()); + } + public void testCreateThreatIntelMonitor() throws IOException { updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", @@ -572,6 +726,26 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { List.of(t1, t2, t3, t4)); } + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(ipIndex), emptyList(), emptyList(), "indexmatch", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of( + new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(ipIndex, List.of("ip"))), + new PerIocTypeScanInputDto(IOCType.HASHES_TYPE, Map.of(hashIndex, List.of("hash"))), + new PerIocTypeScanInputDto(IOCType.DOMAIN_NAME_TYPE, Map.of(domainIndex, List.of("domain"))) + ), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1, t2, t3, t4)); + } + @Override protected boolean preserveIndicesUponCompletion() { return false; From bab0c8cc4cccead368693407c6b37cf8f36b1831 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Wed, 2 Oct 2024 18:37:35 -0700 Subject: [PATCH 2/2] fix constructor for 2.x Signed-off-by: Surya Sashank Nistala --- .../resthandler/ThreatIntelMonitorRestApiIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index ad7b3f969..fe15784c2 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -387,13 +387,13 @@ public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonito Map executeResults = entityAsMap(executeResponse); String matchAllRequest = getMatchAllRequest(); - Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON, false)); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); HashMap totalHits = (HashMap) hits.get("total"); Integer totalHitsVal = (Integer) totalHits.get("value"); assertEquals(totalHitsVal.intValue(), 1); - makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON, false)); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null);