Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Backport #1335 to 2.x #1339

Merged
merged 2 commits into from
Oct 3, 2024
Merged

Backport #1335 to 2.x #1339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eb8433b
Add null check while adding fetched iocs into per-indicator-type map …
eirsep Oct 3, 2024
bab0c8c
fix constructor for 2.x
eirsep Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions ...ain/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,28 +128,28 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) {
String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase();
String concreteIndex = getIndexName(datum);
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)
&& false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty()
) {
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) {
// if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc.
String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0);
List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
for (String field : fields) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>());
iocs.addAll(vals);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
List<String> fieldsConfiguredInMonitorForCurrentIndex = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
if(fieldsConfiguredInMonitorForCurrentIndex != null && false == fieldsConfiguredInMonitorForCurrentIndex.isEmpty()) {
for (String field : fieldsConfiguredInMonitorForCurrentIndex) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
}
}
}
}
Expand Down
178 changes: 176 additions & 2 deletions src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers;
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS;
import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH;

public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase {
Expand Down Expand Up @@ -81,6 +80,71 @@ public String indexSourceConfigsAndIocs(List<String> iocVals) throws IOException
return indexTifSourceConfig(testIocDtos);
}

public String indexSourceConfigsAndIocs(List<String> ipVals, List<String> hashVals, List<String> domainVals) throws IOException {
testIocDtos = new ArrayList<>();
for (int i1 = 0; i1 < ipVals.size(); i1++) {
// create IOCs
STIX2IOCDto stix2IOCDto = new STIX2IOCDto(
"id" + randomAlphaOfLength(3),
"random",
new IOCType(IOCType.IPV4_TYPE),
ipVals.get(i1),
"",
Instant.now(),
Instant.now(),
"",
emptyList(),
"spec",
"configId",
"",
1L
);

testIocDtos.add(stix2IOCDto);
}
for (int i1 = 0; i1 < hashVals.size(); i1++) {
// create IOCs
STIX2IOCDto stix2IOCDto = new STIX2IOCDto(
"id" + randomAlphaOfLength(3),
"random",
new IOCType(IOCType.HASHES_TYPE),
hashVals.get(i1),
"",
Instant.now(),
Instant.now(),
"",
emptyList(),
"spec",
"configId",
"",
1L
);

testIocDtos.add(stix2IOCDto);
}
for (int i1 = 0; i1 < domainVals.size(); i1++) {
// create IOCs
STIX2IOCDto stix2IOCDto = new STIX2IOCDto(
"id" + randomAlphaOfLength(3),
"random",
new IOCType(IOCType.DOMAIN_NAME_TYPE),
domainVals.get(i1),
"",
Instant.now(),
Instant.now(),
"",
emptyList(),
"spec",
"configId",
"",
1L
);

testIocDtos.add(stix2IOCDto);
}
return indexTifSourceConfig(testIocDtos);
}

private String indexTifSourceConfig(List<STIX2IOCDto> testIocDtos) throws IOException {
SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto(
"configId",
Expand All @@ -100,7 +164,7 @@ private String indexTifSourceConfig(List<STIX2IOCDto> testIocDtos) throws IOExce
null,
null,
false,
List.of(IOCType.IPV4_TYPE),
List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE, IOCType.DOMAIN_NAME_TYPE),
true
);

Expand Down Expand Up @@ -251,6 +315,96 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException {
assertEquals(totalHitsVal.intValue(), 0);
}



public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException {
updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1");
Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
Map<String, Object> responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(0, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());
List<String> ipVals = List.of("ip1", "ip2");
List<String> hashVals = List.of("h1", "h2");
List<String> domainVals = List.of("d1", "d2");
String createdId = indexSourceConfigsAndIocs(ipVals, hashVals, domainVals);

String ipIndex = "ipAlias";
createTestAlias(ipIndex, 1, true);
String hashIndex = "hashAlias";
createTestAlias(hashIndex, 1, true);
String domainIndex = "domainAlias";
createTestAlias(domainIndex, 1, true);


/**create monitor */
ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(ipIndex, hashIndex, domainIndex);
Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
Assert.assertEquals(201, response.getStatusLine().getStatusCode());
Map<String, Object> responseBody = asMap(response);

try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
fail();
} catch (Exception e) {
/** creating a second threat intel monitor should fail*/
assertTrue(e.getMessage().contains("already exists"));
}

final String monitorId = responseBody.get("id").toString();
Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId);
Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode());

Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId);
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
int i = 1;
for (String val : ipVals) {
String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(ipIndex, "" + i++, doc);
} catch (IOException e) {
fail();
}
}
for (String val : hashVals) {
String doc = String.format("{\"hash\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(hashIndex, "" + i++, doc);
} catch (IOException e) {
fail();
}
}
for (String val : domainVals) {
String doc = String.format("{\"domain\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(domainIndex, "" + i++, doc);
} catch (IOException e) {
fail();
}
}

executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Map<String, Object> executeResults = entityAsMap(executeResponse);

String matchAllRequest = getMatchAllRequest();
Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
HashMap<String, Object> hits = (HashMap<String, Object>) asMap(searchMonitorResponse).get("hits");
HashMap<String, Object> totalHits = (HashMap<String, Object>) hits.get("total");
Integer totalHitsVal = (Integer) totalHits.get("value");
assertEquals(totalHitsVal.intValue(), 1);
makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));

iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(6, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());

//alerts
List<SearchHit> searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest);
Assert.assertEquals(6, searchHits.size());
}

public void testCreateThreatIntelMonitor() throws IOException {
updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1");
Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Expand Down Expand Up @@ -572,6 +726,26 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) {
List.of(t1, t2, t3, t4));
}

public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) {
ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity");
ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity");
ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity");
ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(ipIndex), emptyList(), emptyList(), "indexmatch", null, "severity");

return new ThreatIntelMonitorDto(
Monitor.NO_ID,
randomAlphaOfLength(10),
List.of(
new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(ipIndex, List.of("ip"))),
new PerIocTypeScanInputDto(IOCType.HASHES_TYPE, Map.of(hashIndex, List.of("hash"))),
new PerIocTypeScanInputDto(IOCType.DOMAIN_NAME_TYPE, Map.of(domainIndex, List.of("domain")))
),
new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()),
false,
null,
List.of(t1, t2, t3, t4));
}

@Override
protected boolean preserveIndicesUponCompletion() {
return false;
Expand Down
Loading