diff --git a/data-prepper-plugins/geoip-processor/build.gradle b/data-prepper-plugins/geoip-processor/build.gradle index e97e28bb4b..7f6d88a885 100644 --- a/data-prepper-plugins/geoip-processor/build.gradle +++ b/data-prepper-plugins/geoip-processor/build.gradle @@ -1,3 +1,10 @@ +plugins{ + id 'de.undercouch.download' version '4.1.2' +} +apply plugin: 'de.undercouch.download' + +import de.undercouch.gradle.tasks.download.Download + /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 @@ -23,8 +30,75 @@ dependencies { testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation project(':data-prepper-test-common') } +def geoIP2='GeoIP2' +def geoLite2= 'GeoLite2' +task downloadFile(type: Download) { + + def urls = [ + 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-City-Test.mmdb', + 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-Country-Test.mmdb', + 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoLite2-ASN-Test.mmdb' + ] + def mmdbFileExtension = '.mmdb' + def baseDirPath = 'src/test/resources/mmdb-file/geo-lite2/' + + urls.each { url -> + src(url) + dest(baseDirPath) + doLast { + + def testFileName = url.substring(url.lastIndexOf('/') + 1) + def testMmdbSubString = testFileName.substring(testFileName.lastIndexOf('-')) + def fileName = testFileName.substring(0, testFileName.length() - testMmdbSubString.length()) + + if(fileName.contains(geoIP2)) { + fileName = fileName.replace(geoIP2, geoLite2) + } + File sourceFile = file(baseDirPath+testFileName) + File destinationFile = file( baseDirPath+fileName+mmdbFileExtension) + sourceFile.renameTo(destinationFile) + + } + + } + + +} +task downloadEnterpriseFile(type: Download) { + dependsOn downloadFile + def urls = [ + 'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-Enterprise-Test.mmdb' + ] + def mmdbFileExtension = '.mmdb' + def baseDirPath = 'src/test/resources/mmdb-file/geo-enterprise/' + + urls.each { url -> + src(url) + def testFileName = url.substring(url.lastIndexOf('/') + 1) + def testMmdbSubString = testFileName.substring(testFileName.lastIndexOf('-')) + def fileName = testFileName.substring(0, testFileName.length() - testMmdbSubString.length()) + dest(baseDirPath+testFileName) + doLast { + if(fileName.contains(geoIP2)) { + fileName = fileName.replace(geoIP2, geoLite2) + } + File sourceFile = file(baseDirPath+testFileName) + File destinationFile = file( baseDirPath+fileName+mmdbFileExtension) + sourceFile.renameTo(destinationFile) + } + + } + +} +/*task processTestResources(type: Copy) { + dependsOn downloadEnterpriseFile + from 'src/test/resources' // Source directory containing test resources + into 'build/resources/test' // Destination directory for processed test resources +}*/ +tasks.test.dependsOn 'processTestResources' +tasks.processTestResources.dependsOn 'downloadEnterpriseFile' test { useJUnitPlatform() } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java index 01dedfb987..b60d09a934 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java @@ -182,7 +182,7 @@ public Map getGeoData(InetAddress inetAddress, List attr enrichData(geoData, ORGANIZATION_NAME, organizationName); break; case NETWORK: - enrichData(geoData, NETWORK, network.toString()); + enrichData(geoData, NETWORK,network!=null? network.toString():null); break; } } @@ -204,7 +204,7 @@ public Map getGeoData(InetAddress inetAddress, List attr } enrichData(geoData, ORGANIZATION_NAME, organizationName); - enrichData(geoData, NETWORK, network.toString()); + enrichData(geoData, NETWORK,network!=null? network.toString():null); } } catch (Exception ex) { throw new EnrichFailedException("Enrichment failed exception" + ex); diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java index cdfd270de0..b8e052d6f9 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java @@ -36,7 +36,7 @@ class GetGeoLite2DataTest { private static final String PATH = "./src/test/resources/mmdb-file/geo-lite2"; - public static final String IP = "2001:4860:4860::8888"; + public static final String IP = "2a02:ec00:0:0:0:0:0:0"; private static final String PREFIX_DIR = "first_database"; private String tempFolderPath = System.getProperty("java.io.tmpdir") + File.separator + "GeoIP"; @Mock @@ -64,8 +64,8 @@ void getGeoDataTest_without_attributes() throws UnknownHostException { GeoIPProcessorService.downloadReady = false; Map geoData = getGeoLite2Data.getGeoData(inetAddress, attributes, tempFolderPath); Assertions.assertNotNull(geoData); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("ip"), equalTo("2001:4860:4860:0:0:0:0:8888")); + assertThat(geoData.get("country_iso_code"), equalTo("FR")); + assertThat(geoData.get("ip"), equalTo(IP)); assertDoesNotThrow(() -> { getGeoLite2Data.closeReader(); }); @@ -88,8 +88,8 @@ void getGeoDataTest_with_attributes() throws UnknownHostException { GeoIPProcessorService.downloadReady = false; Map geoData = getGeoLite2Data.getGeoData(inetAddress, attributes, tempFolderPath); Assertions.assertNotNull(geoData); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("ip"), equalTo("2001:4860:4860:0:0:0:0:8888")); + assertThat(geoData.get("country_name"), equalTo("France")); + assertThat(geoData.get("ip"), equalTo(IP)); assertDoesNotThrow(() -> { getGeoLite2Data.closeReader(); }); diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-enterprise/GeoIP2-Enterprise.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-enterprise/GeoIP2-Enterprise.mmdb deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-ASN.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-ASN.mmdb deleted file mode 100644 index 2397f84d0c..0000000000 Binary files a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-ASN.mmdb and /dev/null differ diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-City.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-City.mmdb deleted file mode 100644 index d954071728..0000000000 Binary files a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-City.mmdb and /dev/null differ diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-Country.mmdb b/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-Country.mmdb deleted file mode 100644 index 737c6f0fee..0000000000 Binary files a/data-prepper-plugins/geoip-processor/src/test/resources/mmdb-file/geo-lite2/GeoLite2-Country.mmdb and /dev/null differ