Fake place creation + geofabrik testing functionality

nominatim-docker-test.yml: added environment variable for the geofabrik query url with GitHub actions secrets. we will see if this works in this commit.

docker-compose.yml: commented out healthcheck because it isn't currently doing anything useful to delay the start. Should remove completely once docker commit is working.

TestNominatim.py: added functionality for geofabrik calls with GH actions secrets.

Added creation of a fake place to use for testing of get_filtered_place.

Added test comparing geofabrik and nominatim hard-coded calls (these may not be possible from within the container so might be removed)

Added test for get_filtered_place in clean_and_resample, which passes in the fake place created above.

Modified test_get_json_reverse to only check the display name for now.

.github/workflows/nominatim-docker-test.yml

@@ -26,5 +26,7 @@ jobs:
     - name: Workflow test
       run: echo Smoke test
 
-    - name: Test nominatim 
+    - name: Test nominatim
+      env:
+        GEOFABRIK_QUERY_URL: https://geocoding.geofabrik.de/${{secrets.GEOFABRIK_API}}
       run: docker-compose -f emission/integrationTests/docker-compose.yml up --exit-code-from web-server

emission/individual_tests/TestNominatim.py

@@ -7,23 +7,51 @@
 from builtins import *
 import unittest
 import unittest.mock as mock
+# import overpy
 import os
 from emission.core.wrapper.trip_old import Coordinate
+import requests
+import emission.core.wrapper.entry as ecwe
+import emission.core.wrapper.wrapperbase as ecww
 import emission.net.ext_service.geocoder.nominatim as eco
+import emission.analysis.intake.cleaning.clean_and_resample as clean
+
 
 #temporarily sets NOMINATIM_QUERY_URL to the environment variable for testing.
 NOMINATIM_QUERY_URL_env = os.environ.get("NOMINATIM_QUERY_URL", "")
 NOMINATIM_QUERY_URL = NOMINATIM_QUERY_URL_env if NOMINATIM_QUERY_URL_env != "" else eco.NOMINATIM_QUERY_URL
-print("query URL in TestNominatim:", NOMINATIM_QUERY_URL)
+GEOFABRIK_QUERY_URL = os.environ.get("GEOFABRIK_QUERY_URL", "")
+
+#Creates a fake place in Rhode Island to use for testing.
+fake_id = "rhodeislander"
+key = "segmentation/raw_place"
+write_ts = 1694344333
+data = {'source': 'FakeTripGenerator','location': {'type': 'Point', 'coordinates': [-71.4128343, 41.8239891]}}
+fake_place = ecwe.Entry.create_fake_entry(fake_id, key, data, write_ts)
 
 class NominatimTest(unittest.TestCase):
     maxDiff = None
+
+    #basic query to check that both nominatim and geofabrik are acting as drop-ins for eachother.
+    def test_geofabrik_and_nominatim(self):
+        nominatim_result =  requests.get(NOMINATIM_QUERY_URL + "/reverse?lat=41.8239891&lon=-71.4128343&format=json")
+        geofabrik_result =  requests.get(GEOFABRIK_QUERY_URL + "/reverse?lat=41.8239891&lon=-71.4128343&format=json")
+        self.assertEqual(nominatim_result, geofabrik_result)
+
+#checks the display name generated by get_filtered place in clean_and_resample.py, which creates a cleaned place from the fake place
+# and reverse geocodes with the coordinates.
+    def test_get_filtered_place(self):
+        raw_result = ecww.WrapperBase.__getattr__(clean.get_filtered_place(fake_place), "data")
+        actual_result = ecww.WrapperBase.__getattr__(raw_result, "display_name")
+        expected_result = "Fulton Street, Providence"
+        self.assertEqual(expected_result, actual_result)
+
     def test_make_url_geo(self):
         expected_result = NOMINATIM_QUERY_URL + "/search?q=Providence%2C+Rhode+Island&format=json"
         actual_result = eco.Geocoder.make_url_geo("Providence, Rhode Island")
         self.assertEqual(expected_result, actual_result)
 
-#we ignore the place_id because it is an internal Nominatim identifier 
+    #we ignore the place_id because it is an internal Nominatim identifier 
     def test_get_json_geo(self):
         expected_result = [{'place_id': mock.ANY, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 121496393, 'boundingbox': ['41.8237547', '41.8243153', '-71.4132816', '-71.4125278'], 'lat': '41.824034499999996', 'lon': '-71.41290469687814', 'display_name': 'Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States', 'class': 'amenity', 'type': 'townhall', 'importance': 1.25001}]
         actual_result = eco.Geocoder.get_json_geo("Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States")
@@ -44,34 +72,34 @@ def test_make_url_reverse(self):
         actual_result = (eco.Geocoder.make_url_reverse(41.8239891, -71.4128343))
         self.assertEqual(expected_result, actual_result)
 
-#we ignore the place_id because it is an internal Nominatim identifier 
+        #tested result was modified to only look at the name returned with the coordinates, rather than the entire dictionary.
     def test_get_json_reverse(self):
-        expected_result = {'place_id': mock.ANY, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 121496393, 'lat': '41.824034499999996', 'lon': '-71.41290469687814', 'display_name': 'Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States', 'address': {'amenity': 'Providence City Hall', 'road': 'Fulton Street', 'neighbourhood': 'Downtown', 'city': 'Providence', 'county': 'Providence County', 'postcode': '02903', 'country': 'United States', 'country_code': 'us'}, 'boundingbox': ['41.8237547', '41.8243153', '-71.4132816', '-71.4125278']}
-        actual_docker = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)
-        # actual_nominatim = "httsp://nominatim.openstreetmap.org/"
-        self.assertEqual(expected_result, actual_docker)
-        # self.assertEqual(actual_docker, actual_nominatim)
+        expected_result = "Providence City Hall"
+        actual_result = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)["display_name"].split(",")[0]
+        self.assertEqual(expected_result, actual_result)
 
     def test_reverse_geocode(self):
         expected_result = "Portugal Parkway, Fox Point, Providence, Providence County, 02906, United States"
         actual_result = eco.Geocoder.reverse_geocode(41.8174476, -71.3903767)
         self.assertEqual(expected_result, actual_result)
-
-#this test was written with the intention of using a ground truth file. Once a fake trip is generated in Rhode island, this section will be modified.  
-    # def test_display_name(self):
-    #     nominatim_reverse_query = NOMINATIM_QUERY_URL + "/reverse?"
-    #     params = {
-    #             "lat" : 41.831174, 
-    #             "lon" : -71.414907,
-    #             "format" : "json"
-    #         }
-    #     encoded_params = urllib.parse.urlencode(params)
-    #     url = nominatim_reverse_query + encoded_params
-    #     request = urllib.request.Request(url)
-    #     response = urllib.request.urlopen(request)
-    #     parsed_response = json.loads(response.read())
-    #     actual_result = str(parsed_response.get("display_name"))
-    #     expected_result = "Rhode Island State Capitol Building, 82, Smith Street, Downtown, Providence, Providence County, 02903, United States"
-    #     self.assertEqual(expected_result, actual_result)
+
+    #a hard-coded nominatim call to compare with our container. 
+    def test_nominatim_api(self):
+        nominatim_url = "http://nominatim.openstreetmap.org/reverse?lat=41.832942092439694&lon=-71.41558148857203&format=json"
+        nominatim_result_raw = requests.get(nominatim_url)
+        nominatim_result = nominatim_result_raw.json()['display_name']
+        # NOMINATIM_QUERY_URL = eco.NOMINATIM_QUERY_URL
+        docker_result = eco.Geocoder.reverse_geocode(41.832942092439694, -71.41558148857203)
+        print("HEYYY")
+        print(docker_result)
+        print("hey2")
+        print(nominatim_result)
+        self.assertEqual(nominatim_result, docker_result)
+
+    # def test_overpass_api(self):
+    #     api = overpy.Overpass()
+    #     result = api.query("""way["name"="Gielgenstraße"](50.7,7.1,50.8,7.25);out;""")
+
+
 if __name__ == '__main__':
     unittest.main()

emission/integrationTests/docker-compose.yml

@@ -44,14 +44,15 @@ services:
     environment:
       - PBF_URL=https://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf
       - REPLICATION_URL=https://download.geofabrik.de/north-america/us/rhode-island-updates/
-      - IMPORT_WIKIPEDIA=false
+      # - IMPORT_WIKIPEDIA=false
     ports:
       - "8080:8080"
-    healthcheck:
-      test: ["CMD", "curl", "-v", "http://localhost:8080"]
-      interval: 1m
-      timeout: 15s
-      retries: 3
+      #healthcheck currently commented out, since it isn't necessary and keeps going the entire time the container is running. 
+    # healthcheck:
+    #   test: ["CMD", "curl", "-v", "http://localhost:8080"]
+    #   interval: 1m
+    #   timeout: 15s
+    #   retries: 3
     deploy:
       replicas: 1
       restart_policy: