Merge pull request #17 from openstreetmap-polska/dev

Release to main

.vscode/launch.json

@@ -12,7 +12,7 @@
             "args": [
                 "main:app",
                 "--workers",
-                "1",
+                "8",
             ],
             "jinja": true,
             "justMyCode": true,

Makefile

@@ -5,6 +5,7 @@ build:
 
 version:
 	sed -i -r "s|VERSION = '([0-9.]+)'|VERSION = '\1.$$(date +%y%m%d)'|g" config.py
+	sed -i -r "s|VERSION_TIMESTAMP = ([0-9.]+)|VERSION_TIMESTAMP = $$(date +%s)|g" config.py
 
 dev-start:
 	docker compose -f docker-compose.dev.yml up -d

Pipfile

@@ -5,27 +5,31 @@ verify_ssl = true
 
 [packages]
 anyio = "*"
+asyncache = "*"
 asyncio = "*"
+cachetools = "*"
 dacite = "*"
 fastapi = "*"
 filelock = "*"
 httpx = "*"
 jinja2 = "*"
 mapbox-vector-tile = "*"
 motor = "*"
+networkx = "*"
 numba = "*"
 numpy = "*"
 orjson = "*"
 psutil = "*"
 pyproj = "*"
+python-dateutil = "*"
 scikit-learn = "*"
 shapely = "*"
 six = "*"
 starlette = "*"
 timezonefinder = "*"
+tqdm = "*"
 uvicorn = "*"
 xmltodict = "*"
-python-dateutil = "*"
 
 [dev-packages]
 autopep8 = "*"

api/v1/countries.py

@@ -1,10 +1,9 @@
 from datetime import timedelta
 from typing import Annotated
-from urllib.parse import quote_plus, unquote_plus
 
 import anyio
 from anyio.streams.memory import MemoryObjectSendStream
-from fastapi import APIRouter, HTTPException, Path, Request, Response
+from fastapi import APIRouter, Path, Request, Response
 from shapely.geometry import Point, mapping
 
 from middlewares.cache_middleware import configure_cache
@@ -16,12 +15,12 @@
 
 
 async def _count_aed_in_country(country: Country, aed_state: AEDState, send_stream: MemoryObjectSendStream) -> None:
-    count = await aed_state.count_aeds(country.geometry)
+    count = await aed_state.count_aeds_by_country_code(country.code)
     await send_stream.send((country, count))
 
 
 @router.get('/countries/names')
-@configure_cache(timedelta(hours=1), stale=timedelta(days=2))
+@configure_cache(timedelta(hours=1), stale=timedelta(days=7))
 async def get_country_names(request: Request, country_state: CountryStateDep, aed_state: AEDStateDep):
     countries = await country_state.get_all_countries()
 
@@ -56,11 +55,7 @@ async def get_country_geojson(request: Request, response: Response, country_code
     if country_code == 'WORLD':
         aeds = await aed_state.get_all_aeds()
     else:
-        country = await country_state.get_country_by_code(country_code)
-        if country is None:
-            raise HTTPException(404, f'Country code {country_code!r} not found')
-
-        aeds = await aed_state.get_aeds_within_geom(country.geometry, group_eps=None)
+        aeds = await aed_state.get_aeds_by_country_code(country_code)
 
     response.headers['Content-Disposition'] = 'attachment'
     response.headers['Content-Type'] = 'application/geo+json'

api/v1/tile.py

@@ -42,7 +42,7 @@ def _tile_to_bbox(z: int, x: int, y: int) -> BBox:
 
 
 async def _count_aed_in_country(country: Country, aed_state: AEDState, send_stream: MemoryObjectSendStream) -> None:
-    count = await aed_state.count_aeds(country.geometry)
+    count = await aed_state.count_aeds_by_country_code(country.code)
     await send_stream.send((country, count))
 
 
@@ -80,13 +80,14 @@ async def _get_tile_country(z: int, bbox: BBox, lang: str, country_state: Countr
     send_stream, receive_stream = anyio.create_memory_object_stream()
     country_count_map = {}
 
-    async with anyio.create_task_group() as tg, send_stream, receive_stream:
-        for country in countries:
-            tg.start_soon(_count_aed_in_country, country, aed_state, send_stream)
+    with print_run_time('Counting AEDs'):
+        async with anyio.create_task_group() as tg, send_stream, receive_stream:
+            for country in countries:
+                tg.start_soon(_count_aed_in_country, country, aed_state, send_stream)
 
-        for _ in range(len(countries)):
-            country, count = await receive_stream.receive()
-            country_count_map[country.name] = (count, abbreviate(count))
+            for _ in range(len(countries)):
+                country, count = await receive_stream.receive()
+                country_count_map[country.name] = (count, abbreviate(count))
 
     return _mvt_encode(bbox, [{
         'name': 'countries',
@@ -113,7 +114,6 @@ async def _get_tile_country(z: int, bbox: BBox, lang: str, country_state: Countr
                     'point_count_abbreviated': country_count_map[country.name][1],
                 },
             } for country in countries
-            if country.label.min_z <= z <= country.label.max_z
         ]
     }])
 

config.py

@@ -4,10 +4,12 @@
 import pymongo
 from motor.core import AgnosticDatabase
 from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo import IndexModel
 from pyproj import Transformer
 
 NAME = 'openaedmap-backend'
 VERSION = '2.0'
+VERSION_TIMESTAMP = 0
 WEBSITE = 'https://openaedmap.org'
 USER_AGENT = f'{NAME}/{VERSION} (+{WEBSITE})'
 
@@ -21,7 +23,7 @@
 AED_UPDATE_DELAY = timedelta(seconds=30)
 AED_REBUILD_THRESHOLD = timedelta(hours=1)
 
-TILE_COUNTRIES_CACHE_MAX_AGE = timedelta(hours=2)
+TILE_COUNTRIES_CACHE_MAX_AGE = timedelta(hours=4)
 TILE_CACHE_STALE = timedelta(days=7)
 
 TILE_COUNTRIES_MAX_Z = 5
@@ -45,7 +47,17 @@
 
 # this is run by a single, primary worker on startup
 async def startup_setup() -> None:
-    await COUNTRY_COLLECTION.create_index([('code', pymongo.ASCENDING)], unique=True)
-    await COUNTRY_COLLECTION.create_index([('geometry', pymongo.GEOSPHERE)])
-    await AED_COLLECTION.create_index([('id', pymongo.ASCENDING)], unique=True)
-    await AED_COLLECTION.create_index([('position', pymongo.GEOSPHERE)])
+    try:
+        await COUNTRY_COLLECTION.drop_index([('code', pymongo.ASCENDING)])
+    except Exception:
+        pass
+
+    await COUNTRY_COLLECTION.create_indexes([
+        IndexModel([('geometry', pymongo.GEOSPHERE)]),
+    ])
+
+    await AED_COLLECTION.create_indexes([
+        IndexModel([('id', pymongo.ASCENDING)], unique=True),
+        IndexModel([('country_codes', pymongo.ASCENDING)]),
+        IndexModel([('position', pymongo.GEOSPHERE)]),
+    ])