diff --git a/CHANGELOG.md b/CHANGELOG.md index b96a2a5b..0dc3b884 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Symlinks from project-specific readme files to main readme [#250](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/250) - Support for Python 3.12 [#234](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/234) +- Extended Datetime Search to search on start_datetime and end_datetime as well as datetime fields. [#182](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/182) ### Changed diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 56afcbc8..993b265a 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -725,7 +725,7 @@ async def update_item( """ item = item.model_dump(mode="json") base_url = str(kwargs["request"].base_url) - now = datetime_type.now(timezone.utc).isoformat().replace("+00:00", "Z") + now = datetime_type.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") item["properties"]["updated"] = now await self.database.check_collection_exists(collection_id) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index da6d6880..1874e451 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -473,8 +473,8 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): return search.filter("terms", collection=collection_ids) @staticmethod - def apply_datetime_filter(search: Search, datetime_search): - """Apply a filter to search based on datetime field. + def apply_datetime_filter(search: Search, datetime_search: dict): + """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: search (Search): The search object to filter. @@ -483,17 +483,109 @@ def apply_datetime_filter(search: Search, datetime_search): Returns: Search: The filtered search object. """ + should = [] + + # If the request is a single datetime return + # items with datetimes equal to the requested datetime OR + # the requested datetime is between their start and end datetimes if "eq" in datetime_search: - search = search.filter( - "term", **{"properties__datetime": datetime_search["eq"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "term", + properties__datetime=datetime_search["eq"], + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["eq"], + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["eq"], + }, + ), + ], + ), + ] ) + + # If the request is a date range return + # items with datetimes within the requested date range OR + # their startdatetime ithin the requested date range OR + # their enddatetime ithin the requested date range OR + # the requested daterange within their start and end datetimes else: - search = search.filter( - "range", properties__datetime={"lte": datetime_search["lte"]} - ) - search = search.filter( - "range", properties__datetime={"gte": datetime_search["gte"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "range", + properties__datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["gte"] + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["lte"] + }, + ), + ], + ), + ] ) + + search = search.query(Q("bool", filter=[Q("bool", should=should)])) + return search @staticmethod diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 778cfe03..8f9f0000 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -512,7 +512,7 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str] @staticmethod def apply_datetime_filter(search: Search, datetime_search): - """Apply a filter to search based on datetime field. + """Apply a filter to search based on datetime field, start_datetime, and end_datetime fields. Args: search (Search): The search object to filter. @@ -521,17 +521,109 @@ def apply_datetime_filter(search: Search, datetime_search): Returns: Search: The filtered search object. """ + should = [] + + # If the request is a single datetime return + # items with datetimes equal to the requested datetime OR + # the requested datetime is between their start and end datetimes if "eq" in datetime_search: - search = search.filter( - "term", **{"properties__datetime": datetime_search["eq"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "term", + properties__datetime=datetime_search["eq"], + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["eq"], + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["eq"], + }, + ), + ], + ), + ] ) + + # If the request is a date range return + # items with datetimes within the requested date range OR + # their startdatetime ithin the requested date range OR + # their enddatetime ithin the requested date range OR + # the requested daterange within their start and end datetimes else: - search = search.filter( - "range", properties__datetime={"lte": datetime_search["lte"]} - ) - search = search.filter( - "range", properties__datetime={"gte": datetime_search["gte"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "range", + properties__datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["gte"] + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["lte"] + }, + ), + ], + ), + ] ) + + search = search.query(Q("bool", filter=[Q("bool", should=should)])) + return search @staticmethod diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 64545807..0533a483 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1,5 +1,6 @@ import uuid -from datetime import datetime, timedelta, timezone +from copy import deepcopy +from datetime import datetime, timedelta import pytest @@ -205,7 +206,13 @@ async def test_app_fields_extension_return_all_properties( feature = resp_json["features"][0] assert len(feature["properties"]) >= len(item["properties"]) for expected_prop, expected_value in item["properties"].items(): - if expected_prop in ("datetime", "created", "updated"): + if expected_prop in ( + "datetime", + "start_datetime", + "end_datetime", + "created", + "updated", + ): assert feature["properties"][expected_prop][0:19] == expected_value[0:19] else: assert feature["properties"][expected_prop] == expected_value @@ -259,9 +266,9 @@ async def test_app_sort_extension_get_asc(app_client, txn_client, ctx): second_item["id"] = "another-item" another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" - ).replace(tzinfo=timezone.utc) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + ) - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) await create_item(txn_client, second_item) @@ -281,10 +288,11 @@ async def test_app_sort_extension_get_desc(app_client, txn_client, ctx): second_item["id"] = "another-item" another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" - ).replace(tzinfo=timezone.utc) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + ) - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) + await create_item(txn_client, second_item) resp = await app_client.get("/search?sortby=-properties.datetime") @@ -302,10 +310,11 @@ async def test_app_sort_extension_post_asc(app_client, txn_client, ctx): second_item["id"] = "another-item" another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" - ).replace(tzinfo=timezone.utc) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + ) - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) + await create_item(txn_client, second_item) params = { @@ -327,9 +336,9 @@ async def test_app_sort_extension_post_desc(app_client, txn_client, ctx): second_item["id"] = "another-item" another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" - ).replace(tzinfo=timezone.utc) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + ) - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) await create_item(txn_client, second_item) @@ -410,7 +419,22 @@ async def test_search_point_does_not_intersect(app_client, ctx): @pytest.mark.asyncio -async def test_datetime_non_interval(app_client, ctx): +async def test_datetime_response_format(app_client, txn_client, ctx): + first_item = dict(ctx.item) + + second_item = deepcopy(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = deepcopy(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + dt_formats = [ "2020-02-12T12:30:22+00:00", "2020-02-12T12:30:22.00Z", @@ -431,6 +455,150 @@ async def test_datetime_non_interval(app_client, ctx): assert resp_json["features"][0]["properties"]["datetime"][0:19] == dt[0:19] +@pytest.mark.asyncio +async def test_datetime_non_interval(app_client, txn_client, ctx): + first_item = dict(ctx.item) + + second_item = deepcopy(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = deepcopy(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + dt_formats = [ + "2020-02-12T12:30:22+00:00", + "2020-02-12T12:30:22.00Z", + "2020-02-12T12:30:22Z", + "2020-02-12T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 3 + + +@pytest.mark.asyncio +async def test_datetime_interval(app_client, txn_client, ctx): + first_item = dict(ctx.item) + + second_item = deepcopy(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = deepcopy(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + dt_formats = [ + "2020-02-06T12:30:22+00:00/2020-02-13T12:30:22+00:00", + "2020-02-12T12:30:22.00Z/2020-02-20T12:30:22.00Z", + "2020-02-12T12:30:22Z/2020-02-13T12:30:22Z", + "2020-02-06T12:30:22.00+00:00/2020-02-20T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 3 + + +@pytest.mark.asyncio +async def test_datetime_bad_non_interval(app_client, txn_client, ctx): + first_item = dict(ctx.item) + + second_item = deepcopy(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = deepcopy(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + dt_formats = [ + "2020-02-06T12:30:22+00:00", + "2020-02-06T12:30:22.00Z", + "2020-02-06T12:30:22Z", + "2020-02-06T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 0 + + +@pytest.mark.asyncio +async def test_datetime_bad_interval(app_client, txn_client, ctx): + first_item = dict(ctx.item) + + second_item = deepcopy(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = deepcopy(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + dt_formats = [ + "1920-02-04T12:30:22+00:00/1920-02-06T12:30:22+00:00", + "1920-02-04T12:30:22.00Z/1920-02-06T12:30:22.00Z", + "1920-02-04T12:30:22Z/1920-02-06T12:30:22Z", + "1920-02-04T12:30:22.00+00:00/1920-02-06T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 0 + + @pytest.mark.asyncio async def test_bbox_3d(app_client, ctx): australia_bbox = [106.343365, -47.199523, 0.1, 168.218365, -19.437288, 0.1] diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index 904adbbf..7837d1fc 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -392,6 +392,25 @@ async def test_item_search_temporal_window_post(app_client, ctx, load_test_data) assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio +async def test_item_search_temporal_intersecting_window_post(app_client, ctx): + """Test POST search with two-tailed spatio-temporal query (core)""" + test_item = ctx.item + + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) + item_date_before = item_date - timedelta(days=10) + item_date_after = item_date - timedelta(days=2) + + params = { + "collections": [test_item["collection"]], + "intersects": test_item["geometry"], + "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}", + } + resp = await app_client.post("/search", json=params) + resp_json = resp.json() + assert resp_json["features"][0]["id"] == test_item["id"] + + @pytest.mark.asyncio async def test_item_search_temporal_open_window(app_client, ctx): """Test POST search with open spatio-temporal query (core)"""