From 2f8d666a0b6b07521d4645a63cce23e1326417be Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 5 Jan 2024 10:23:28 +0000 Subject: [PATCH 01/19] Extending datetime search to include start_datetime and end_datetime. --- .../elasticsearch/database_logic.py | 98 +++++++++++++++++-- 1 file changed, 90 insertions(+), 8 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 336c8d07..1e605964 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -388,18 +388,100 @@ def apply_datetime_filter(search: Search, datetime_search): Returns: Search: The filtered search object. """ + should = [] + if "eq" in datetime_search: - search = search.filter( - "term", **{"properties__datetime": datetime_search["eq"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "term", + properties__datetime=datetime_search["eq"], + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["eq"], + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["eq"], + }, + ), + ], + ), + ] ) + else: - search = search.filter( - "range", properties__datetime={"lte": datetime_search["lte"]} - ) - search = search.filter( - "range", properties__datetime={"gte": datetime_search["gte"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "range", + properties__datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["gte"] + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["lte"] + }, + ), + ], + ), + ] ) - return search + + search = search.query(Q("bool", filter=[Q("bool", should=should)])) @staticmethod def apply_bbox_filter(search: Search, bbox: List): From 7b33232677e501b85ad44adf3111cf547b2bc107 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 5 Jan 2024 15:12:47 +0000 Subject: [PATCH 02/19] Removing _return_date bounding constants. --- .../stac_fastapi/elasticsearch/core.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 4fb9f174..4f4d5ce2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -289,22 +289,10 @@ def _return_date(interval_str): datetime = f"{intervals[0][0:19]}Z" return {"eq": datetime} else: - start_date = intervals[0] - end_date = intervals[1] - if ".." not in intervals: - start_date = f"{start_date[0:19]}Z" - end_date = f"{end_date[0:19]}Z" - elif start_date != "..": - start_date = f"{start_date[0:19]}Z" - end_date = "2200-12-01T12:31:12Z" - elif end_date != "..": - start_date = "1900-10-01T00:00:00Z" - end_date = f"{end_date[0:19]}Z" - else: - start_date = "1900-10-01T00:00:00Z" - end_date = "2200-12-01T12:31:12Z" + start_date = f"{intervals[0][0:19]}Z" if intervals[0] != ".." else None + end_date = f"{intervals[1][0:19]}Z" if intervals[1] != ".." else None - return {"lte": end_date, "gte": start_date} + return {"lte": end_date, "gte": start_date} async def get_search( self, @@ -457,9 +445,9 @@ async def post_search( ) if search_request.query: - for (field_name, expr) in search_request.query.items(): + for field_name, expr in search_request.query.items(): field = "properties__" + field_name - for (op, value) in expr.items(): + for op, value in expr.items(): search = self.database.apply_stacql_filter( search=search, op=op, field=field, value=value ) From 0a6616e598c12db7e6c68ddb8cf8de26c6dd55d9 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 08:02:33 +0000 Subject: [PATCH 03/19] Setting elasticsearch hostname for dockercompose. --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db3352fb..03698654 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: - RELOAD=true - ENVIRONMENT=local - WEB_CONCURRENCY=10 - - ES_HOST=172.17.0.1 + - ES_HOST=elasticsearch - ES_PORT=9200 - ES_USE_SSL=false - ES_VERIFY_CERTS=false @@ -32,6 +32,7 @@ services: elasticsearch: container_name: es-container image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0} + hostname: elasticsearch environment: ES_JAVA_OPTS: -Xms512m -Xmx1g volumes: From 53d44ce06f76584e800cbc48754696225d8bc79a Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 08:04:06 +0000 Subject: [PATCH 04/19] Adding extra test for start_datetime/end_datetime intersecting searches. --- .../elasticsearch/tests/api/test_api.py | 8 +++++++- .../elasticsearch/tests/data/test_item.json | 2 ++ .../tests/resources/test_item.py | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 74f0bb55..941cdf20 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -191,7 +191,13 @@ async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_c feature = resp_json["features"][0] assert len(feature["properties"]) >= len(item["properties"]) for expected_prop, expected_value in item["properties"].items(): - if expected_prop in ("datetime", "created", "updated"): + if expected_prop in ( + "datetime", + "start_datetime", + "end_datetime", + "created", + "updated", + ): assert feature["properties"][expected_prop][0:19] == expected_value[0:19] else: assert feature["properties"][expected_prop] == expected_value diff --git a/stac_fastapi/elasticsearch/tests/data/test_item.json b/stac_fastapi/elasticsearch/tests/data/test_item.json index 2b7fdd86..3a6ea961 100644 --- a/stac_fastapi/elasticsearch/tests/data/test_item.json +++ b/stac_fastapi/elasticsearch/tests/data/test_item.json @@ -35,6 +35,8 @@ }, "properties": { "datetime": "2020-02-12T12:30:22Z", + "start_datetime": "2020-02-08T12:30:22Z", + "end_datetime": "2020-02-16T12:30:22Z", "landsat:scene_id": "LC82081612020043LGN00", "landsat:row": "161", "gsd": 15, diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_fastapi/elasticsearch/tests/resources/test_item.py index 5b382873..1dc25992 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_item.py @@ -385,6 +385,25 @@ async def test_item_search_temporal_window_post(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio +async def test_item_search_temporal_intersecting_window_post(app_client, ctx): + """Test POST search with two-tailed spatio-temporal query (core)""" + test_item = ctx.item + + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) + item_date_before = item_date - timedelta(days=10) + item_date_after = item_date - timedelta(days=2) + + params = { + "collections": [test_item["collection"]], + "intersects": test_item["geometry"], + "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}", + } + resp = await app_client.post("/search", json=params) + resp_json = resp.json() + assert resp_json["features"][0]["id"] == test_item["id"] + + @pytest.mark.asyncio @pytest.mark.skip(reason="KeyError: 'features") async def test_item_search_temporal_open_window(app_client, ctx): From 18d4be1cbef0b8de1e47b251ae63c70898b3d806 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 08:51:40 +0000 Subject: [PATCH 05/19] Fixing broken tests. --- .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 1e605964..0af6f5c1 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -483,6 +483,8 @@ def apply_datetime_filter(search: Search, datetime_search): search = search.query(Q("bool", filter=[Q("bool", should=should)])) + return search + @staticmethod def apply_bbox_filter(search: Search, bbox: List): """Filter search results based on bounding box. From 7eddfee11eb643fcae7b9f4c58cd88ab8c8a7e21 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 13:50:32 +0000 Subject: [PATCH 06/19] Updating apply_datetime_filter doc string. --- .../stac_fastapi/elasticsearch/database_logic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 0af6f5c1..204a5da0 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -378,8 +378,8 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): return search.filter("terms", collection=collection_ids) @staticmethod - def apply_datetime_filter(search: Search, datetime_search): - """Apply a filter to search based on datetime field. + def apply_datetime_filter(search: Search, datetime_search: dict): + """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: search (Search): The search object to filter. From 65ebbea853fbcb3df7d9a8be54428f6ce57ce904 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 13:55:07 +0000 Subject: [PATCH 07/19] Adding changes to change log. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94bec4de..da86de52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +- Extended Datetime Search to search on start_datetime and end_datetime as well as datetime fields. [#182](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/182) + - Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169) ### Fixed From 5143f00b13e62fa7284328972375a4a9175f1e5e Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Tue, 13 Feb 2024 11:31:04 +0000 Subject: [PATCH 08/19] Adding extra tests. --- .../elasticsearch/tests/api/test_api.py | 165 +++++++++++++++++- 1 file changed, 164 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 941cdf20..1d0900bd 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -402,7 +402,22 @@ async def test_search_point_does_not_intersect(app_client, ctx): @pytest.mark.asyncio -async def test_datetime_non_interval(app_client, ctx): +async def test_datetime_response_format(app_client, txn_client, ctx): + first_item = ctx.item + + second_item = dict(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = dict(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + dt_formats = [ "2020-02-12T12:30:22+00:00", "2020-02-12T12:30:22.00Z", @@ -423,6 +438,154 @@ async def test_datetime_non_interval(app_client, ctx): assert resp_json["features"][0]["properties"]["datetime"][0:19] == dt[0:19] +@pytest.mark.asyncio +async def test_datetime_non_interval(app_client, txn_client, ctx): + first_item = ctx.item + + second_item = dict(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = dict(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + dt_formats = [ + "2020-02-12T12:30:22+00:00", + "2020-02-12T12:30:22.00Z", + "2020-02-12T12:30:22Z", + "2020-02-12T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 3 + + +@pytest.mark.asyncio +async def test_datetime_interval(app_client, txn_client, ctx): + first_item = ctx.item + + second_item = dict(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = dict(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + print("CREATED ITEMS") + + dt_formats = [ + "2020-02-06T12:30:22+00:00/2020-02-13T12:30:22+00:00", + "2020-02-12T12:30:22.00Z/2020-02-20T12:30:22.00Z", + "2020-02-12T12:30:22Z/2020-02-13T12:30:22Z", + "2020-02-06T12:30:22.00+00:00/2020-02-20T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 3 + + +@pytest.mark.asyncio +async def test_datetime_bad_non_interval(app_client, txn_client, ctx): + first_item = ctx.item + + second_item = dict(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = dict(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + dt_formats = [ + "2020-02-06T12:30:22+00:00", + "2020-02-06:30:22.00Z", + "2020-02-06:30:22Z", + "2020-02-06T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 0 + + +@pytest.mark.asyncio +async def test_datetime_bad_interval(app_client, txn_client, ctx): + first_item = ctx.item + + second_item = dict(first_item) + second_item["id"] = "second-item" + second_item["properties"]["datetime"] = None + + await create_item(txn_client, second_item) + + third_item = dict(first_item) + third_item["id"] = "third-item" + del third_item["properties"]["start_datetime"] + del third_item["properties"]["end_datetime"] + + await create_item(txn_client, third_item) + + print("CREATED ITEMS") + + dt_formats = [ + "1920-02-04T12:30:22+00:00/1920-02-06T12:30:22+00:00", + "1920-02-04T12:30:22.00Z/1920-02-06T12:30:22.00Z", + "1920-02-04T12:30:22Z/1920-02-06T12:30:22Z", + "1920-02-04T12:30:22.00+00:00/1920-02-06T12:30:22.00+00:00", + ] + + for dt in dt_formats: + params = { + "datetime": dt, + "collections": [ctx.item["collection"]], + } + + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 0 + + @pytest.mark.asyncio async def test_bbox_3d(app_client, ctx): australia_bbox = [106.343365, -47.199523, 0.1, 168.218365, -19.437288, 0.1] From 8ed09285711990e7e0993f1027907ae7cbb69075 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Tue, 9 Apr 2024 09:59:11 +0100 Subject: [PATCH 09/19] Adding datetime extension to opensearch. --- CHANGELOG.md | 4 +- .../elasticsearch/database_logic.py | 8 ++ .../stac_fastapi/opensearch/database_logic.py | 109 ++++++++++++++++-- stac_fastapi/tests/api/test_api.py | 4 - 4 files changed, 110 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff267343..61415191 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - use index templates for Collection and Item indices [#208](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions/208) - Added API `title`, `version`, and `description` parameters from environment variables `STAC_FASTAPI_TITLE`, `STAC_FASTAPI_VERSION` and `STAC_FASTAPI_DESCRIPTION`, respectively. [#207](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/207) - Added a `STAC_FASTAPI_ROOT_PATH` environment variable to define the root path. Useful when working with an API gateway or load balancer. [#221](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/221) - +- Extended Datetime Search to search on start_datetime and end_datetime as well as datetime fields. [#182](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/182) ### Changed @@ -62,8 +62,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed -- Extended Datetime Search to search on start_datetime and end_datetime as well as datetime fields. [#182](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/182) - - Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169) - Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index c509afc9..950fd5ed 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -409,6 +409,9 @@ def apply_datetime_filter(search: Search, datetime_search: dict): """ should = [] + # If the request is a single datetime return + # items with datetimes equal to the requested datetime OR + # the requested datetime is between their start and end datetimes if "eq" in datetime_search: should.extend( [ @@ -441,6 +444,11 @@ def apply_datetime_filter(search: Search, datetime_search: dict): ] ) + # If the request is a date range return + # items with datetimes within the requested date range OR + # their startdatetime ithin the requested date range OR + # their enddatetime ithin the requested date range OR + # the requested daterange within their start and end datetimes else: should.extend( [ diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 95129f27..f6ee8023 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -1,4 +1,5 @@ """Database logic.""" + import asyncio import logging import os @@ -425,7 +426,7 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): @staticmethod def apply_datetime_filter(search: Search, datetime_search): - """Apply a filter to search based on datetime field. + """Apply a filter to search based on datetime field, start_datetime, and end_datetime fields. Args: search (Search): The search object to filter. @@ -434,17 +435,109 @@ def apply_datetime_filter(search: Search, datetime_search): Returns: Search: The filtered search object. """ + should = [] + + # If the request is a single datetime return + # items with datetimes equal to the requested datetime OR + # the requested datetime is between their start and end datetimes if "eq" in datetime_search: - search = search.filter( - "term", **{"properties__datetime": datetime_search["eq"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "term", + properties__datetime=datetime_search["eq"], + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["eq"], + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["eq"], + }, + ), + ], + ), + ] ) + + # If the request is a date range return + # items with datetimes within the requested date range OR + # their startdatetime ithin the requested date range OR + # their enddatetime ithin the requested date range OR + # the requested daterange within their start and end datetimes else: - search = search.filter( - "range", properties__datetime={"lte": datetime_search["lte"]} - ) - search = search.filter( - "range", properties__datetime={"gte": datetime_search["gte"]} + should.extend( + [ + Q( + "bool", + filter=[ + Q( + "range", + properties__datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + filter=[ + Q( + "range", + properties__start_datetime={ + "lte": datetime_search["gte"] + }, + ), + Q( + "range", + properties__end_datetime={ + "gte": datetime_search["lte"] + }, + ), + ], + ), + ] ) + + search = search.query(Q("bool", filter=[Q("bool", should=should)])) + return search @staticmethod diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 1d0900bd..c6b23fbc 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -491,8 +491,6 @@ async def test_datetime_interval(app_client, txn_client, ctx): await create_item(txn_client, third_item) - print("CREATED ITEMS") - dt_formats = [ "2020-02-06T12:30:22+00:00/2020-02-13T12:30:22+00:00", "2020-02-12T12:30:22.00Z/2020-02-20T12:30:22.00Z", @@ -565,8 +563,6 @@ async def test_datetime_bad_interval(app_client, txn_client, ctx): await create_item(txn_client, third_item) - print("CREATED ITEMS") - dt_formats = [ "1920-02-04T12:30:22+00:00/1920-02-06T12:30:22+00:00", "1920-02-04T12:30:22.00Z/1920-02-06T12:30:22.00Z", From 5212768634ad24372dba23867b494bb031c8e864 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 13 May 2024 09:50:01 +0100 Subject: [PATCH 10/19] Remove unneeded required install. --- stac_fastapi/elasticsearch/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 809d7833..424705d1 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,6 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==2.4.1", "elasticsearch[async]==8.11.0", "elasticsearch-dsl==8.11.0", "uvicorn", From 0c1e08fef01e195095867b374741e260a673f2e2 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Wed, 22 May 2024 08:47:50 +0100 Subject: [PATCH 11/19] Update stac_pydantic requirement. --- stac_fastapi/core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 7f8d0b31..6cf00d97 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -9,7 +9,7 @@ "fastapi-slim", "attrs>=23.2.0", "pydantic[dotenv]", - "stac_pydantic>=3", + "stac_pydantic>=3.1.0", "stac-fastapi.types==3.0.0a", "stac-fastapi.api==3.0.0a", "stac-fastapi.extensions==3.0.0a", From c69c9e57d0ecf7dbf9180e5d45386a464156b5c3 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Thu, 23 May 2024 13:44:53 +0100 Subject: [PATCH 12/19] Fixing datetime type issue. --- stac_fastapi/tests/api/test_api.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 562960e7..e3955c0c 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1,5 +1,5 @@ import uuid -from datetime import timedelta +from datetime import datetime, timedelta import pytest @@ -251,7 +251,9 @@ async def test_app_sort_extension_get_asc(app_client, txn_client, ctx): second_item = dict(first_item) second_item["id"] = "another-item" - another_item_date = first_item["properties"]["datetime"] - timedelta(days=1) + another_item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) - timedelta(days=1) second_item["properties"]["datetime"] = another_item_date.isoformat().replace( "+00:00", "Z" ) @@ -271,7 +273,9 @@ async def test_app_sort_extension_get_desc(app_client, txn_client, ctx): second_item = dict(first_item) second_item["id"] = "another-item" - another_item_date = first_item["properties"]["datetime"] - timedelta(days=1) + another_item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) - timedelta(days=1) second_item["properties"]["datetime"] = another_item_date.isoformat().replace( "+00:00", "Z" ) @@ -290,7 +294,9 @@ async def test_app_sort_extension_post_asc(app_client, txn_client, ctx): second_item = dict(first_item) second_item["id"] = "another-item" - another_item_date = first_item["properties"]["datetime"] - timedelta(days=1) + another_item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) - timedelta(days=1) second_item["properties"]["datetime"] = another_item_date.isoformat().replace( "+00:00", "Z" ) @@ -313,7 +319,9 @@ async def test_app_sort_extension_post_desc(app_client, txn_client, ctx): second_item = dict(first_item) second_item["id"] = "another-item" - another_item_date = first_item["properties"]["datetime"] - timedelta(days=1) + another_item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) - timedelta(days=1) second_item["properties"]["datetime"] = another_item_date.isoformat().replace( "+00:00", "Z" ) From 705bcb14ed8112bae2db312a85d025297c0b8840 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Thu, 23 May 2024 15:47:27 +0100 Subject: [PATCH 13/19] Fix for datetime format. --- stac_fastapi/core/stac_fastapi/core/core.py | 2 +- stac_fastapi/tests/api/test_api.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 1d08a583..29409d5c 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -707,7 +707,7 @@ async def update_item( """ item = item.model_dump(mode="json") base_url = str(kwargs["request"].base_url) - now = datetime_type.now(timezone.utc).isoformat().replace("+00:00", "Z") + now = datetime_type.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") item["properties"]["updated"] = now await self.database.check_collection_exists(collection_id) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index e3955c0c..d33df2ee 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -254,8 +254,8 @@ async def test_app_sort_extension_get_asc(app_client, txn_client, ctx): another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" ) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) await create_item(txn_client, second_item) @@ -276,9 +276,10 @@ async def test_app_sort_extension_get_desc(app_client, txn_client, ctx): another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" ) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) + await create_item(txn_client, second_item) resp = await app_client.get("/search?sortby=-properties.datetime") @@ -297,9 +298,10 @@ async def test_app_sort_extension_post_asc(app_client, txn_client, ctx): another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" ) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) + await create_item(txn_client, second_item) params = { @@ -322,8 +324,8 @@ async def test_app_sort_extension_post_desc(app_client, txn_client, ctx): another_item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" ) - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.isoformat().replace( - "+00:00", "Z" + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" ) await create_item(txn_client, second_item) From d15173e9132fea5401adb89de5d40b31adc1287c Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 24 May 2024 10:09:45 +0100 Subject: [PATCH 14/19] Pinning specific version of stac pydantic. --- stac_fastapi/core/setup.py | 2 +- stac_fastapi/opensearch/setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 6cf00d97..a69650c0 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -9,10 +9,10 @@ "fastapi-slim", "attrs>=23.2.0", "pydantic[dotenv]", - "stac_pydantic>=3.1.0", "stac-fastapi.types==3.0.0a", "stac-fastapi.api==3.0.0a", "stac-fastapi.extensions==3.0.0a", + "stac_pydantic @ git+ssh://git@github.com/stac-utils/stac-pydantic@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", "orjson", "overrides", "geojson-pydantic", diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 7c5cc6f0..cae0e5b1 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -11,6 +11,7 @@ "opensearch-py[async]==2.4.2", "uvicorn", "starlette", + "stac_pydantic @ git+ssh://git@github.com/stac-utils/stac-pydantic@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", ] extra_reqs = { From 4f31ac74e742342684d152d5b9aee738b00fe875 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 24 May 2024 10:15:29 +0100 Subject: [PATCH 15/19] Adding missing .git. --- stac_fastapi/core/setup.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index a69650c0..9def7484 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -12,7 +12,7 @@ "stac-fastapi.types==3.0.0a", "stac-fastapi.api==3.0.0a", "stac-fastapi.extensions==3.0.0a", - "stac_pydantic @ git+ssh://git@github.com/stac-utils/stac-pydantic@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", + "stac_pydantic @ git+ssh://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", "orjson", "overrides", "geojson-pydantic", diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index cae0e5b1..b8e3b1ba 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -11,7 +11,7 @@ "opensearch-py[async]==2.4.2", "uvicorn", "starlette", - "stac_pydantic @ git+ssh://git@github.com/stac-utils/stac-pydantic@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", + "stac_pydantic @ git+https://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", ] extra_reqs = { From 29ea9ba81bc260343c27419f0092486969446f7a Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 24 May 2024 10:23:38 +0100 Subject: [PATCH 16/19] Switch to https. --- stac_fastapi/core/setup.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 9def7484..5cce1854 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -12,7 +12,7 @@ "stac-fastapi.types==3.0.0a", "stac-fastapi.api==3.0.0a", "stac-fastapi.extensions==3.0.0a", - "stac_pydantic @ git+ssh://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", + "stac-pydantic @ git+https://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", "orjson", "overrides", "geojson-pydantic", diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index b8e3b1ba..7525a096 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -11,7 +11,7 @@ "opensearch-py[async]==2.4.2", "uvicorn", "starlette", - "stac_pydantic @ git+https://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", + "stac-pydantic @ git+https://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", ] extra_reqs = { From 44fd039ff96a989483913b3165858175123701a4 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 24 May 2024 11:16:17 +0100 Subject: [PATCH 17/19] Removing stac pydantic requirements. --- stac_fastapi/core/setup.py | 1 - stac_fastapi/opensearch/setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 5cce1854..a059dd11 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -12,7 +12,6 @@ "stac-fastapi.types==3.0.0a", "stac-fastapi.api==3.0.0a", "stac-fastapi.extensions==3.0.0a", - "stac-pydantic @ git+https://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", "orjson", "overrides", "geojson-pydantic", diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 7525a096..7c5cc6f0 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -11,7 +11,6 @@ "opensearch-py[async]==2.4.2", "uvicorn", "starlette", - "stac-pydantic @ git+https://git@github.com/stac-utils/stac-pydantic.git@1417932609410b40ece93db9ff6551da4a17d104#egg=stac_pydantic", ] extra_reqs = { From d003db164cc07f9b0e5d4ba94d8f2a0f9455ad30 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 3 Jun 2024 08:46:43 +0100 Subject: [PATCH 18/19] Using deepcopy for multiple item creation. --- stac_fastapi/tests/api/test_api.py | 31 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index d33df2ee..b4b4839c 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1,4 +1,5 @@ import uuid +from copy import deepcopy from datetime import datetime, timedelta import pytest @@ -407,15 +408,15 @@ async def test_search_point_does_not_intersect(app_client, ctx): @pytest.mark.asyncio async def test_datetime_response_format(app_client, txn_client, ctx): - first_item = ctx.item + first_item = dict(ctx.item) - second_item = dict(first_item) + second_item = deepcopy(first_item) second_item["id"] = "second-item" second_item["properties"]["datetime"] = None await create_item(txn_client, second_item) - third_item = dict(first_item) + third_item = deepcopy(first_item) third_item["id"] = "third-item" del third_item["properties"]["start_datetime"] del third_item["properties"]["end_datetime"] @@ -444,15 +445,15 @@ async def test_datetime_response_format(app_client, txn_client, ctx): @pytest.mark.asyncio async def test_datetime_non_interval(app_client, txn_client, ctx): - first_item = ctx.item + first_item = dict(ctx.item) - second_item = dict(first_item) + second_item = deepcopy(first_item) second_item["id"] = "second-item" second_item["properties"]["datetime"] = None await create_item(txn_client, second_item) - third_item = dict(first_item) + third_item = deepcopy(first_item) third_item["id"] = "third-item" del third_item["properties"]["start_datetime"] del third_item["properties"]["end_datetime"] @@ -480,15 +481,15 @@ async def test_datetime_non_interval(app_client, txn_client, ctx): @pytest.mark.asyncio async def test_datetime_interval(app_client, txn_client, ctx): - first_item = ctx.item + first_item = dict(ctx.item) - second_item = dict(first_item) + second_item = deepcopy(first_item) second_item["id"] = "second-item" second_item["properties"]["datetime"] = None await create_item(txn_client, second_item) - third_item = dict(first_item) + third_item = deepcopy(first_item) third_item["id"] = "third-item" del third_item["properties"]["start_datetime"] del third_item["properties"]["end_datetime"] @@ -516,15 +517,15 @@ async def test_datetime_interval(app_client, txn_client, ctx): @pytest.mark.asyncio async def test_datetime_bad_non_interval(app_client, txn_client, ctx): - first_item = ctx.item + first_item = dict(ctx.item) - second_item = dict(first_item) + second_item = deepcopy(first_item) second_item["id"] = "second-item" second_item["properties"]["datetime"] = None await create_item(txn_client, second_item) - third_item = dict(first_item) + third_item = deepcopy(first_item) third_item["id"] = "third-item" del third_item["properties"]["start_datetime"] del third_item["properties"]["end_datetime"] @@ -552,15 +553,15 @@ async def test_datetime_bad_non_interval(app_client, txn_client, ctx): @pytest.mark.asyncio async def test_datetime_bad_interval(app_client, txn_client, ctx): - first_item = ctx.item + first_item = dict(ctx.item) - second_item = dict(first_item) + second_item = deepcopy(first_item) second_item["id"] = "second-item" second_item["properties"]["datetime"] = None await create_item(txn_client, second_item) - third_item = dict(first_item) + third_item = deepcopy(first_item) third_item["id"] = "third-item" del third_item["properties"]["start_datetime"] del third_item["properties"]["end_datetime"] From 0d89769155ba4c4206375a0c30e36ff5045115e7 Mon Sep 17 00:00:00 2001 From: rhysrevans3 <34507919+rhysrevans3@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:56:40 +0100 Subject: [PATCH 19/19] Fixing dt format error. --- stac_fastapi/tests/api/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index b4b4839c..0c3109c5 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -534,8 +534,8 @@ async def test_datetime_bad_non_interval(app_client, txn_client, ctx): dt_formats = [ "2020-02-06T12:30:22+00:00", - "2020-02-06:30:22.00Z", - "2020-02-06:30:22Z", + "2020-02-06T12:30:22.00Z", + "2020-02-06T12:30:22Z", "2020-02-06T12:30:22.00+00:00", ]