From f48289dd0c79de01a967fb3ccbd7d39f05ea656f Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Thu, 5 Oct 2023 11:39:09 +0200 Subject: [PATCH 1/5] use Elasticsearch aliases with number suffix in index name --- .../elasticsearch/database_logic.py | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6a7a1f27..4c0142c2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -25,7 +25,7 @@ COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") -DEFAULT_INDICES = f"*,-*kibana*,-{COLLECTIONS_INDEX}" +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*" DEFAULT_SORT = { "properties.datetime": {"order": "desc"}, @@ -150,7 +150,7 @@ def indices(collection_ids: Optional[List[str]]) -> str: A string of comma-separated index names. If `collection_ids` is None, returns the default indices. """ if collection_ids is None: - return DEFAULT_INDICES + return ITEM_INDICES else: return ",".join([f"{ITEMS_INDEX_PREFIX}{c.strip()}" for c in collection_ids]) @@ -164,7 +164,8 @@ async def create_collection_index() -> None: client = AsyncElasticsearchSettings().create_client await client.indices.create( - index=COLLECTIONS_INDEX, + index=f"{COLLECTIONS_INDEX}-000001", + aliases={COLLECTIONS_INDEX: {}}, mappings=ES_COLLECTIONS_MAPPINGS, ignore=400, # ignore 400 already exists code ) @@ -183,9 +184,11 @@ async def create_item_index(collection_id: str): """ client = AsyncElasticsearchSettings().create_client + index_name = index_by_collection_id(collection_id) await client.indices.create( - index=index_by_collection_id(collection_id), + index=f"{index_by_collection_id(collection_id)}-000001", + aliases={index_name: {}}, mappings=ES_ITEMS_MAPPINGS, settings=ES_ITEMS_SETTINGS, ignore=400, # ignore 400 already exists code @@ -201,7 +204,14 @@ async def delete_item_index(collection_id: str): """ client = AsyncElasticsearchSettings().create_client - await client.indices.delete(index=index_by_collection_id(collection_id)) + name = index_by_collection_id(collection_id) + resolved = await client.indices.resolve_index(name=name) + if "aliases" in resolved and resolved["aliases"]: + [alias] = resolved["aliases"] + await client.indices.delete_alias(index=alias["indices"], name=alias["name"]) + await client.indices.delete(index=alias["indices"]) + else: + await client.indices.delete(index=name) await client.close() @@ -759,14 +769,11 @@ async def bulk_async( `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True, the index is refreshed after the bulk insert. The function does not return any value. """ - await asyncio.get_event_loop().run_in_executor( - None, - lambda: helpers.bulk( - self.sync_client, - mk_actions(collection_id, processed_items), - refresh=refresh, - raise_on_error=False, - ), + await helpers.async_bulk( + self.client, + mk_actions(collection_id, processed_items), + refresh=refresh, + raise_on_error=False, ) def bulk_sync( @@ -797,7 +804,7 @@ def bulk_sync( async def delete_items(self) -> None: """Danger. this is only for tests.""" await self.client.delete_by_query( - index=DEFAULT_INDICES, + index=ITEM_INDICES, body={"query": {"match_all": {}}}, wait_for_completion=True, ) From 920f121c4d0e77ff3a721c773ebaa00cc0ec1e17 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Thu, 5 Oct 2023 11:42:51 +0200 Subject: [PATCH 2/5] add changes to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c743ba6..7ab9c443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) ### Changed +- Use aliases on Elasticsearch indices, add number suffix in index name. ### Fixed - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) From 4904f84c08c64e42cdf5f180d342371e485ecdbf Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 17 Oct 2023 10:46:30 +0200 Subject: [PATCH 3/5] re-add exclusion of collections index and internal indices to pattern of item indices in order to ensure backwards compatibility --- .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 22f40fed..126ab652 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -39,7 +39,7 @@ ":", } -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*" +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}" DEFAULT_SORT = { "properties.datetime": {"order": "desc"}, From e1d6232bcc4485176f610ea778dfafdf6de41145 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 17 Oct 2023 10:48:49 +0200 Subject: [PATCH 4/5] use wildcard in exclusion of collections index to support aliases with numbered indices --- .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 126ab652..702225ce 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -39,7 +39,7 @@ ":", } -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}" +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" DEFAULT_SORT = { "properties.datetime": {"order": "desc"}, From eade0b4176aec064b3b116003359e9255785025c Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 7 Nov 2023 14:36:38 +0100 Subject: [PATCH 5/5] add link to PR in CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4a7b84..c99c10f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) ### Changed -- Use aliases on Elasticsearch indices, add number suffix in index name. + +- Use aliases on Elasticsearch indices, add number suffix in index name. [#152](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/152) + ### Fixed - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132)