integration-tests: Handles network issues when fetching corpora

delb-xml · May 12, 2024 · 69038de · 69038de
1 parent 2239b76
commit 69038de
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/integration-tests/fetch-corpora.py b/integration-tests/fetch-corpora.py
@@ -30,6 +30,7 @@
 from typing import Final, NamedTuple
 
 from httpx import AsyncClient, HTTPError
+from tenacity import retry, wait_random_exponential
 
 import delb
 
@@ -243,6 +244,7 @@ class Archive(NamedTuple):
 http_client: Final = AsyncClient()
 
 
+@retry(wait=wait_random_exponential(multiplier=1, max=120))
 async def fetch_resource(url: str, destination: io.BufferedWriter) -> bool:
     async with http_client.stream("GET", url, follow_redirects=True) as response:
         try:

diff --git a/pyproject.toml b/pyproject.toml
@@ -196,6 +196,7 @@ coverage-report = """
 [tool.hatch.envs.integration-tests]
 dependencies = [
     "pytest-httpx",
+    "tenacity",
     "tqdm",
 ]
 [tool.hatch.envs.integration-tests.scripts]