diff --git a/integration-tests/fetch-corpora.py b/integration-tests/fetch-corpora.py index 9aafe41..4e44b75 100755 --- a/integration-tests/fetch-corpora.py +++ b/integration-tests/fetch-corpora.py @@ -30,6 +30,7 @@ from typing import Final, NamedTuple from httpx import AsyncClient, HTTPError +from tenacity import retry, wait_random_exponential import delb @@ -243,6 +244,7 @@ class Archive(NamedTuple): http_client: Final = AsyncClient() +@retry(wait=wait_random_exponential(multiplier=1, max=120)) async def fetch_resource(url: str, destination: io.BufferedWriter) -> bool: async with http_client.stream("GET", url, follow_redirects=True) as response: try: diff --git a/pyproject.toml b/pyproject.toml index 730cbb2..56e925e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,7 @@ coverage-report = """ [tool.hatch.envs.integration-tests] dependencies = [ "pytest-httpx", + "tenacity", "tqdm", ] [tool.hatch.envs.integration-tests.scripts]