Fix/deal with no leaderboard

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
   pytest:
     strategy:
       matrix:
-        os: [windows-latest, macos-latest-xlarge, ubuntu-latest]
+        os: [macos-latest-xlarge, ubuntu-latest]
         python-version: ["3.11"]
     runs-on: ${{ matrix.os }}
     steps:

README.md

@@ -9,7 +9,7 @@ ______________________________________________________________________
 [![Documentation](https://img.shields.io/badge/docs-passing-green)](https://alexandrainst.github.io/alexandra_ai_eval/alexandra_ai_eval.html)
 [![License](https://img.shields.io/github/license/alexandrainst/alexandra_ai_eval)](https://github.com/alexandrainst/alexandra_ai_eval/blob/main/LICENSE)
 [![LastCommit](https://img.shields.io/github/last-commit/alexandrainst/alexandra_ai_eval)](https://github.com/alexandrainst/alexandra_ai_eval/commits/main)
-[![Code Coverage](https://img.shields.io/badge/Coverage-75%25-yellowgreen.svg)](https://github.com/alexandrainst/alexandra_ai_eval/tree/main/tests)
+[![Code Coverage](https://img.shields.io/badge/Coverage-74%25-yellow.svg)](https://github.com/alexandrainst/alexandra_ai_eval/tree/main/tests)
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.0-4baaaa.svg)](https://github.com/alexandrainst/alexandra_ai_eval/blob/main/CODE_OF_CONDUCT.md)
 
 

src/alexandra_ai_eval/cli.py

@@ -68,11 +68,16 @@
 )
 @click.option(
     "--no-save-results",
-    "-ns",
     is_flag=True,
     show_default=True,
     help="Whether results should not be stored to disk.",
 )
+@click.option(
+    "--no-send-results-to-leaderboard",
+    is_flag=True,
+    show_default=True,
+    help="Whether results should not be sent to the leaderboard.",
+)
 @click.option(
     "--raise-error-on-invalid-model",
     "-r",
@@ -126,6 +131,7 @@ def evaluate(
     country_code: str,
     no_progress_bar: bool,
     no_save_results: bool,
+    no_send_results_to_leaderboard: bool,
     raise_error_on_invalid_model: bool,
     cache_dir: str,
     prefer_device: str,
@@ -155,6 +161,7 @@ def evaluate(
     evaluator = Evaluator(
         progress_bar=(not no_progress_bar),
         save_results=(not no_save_results),
+        send_results_to_leaderboard=(not no_send_results_to_leaderboard),
         raise_error_on_invalid_model=raise_error_on_invalid_model,
         cache_dir=cache_dir,
         token=auth,

src/alexandra_ai_eval/evaluator.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 import pandas as pd
-from requests.exceptions import ConnectionError, RequestException
+from requests.exceptions import ConnectionError
 from tabulate import tabulate
 
 from .config import EvaluationConfig, TaskConfig
@@ -131,9 +131,7 @@ def __init__(
         self.send_results_to_leaderboard = send_results_to_leaderboard
         self.leaderboard_url = leaderboard_url
         self.leaderboard_client = (
-            Session(
-                base_url=self.leaderboard_url,
-            )
+            Session(base_url=self.leaderboard_url, num_retries=1)
             if self.send_results_to_leaderboard
             else None
         )
@@ -210,7 +208,7 @@ def evaluate(
                 if all(self._send_results_to_leaderboard()):
                     logger.info("Successfully sent results to leaderboard.")
                 else:
-                    raise RequestException("Failed to send result(s) to leaderboard.")
+                    logger.warning("Failed to send results to leaderboard.")
 
             return self.evaluation_results
 
@@ -322,7 +320,10 @@ def _send_results_to_leaderboard(self) -> list[bool]:
         )
         assert self.leaderboard_client is not None, error_if_client_is_none
 
-        self.leaderboard_client.check_connection()
+        try:
+            self.leaderboard_client.check_connection(timeout=2)
+        except ConnectionError:
+            return [False]
 
         status: list[bool] = list()
 

src/alexandra_ai_eval/leaderboard_utils.py

@@ -5,18 +5,26 @@
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
+import time
+import sys
 
 from .task_configs import get_all_task_configs
 
 
 class Session(requests.Session):
-    """A requests session that automatically adds the API key to the headers."""
+    """A requests session that automatically adds the API key to the headers.
 
-    def __init__(self, base_url: str):
+    Args:
+        base_url:
+            The base url of the leaderboard.
+        num_retries:
+            The number of retries to make if the connection fails.
+    """
+
+    def __init__(self, base_url: str, num_retries: int):
         super().__init__()
         self.base_url = base_url
-        retry = Retry(connect=3, backoff_factor=0.5)
-        adapter = HTTPAdapter(max_retries=retry)
+        adapter = HTTPAdapter(max_retries=Retry(total=num_retries))
         self.mount("http://", adapter)
         self.mount("https://", adapter)
 
@@ -170,17 +178,34 @@ def post_model_to_task(
         response.close()
         return response_json
 
-    def check_connection(self, timeout: int = 5):
+    def check_connection(self, timeout: int):
         """Checks whether we can establish a connection to the leaderboard.
 
         Args:
             timeout:
-                Timeout in seconds. Defaults to 5.
+                Timeout in seconds.
 
         Raises:
             requests.exceptions.ConnectionError: Failed to establish a connection.
             requests.exceptions.ReadTimeout: Connection timed out after 5 seconds.
             requests.exceptions.HTTPError: Non 200 response.
         """
-        response = self.get(self.base_url, timeout=timeout)
-        response.raise_for_status()
+
+        def trace_function(frame, event, arg):
+            """Trace function that raises TimeoutError if the connection times out."""
+            if time.time() - start > timeout:
+                raise TimeoutError("Connection to leaderboard timed out.")
+            return trace_function
+
+        # We need to set a trace function to raise a TimeoutError if the connection
+        # times out. See https://stackoverflow.com/a/71453648/3154226
+        start = time.time()
+        sys.settrace(trace_function)
+
+        try:
+            response = self.get(url=self.base_url, timeout=timeout)
+            response.raise_for_status()
+        except Exception as e:
+            raise e
+        finally:
+            sys.settrace(None)

src/alexandra_ai_eval/utils.py

@@ -137,6 +137,7 @@ def block_terminal_output():
     logging.getLogger("absl").setLevel(logging.ERROR)
     logging.getLogger("datasets").setLevel(logging.ERROR)
     logging.getLogger("codecarbon").setLevel(logging.ERROR)
+    logging.getLogger("urllib3").setLevel(logging.ERROR)
 
     # Disable `wasabi` logging, used in spaCy
     wasabi_msg.no_print = True

tests/test_cli.py

@@ -22,6 +22,7 @@ def test_cli_param_names(params):
         "country_code",
         "no_progress_bar",
         "no_save_results",
+        "no_send_results_to_leaderboard",
         "raise_error_on_invalid_model",
         "cache_dir",
         "prefer_device",
@@ -41,6 +42,7 @@ def test_cli_param_types(params):
     assert isinstance(params["country_code"], Choice)
     assert params["no_progress_bar"] == BOOL
     assert params["no_save_results"] == BOOL
+    assert params["no_send_results_to_leaderboard"] == BOOL
     assert params["raise_error_on_invalid_model"] == BOOL
     assert params["cache_dir"] == STRING
     assert isinstance(params["prefer_device"], Choice)