From f657ac0086edb1d3483b72d1948e3fa1201d39ff Mon Sep 17 00:00:00 2001 From: ugyballoons Date: Wed, 10 Jul 2024 17:06:21 +0100 Subject: [PATCH 1/5] First pass creating all page tests --- tests/handlers/external_test.py | 64 ++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/tests/handlers/external_test.py b/tests/handlers/external_test.py index af8dd79c..004908a2 100644 --- a/tests/handlers/external_test.py +++ b/tests/handlers/external_test.py @@ -10,13 +10,17 @@ from fastapi import FastAPI from httpx import AsyncClient from lsst.ts.rubintv.background.currentpoller import CurrentPoller -from lsst.ts.rubintv.models.models import Location +from lsst.ts.rubintv.background.historicaldata import HistoricalPoller +from lsst.ts.rubintv.config import config +from lsst.ts.rubintv.models.models import Camera, Location, get_current_day_obs from lsst.ts.rubintv.models.models_helpers import find_first from lsst.ts.rubintv.models.models_init import ModelsInitiator from ..mockdata import RubinDataMocker m = ModelsInitiator() +app_name = config.name +day_obs = get_current_day_obs().isoformat() @pytest.mark.asyncio @@ -25,7 +29,7 @@ async def test_get_home( ) -> None: """Test that home page has links to every location""" client, app, mocker = mocked_client - response = await client.get("/rubintv/") + response = await client.get(f"/{app_name}/") html = await response.aread() parsed = BeautifulSoup(html, "html.parser") locations = m.locations @@ -51,7 +55,7 @@ async def test_get_location( groups = location.camera_groups.values() camera_names = list(chain(*groups)) - response = await client.get(f"/rubintv/{location_name}") + response = await client.get(f"/{app_name}/{location_name}") html = await response.aread() parsed = BeautifulSoup(html, "html.parser") @@ -78,11 +82,12 @@ async def test_current_channels( loc_cam = f"{location.name}/{camera.name}" for seq_chan in camera.seq_channels(): url = ( - f"/rubintv/{location.name}/{camera.name}" + f"/{app_name}/{location.name}/{camera.name}" f"/current/{seq_chan.name}" ) response = await client.get(url) - assert response.status_code == 200 + assert response.is_success + html = await response.aread() parsed = BeautifulSoup(html, "html.parser") if mocker.empty_channel[loc_cam] == seq_chan.name: @@ -91,3 +96,52 @@ async def test_current_channels( else: assert parsed.select(".event-info") assert not parsed.select(".event-error") + + +@pytest.mark.asyncio +async def test_all_endpoints( + mocked_client: tuple[AsyncClient, FastAPI, RubinDataMocker] +) -> None: + + client, app, mocker = mocked_client + + locations = [loc.name for loc in m.locations] + summit: Location = find_first(m.locations, "name", "summit-usdf") + assert isinstance(summit, Location) + + cameras = [f"{summit.name}/{cam.name}" for cam in summit.cameras if cam.online] + camera_historical = [f"{cam}/historical" for cam in cameras] + camera_dates = [f"{cam}/date/{day_obs}" for cam in cameras if cam] + night_reports = [f"{cam}/night_report" for cam in cameras] + night_reports = [f"{cam}/night_report/{day_obs}" for cam in cameras] + + auxtel: Camera = find_first(summit.cameras, "name", "auxtel") + assert isinstance(auxtel, Camera) + + channels = [ + f"{summit.name}/{auxtel.name}/current/{chan.name}" + for chan in auxtel.seq_channels() + ] + + page_rel_urls = [ + "admin", + *locations, + *cameras, + *camera_dates, + *camera_historical, + *night_reports, + *channels, + ] + + for url_frag in page_rel_urls: + url = f"/{app_name}/{url_frag}" + print(f"Looking for: {url}") + res = await client.get(url) + assert res.is_success + + hp: HistoricalPoller = app.state.historical + while hp._have_downloaded is False: + await asyncio.sleep(0.1) + + # make sure we've reached here! + assert True From 4edaeb42b623e7e8834a15959b3143b4f3b74861 Mon Sep 17 00:00:00 2001 From: ugyballoons Date: Thu, 18 Jul 2024 20:10:11 +0100 Subject: [PATCH 2/5] Refactor date validation --- python/lsst/ts/rubintv/handlers/api.py | 16 +++++++--------- .../lsst/ts/rubintv/handlers/handlers_helpers.py | 9 +++++++++ python/lsst/ts/rubintv/handlers/pages.py | 16 +++++++--------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/python/lsst/ts/rubintv/handlers/api.py b/python/lsst/ts/rubintv/handlers/api.py index 163f354c..25a52989 100644 --- a/python/lsst/ts/rubintv/handlers/api.py +++ b/python/lsst/ts/rubintv/handlers/api.py @@ -7,12 +7,13 @@ from lsst.ts.rubintv.background.historicaldata import HistoricalPoller from lsst.ts.rubintv.config import rubintv_logger from lsst.ts.rubintv.handlers.handlers_helpers import ( + date_validation, get_camera_current_data, get_camera_events_for_date, get_current_night_report_payload, ) from lsst.ts.rubintv.models.models import Camera, Event, Location, NightReport -from lsst.ts.rubintv.models.models_helpers import date_str_to_date, find_first +from lsst.ts.rubintv.models.models_helpers import find_first api_router = APIRouter() """FastAPI router for all external handlers.""" @@ -109,10 +110,9 @@ async def get_camera_events_for_date_api( location_name: str, camera_name: str, date_str: str, request: Request ) -> dict: location, camera = await get_location_camera(location_name, camera_name, request) - try: - day_obs = date_str_to_date(date_str) - except ValueError: - raise HTTPException(status_code=404, detail="Invalid date.") + + day_obs = date_validation(date_str) + data = await get_camera_events_for_date(location, camera, day_obs, request) if data: channel_data, per_day, metadata, nr_exists = data @@ -200,10 +200,8 @@ async def get_night_report_for_date( location_name: str, camera_name: str, date_str: str, request: Request ) -> NightReport: location, camera = await get_location_camera(location_name, camera_name, request) - try: - day_obs = date_str_to_date(date_str) - except ValueError: - raise HTTPException(status_code=404, detail="Invalid date.") + + day_obs = date_validation(date_str) historical: HistoricalPoller = request.app.state.historical if await historical.is_busy(): diff --git a/python/lsst/ts/rubintv/handlers/handlers_helpers.py b/python/lsst/ts/rubintv/handlers/handlers_helpers.py index 10defd6d..3a53bee0 100644 --- a/python/lsst/ts/rubintv/handlers/handlers_helpers.py +++ b/python/lsst/ts/rubintv/handlers/handlers_helpers.py @@ -15,6 +15,7 @@ NightReport, get_current_day_obs, ) +from lsst.ts.rubintv.models.models_helpers import date_str_to_date from starlette.requests import HTTPConnection logger = rubintv_logger() @@ -131,3 +132,11 @@ async def get_prev_next_event( raise HTTPException(423, "Historical data is being processed") nxt, prv = await hp.get_next_prev_event(location, camera, event) return {"next": nxt, "prev": prv} + + +def date_validation(date_str: str) -> date: + try: + day_obs = date_str_to_date(date_str) + except ValueError: + raise HTTPException(status_code=404, detail="Invalid date.") + return day_obs diff --git a/python/lsst/ts/rubintv/handlers/pages.py b/python/lsst/ts/rubintv/handlers/pages.py index f1eae3dd..9ec9160f 100644 --- a/python/lsst/ts/rubintv/handlers/pages.py +++ b/python/lsst/ts/rubintv/handlers/pages.py @@ -13,6 +13,7 @@ get_specific_channel_event, ) from lsst.ts.rubintv.handlers.handlers_helpers import ( + date_validation, get_camera_calendar, get_camera_current_data, get_camera_events_for_date, @@ -28,7 +29,7 @@ to_dict, ) from lsst.ts.rubintv.models.models import Channel, Event, Location, NightReport -from lsst.ts.rubintv.models.models_helpers import date_str_to_date, find_first +from lsst.ts.rubintv.models.models_helpers import find_first from lsst.ts.rubintv.templates_init import get_templates __all__ = ["get_home", "pages_router", "templates"] @@ -162,10 +163,9 @@ async def get_camera_for_date_page( location, camera = await get_location_camera(location_name, camera_name, request) if not camera.online: raise HTTPException(404, "Camera not online.") - try: - day_obs = date_str_to_date(date_str) - except ValueError: - raise HTTPException(status_code=404, detail="Invalid date.") + + day_obs = date_validation(date_str) + historical_busy = False nr_exists = False metadata: dict = {} @@ -326,10 +326,8 @@ async def get_historical_night_report_page( request: Request, ) -> Response: location, camera = await get_location_camera(location_name, camera_name, request) - try: - day_obs = date_str_to_date(date_str) - except ValueError: - raise HTTPException(status_code=404, detail="Invalid date.") + + day_obs = date_validation(date_str) night_report: NightReport night_report, historical_busy = await try_historical_call( From 1b6c4c8bbeef2e25dd19bef992facef2736f95ea Mon Sep 17 00:00:00 2001 From: ugyballoons Date: Thu, 18 Jul 2024 20:38:24 +0100 Subject: [PATCH 3/5] Add to external tests --- tests/handlers/external_test.py | 62 ++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/tests/handlers/external_test.py b/tests/handlers/external_test.py index 004908a2..920d4a73 100644 --- a/tests/handlers/external_test.py +++ b/tests/handlers/external_test.py @@ -109,11 +109,12 @@ async def test_all_endpoints( summit: Location = find_first(m.locations, "name", "summit-usdf") assert isinstance(summit, Location) - cameras = [f"{summit.name}/{cam.name}" for cam in summit.cameras if cam.online] - camera_historical = [f"{cam}/historical" for cam in cameras] - camera_dates = [f"{cam}/date/{day_obs}" for cam in cameras if cam] - night_reports = [f"{cam}/night_report" for cam in cameras] - night_reports = [f"{cam}/night_report/{day_obs}" for cam in cameras] + all_cams = [f"{summit.name}/{cam.name}" for cam in summit.cameras] + online_cams = [f"{summit.name}/{cam.name}" for cam in summit.cameras if cam.online] + camera_historical = [f"{cam}/historical" for cam in online_cams] + camera_dates = [f"{cam}/date/{day_obs}" for cam in online_cams if cam] + nr_current = [f"{cam}/night_report" for cam in online_cams] + nr_dates = [f"{cam}/night_report/{day_obs}" for cam in online_cams] auxtel: Camera = find_first(summit.cameras, "name", "auxtel") assert isinstance(auxtel, Camera) @@ -125,23 +126,58 @@ async def test_all_endpoints( page_rel_urls = [ "admin", + *all_cams, *locations, - *cameras, *camera_dates, *camera_historical, - *night_reports, + *nr_current, + *nr_dates, *channels, ] + hp: HistoricalPoller = app.state.historical + while hp._have_downloaded is False: + await asyncio.sleep(0.1) + for url_frag in page_rel_urls: url = f"/{app_name}/{url_frag}" - print(f"Looking for: {url}") res = await client.get(url) assert res.is_success - hp: HistoricalPoller = app.state.historical - while hp._have_downloaded is False: - await asyncio.sleep(0.1) - # make sure we've reached here! - assert True +@pytest.mark.asyncio +async def test_request_invalid_dates( + mocked_client: tuple[AsyncClient, FastAPI, RubinDataMocker] +) -> None: + + client, app, mocker = mocked_client + + summit: Location = find_first(m.locations, "name", "summit-usdf") + assert isinstance(summit, Location) + + online_cams = [f"{summit.name}/{cam.name}" for cam in summit.cameras if cam.online] + cam_date_words = [f"{cam}/date/not-a-date" for cam in online_cams if cam] + cam_invalid_day = [f"{cam}/date/2000-49-10" for cam in online_cams if cam] + cam_invalid_month = [f"{cam}/date/2000-27-13" for cam in online_cams if cam] + cam_invalid_year = [f"{cam}/date/20001-27-12" for cam in online_cams if cam] + cam_empty_year = [f"{cam}/date/1969-01-01" for cam in online_cams if cam] + + invalid_urls = [ + *cam_date_words, + *cam_invalid_day, + *cam_invalid_month, + *cam_invalid_year, + ] + + for url_frag in invalid_urls: + url = f"/{app_name}/{url_frag}" + res = await client.get(url) + assert res.is_error + + # TODO: This needs not to be just a success, but a page with a + # 'nothing for this day' message. + # See DM-45327 https://rubinobs.atlassian.net/browse/DM-45327 + for url_frag in cam_empty_year: + url = f"/{app_name}/{url_frag}" + res = await client.get(url) + assert res.is_success From cf2cb0e2b37ec3c75d9f77e06930b353123f1164 Mon Sep 17 00:00:00 2001 From: ugyballoons Date: Thu, 18 Jul 2024 21:39:54 +0100 Subject: [PATCH 4/5] Add homepage to test --- tests/handlers/external_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/handlers/external_test.py b/tests/handlers/external_test.py index 920d4a73..53433cfe 100644 --- a/tests/handlers/external_test.py +++ b/tests/handlers/external_test.py @@ -125,6 +125,7 @@ async def test_all_endpoints( ] page_rel_urls = [ + "", "admin", *all_cams, *locations, From 5aaf5ec8df32f9397b1d0f2837b452d7d9a66a59 Mon Sep 17 00:00:00 2001 From: ugyballoons Date: Tue, 3 Sep 2024 15:55:11 +0100 Subject: [PATCH 5/5] Tidy up as per PR recommendations --- tests/handlers/external_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/handlers/external_test.py b/tests/handlers/external_test.py index 53433cfe..99ca9be9 100644 --- a/tests/handlers/external_test.py +++ b/tests/handlers/external_test.py @@ -112,7 +112,7 @@ async def test_all_endpoints( all_cams = [f"{summit.name}/{cam.name}" for cam in summit.cameras] online_cams = [f"{summit.name}/{cam.name}" for cam in summit.cameras if cam.online] camera_historical = [f"{cam}/historical" for cam in online_cams] - camera_dates = [f"{cam}/date/{day_obs}" for cam in online_cams if cam] + camera_dates = [f"{cam}/date/{day_obs}" for cam in online_cams] nr_current = [f"{cam}/night_report" for cam in online_cams] nr_dates = [f"{cam}/night_report/{day_obs}" for cam in online_cams] @@ -175,9 +175,6 @@ async def test_request_invalid_dates( res = await client.get(url) assert res.is_error - # TODO: This needs not to be just a success, but a page with a - # 'nothing for this day' message. - # See DM-45327 https://rubinobs.atlassian.net/browse/DM-45327 for url_frag in cam_empty_year: url = f"/{app_name}/{url_frag}" res = await client.get(url)