rest_api paginator: Allows using json_response in place of json_link (#…

…533) * rest_api paginator: Allows using json_response in place of json_link
dlt-hub · Jul 24, 2024 · 123c5be · 123c5be
1 parent ee68404
commit 123c5be
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 3 deletions.
diff --git a/sources/rest_api/config_setup.py b/sources/rest_api/config_setup.py
@@ -37,7 +37,7 @@
 try:
     from dlt.sources.helpers.rest_client.paginators import JSONLinkPaginator
 except ImportError:
-    from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator # type: ignore
+    from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator  # type: ignore
 
 from dlt.sources.helpers.rest_client.detector import single_entity_path
 from dlt.sources.helpers.rest_client.exceptions import IgnoreResponseException
@@ -66,6 +66,7 @@
 
 PAGINATOR_MAP: Dict[PaginatorType, Type[BasePaginator]] = {
     "json_link": JSONLinkPaginator,
+    "json_response": JSONLinkPaginator,  # deprecated. Use json_link instead. Will be removed in upcoming release
     "header_link": HeaderLinkPaginator,
     "auto": None,
     "single_page": SinglePagePaginator,

diff --git a/sources/rest_api/typing.py b/sources/rest_api/typing.py
@@ -38,7 +38,7 @@
 try:
     from dlt.sources.helpers.rest_client.paginators import JSONLinkPaginator
 except ImportError:
-    from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator # type: ignore
+    from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator  # type: ignore
 
 from dlt.sources.helpers.rest_client.auth import (
     AuthConfigBase,
@@ -49,6 +49,7 @@
 
 PaginatorType = Literal[
     "json_link",
+    "json_response",  # deprecated. Use json_link instead. Will be removed in upcoming release
     "header_link",
     "auto",
     "single_page",

diff --git a/tests/rest_api/test_configurations.py b/tests/rest_api/test_configurations.py
@@ -60,7 +60,9 @@
 try:
     from dlt.sources.helpers.rest_client.paginators import JSONLinkPaginator
 except ImportError:
-    from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator
+    from dlt.sources.helpers.rest_client.paginators import (
+        JSONResponsePaginator as JSONLinkPaginator,
+    )
 
 
 from dlt.sources.helpers.rest_client.auth import (
@@ -162,6 +164,30 @@ def test_page_number_paginator_creation() -> None:
         pytest.fail("DictValidationException was unexpectedly raised")
 
 
+def test_allow_deprecated_json_response_paginator(mock_api_server) -> None:
+    """
+    Delete this test as soon as we stop supporting the deprecated key json_response
+    for the JSONLinkPaginator
+    """
+    config: RESTAPIConfig = { # type: ignore
+        "client": {"base_url": "https://api.example.com"},
+        "resources": [
+            {
+                "name": "posts",
+                "endpoint": {
+                    "path": "posts",
+                    "paginator": {
+                        "type": "json_response",
+                        "next_url_path": "links.next",
+                    },
+                },
+            },
+        ],
+    }
+
+    rest_api_source(config)
+
+
 @pytest.mark.parametrize("auth_type", get_args(AuthType))
 @pytest.mark.parametrize(
     "section", ("SOURCES__REST_API__CREDENTIALS", "SOURCES__CREDENTIALS", "CREDENTIALS")