Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rest_api paginator: Allows using json_response in place of json_link #533

Merged
merged 2 commits into from
Jul 24, 2024
Merged

rest_api paginator: Allows using json_response in place of json_link #533

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f861c08
rest_api paginator: Allows using json_response in place of json_link
willi-mueller Jul 24, 2024
ac6be03
renames test for clarity
willi-mueller Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion sources/rest_api/config_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
try:
from dlt.sources.helpers.rest_client.paginators import JSONLinkPaginator
except ImportError:
from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator # type: ignore
from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator # type: ignore

from dlt.sources.helpers.rest_client.detector import single_entity_path
from dlt.sources.helpers.rest_client.exceptions import IgnoreResponseException
Expand Down Expand Up @@ -66,6 +66,7 @@

PAGINATOR_MAP: Dict[PaginatorType, Type[BasePaginator]] = {
"json_link": JSONLinkPaginator,
"json_response": JSONLinkPaginator, # deprecated. Use json_link instead. Will be removed in upcoming release
"header_link": HeaderLinkPaginator,
"auto": None,
"single_page": SinglePagePaginator,
Expand Down
3 changes: 2 additions & 1 deletion sources/rest_api/typing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
try:
from dlt.sources.helpers.rest_client.paginators import JSONLinkPaginator
except ImportError:
from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator # type: ignore
from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator # type: ignore

from dlt.sources.helpers.rest_client.auth import (
AuthConfigBase,
Expand All @@ -49,6 +49,7 @@

PaginatorType = Literal[
"json_link",
"json_response", # deprecated. Use json_link instead. Will be removed in upcoming release
"header_link",
"auto",
"single_page",
Expand Down
28 changes: 27 additions & 1 deletion tests/rest_api/test_configurations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@
try:
from dlt.sources.helpers.rest_client.paginators import JSONLinkPaginator
except ImportError:
from dlt.sources.helpers.rest_client.paginators import JSONResponsePaginator as JSONLinkPaginator
from dlt.sources.helpers.rest_client.paginators import (
JSONResponsePaginator as JSONLinkPaginator,
)


from dlt.sources.helpers.rest_client.auth import (
Expand Down Expand Up @@ -162,6 +164,30 @@ def test_page_number_paginator_creation() -> None:
pytest.fail("DictValidationException was unexpectedly raised")


def test_json_link_paginator(mock_api_server) -> None:
"""
Delete this test as soon as we stop supporting the deprecated key json_response
for the JSONLinkPaginator
"""
config: RESTAPIConfig = { # type: ignore
"client": {"base_url": "https://api.example.com"},
"resources": [
{
"name": "posts",
"endpoint": {
"path": "posts",
"paginator": {
"type": "json_response",
"next_url_path": "links.next",
},
},
},
],
}

rest_api_source(config)


@pytest.mark.parametrize("auth_type", get_args(AuthType))
@pytest.mark.parametrize(
"section", ("SOURCES__REST_API__CREDENTIALS", "SOURCES__CREDENTIALS", "CREDENTIALS")
Expand Down
Loading