rest_api: changes incremental configuration from transform to convert

sources/rest_api/config_setup.py

@@ -1,3 +1,4 @@
+import warnings
 from copy import copy
 from typing import (
     Type,
@@ -177,7 +178,7 @@ def setup_incremental_object(
         raise ValueError(
             f"Only a single incremental parameter is allower per endpoint. Found: {incremental_params}"
         )
-    transform: Optional[Callable[..., Any]]
+    convert: Optional[Callable[..., Any]]
     for param_name, param_config in request_params.items():
         if isinstance(param_config, dlt.sources.incremental):
             if param_config.end_value is not None:
@@ -190,31 +191,48 @@ def setup_incremental_object(
                 raise ValueError(
                     f"Only start_param and initial_value are allowed in the configuration of param: {param_name}. To set end_value too use the incremental configuration at the resource level. See https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api#incremental-loading"
                 )
-            transform = param_config.get("transform", None)
-            config = exclude_keys(param_config, {"type", "transform"})
+            convert = parse_convert_or_deprecated_transform(param_config)
+
+            config = exclude_keys(param_config, {"type", "convert", "transform"})
             # TODO: implement param type to bind incremental to
             return (
                 dlt.sources.incremental(**config),
                 IncrementalParam(start=param_name, end=None),
-                transform,
+                convert,
             )
     if incremental_config:
-        transform = incremental_config.get("transform", None)
+        convert = parse_convert_or_deprecated_transform(incremental_config)
         config = exclude_keys(
-            incremental_config, {"start_param", "end_param", "transform"}
+            incremental_config, {"start_param", "end_param", "convert", "transform"}
         )
         return (
             dlt.sources.incremental(**config),
             IncrementalParam(
                 start=incremental_config["start_param"],
                 end=incremental_config.get("end_param"),
             ),
-            transform,
+            convert,
         )
 
     return None, None, None
 
 
+def parse_convert_or_deprecated_transform(
+    config: Union[IncrementalConfig, Dict[str, Any]]
+) -> Optional[Callable[..., Any]]:
+    convert = config.get("convert", None)
+    deprecated_transform = config.get("transform", None)
+    if deprecated_transform:
+        warnings.warn(
+            "The key `transform` is deprecated in the incremental configuration and it will be removed. "
+            "Use `convert` instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        convert = deprecated_transform
+    return convert
+
+
 def make_parent_key_name(resource_name: str, field_name: str) -> str:
     return f"_{resource_name}_{field_name}"
 

sources/rest_api/typing.py

@@ -182,7 +182,7 @@ class IncrementalArgs(TypedDict, total=False):
     primary_key: Optional[TTableHintTemplate[TColumnNames]]
     end_value: Optional[str]
     row_order: Optional[TSortOrder]
-    transform: Optional[Callable[..., Any]]
+    convert: Optional[Callable[..., Any]]
 
 
 class IncrementalConfig(IncrementalArgs, total=False):

tests/rest_api/test_configurations.py

@@ -740,7 +740,7 @@ def test_constructs_incremental_from_request_param_with_incremental_object(
     assert incremental_with_init == incremental_obj
 
 
-def test_constructs_incremental_from_request_param_with_transform(
+def test_constructs_incremental_from_request_param_with_convert(
     incremental_with_init,
 ) -> None:
     def epoch_to_datetime(epoch: str):
@@ -751,15 +751,15 @@ def epoch_to_datetime(epoch: str):
             "type": "incremental",
             "cursor_path": "updated_at",
             "initial_value": "2024-01-01T00:00:00Z",
-            "transform": epoch_to_datetime,
+            "convert": epoch_to_datetime,
         }
     }
 
-    (incremental_obj, incremental_param, transform) = setup_incremental_object(
+    (incremental_obj, incremental_param, convert) = setup_incremental_object(
         param_config, None
     )
     assert incremental_param == IncrementalParam(start="since", end=None)
-    assert transform == epoch_to_datetime
+    assert convert == epoch_to_datetime
 
     assert incremental_with_init == incremental_obj
 
@@ -830,7 +830,7 @@ def test_constructs_incremental_from_endpoint_config_incremental(
     assert incremental_with_init == incremental_obj
 
 
-def test_constructs_incremental_from_endpoint_config_incremental_with_transform(
+def test_constructs_incremental_from_endpoint_config_incremental_with_convert(
     incremental_with_init_and_end,
 ) -> None:
     def epoch_to_datetime(epoch):
@@ -842,18 +842,18 @@ def epoch_to_datetime(epoch):
         "cursor_path": "updated_at",
         "initial_value": "2024-01-01T00:00:00Z",
         "end_value": "2024-06-30T00:00:00Z",
-        "transform": epoch_to_datetime,
+        "convert": epoch_to_datetime,
     }
 
-    (incremental_obj, incremental_param, transform) = setup_incremental_object(
+    (incremental_obj, incremental_param, convert) = setup_incremental_object(
         {}, resource_config_incremental
     )
     assert incremental_param == IncrementalParam(start="since", end="until")
-    assert transform == epoch_to_datetime
+    assert convert == epoch_to_datetime
     assert incremental_with_init_and_end == incremental_obj
 
 
-def test_calls_transform_from_endpoint_config_incremental(mocker) -> None:
+def test_calls_convert_from_endpoint_config_incremental(mocker) -> None:
     def epoch_to_date(epoch: str):
         return pendulum.from_timestamp(int(epoch)).to_date_string()
 
@@ -869,7 +869,7 @@ def epoch_to_date(epoch: str):
     assert callback.call_args_list[0].args == ("1",)
 
 
-def test_calls_transform_from_request_param(mocker) -> None:
+def test_calls_convert_from_request_param(mocker) -> None:
     def epoch_to_datetime(epoch: str):
         return pendulum.from_timestamp(int(epoch)).to_date_string()
 
@@ -882,7 +882,7 @@ def epoch_to_datetime(epoch: str):
         "cursor_path": "updated_at",
         "initial_value": str(start),
         "end_value": str(one_day_later),
-        "transform": callback,
+        "convert": callback,
     }
 
     (incremental_obj, incremental_param, _) = setup_incremental_object(
@@ -898,7 +898,7 @@ def epoch_to_datetime(epoch: str):
     assert callback.call_args_list[1].args == (str(one_day_later),)
 
 
-def test_default_transform_is_identity(mocker) -> None:
+def test_default_convert_is_identity() -> None:
     start = 1
     one_day_later = 60 * 60 * 24
     incremental_config: IncrementalConfig = {
@@ -920,6 +920,59 @@ def test_default_transform_is_identity(mocker) -> None:
     assert created_param == {"since": str(start), "until": str(one_day_later)}
 
 
+def test_incremental_param_transform_is_deprecated(incremental_with_init) -> None:
+    """Tests that deprecated interface works but issues deprecation warning"""
+
+    def epoch_to_datetime(epoch: str):
+        return pendulum.from_timestamp(int(epoch))
+
+    param_config = {
+        "since": {
+            "type": "incremental",
+            "cursor_path": "updated_at",
+            "initial_value": "2024-01-01T00:00:00Z",
+            "transform": epoch_to_datetime,
+        }
+    }
+
+    with pytest.deprecated_call():
+        (incremental_obj, incremental_param, convert) = setup_incremental_object(
+            param_config, None
+        )
+
+        assert incremental_param == IncrementalParam(start="since", end=None)
+        assert convert == epoch_to_datetime
+
+        assert incremental_with_init == incremental_obj
+
+
+def test_incremental_endpoint_config_transform_is_deprecated(
+    mocker,
+    incremental_with_init_and_end,
+) -> None:
+    """Tests that deprecated interface works but issues deprecation warning"""
+
+    def epoch_to_datetime(epoch):
+        return pendulum.from_timestamp(int(epoch))
+
+    resource_config_incremental: IncrementalConfig = {
+        "start_param": "since",
+        "end_param": "until",
+        "cursor_path": "updated_at",
+        "initial_value": "2024-01-01T00:00:00Z",
+        "end_value": "2024-06-30T00:00:00Z",
+        "transform": epoch_to_datetime,
+    }
+
+    with pytest.deprecated_call():
+        (incremental_obj, incremental_param, convert) = setup_incremental_object(
+            {}, resource_config_incremental
+        )
+        assert incremental_param == IncrementalParam(start="since", end="until")
+        assert convert == epoch_to_datetime
+        assert incremental_with_init_and_end == incremental_obj
+
+
 def test_resource_hints_are_passed_to_resource_constructor() -> None:
     config: RESTAPIConfig = {
         "client": {"base_url": "https://api.example.com"},

tests/rest_api/test_rest_api_source_offline.py

@@ -433,7 +433,7 @@ def add_field(response: Response, *args, **kwargs) -> Response:
     assert all(record["custom_field"] == "foobar" for record in data)
 
 
-def test_posts_with_inremental_date_transformation(mock_api_server) -> None:
+def test_posts_with_inremental_date_conversion(mock_api_server) -> None:
     start_time = pendulum.from_timestamp(1)
     one_day_later = start_time.add(days=1)
     config: RESTAPIConfig = {
@@ -449,7 +449,7 @@ def test_posts_with_inremental_date_transformation(mock_api_server) -> None:
                         "cursor_path": "updated_at",
                         "initial_value": str(start_time.int_timestamp),
                         "end_value": str(one_day_later.int_timestamp),
-                        "transform": lambda epoch: pendulum.from_timestamp(
+                        "convert": lambda epoch: pendulum.from_timestamp(
                             int(epoch)
                         ).to_date_string(),
                     },