diff --git a/dlt/common/libs/pyarrow.py b/dlt/common/libs/pyarrow.py index 805b43b163..14e31b4603 100644 --- a/dlt/common/libs/pyarrow.py +++ b/dlt/common/libs/pyarrow.py @@ -245,7 +245,12 @@ def should_normalize_arrow_schema( naming: NamingConvention, add_load_id: bool = False, ) -> Tuple[bool, Mapping[str, str], Dict[str, str], Dict[str, bool], bool, TTableSchemaColumns]: + """Figure out if any of the normalization steps must be executed. This prevents + from rewriting arrow tables when no changes are needed. Refer to `normalize_py_arrow_item` + for a list of normalizations. Note that `column` must be already normalized. + """ rename_mapping = get_normalized_arrow_fields_mapping(schema, naming) + # no clashes in rename ensured above rev_mapping = {v: k for k, v in rename_mapping.items()} nullable_mapping = {k: is_nullable_column(v) for k, v in columns.items()} # All fields from arrow schema that have nullable set to different value than in columns @@ -301,7 +306,8 @@ def normalize_py_arrow_item( caps: DestinationCapabilitiesContext, load_id: Optional[str] = None, ) -> TAnyArrowItem: - """Normalize arrow `item` schema according to the `columns`. + """Normalize arrow `item` schema according to the `columns`. Note that + columns must be already normalized. 1. arrow schema field names will be normalized according to `naming` 2. arrows columns will be reordered according to `columns` @@ -366,13 +372,14 @@ def normalize_py_arrow_item( def get_normalized_arrow_fields_mapping(schema: pyarrow.Schema, naming: NamingConvention) -> StrStr: """Normalizes schema field names and returns mapping from original to normalized name. Raises on name collisions""" - norm_f = naming.normalize_identifier + # use normalize_path to be compatible with how regular columns are normalized in dlt.Schema + norm_f = naming.normalize_path name_mapping = {n.name: norm_f(n.name) for n in schema} # verify if names uniquely normalize normalized_names = set(name_mapping.values()) if len(name_mapping) != len(normalized_names): raise NameNormalizationCollision( - f"Arrow schema fields normalized from {list(name_mapping.keys())} to" + f"Arrow schema fields normalized from:\n{list(name_mapping.keys())}:\nto:\n" f" {list(normalized_names)}" ) return name_mapping diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md index 4236d656eb..acc223f54d 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md @@ -198,12 +198,15 @@ def _double_as_decimal_adapter(table: sa.Table) -> None: sql_alchemy_source = sql_database( "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", backend="pyarrow", + backend_kwargs={"tz": "UTC"}, table_adapter_callback=_double_as_decimal_adapter ).with_resources("family", "genome") info = pipeline.run(sql_alchemy_source) print(info) ``` +For more information on the `tz` parameter within `backend_kwargs` supported by PyArrow, please refer to the +[official documentation.](https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html) ### Pandas diff --git a/poetry.lock b/poetry.lock index 8e3c8a2855..f66c41aff6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8714,7 +8714,7 @@ typing-extensions = "*" name = "sqlglot" version = "25.24.5" description = "An easily customizable SQL parser and transpiler" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "sqlglot-25.24.5-py3-none-any.whl", hash = "sha256:f8a8870d1f5cdd2e2dc5c39a5030a0c7b0a91264fb8972caead3dac8e8438873"}, @@ -9861,4 +9861,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "11385c8ff3ce09de74da03658d4e81cdc3bd991556d715d69dbc1e17b54a1d91" +content-hash = "97666ad4613f07d95c5388bae41befe6cc10c88d02ee8f1cee27b161e13729f1" diff --git a/pyproject.toml b/pyproject.toml index 2bd28cdb36..cb5ba4f095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,6 +194,7 @@ pandas = [ {version = ">2.1", markers = "python_version >= '3.12'"}, {version = "<2.1", markers = "python_version < '3.12'"} ] +sqlglot = {version = ">=20.0.0"} [tool.poetry.group.airflow] optional = true diff --git a/tests/pipeline/cases/github_events.json b/tests/pipeline/cases/github_events.json new file mode 100644 index 0000000000..3f807b61ad --- /dev/null +++ b/tests/pipeline/cases/github_events.json @@ -0,0 +1,12582 @@ +[ + { + "id": 14597260375, + "node_id": "SE_lADOGvRYu86YjcEdzwAAAANmEIRX", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597260375", + "actor": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:55:04Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14597260359, + "node_id": "MEE_lADOGvRYu86YjcEdzwAAAANmEIRH", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597260359", + "actor": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:55:04Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14597095810, + "node_id": "CE_lADOGvRYu86VFupkzwAAAANmDgGC", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597095810", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:37:21Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1778", + "id": 2501306980, + "node_id": "I_kwDOGvRYu86VFupk", + "number": 1778, + "title": "1.0.0 announcement and release notes", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6869013926, + "node_id": "LA_kwDOGvRYu88AAAABmWzVpg", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/sprint", + "name": "sprint", + "color": "94B027", + "default": false, + "description": "Marks group of tasks with core team focus at this moment" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-09-02T16:27:23Z", + "updated_at": "2024-10-10T20:37:21Z", + "closed_at": "2024-10-10T20:37:21Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "## Why `1.0`?\r\nWe are releasing `1.0.0` version of `dlt`. In the last 2 years we've got quite stable (in term of our API, internal migrations and major bugs being rare) and feature complete. There so many production deployments that even with our obsessive approach to testing (you can always write more test cases!) we are pretty confident `dlt` is now \"stable\" and ready for production.\r\n\r\n## What is coming if full release\r\n1. We move [sql database](https://dlthub.com/docs/dlt-ecosystem/verified-sources/sql_database) [filesystem/buckets](https://dlthub.com/docs/dlt-ecosystem/verified-sources/filesystem) and [rest api](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api) sources to the core library to make them easily available, stabilize the APIs and run tons of additional tests.\r\n2. Our documentation gets a big update: additional tutorials on syncing the databases, working with buckets and file readers and using rest api toolkit to declare pipelines loading data from REST APIs\r\n\r\nOn top of that we will plan a few quick follow-up features:\r\n1. Define hints for nested tables/resources (currently only root table can be conveniently hinted) dlt-hub/dlt#1647 \r\n2. Define cross-table references dlt-hub/dlt#1713 \r\n3. SQL Alchemy destination is coming with SQLLite and MySQL fully tested (and optimized). You'll be able to bring your own settings to finetune other dialects (#1734 and dlt-hub/dlt#21 )\r\n4. We will finally stabilize dlt traces, expose a core source and a data contract (schema) so loading dlt metadata is easy and predictable\r\n\r\n**Deprecations and Breaking Changes**\r\n1. Load packages with failed jobs (terminally) will be automatically aborted with an exception. Currently user had to detect this in code (this behavior will be still available). https://github.com/dlt-hub/dlt/issues/1749\r\n2. To use `iceberg` table format on Athena destination, set the `table_format` to `iceberg` on all your resources instead of `force_iceberg` flag in destination configuration. This flag is deprecated but will be still observed for backward compatibility.\r\n3. `complex` type is deprecated and superseded by `json` dlt-hub/dlt#1673 \r\n\r\nInternal or obscure changes:\r\n1. A few column hints (`foregin_key` and `index`) that were not documented and have no real use, will be removed.\r\n2. if primary key was used in nested table, linking was not created in relational.py. now linking is skipped when nested row is fitted into table that is not nested (does not have a parent). a rare case of someone that does not want `dlt` linking\r\n4. removes generate_dlt_id from json relational normalizer config\r\n5. deprecates `skip_complex_types` in `dlt` Pydantic config, asks to use `skip_nested_types`\r\n6. when extracting a list of standalone resources, they will be grouped in smallest possible number of source (previously: each resource was extracted in a single source, including transformers, dlt-hub/dlt#1535 \r\n7. secrets (TSecretValue and configs deriving from Credentials) won't be saved to trace dumps dlt-hub/dlt#1687 \r\n\r\n**dlt schema engine migration**\r\nIf you run this version against existing dataset in a destination, schema in `_dlt_version` will be migrated to engine v10. Same applies to local pipeline working dir. You can restore the old schema by deleting the migrated version from the version table.\r\n\r\n### New Versioning Scheme\r\nWe'll follow classical `major.minor.patch` scheme. Where\r\n* `major` means breaking changes and removed deprecations\r\n* `minor` new features, sometimes automatic migrations\r\n* `patch` bug fixes\r\n\r\n## Version rollout plan\r\n1. `0.5.x` will be still supported: docs will be available and major bugs fixed\r\n4. We plan an alpha release with sources merged in the core and docs updates early next week.\r\n9. We plan `1.0.0` release in the second / third week of September\r\n10. Each next week we'll release one of follow-up features\r\n11. Track our progress here: https://github.com/orgs/dlt-hub/projects/9/views/3\r\n\r\n## 0.9.9a1 pre-release available\r\nThis pre-release brings **sql**, **filesystem** and **rest_api** sources to the core and introduces 95% of the breaking changes and the deprecations. New documentation is not yet available. ⚠️ do not deploy in production ⚠️ will migrate existing schemas - try on fresh datasets\r\ntry\r\n```py\r\nfrom dlt.sources.sql_database import sql_table\r\n```\r\nor \r\n```sh\r\ndlt init sql_database duckdb\r\n```\r\nto start a new project\r\n\r\n### breaking changes and warnings\r\n**Deprecations and Breaking Changes**\r\n1. Load packages with failed jobs (terminally) will be automatically aborted with an exception. Currently user had to detect this in code (this behavior will be still available). https://github.com/dlt-hub/dlt/issues/1749\r\n2. To use `iceberg` table format on Athena destination, set the `table_format` to `iceberg` on all your resources instead of `force_iceberg` flag in destination configuration. This flag is deprecated but will be still observed for backward compatibility.\r\n3. Will migrate schemas to engine v. 10. this is irreversible\r\n\r\nInternal or obscure features:\r\n1. A few column hints (`foregin_key` and `index`) that were not documented and have no real use, will be removed.\r\n2. if primary key was used in nested table, linking was not created in relational.py. now linking is skipped when nested row is fitted into table that is not nested (does not have a parent). a rare case of someone that does not want `dlt` linking\r\n4. removes generate_dlt_id from json relational normalizer config\r\n7. deprecates `skip_complex_types` in `dlt` Pydantic config, asks to use `skip_nested_types`\r\n8. if a list of resources is passed to `run` method, those will be evaluated in a single ad-hoc source. previously each resource was evaluated separately (serialized). https://github.com/dlt-hub/dlt/pull/1535\r\n\r\n### Other features\r\n* Feat/1492 extend timestamp config to handle naive timestamps (without timezone) by @donotpush in https://github.com/dlt-hub/dlt/pull/1669\r\n* Fix/1571 Incremental: Optionally load or ignore/exclude/include records with `cursor_path` missing or None value by @willi-mueller in https://github.com/dlt-hub/dlt/pull/1576\r\n* Don't use Custom Embedding Functions on LanceDB by @Pipboyguy in https://github.com/dlt-hub/dlt/pull/1771\r\n* sets default concurrency for blob upload for adlfs to 1 to avoid massive memory usage on large files by @rudolfix in https://github.com/dlt-hub/dlt/pull/1779\r\n* Fix/1790 support incremental load with arrow when cursor column is not nullable by @willi-mueller in https://github.com/dlt-hub/dlt/pull/1791\r\n* controls row group size and empty tables in memory buffer when writing parquet by @rudolfix in https://github.com/dlt-hub/dlt/pull/1782\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/reactions", + "total_count": 8, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 8, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14597057570, + "node_id": "RDE_lADOGvRYu86X241OzwAAAANmDWwi", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597057570", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_dismissed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:33:28Z", + "dismissed_review": { + "state": "approved", + "review_id": 2333616749, + "dismissal_message": null, + "dismissal_commit_id": "a5864b168f32b1a4f27f39e540d957aed4dbef8d" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1871", + "id": 2547748174, + "node_id": "PR_kwDOGvRYu858pQIE", + "number": 1871, + "title": "Feat/557 rest api add oauth2clientcredentials to built in auth methods", + "user": { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-09-25T11:39:26Z", + "updated_at": "2024-10-10T20:38:23Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1871", + "html_url": "https://github.com/dlt-hub/dlt/pull/1871", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1871.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1871.patch", + "merged_at": null + }, + "body": "### Description\r\n- Adds `oauth2_client_credentials` to auth shorthand in the RESTAPIConfig\r\n- masks the secrets of aouth2 config dict during dict validation\r\n\r\n### Related Issues\r\n\r\n- Resolves #1870 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14596938143, + "node_id": "PVTISC_lADOGvRYu86SVTahzwAAAANmC5mf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596938143", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:21:45Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1670", + "id": 2455058081, + "node_id": "I_kwDOGvRYu86SVTah", + "number": 1670, + "title": "CLI Plugins", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-08-08T07:15:45Z", + "updated_at": "2024-10-10T20:21:43Z", + "closed_at": "2024-10-10T20:21:43Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We need a way to extend the cli. Here we want to extend the existing plugin interface to allow for cli extensions to be added to dlt. We might also want to convert all the already existing commands into plugins, but this may also be the next step.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596937091, + "node_id": "CE_lADOGvRYu86SVTahzwAAAANmC5WD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596937091", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:21:43Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1670", + "id": 2455058081, + "node_id": "I_kwDOGvRYu86SVTah", + "number": 1670, + "title": "CLI Plugins", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-08-08T07:15:45Z", + "updated_at": "2024-10-10T20:21:43Z", + "closed_at": "2024-10-10T20:21:43Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We need a way to extend the cli. Here we want to extend the existing plugin interface to allow for cli extensions to be added to dlt. We might also want to convert all the already existing commands into plugins, but this may also be the next step.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596898077, + "node_id": "PVTISC_lADOGvRYu86KBei_zwAAAANmCv0d", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596898077", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:17:54Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1408", + "id": 2315643071, + "node_id": "I_kwDOGvRYu86KBei_", + "number": 1408, + "title": "Documentation: update installation command in destination docs", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + }, + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 3767923859, + "node_id": "LA_kwDOGvRYu87glfST", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/good%20first%20issue", + "name": "good first issue", + "color": "7057ff", + "default": true, + "description": "Good for newcomers" + }, + { + "id": 3767923861, + "node_id": "LA_kwDOGvRYu87glfSV", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/help%20wanted", + "name": "help wanted", + "color": "008672", + "default": true, + "description": "Extra attention is needed" + } + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-05-24T15:10:08Z", + "updated_at": "2024-10-10T20:17:52Z", + "closed_at": "2024-10-10T20:17:52Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Background\r\n\r\nThe current documentation for installing packages with extras (e.g., pip install dlt[duckdb]) does not use quotes around the command in some of [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/). Example: [Google BigQuery](https://dlthub.com/docs/dlt-ecosystem/destinations/bigquery)\r\n\r\nThis can lead to potential errors or unexpected behavior in certain shell environments due to the way special characters, like square brackets, are interpreted.\r\n\r\n### Task\r\n\r\nUpdate the installation command so it uses quotes (e.g., `pip install \"dlt[duckdb]\"`):\r\n1. In our [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/).\r\n2. Search other documentation pages and update installation command accordingly.\r\n\r\n### Relevant Issues\r\n- #1404.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596897630, + "node_id": "CE_lADOGvRYu86KBei_zwAAAANmCvte", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596897630", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:17:52Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1408", + "id": 2315643071, + "node_id": "I_kwDOGvRYu86KBei_", + "number": 1408, + "title": "Documentation: update installation command in destination docs", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + }, + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 3767923859, + "node_id": "LA_kwDOGvRYu87glfST", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/good%20first%20issue", + "name": "good first issue", + "color": "7057ff", + "default": true, + "description": "Good for newcomers" + }, + { + "id": 3767923861, + "node_id": "LA_kwDOGvRYu87glfSV", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/help%20wanted", + "name": "help wanted", + "color": "008672", + "default": true, + "description": "Extra attention is needed" + } + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-05-24T15:10:08Z", + "updated_at": "2024-10-10T20:17:52Z", + "closed_at": "2024-10-10T20:17:52Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Background\r\n\r\nThe current documentation for installing packages with extras (e.g., pip install dlt[duckdb]) does not use quotes around the command in some of [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/). Example: [Google BigQuery](https://dlthub.com/docs/dlt-ecosystem/destinations/bigquery)\r\n\r\nThis can lead to potential errors or unexpected behavior in certain shell environments due to the way special characters, like square brackets, are interpreted.\r\n\r\n### Task\r\n\r\nUpdate the installation command so it uses quotes (e.g., `pip install \"dlt[duckdb]\"`):\r\n1. In our [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/).\r\n2. Search other documentation pages and update installation command accordingly.\r\n\r\n### Relevant Issues\r\n- #1404.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596857733, + "node_id": "CE_lADOGvRYu86FYThGzwAAAANmCl-F", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596857733", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:13:50Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1213", + "id": 2237741126, + "node_id": "I_kwDOGvRYu86FYThG", + "number": 1213, + "title": "Intermittent CI failure for `test_limit_edge_cases` in async and sync resource limits", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sultaniman", + "id": 354868, + "node_id": "MDQ6VXNlcjM1NDg2OA==", + "avatar_url": "https://avatars.githubusercontent.com/u/354868?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sultaniman", + "html_url": "https://github.com/sultaniman", + "followers_url": "https://api.github.com/users/sultaniman/followers", + "following_url": "https://api.github.com/users/sultaniman/following{/other_user}", + "gists_url": "https://api.github.com/users/sultaniman/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sultaniman/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sultaniman/subscriptions", + "organizations_url": "https://api.github.com/users/sultaniman/orgs", + "repos_url": "https://api.github.com/users/sultaniman/repos", + "events_url": "https://api.github.com/users/sultaniman/events{/privacy}", + "received_events_url": "https://api.github.com/users/sultaniman/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sultaniman", + "id": 354868, + "node_id": "MDQ6VXNlcjM1NDg2OA==", + "avatar_url": "https://avatars.githubusercontent.com/u/354868?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sultaniman", + "html_url": "https://github.com/sultaniman", + "followers_url": "https://api.github.com/users/sultaniman/followers", + "following_url": "https://api.github.com/users/sultaniman/following{/other_user}", + "gists_url": "https://api.github.com/users/sultaniman/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sultaniman/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sultaniman/subscriptions", + "organizations_url": "https://api.github.com/users/sultaniman/orgs", + "repos_url": "https://api.github.com/users/sultaniman/repos", + "events_url": "https://api.github.com/users/sultaniman/events{/privacy}", + "received_events_url": "https://api.github.com/users/sultaniman/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-04-11T13:33:19Z", + "updated_at": "2024-10-10T20:13:50Z", + "closed_at": "2024-10-10T20:13:50Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "## Description\r\n\r\nCI tests intermittently fail on `test_limit_edge_cases` with a limit of 10. `async_list` unexpectedly contains an extra item compared to the `sync_list`. Race condition?\r\n\r\nTest file: tests/extract/test_sources.py\r\nTest case: test_limit_edge_cases[10]\r\nCI link: https://github.com/dlt-hub/dlt/actions/runs/8646067265/job/23704568405?pr=1211#step:8:1018\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596844794, + "node_id": "CE_lADOGvRYu86Dctm8zwAAAANmCiz6", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596844794", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:12:29Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1145", + "id": 2205342140, + "node_id": "I_kwDOGvRYu86Dctm8", + "number": 1145, + "title": "Docs: Enable type checking for embedded snippets", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-03-25T09:57:05Z", + "updated_at": "2024-10-10T20:12:29Z", + "closed_at": "2024-10-10T20:12:29Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Documentation description\n\nCheck embedded snippets with mypy\n\n### Are you a dlt user?\n\nNone", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596848579, + "node_id": "PVTISC_lADOGvRYu86Cqci5zwAAAANmCjvD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596848579", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:12:03Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1102", + "id": 2192165049, + "node_id": "I_kwDOGvRYu86Cqci5", + "number": 1102, + "title": "Prevent running 2 pipelines with the same name at the same time on the same machine", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-18T13:10:05Z", + "updated_at": "2024-10-10T20:12:01Z", + "closed_at": "2024-10-10T20:12:01Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### dlt version\n\n0.4.6\n\n### Describe the problem\n\nWhen running 2 pipelines with the same name at the same time on the same machine, there will be race conditions with the load packages, as pipelines are designed to pick up and complete incomplete load packages lying around in their local folder. The solution from the user side is to use different names for each pipeline. While this is clear if you are familiar with the inner workings of dlt, this is not really super obvious or intuitive, so we should attempt to prevent this scenario or at least print a warning to stdout.\r\n\r\nThis solution should also work if 2 pipelines are started from two different python scripts running in 2 different processes, so we need some kind of interprocess communication for this. Possibilities are:\r\n\r\n- Place a marker as a file into the pipeline local directory which is removed after the pipeline exits (either successfully or unsuccessfully). We can lock this file to make it \"thread safe\" between processes. We would need a way for the user to clear this lock if somehow the lock is not removed after the pipeline exits (container crash etc.) If a pipeline gets started and finds the lock, it will print a warning to console and exit without doing anything. The warning should contain info on parallel pipeline runs and instructions on how to clear the marker.\r\n- Use multiprocessing.Lock() with pipeline name as key (will not work if different containers are accessing the same \"local storage\" mounted to them)\r\n- ...\n\n### Expected behavior\n\n_No response_\n\n### Steps to reproduce\n\nRun the same pipeline twice at the same time.\n\n### Operating system\n\nmacOS\n\n### Runtime environment\n\nLocal\n\n### Python version\n\n3.10\n\n### dlt data source\n\n_No response_\n\n### dlt destination\n\n_No response_\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/reactions", + "total_count": 2, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 2 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596840441, + "node_id": "CE_lADOGvRYu86Cqci5zwAAAANmChv5", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596840441", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:12:01Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1102", + "id": 2192165049, + "node_id": "I_kwDOGvRYu86Cqci5", + "number": 1102, + "title": "Prevent running 2 pipelines with the same name at the same time on the same machine", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-18T13:10:05Z", + "updated_at": "2024-10-10T20:12:01Z", + "closed_at": "2024-10-10T20:12:01Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### dlt version\n\n0.4.6\n\n### Describe the problem\n\nWhen running 2 pipelines with the same name at the same time on the same machine, there will be race conditions with the load packages, as pipelines are designed to pick up and complete incomplete load packages lying around in their local folder. The solution from the user side is to use different names for each pipeline. While this is clear if you are familiar with the inner workings of dlt, this is not really super obvious or intuitive, so we should attempt to prevent this scenario or at least print a warning to stdout.\r\n\r\nThis solution should also work if 2 pipelines are started from two different python scripts running in 2 different processes, so we need some kind of interprocess communication for this. Possibilities are:\r\n\r\n- Place a marker as a file into the pipeline local directory which is removed after the pipeline exits (either successfully or unsuccessfully). We can lock this file to make it \"thread safe\" between processes. We would need a way for the user to clear this lock if somehow the lock is not removed after the pipeline exits (container crash etc.) If a pipeline gets started and finds the lock, it will print a warning to console and exit without doing anything. The warning should contain info on parallel pipeline runs and instructions on how to clear the marker.\r\n- Use multiprocessing.Lock() with pipeline name as key (will not work if different containers are accessing the same \"local storage\" mounted to them)\r\n- ...\n\n### Expected behavior\n\n_No response_\n\n### Steps to reproduce\n\nRun the same pipeline twice at the same time.\n\n### Operating system\n\nmacOS\n\n### Runtime environment\n\nLocal\n\n### Python version\n\n3.10\n\n### dlt data source\n\n_No response_\n\n### dlt destination\n\n_No response_\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/reactions", + "total_count": 2, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 2 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596832859, + "node_id": "PVTISC_lADOGvRYu86BzSUazwAAAANmCf5b", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596832859", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:11:14Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1068", + "id": 2177705242, + "node_id": "I_kwDOGvRYu86BzSUa", + "number": 1068, + "title": "Clarification and Documentation on Lookback Windows in Incremental Loads", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 6613777925, + "node_id": "LA_kwDOGvRYu88AAAABijY-BQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/community", + "name": "community", + "color": "E99695", + "default": false, + "description": "This issue came from slack community workspace" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "VioletM", + "id": 9139429, + "node_id": "MDQ6VXNlcjkxMzk0Mjk=", + "avatar_url": "https://avatars.githubusercontent.com/u/9139429?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/VioletM", + "html_url": "https://github.com/VioletM", + "followers_url": "https://api.github.com/users/VioletM/followers", + "following_url": "https://api.github.com/users/VioletM/following{/other_user}", + "gists_url": "https://api.github.com/users/VioletM/gists{/gist_id}", + "starred_url": "https://api.github.com/users/VioletM/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/VioletM/subscriptions", + "organizations_url": "https://api.github.com/users/VioletM/orgs", + "repos_url": "https://api.github.com/users/VioletM/repos", + "events_url": "https://api.github.com/users/VioletM/events{/privacy}", + "received_events_url": "https://api.github.com/users/VioletM/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "AstrakhantsevaAA", + "id": 20367975, + "node_id": "MDQ6VXNlcjIwMzY3OTc1", + "avatar_url": "https://avatars.githubusercontent.com/u/20367975?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/AstrakhantsevaAA", + "html_url": "https://github.com/AstrakhantsevaAA", + "followers_url": "https://api.github.com/users/AstrakhantsevaAA/followers", + "following_url": "https://api.github.com/users/AstrakhantsevaAA/following{/other_user}", + "gists_url": "https://api.github.com/users/AstrakhantsevaAA/gists{/gist_id}", + "starred_url": "https://api.github.com/users/AstrakhantsevaAA/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/AstrakhantsevaAA/subscriptions", + "organizations_url": "https://api.github.com/users/AstrakhantsevaAA/orgs", + "repos_url": "https://api.github.com/users/AstrakhantsevaAA/repos", + "events_url": "https://api.github.com/users/AstrakhantsevaAA/events{/privacy}", + "received_events_url": "https://api.github.com/users/AstrakhantsevaAA/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-10T12:53:49Z", + "updated_at": "2024-10-10T20:11:13Z", + "closed_at": "2024-10-10T20:11:12Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Feature description\r\n\r\nDocumentation and examples are needed to clarify the behavior of incremental resources when implementing lookback windows. Currently, records older than incremental.last_value are dropped, which can be counterintuitive, especially when write_disposition is set to merge. This behavior can be implemented using \"last_value_func\". To be documented in the incremental loading documentation.\r\n\r\n\r\nSlack thread: [here](https://dlthub-community.slack.com/archives/C04DQA7JJN6/p1709196207045829)\r\n\r\n### Are you a dlt user?\r\n\r\nYes, I'm already a dlt user.\r\n\r\n### Use case\r\n\r\nDevelopers implementing incremental loads with a requirement for lookback windows face challenges due to the absence of clear documentation on handling records older than the last value.\r\n\r\n### Proposed solution\r\n\r\nUpdate documentation to include specific examples of implementing lookback windows.\r\n\r\n### Related issues\r\n\r\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596832480, + "node_id": "CE_lADOGvRYu86BzSUazwAAAANmCfzg", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596832480", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:11:13Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1068", + "id": 2177705242, + "node_id": "I_kwDOGvRYu86BzSUa", + "number": 1068, + "title": "Clarification and Documentation on Lookback Windows in Incremental Loads", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 6613777925, + "node_id": "LA_kwDOGvRYu88AAAABijY-BQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/community", + "name": "community", + "color": "E99695", + "default": false, + "description": "This issue came from slack community workspace" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "VioletM", + "id": 9139429, + "node_id": "MDQ6VXNlcjkxMzk0Mjk=", + "avatar_url": "https://avatars.githubusercontent.com/u/9139429?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/VioletM", + "html_url": "https://github.com/VioletM", + "followers_url": "https://api.github.com/users/VioletM/followers", + "following_url": "https://api.github.com/users/VioletM/following{/other_user}", + "gists_url": "https://api.github.com/users/VioletM/gists{/gist_id}", + "starred_url": "https://api.github.com/users/VioletM/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/VioletM/subscriptions", + "organizations_url": "https://api.github.com/users/VioletM/orgs", + "repos_url": "https://api.github.com/users/VioletM/repos", + "events_url": "https://api.github.com/users/VioletM/events{/privacy}", + "received_events_url": "https://api.github.com/users/VioletM/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "AstrakhantsevaAA", + "id": 20367975, + "node_id": "MDQ6VXNlcjIwMzY3OTc1", + "avatar_url": "https://avatars.githubusercontent.com/u/20367975?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/AstrakhantsevaAA", + "html_url": "https://github.com/AstrakhantsevaAA", + "followers_url": "https://api.github.com/users/AstrakhantsevaAA/followers", + "following_url": "https://api.github.com/users/AstrakhantsevaAA/following{/other_user}", + "gists_url": "https://api.github.com/users/AstrakhantsevaAA/gists{/gist_id}", + "starred_url": "https://api.github.com/users/AstrakhantsevaAA/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/AstrakhantsevaAA/subscriptions", + "organizations_url": "https://api.github.com/users/AstrakhantsevaAA/orgs", + "repos_url": "https://api.github.com/users/AstrakhantsevaAA/repos", + "events_url": "https://api.github.com/users/AstrakhantsevaAA/events{/privacy}", + "received_events_url": "https://api.github.com/users/AstrakhantsevaAA/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-10T12:53:49Z", + "updated_at": "2024-10-10T20:11:13Z", + "closed_at": "2024-10-10T20:11:12Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Feature description\r\n\r\nDocumentation and examples are needed to clarify the behavior of incremental resources when implementing lookback windows. Currently, records older than incremental.last_value are dropped, which can be counterintuitive, especially when write_disposition is set to merge. This behavior can be implemented using \"last_value_func\". To be documented in the incremental loading documentation.\r\n\r\n\r\nSlack thread: [here](https://dlthub-community.slack.com/archives/C04DQA7JJN6/p1709196207045829)\r\n\r\n### Are you a dlt user?\r\n\r\nYes, I'm already a dlt user.\r\n\r\n### Use case\r\n\r\nDevelopers implementing incremental loads with a requirement for lookback windows face challenges due to the absence of clear documentation on handling records older than the last value.\r\n\r\n### Proposed solution\r\n\r\nUpdate documentation to include specific examples of implementing lookback windows.\r\n\r\n### Related issues\r\n\r\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596822697, + "node_id": "CE_lADOGvRYu86Br_qwzwAAAANmCdap", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596822697", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:10:13Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1067", + "id": 2175793840, + "node_id": "I_kwDOGvRYu86Br_qw", + "number": 1067, + "title": "CI fails on forks: deploy docs workflow run", + "user": { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-08T10:59:09Z", + "updated_at": "2024-10-10T20:10:12Z", + "closed_at": "2024-10-10T20:10:12Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### dlt version\n\n0.4.6\n\n### Describe the problem\n\nIn my fork of this dlt repository I have now a daily workflow run which has been failing since at least the commit 9410bc4.\r\n\r\nI just manually tried it with the latest commit 3761335e6\r\n\r\nError message:\r\n```\r\nRun curl -X POST\r\ncurl: no URL specified!\r\ncurl: try 'curl --help' or 'curl --manual' for more information\r\nError: Process completed with exit code 2.\r\n```\r\n\r\nLink to failed run: https://github.com/willi-mueller/dlt/actions/runs/8197767018/job/22420227197\r\n\n\n### Expected behavior\n\nDo not run the workflow `deploy docs` on forks. I have no intention to build and deploy documentation.\n\n### Steps to reproduce\n\n1. fork the dlt repo\r\n1. wait until 2am UTC for CRON to trigger or manually execute the .github/workflows/deploy_docs.yml\n\n### Operating system\n\nLinux\n\n### Runtime environment\n\nOther\n\n### Python version\n\n3.11\n\n### dlt data source\n\n_No response_\n\n### dlt destination\n\n_No response_\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596639582, + "node_id": "PVTISC_lADOGvRYu85jLx1XzwAAAANmBwte", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596639582", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:01:16Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/252", + "id": 1664032087, + "node_id": "I_kwDOGvRYu85jLx1X", + "number": 252, + "title": "add something like CLA assistant to contribution process", + "user": { + "login": "TyDunn", + "id": 13314504, + "node_id": "MDQ6VXNlcjEzMzE0NTA0", + "avatar_url": "https://avatars.githubusercontent.com/u/13314504?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TyDunn", + "html_url": "https://github.com/TyDunn", + "followers_url": "https://api.github.com/users/TyDunn/followers", + "following_url": "https://api.github.com/users/TyDunn/following{/other_user}", + "gists_url": "https://api.github.com/users/TyDunn/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TyDunn/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TyDunn/subscriptions", + "organizations_url": "https://api.github.com/users/TyDunn/orgs", + "repos_url": "https://api.github.com/users/TyDunn/repos", + "events_url": "https://api.github.com/users/TyDunn/events{/privacy}", + "received_events_url": "https://api.github.com/users/TyDunn/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-04-12T07:48:52Z", + "updated_at": "2024-10-10T20:01:14Z", + "closed_at": "2024-10-10T20:01:14Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "add something like add something like [Contributor License Agreements (CLA) assistant](https://cla-assistant.io/) to the `dlt` and `pipelines` repo contribution process", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596639022, + "node_id": "CE_lADOGvRYu85jLx1XzwAAAANmBwku", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596639022", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:01:14Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/252", + "id": 1664032087, + "node_id": "I_kwDOGvRYu85jLx1X", + "number": 252, + "title": "add something like CLA assistant to contribution process", + "user": { + "login": "TyDunn", + "id": 13314504, + "node_id": "MDQ6VXNlcjEzMzE0NTA0", + "avatar_url": "https://avatars.githubusercontent.com/u/13314504?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TyDunn", + "html_url": "https://github.com/TyDunn", + "followers_url": "https://api.github.com/users/TyDunn/followers", + "following_url": "https://api.github.com/users/TyDunn/following{/other_user}", + "gists_url": "https://api.github.com/users/TyDunn/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TyDunn/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TyDunn/subscriptions", + "organizations_url": "https://api.github.com/users/TyDunn/orgs", + "repos_url": "https://api.github.com/users/TyDunn/repos", + "events_url": "https://api.github.com/users/TyDunn/events{/privacy}", + "received_events_url": "https://api.github.com/users/TyDunn/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-04-12T07:48:52Z", + "updated_at": "2024-10-10T20:01:14Z", + "closed_at": "2024-10-10T20:01:14Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "add something like add something like [Contributor License Agreements (CLA) assistant](https://cla-assistant.io/) to the `dlt` and `pipelines` repo contribution process", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596628568, + "node_id": "RTE_lADOGvRYu85i8U9VzwAAAANmBuBY", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596628568", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:00:39Z", + "rename": { + "from": "post merge improvements rollup", + "to": "add hints and columns propagation" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/242", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/242", + "id": 1659981653, + "node_id": "I_kwDOGvRYu85i8U9V", + "number": 242, + "title": "add hints and columns propagation", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6356257098, + "node_id": "LA_kwDOGvRYu88AAAABetzJSg", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/tech-debt", + "name": "tech-debt", + "color": "000000", + "default": false, + "description": "Leftovers from previous sprint that should be fixed over time" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-04-09T19:41:57Z", + "updated_at": "2024-10-10T20:00:39Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nSeveral things could be much better and several tests are missing. Let's try to fix those one by one\r\n\r\n**Tasks**\r\n1. * [x] when `replace` write disposition is used on an existing resource, the resource state is not reset before loading. reset the resource state see #214 - we have the layout implemented\r\n2. * [ ] ~the loader needs refactoring. we should split it into `stage` (load files to bucket), `load` (does what it does now but without merging) and `merge` which generates and executes merge transformations.~\r\n3. * [x] right now, propagation of root key is enabled for merge tables in a data normalizer. we need something more clever ie. to enable it when no root key is propagated (user can set up this and use something else than dlt default)\r\n4. * [ ] move column propagation into table definition, right now those are in normalizer config. still, the propagation is done by normalizer so those elements should be validated by it\r\n5. * [ ] propagated columns should inherit the hints from parent table or such hint could be specified in the propagation definition\r\n\r\n**Tests**\r\nwe have end to end tests, but following unit tests are missing:\r\n\r\n- * partial columns: column inference when there's no data type, merging of partial columns, non null coercion on partial columns (hmmm this should not happen...)\r\n- test sql jobs on dummy and destinations\r\n- test followup tasks (all file tasks completed, failed and completed tasks allowed)\r\n- LoadJob: job_id and job_file_info (dummy)\r\n- sql client test truncate\r\n- test staging in sql_client_impl: init_storage, truncate, update schemas selectively for tables\r\n- partial columns are not updated (without data type)\r\n\r\n- * we are not saving the default values, check the flows with schema export/import\r\n- * table, column diff and merge\r\n- get_child_tables and get_top_level table\r\n- loader: list jobs for table, add new job\r\n- merge generation for different cases, check with sqlfluff: case without child tables, case without any keys, case with single merge and single primary, case with compound keys", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14596615003, + "node_id": "PVTISC_lADOGvRYu85fM3chzwAAAANmBqtb", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596615003", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:59:45Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/152", + "id": 1597208353, + "node_id": "I_kwDOGvRYu85fM3ch", + "number": 152, + "title": "Make the dlt resources failure resilient.", + "user": { + "login": "redicane", + "id": 55678053, + "node_id": "MDQ6VXNlcjU1Njc4MDUz", + "avatar_url": "https://avatars.githubusercontent.com/u/55678053?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/redicane", + "html_url": "https://github.com/redicane", + "followers_url": "https://api.github.com/users/redicane/followers", + "following_url": "https://api.github.com/users/redicane/following{/other_user}", + "gists_url": "https://api.github.com/users/redicane/gists{/gist_id}", + "starred_url": "https://api.github.com/users/redicane/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/redicane/subscriptions", + "organizations_url": "https://api.github.com/users/redicane/orgs", + "repos_url": "https://api.github.com/users/redicane/repos", + "events_url": "https://api.github.com/users/redicane/events{/privacy}", + "received_events_url": "https://api.github.com/users/redicane/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2023-02-23T16:56:02Z", + "updated_at": "2024-10-10T19:59:44Z", + "closed_at": "2024-10-10T19:59:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "It would improve user experience if a single resource failing wouldn't stop all the other resources from running. This could be an option as well.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596614777, + "node_id": "CE_lADOGvRYu85fM3chzwAAAANmBqp5", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596614777", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:59:44Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/152", + "id": 1597208353, + "node_id": "I_kwDOGvRYu85fM3ch", + "number": 152, + "title": "Make the dlt resources failure resilient.", + "user": { + "login": "redicane", + "id": 55678053, + "node_id": "MDQ6VXNlcjU1Njc4MDUz", + "avatar_url": "https://avatars.githubusercontent.com/u/55678053?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/redicane", + "html_url": "https://github.com/redicane", + "followers_url": "https://api.github.com/users/redicane/followers", + "following_url": "https://api.github.com/users/redicane/following{/other_user}", + "gists_url": "https://api.github.com/users/redicane/gists{/gist_id}", + "starred_url": "https://api.github.com/users/redicane/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/redicane/subscriptions", + "organizations_url": "https://api.github.com/users/redicane/orgs", + "repos_url": "https://api.github.com/users/redicane/repos", + "events_url": "https://api.github.com/users/redicane/events{/privacy}", + "received_events_url": "https://api.github.com/users/redicane/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2023-02-23T16:56:02Z", + "updated_at": "2024-10-10T19:59:44Z", + "closed_at": "2024-10-10T19:59:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "It would improve user experience if a single resource failing wouldn't stop all the other resources from running. This could be an option as well.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596608003, + "node_id": "PVTISC_lADOGvRYu85fLeYFzwAAAANmBpAD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596608003", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:59:00Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/150", + "id": 1596843525, + "node_id": "I_kwDOGvRYu85fLeYF", + "number": 150, + "title": "implement merge code update for `dlt init`", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-02-23T13:13:43Z", + "updated_at": "2024-10-10T19:58:58Z", + "closed_at": "2024-10-10T19:58:58Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Motivation**\r\nWe want to assist our users with pipeline code updates but also allow them to hack the pipeline code. Having the git commit history and information on the moment the branching happen we can improve the update strategy.\r\n\r\nThe strategy in #149 looks like `fast-forward` and will fail when it is impossible (any changes were made). for simple changes we are able to merge it with master branch. **the user can contribute back their changes to the contrib repo - this will also enable this merge**\r\n\r\n**Tasks**\r\nThe idea is for each file to create a branch in the `pipelines` repo, commit local changes to it and then try to merge changes into master. if that happens we use master branch if not we revert to fast-forward.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596607408, + "node_id": "CE_lADOGvRYu85fLeYFzwAAAANmBo2w", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596607408", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:58:58Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/150", + "id": 1596843525, + "node_id": "I_kwDOGvRYu85fLeYF", + "number": 150, + "title": "implement merge code update for `dlt init`", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-02-23T13:13:43Z", + "updated_at": "2024-10-10T19:58:58Z", + "closed_at": "2024-10-10T19:58:58Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Motivation**\r\nWe want to assist our users with pipeline code updates but also allow them to hack the pipeline code. Having the git commit history and information on the moment the branching happen we can improve the update strategy.\r\n\r\nThe strategy in #149 looks like `fast-forward` and will fail when it is impossible (any changes were made). for simple changes we are able to merge it with master branch. **the user can contribute back their changes to the contrib repo - this will also enable this merge**\r\n\r\n**Tasks**\r\nThe idea is for each file to create a branch in the `pipelines` repo, commit local changes to it and then try to merge changes into master. if that happens we use master branch if not we revert to fast-forward.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14595288825, + "node_id": "HRFPE_lADOGvRYu86Y5CMnzwAAAANl8m75", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14595288825", + "actor": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:55:00Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "id": 2565088039, + "node_id": "PR_kwDOGvRYu859jl-r", + "number": 1925, + "title": "Add `references` table hint", + "user": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-03T22:16:14Z", + "updated_at": "2024-10-10T18:23:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1925", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1925.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1925.patch", + "merged_at": null + }, + "body": "\r\n\r\n\r\n### Description\r\nAdds new `references` table hint. Can be added via `@resource` decorator and `apply_hints` \r\nTakes a list of \"foreign key\" references.\r\n\r\nSQLAlchemy database source automatically generates the hint from tabel foreign keys\r\n\r\n\r\n\r\n### Related Issues\r\n#1713 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594890035, + "node_id": "HRDE_lADOGvRYu86Zht_fzwAAAANl7Fkz", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594890035", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:18:24Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594889824, + "node_id": "REFE_lADOGvRYu86Zht_fzwAAAANl7Fhg", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594889824", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "e9efa4fc930c17db8102c9d36929514948854d92", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/e9efa4fc930c17db8102c9d36929514948854d92", + "created_at": "2024-10-10T17:18:23Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594889577, + "node_id": "CE_lADOGvRYu86Zht_fzwAAAANl7Fdp", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594889577", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:18:22Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594889556, + "node_id": "ME_lADOGvRYu86Zht_fzwAAAANl7FdU", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594889556", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "e9efa4fc930c17db8102c9d36929514948854d92", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/e9efa4fc930c17db8102c9d36929514948854d92", + "created_at": "2024-10-10T17:18:21Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594817912, + "node_id": "RFRE_lADOGvRYu86ZvoRgzwAAAANl6z94", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594817912", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "ready_for_review", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:11:35Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1945", + "id": 2579399776, + "node_id": "PR_kwDOGvRYu85-QCGz", + "number": 1945, + "title": "WIP: dataset factory", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T17:07:43Z", + "updated_at": "2024-10-10T18:28:32Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1945", + "html_url": "https://github.com/dlt-hub/dlt/pull/1945", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1945.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1945.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\nThis PR is an example implementation of a dataset factory to build datasets without a pipeline object.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594112193, + "node_id": "AE_lADOGvRYu86ZvNfazwAAAANl4HrB", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594112193", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "assigned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T16:12:15Z", + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assigner": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1944", + "id": 2579290074, + "node_id": "PR_kwDOGvRYu85-Pr4P", + "number": 1944, + "title": "unifies run configuration and run context", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T16:12:12Z", + "updated_at": "2024-10-10T16:12:38Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": true, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1944", + "html_url": "https://github.com/dlt-hub/dlt/pull/1944", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1944.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1944.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\nintegrates RunConfiguration and logger instances into run context\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592486817, + "node_id": "RFRE_lADOGvRYu86Zht_fzwAAAANlx62h", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592486817", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "ready_for_review", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:27:49Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592433695, + "node_id": "RTE_lADOGvRYu86Zht_fzwAAAANlxt4f", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592433695", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:24:52Z", + "rename": { + "from": "WIP: Pluggable Cli Commands", + "to": "Pluggable Cli Commands" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592370291, + "node_id": "PVTISC_lADOGvRYu86ZuKuEzwAAAANlxeZz", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592370291", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:21:28Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1943", + "id": 2579016580, + "node_id": "I_kwDOGvRYu86ZuKuE", + "number": 1943, + "title": "Allow to create datasets outside of pipeline context", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-10T14:21:27Z", + "updated_at": "2024-10-10T14:24:17Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We want to be able to create datasets (data accessors) outside of the context of a pipeline. For this the user would have to supply a destination with credentials, the dataset_name of the dataset in the destination as well as which schema to use. If no schema is given, the dataset should look up the most recent schema in the destination dataset and use that. Optionally we could print a warning of more than one schema name is found.\n\nQuestions:\n* If the user wants to construct two datasets to the same destination type but with separate credentials we need to namespace the credentials somehow. We could use the destination name for this, but would have to add that as an argument to the dataset contructor...", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592370260, + "node_id": "ATPVTE_lADOGvRYu86ZuKuEzwAAAANlxeZU", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592370260", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "added_to_project_v2", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:21:28Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1943", + "id": 2579016580, + "node_id": "I_kwDOGvRYu86ZuKuE", + "number": 1943, + "title": "Allow to create datasets outside of pipeline context", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-10T14:21:27Z", + "updated_at": "2024-10-10T14:24:17Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We want to be able to create datasets (data accessors) outside of the context of a pipeline. For this the user would have to supply a destination with credentials, the dataset_name of the dataset in the destination as well as which schema to use. If no schema is given, the dataset should look up the most recent schema in the destination dataset and use that. Optionally we could print a warning of more than one schema name is found.\n\nQuestions:\n* If the user wants to construct two datasets to the same destination type but with separate credentials we need to namespace the credentials somehow. We could use the destination name for this, but would have to add that as an argument to the dataset contructor...", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592369010, + "node_id": "AE_lADOGvRYu86ZuKuEzwAAAANlxeFy", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592369010", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "assigned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:21:28Z", + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assigner": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1943", + "id": 2579016580, + "node_id": "I_kwDOGvRYu86ZuKuE", + "number": 1943, + "title": "Allow to create datasets outside of pipeline context", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-10T14:21:27Z", + "updated_at": "2024-10-10T14:24:17Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We want to be able to create datasets (data accessors) outside of the context of a pipeline. For this the user would have to supply a destination with credentials, the dataset_name of the dataset in the destination as well as which schema to use. If no schema is given, the dataset should look up the most recent schema in the destination dataset and use that. Optionally we could print a warning of more than one schema name is found.\n\nQuestions:\n* If the user wants to construct two datasets to the same destination type but with separate credentials we need to namespace the credentials somehow. We could use the destination name for this, but would have to add that as an argument to the dataset contructor...", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592123629, + "node_id": "HRDE_lADOGvRYu86ZkDJYzwAAAANlwiLt", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592123629", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:08:13Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592123392, + "node_id": "REFE_lADOGvRYu86ZkDJYzwAAAANlwiIA", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592123392", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "47633c639e87b5263dafc66d73da600b65881583", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/47633c639e87b5263dafc66d73da600b65881583", + "created_at": "2024-10-10T14:08:13Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592122813, + "node_id": "CE_lADOGvRYu86ZkDJYzwAAAANlwh-9", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592122813", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:08:11Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592122786, + "node_id": "ME_lADOGvRYu86ZkDJYzwAAAANlwh-i", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592122786", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "47633c639e87b5263dafc66d73da600b65881583", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/47633c639e87b5263dafc66d73da600b65881583", + "created_at": "2024-10-10T14:08:10Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591847735, + "node_id": "HRFPE_lADOGvRYu86Zht_fzwAAAANlve03", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591847735", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:53:48Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591823971, + "node_id": "SE_lADOGvRYu86Zht_fzwAAAANlvZBj", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591823971", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:52:25Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591823947, + "node_id": "MEE_lADOGvRYu86Zht_fzwAAAANlvZBL", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591823947", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:52:25Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591300024, + "node_id": "SE_lADOGvRYu86ZVp2rzwAAAANltZG4", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591300024", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:21:14Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591300000, + "node_id": "MEE_lADOGvRYu86ZVp2rzwAAAANltZGg", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591300000", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:21:14Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14589694303, + "node_id": "AE_lADOGvRYu86ZVp2rzwAAAANlnRFf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14589694303", + "actor": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "event": "assigned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T11:32:12Z", + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assigner": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14589691203, + "node_id": "SE_lADOGvRYu86ZVp2rzwAAAANlnQVD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14589691203", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T11:31:56Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14589691184, + "node_id": "MEE_lADOGvRYu86ZVp2rzwAAAANlnQUw", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14589691184", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T11:31:56Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14586091972, + "node_id": "REFE_lADOGvRYu86Zp74qzwAAAANlZhnE", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14586091972", + "actor": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "117220be4c896ba0b507f1634d61adbd90c4f0ac", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/117220be4c896ba0b507f1634d61adbd90c4f0ac", + "created_at": "2024-10-10T07:52:44Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "id": 2577907242, + "node_id": "PR_kwDOGvRYu85-LRJj", + "number": 1942, + "title": "Update url in deploy-with-airflow-composer.md", + "user": { + "login": "FriedrichtenHagen", + "id": 108153620, + "node_id": "U_kgDOBnJLFA", + "avatar_url": "https://avatars.githubusercontent.com/u/108153620?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/FriedrichtenHagen", + "html_url": "https://github.com/FriedrichtenHagen", + "followers_url": "https://api.github.com/users/FriedrichtenHagen/followers", + "following_url": "https://api.github.com/users/FriedrichtenHagen/following{/other_user}", + "gists_url": "https://api.github.com/users/FriedrichtenHagen/gists{/gist_id}", + "starred_url": "https://api.github.com/users/FriedrichtenHagen/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/FriedrichtenHagen/subscriptions", + "organizations_url": "https://api.github.com/users/FriedrichtenHagen/orgs", + "repos_url": "https://api.github.com/users/FriedrichtenHagen/repos", + "events_url": "https://api.github.com/users/FriedrichtenHagen/events{/privacy}", + "received_events_url": "https://api.github.com/users/FriedrichtenHagen/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T07:28:18Z", + "updated_at": "2024-10-10T07:52:43Z", + "closed_at": "2024-10-10T07:52:43Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1942", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1942.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1942.patch", + "merged_at": "2024-10-10T07:52:43Z" + }, + "body": "\r\n\r\n### Description\r\nThe original urls is broken. I assume this is the correct page to link to?\r\n\r\n\r\n\r\n---\r\n\r\n\r\njust a small url\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14586091579, + "node_id": "CE_lADOGvRYu86Zp74qzwAAAANlZhg7", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14586091579", + "actor": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T07:52:43Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "id": 2577907242, + "node_id": "PR_kwDOGvRYu85-LRJj", + "number": 1942, + "title": "Update url in deploy-with-airflow-composer.md", + "user": { + "login": "FriedrichtenHagen", + "id": 108153620, + "node_id": "U_kgDOBnJLFA", + "avatar_url": "https://avatars.githubusercontent.com/u/108153620?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/FriedrichtenHagen", + "html_url": "https://github.com/FriedrichtenHagen", + "followers_url": "https://api.github.com/users/FriedrichtenHagen/followers", + "following_url": "https://api.github.com/users/FriedrichtenHagen/following{/other_user}", + "gists_url": "https://api.github.com/users/FriedrichtenHagen/gists{/gist_id}", + "starred_url": "https://api.github.com/users/FriedrichtenHagen/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/FriedrichtenHagen/subscriptions", + "organizations_url": "https://api.github.com/users/FriedrichtenHagen/orgs", + "repos_url": "https://api.github.com/users/FriedrichtenHagen/repos", + "events_url": "https://api.github.com/users/FriedrichtenHagen/events{/privacy}", + "received_events_url": "https://api.github.com/users/FriedrichtenHagen/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T07:28:18Z", + "updated_at": "2024-10-10T07:52:43Z", + "closed_at": "2024-10-10T07:52:43Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1942", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1942.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1942.patch", + "merged_at": "2024-10-10T07:52:43Z" + }, + "body": "\r\n\r\n### Description\r\nThe original urls is broken. I assume this is the correct page to link to?\r\n\r\n\r\n\r\n---\r\n\r\n\r\njust a small url\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14586091560, + "node_id": "ME_lADOGvRYu86Zp74qzwAAAANlZhgo", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14586091560", + "actor": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "117220be4c896ba0b507f1634d61adbd90c4f0ac", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/117220be4c896ba0b507f1634d61adbd90c4f0ac", + "created_at": "2024-10-10T07:52:43Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "id": 2577907242, + "node_id": "PR_kwDOGvRYu85-LRJj", + "number": 1942, + "title": "Update url in deploy-with-airflow-composer.md", + "user": { + "login": "FriedrichtenHagen", + "id": 108153620, + "node_id": "U_kgDOBnJLFA", + "avatar_url": "https://avatars.githubusercontent.com/u/108153620?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/FriedrichtenHagen", + "html_url": "https://github.com/FriedrichtenHagen", + "followers_url": "https://api.github.com/users/FriedrichtenHagen/followers", + "following_url": "https://api.github.com/users/FriedrichtenHagen/following{/other_user}", + "gists_url": "https://api.github.com/users/FriedrichtenHagen/gists{/gist_id}", + "starred_url": "https://api.github.com/users/FriedrichtenHagen/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/FriedrichtenHagen/subscriptions", + "organizations_url": "https://api.github.com/users/FriedrichtenHagen/orgs", + "repos_url": "https://api.github.com/users/FriedrichtenHagen/repos", + "events_url": "https://api.github.com/users/FriedrichtenHagen/events{/privacy}", + "received_events_url": "https://api.github.com/users/FriedrichtenHagen/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T07:28:18Z", + "updated_at": "2024-10-10T07:52:43Z", + "closed_at": "2024-10-10T07:52:43Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1942", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1942.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1942.patch", + "merged_at": "2024-10-10T07:52:43Z" + }, + "body": "\r\n\r\n### Description\r\nThe original urls is broken. I assume this is the correct page to link to?\r\n\r\n\r\n\r\n---\r\n\r\n\r\njust a small url\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14581256918, + "node_id": "HRDE_lADOGvRYu86YnR5ezwAAAANlHFLW", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14581256918", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T00:33:27Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1914", + "id": 2560433758, + "node_id": "PR_kwDOGvRYu859UYar", + "number": 1914, + "title": "fix: UUIDs are not an unknown data type (logging)", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-02T00:32:21Z", + "updated_at": "2024-10-10T00:33:27Z", + "closed_at": "2024-10-02T11:30:18Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1914", + "html_url": "https://github.com/dlt-hub/dlt/pull/1914", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1914.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1914.patch", + "merged_at": "2024-10-02T11:30:18Z" + }, + "body": "### Description\r\n- Fixes #1862 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/reactions", + "total_count": 1, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 1, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14578955196, + "node_id": "RTE_lADOGvRYu86Zl6ULzwAAAANk-TO8", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14578955196", + "actor": { + "login": "francescomucio", + "id": 3058143, + "node_id": "MDQ6VXNlcjMwNTgxNDM=", + "avatar_url": "https://avatars.githubusercontent.com/u/3058143?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/francescomucio", + "html_url": "https://github.com/francescomucio", + "followers_url": "https://api.github.com/users/francescomucio/followers", + "following_url": "https://api.github.com/users/francescomucio/following{/other_user}", + "gists_url": "https://api.github.com/users/francescomucio/gists{/gist_id}", + "starred_url": "https://api.github.com/users/francescomucio/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/francescomucio/subscriptions", + "organizations_url": "https://api.github.com/users/francescomucio/orgs", + "repos_url": "https://api.github.com/users/francescomucio/repos", + "events_url": "https://api.github.com/users/francescomucio/events{/privacy}", + "received_events_url": "https://api.github.com/users/francescomucio/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T20:06:31Z", + "rename": { + "from": "added extended jsonpath_ng parser", + "to": "Added extended jsonpath_ng parser" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1941", + "id": 2576852235, + "node_id": "PR_kwDOGvRYu85-IGPX", + "number": 1941, + "title": "Added extended jsonpath_ng parser", + "user": { + "login": "francescomucio", + "id": 3058143, + "node_id": "MDQ6VXNlcjMwNTgxNDM=", + "avatar_url": "https://avatars.githubusercontent.com/u/3058143?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/francescomucio", + "html_url": "https://github.com/francescomucio", + "followers_url": "https://api.github.com/users/francescomucio/followers", + "following_url": "https://api.github.com/users/francescomucio/following{/other_user}", + "gists_url": "https://api.github.com/users/francescomucio/gists{/gist_id}", + "starred_url": "https://api.github.com/users/francescomucio/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/francescomucio/subscriptions", + "organizations_url": "https://api.github.com/users/francescomucio/orgs", + "repos_url": "https://api.github.com/users/francescomucio/repos", + "events_url": "https://api.github.com/users/francescomucio/events{/privacy}", + "received_events_url": "https://api.github.com/users/francescomucio/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T20:06:22Z", + "updated_at": "2024-10-09T20:06:51Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1941", + "html_url": "https://github.com/dlt-hub/dlt/pull/1941", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1941.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1941.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\nThis PR introduce the `jsonpath_ng` extended parser, which is more compliant with the JsonPath standard.\r\n\r\nAn example of the lack of features of the standard parser is here: [Filters not implmented](https://github.com/h2non/jsonpath-ng/issues/8)\r\n\r\n\r\n### Related Issues\r\n\r\n\r\n### Additional Context\r\nThis was discussed in the slack community [here](https://dlthub-community.slack.com/archives/C04DQA7JJN6/p1728335837644989).\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14577104073, + "node_id": "PVTISC_lADOGvRYu86Yz-aszwAAAANk3PTJ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577104073", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14577103536, + "node_id": "SE_lADOGvRYu86Yz-aszwAAAANk3PKw", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577103536", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14577103521, + "node_id": "MEE_lADOGvRYu86Yz-aszwAAAANk3PKh", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577103521", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14577103279, + "node_id": "CE_lADOGvRYu86Yz-aszwAAAANk3PGv", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577103279", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:30Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14576187113, + "node_id": "HRFPE_lADOGvRYu86Zht_fzwAAAANkzvbp", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14576187113", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T16:40:17Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575730719, + "node_id": "RRE_lADOGvRYu86ZkDJYzwAAAANkyAAf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575730719", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_requested", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T16:08:00Z", + "review_requester": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "requested_reviewer": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575596652, + "node_id": "SE_lADOGvRYu86F_x_yzwAAAANkxfRs", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575596652", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:59:15Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1234", + "id": 2248089586, + "node_id": "I_kwDOGvRYu86F_x_y", + "number": 1234, + "title": "support bulk copy for mssql and synapse", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6597295039, + "node_id": "LA_kwDOGvRYu88AAAABiTq7vw", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/destination", + "name": "destination", + "color": "C5DEF5", + "default": false, + "description": "Issue related to new destinations" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2024-04-17T11:44:54Z", + "updated_at": "2024-10-09T15:59:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nThe way we insert rows into mssql is ineffective. We should switch to bulk copy. MS odbc drivers come with `bcp` command that we can use.\r\nhttps://github.com/yehoshuadimarsky/bcpandas/blob/master/bcpandas/utils.py\r\nis a project that does that quite well\r\n\r\n**Tasks**\r\n1. * [ ] allow `mssql` and `synapse` to handle `csv`\r\n2. * [ ] use `bcp` in copy jobs\r\n3. * [ ] looks like that we can pass DSN as credentials and specify file format to copy on the fly: https://learn.microsoft.com/en-us/sql/tools/bcp-utility?view=sql-server-ver16\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575596643, + "node_id": "MEE_lADOGvRYu86F_x_yzwAAAANkxfRj", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575596643", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:59:15Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1234", + "id": 2248089586, + "node_id": "I_kwDOGvRYu86F_x_y", + "number": 1234, + "title": "support bulk copy for mssql and synapse", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6597295039, + "node_id": "LA_kwDOGvRYu88AAAABiTq7vw", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/destination", + "name": "destination", + "color": "C5DEF5", + "default": false, + "description": "Issue related to new destinations" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2024-04-17T11:44:54Z", + "updated_at": "2024-10-09T15:59:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nThe way we insert rows into mssql is ineffective. We should switch to bulk copy. MS odbc drivers come with `bcp` command that we can use.\r\nhttps://github.com/yehoshuadimarsky/bcpandas/blob/master/bcpandas/utils.py\r\nis a project that does that quite well\r\n\r\n**Tasks**\r\n1. * [ ] allow `mssql` and `synapse` to handle `csv`\r\n2. * [ ] use `bcp` in copy jobs\r\n3. * [ ] looks like that we can pass DSN as credentials and specify file format to copy on the fly: https://learn.microsoft.com/en-us/sql/tools/bcp-utility?view=sql-server-ver16\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203935, + "node_id": "HRDE_lADOGvRYu86Zisg3zwAAAANkv_Zf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203935", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:34:41Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203861, + "node_id": "REFE_lADOGvRYu86Zisg3zwAAAANkv_YV", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203861", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "7ac2ae1a473f515c048bab330d55e766218c59a6", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/7ac2ae1a473f515c048bab330d55e766218c59a6", + "created_at": "2024-10-09T15:34:41Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203206, + "node_id": "CE_lADOGvRYu86Zisg3zwAAAANkv_OG", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203206", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:34:38Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203173, + "node_id": "ME_lADOGvRYu86Zisg3zwAAAANkv_Nl", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203173", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "7ac2ae1a473f515c048bab330d55e766218c59a6", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/7ac2ae1a473f515c048bab330d55e766218c59a6", + "created_at": "2024-10-09T15:34:38Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14574656183, + "node_id": "HRFPE_lADOGvRYu86Y5CMnzwAAAANkt5q3", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14574656183", + "actor": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:00:51Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "id": 2565088039, + "node_id": "PR_kwDOGvRYu859jl-r", + "number": 1925, + "title": "Add `references` table hint", + "user": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-03T22:16:14Z", + "updated_at": "2024-10-10T18:23:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1925", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1925.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1925.patch", + "merged_at": null + }, + "body": "\r\n\r\n\r\n### Description\r\nAdds new `references` table hint. Can be added via `@resource` decorator and `apply_hints` \r\nTakes a list of \"foreign key\" references.\r\n\r\nSQLAlchemy database source automatically generates the hint from tabel foreign keys\r\n\r\n\r\n\r\n### Related Issues\r\n#1713 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14572539818, + "node_id": "RRE_lADOGvRYu86Zht_fzwAAAANkl0-q", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14572539818", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_requested", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T12:55:59Z", + "review_requester": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "requested_reviewer": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569621330, + "node_id": "LE_lADOGvRYu86ZgSClzwAAAANkasdS", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569621330", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "labeled", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:40:07Z", + "label": { + "name": "bug", + "color": "d73a4a" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1937", + "id": 2575376549, + "node_id": "I_kwDOGvRYu86ZgSCl", + "number": 1937, + "title": "Arrow Normalization Problem on Github events", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-09T09:39:58Z", + "updated_at": "2024-10-09T09:40:07Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "When loading github events into one pipeline and piping them into the next one with arrow tables, there is a normalization collission. I believe this has to do with column names that have double underscores, but I am not sure. The code below reproduces this. Disabling normalization on pipeline 2 will make this error go away as does using items.fetchall() instead of iterating over arrow tables.\n\nThe desired behavior is that this passes. I am not sure wether this is a bug or some sideeffect that makes sense in the bigger picture, this will need to be investigated.\n\n```\nimport dlt, os\nimport requests\n\n\nURL = \"https://api.github.com/repos/dlt-hub/dlt/issues/events?per_page=100\"\n\nif __name__ == \"__main__\":\n \n os.environ[\"BUCKET_URL\"] = \"_storage/dave\"\n \n items = requests.get(URL).json()\n \n p1 = dlt.pipeline(\"dave2\", destination=\"filesystem\", dev_mode=True)\n p1.run(items, table_name=\"items\", loader_file_format=\"parquet\")\n \n p2 = dlt.pipeline(\"dave3\", destination=\"duckdb\", dev_mode=True)\n # NOTE: using items.fetchall() will work\n p2.run(p1._dataset().items.arrow(), table_name=\"items\", loader_file_format=\"parquet\")\n```", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569619871, + "node_id": "ATPVTE_lADOGvRYu86ZgSClzwAAAANkasGf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569619871", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "added_to_project_v2", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:39:58Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1937", + "id": 2575376549, + "node_id": "I_kwDOGvRYu86ZgSCl", + "number": 1937, + "title": "Arrow Normalization Problem on Github events", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-09T09:39:58Z", + "updated_at": "2024-10-09T09:40:07Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "When loading github events into one pipeline and piping them into the next one with arrow tables, there is a normalization collission. I believe this has to do with column names that have double underscores, but I am not sure. The code below reproduces this. Disabling normalization on pipeline 2 will make this error go away as does using items.fetchall() instead of iterating over arrow tables.\n\nThe desired behavior is that this passes. I am not sure wether this is a bug or some sideeffect that makes sense in the bigger picture, this will need to be investigated.\n\n```\nimport dlt, os\nimport requests\n\n\nURL = \"https://api.github.com/repos/dlt-hub/dlt/issues/events?per_page=100\"\n\nif __name__ == \"__main__\":\n \n os.environ[\"BUCKET_URL\"] = \"_storage/dave\"\n \n items = requests.get(URL).json()\n \n p1 = dlt.pipeline(\"dave2\", destination=\"filesystem\", dev_mode=True)\n p1.run(items, table_name=\"items\", loader_file_format=\"parquet\")\n \n p2 = dlt.pipeline(\"dave3\", destination=\"duckdb\", dev_mode=True)\n # NOTE: using items.fetchall() will work\n p2.run(p1._dataset().items.arrow(), table_name=\"items\", loader_file_format=\"parquet\")\n```", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569619869, + "node_id": "PVTISC_lADOGvRYu86ZgSClzwAAAANkasGd", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569619869", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:39:58Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1937", + "id": 2575376549, + "node_id": "I_kwDOGvRYu86ZgSCl", + "number": 1937, + "title": "Arrow Normalization Problem on Github events", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-09T09:39:58Z", + "updated_at": "2024-10-09T09:40:07Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "When loading github events into one pipeline and piping them into the next one with arrow tables, there is a normalization collission. I believe this has to do with column names that have double underscores, but I am not sure. The code below reproduces this. Disabling normalization on pipeline 2 will make this error go away as does using items.fetchall() instead of iterating over arrow tables.\n\nThe desired behavior is that this passes. I am not sure wether this is a bug or some sideeffect that makes sense in the bigger picture, this will need to be investigated.\n\n```\nimport dlt, os\nimport requests\n\n\nURL = \"https://api.github.com/repos/dlt-hub/dlt/issues/events?per_page=100\"\n\nif __name__ == \"__main__\":\n \n os.environ[\"BUCKET_URL\"] = \"_storage/dave\"\n \n items = requests.get(URL).json()\n \n p1 = dlt.pipeline(\"dave2\", destination=\"filesystem\", dev_mode=True)\n p1.run(items, table_name=\"items\", loader_file_format=\"parquet\")\n \n p2 = dlt.pipeline(\"dave3\", destination=\"duckdb\", dev_mode=True)\n # NOTE: using items.fetchall() will work\n p2.run(p1._dataset().items.arrow(), table_name=\"items\", loader_file_format=\"parquet\")\n```", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569532803, + "node_id": "HRDE_lADOGvRYu86YQSpKzwAAAANkaW2D", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569532803", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:34:35Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569532304, + "node_id": "REFE_lADOGvRYu86YQSpKzwAAAANkaWuQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569532304", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "c87e399c7ddac0e41f6908013ab1696ed0bac374", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/c87e399c7ddac0e41f6908013ab1696ed0bac374", + "created_at": "2024-10-09T09:34:33Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569531724, + "node_id": "CE_lADOGvRYu86YQSpKzwAAAANkaWlM", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569531724", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:34:31Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569531700, + "node_id": "ME_lADOGvRYu86YQSpKzwAAAANkaWk0", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569531700", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "c87e399c7ddac0e41f6908013ab1696ed0bac374", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/c87e399c7ddac0e41f6908013ab1696ed0bac374", + "created_at": "2024-10-09T09:34:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14558332459, + "node_id": "REFE_lADOGvRYu86Yz-aszwAAAANjvoYr", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14558332459", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "228de289cbe4eb74338a17e4c115eb06f27d48be", + "commit_url": "https://api.github.com/repos/neuromantik33/verified-sources/commits/228de289cbe4eb74338a17e4c115eb06f27d48be", + "created_at": "2024-10-08T14:50:34Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14557499745, + "node_id": "SE_lADOGvRYu86WeZczzwAAAANjsdFh", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14557499745", + "actor": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T14:01:26Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "id": 2524550963, + "node_id": "PR_kwDOGvRYu857a2ja", + "number": 1805, + "title": "Added deploy with modal.", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 10, + "created_at": "2024-09-13T11:00:25Z", + "updated_at": "2024-10-10T12:56:26Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1805", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1805.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1805.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\n\r\nAdded deploy with modal", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14557499724, + "node_id": "MEE_lADOGvRYu86WeZczzwAAAANjsdFM", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14557499724", + "actor": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T14:01:26Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "id": 2524550963, + "node_id": "PR_kwDOGvRYu857a2ja", + "number": 1805, + "title": "Added deploy with modal.", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 10, + "created_at": "2024-09-13T11:00:25Z", + "updated_at": "2024-10-10T12:56:26Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1805", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1805.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1805.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\n\r\nAdded deploy with modal", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556001138, + "node_id": "HRDE_lADOGvRYu86NDn_GzwAAAANjmvNy", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556001138", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:31:00Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556001075, + "node_id": "PVTISC_lADOGvRYu86CdI5KzwAAAANjmvMz", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556001075", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:30:59Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1095", + "id": 2188676682, + "node_id": "I_kwDOGvRYu86CdI5K", + "number": 1095, + "title": "access data after load load as dataframes with ibis", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 9, + "created_at": "2024-03-15T14:36:38Z", + "updated_at": "2024-10-08T12:30:57Z", + "closed_at": "2024-10-08T12:30:57Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nibis https://github.com/ibis-project/ibis is a library that translates dataframe expressions into SQL statement and then executes them in the destination. they do nice work of compiling final SQL statement with sqlglot (so probably resultant SQL is quite optimized)\r\n\r\nWe have large overlap in destinations and we were looking for decent dataframe -> sql thing from the very start. it seems that's it: we can easily build a helper that exposes any dlt dataset as dataframe, share credentials etc.\r\n\r\n**Implementation**\r\nWe can integrate deeply or via a helper scheme. In case of helper, we allow users to get `ibis` connection from `dlt` destination and/or pipeline. The UX will be similar to `dbt` helper.\r\n\r\nDeep integration means that we expose the loaded data from the `Pipeline`, `DltSource` and `DltResource` instances. ie.\r\n```\r\npipeline.run(source, destination=\"bigquery\", dataset_name=\"data\")\r\n\r\n# duckdb-like interface\r\n\r\n# get rows dbapi style\r\nfor row in pipeline.dataset.sql(\"SELECT * FROM table\"):\r\n ...\r\n\r\n# get materialized dataframes\r\nfor batch in pipeline.dataset.sql(\"SELECT * FROM table\").arrow(batch_size=10000):\r\n ...\r\n\r\n# get lazy dataframe via ibis\r\nibis_table = pipeline.dataset.df(table_name)\r\nibis_connection = ibis_table = pipeline.dataset.df()\r\n\r\n# we can expose the `dataset` property of the pipeline via source (and internal resources as well), in that case we automatically bind \r\nto right schema\r\n\r\nibis_table = resource.dataset.df()\r\nibis_connection = source.dataset.df()\r\n```\r\n\r\nImplementation is straightforward for sql-like destinations. We won't support vector databases.\r\nIt would be really interesting to support filesystem destination as above. ie. by registering the json and parquet files in temporary duckdb database and then exposing the database for `ibis` and `sql` access methods\r\n\r\n** Ibis Connection sharing**\r\nWe are discussing a connection sharing approach with ibis here: https://github.com/ibis-project/ibis/issues/8877. As mentioned in the comments there, we could build it in a way that we manage the connection and ibis provides backends that accept an open connection and DO NOT need any addtionally dependencies.\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/reactions", + "total_count": 6, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 6, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14556000740, + "node_id": "REFE_lADOGvRYu86NDn_GzwAAAANjmvHk", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000740", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "created_at": "2024-10-08T12:30:58Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556000617, + "node_id": "CE_lADOGvRYu86CdI5KzwAAAANjmvFp", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000617", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:30:57Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1095", + "id": 2188676682, + "node_id": "I_kwDOGvRYu86CdI5K", + "number": 1095, + "title": "access data after load load as dataframes with ibis", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 9, + "created_at": "2024-03-15T14:36:38Z", + "updated_at": "2024-10-08T12:30:57Z", + "closed_at": "2024-10-08T12:30:57Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nibis https://github.com/ibis-project/ibis is a library that translates dataframe expressions into SQL statement and then executes them in the destination. they do nice work of compiling final SQL statement with sqlglot (so probably resultant SQL is quite optimized)\r\n\r\nWe have large overlap in destinations and we were looking for decent dataframe -> sql thing from the very start. it seems that's it: we can easily build a helper that exposes any dlt dataset as dataframe, share credentials etc.\r\n\r\n**Implementation**\r\nWe can integrate deeply or via a helper scheme. In case of helper, we allow users to get `ibis` connection from `dlt` destination and/or pipeline. The UX will be similar to `dbt` helper.\r\n\r\nDeep integration means that we expose the loaded data from the `Pipeline`, `DltSource` and `DltResource` instances. ie.\r\n```\r\npipeline.run(source, destination=\"bigquery\", dataset_name=\"data\")\r\n\r\n# duckdb-like interface\r\n\r\n# get rows dbapi style\r\nfor row in pipeline.dataset.sql(\"SELECT * FROM table\"):\r\n ...\r\n\r\n# get materialized dataframes\r\nfor batch in pipeline.dataset.sql(\"SELECT * FROM table\").arrow(batch_size=10000):\r\n ...\r\n\r\n# get lazy dataframe via ibis\r\nibis_table = pipeline.dataset.df(table_name)\r\nibis_connection = ibis_table = pipeline.dataset.df()\r\n\r\n# we can expose the `dataset` property of the pipeline via source (and internal resources as well), in that case we automatically bind \r\nto right schema\r\n\r\nibis_table = resource.dataset.df()\r\nibis_connection = source.dataset.df()\r\n```\r\n\r\nImplementation is straightforward for sql-like destinations. We won't support vector databases.\r\nIt would be really interesting to support filesystem destination as above. ie. by registering the json and parquet files in temporary duckdb database and then exposing the database for `ibis` and `sql` access methods\r\n\r\n** Ibis Connection sharing**\r\nWe are discussing a connection sharing approach with ibis here: https://github.com/ibis-project/ibis/issues/8877. As mentioned in the comments there, we could build it in a way that we manage the connection and ibis provides backends that accept an open connection and DO NOT need any addtionally dependencies.\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/reactions", + "total_count": 6, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 6, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14556000376, + "node_id": "CE_lADOGvRYu86NDn_GzwAAAANjmvB4", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000376", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:30:56Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556000328, + "node_id": "ME_lADOGvRYu86NDn_GzwAAAANjmvBI", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000328", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "created_at": "2024-10-08T12:30:56Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553418544, + "node_id": "HRFPE_lADOGvRYu86NDn_GzwAAAANjc4sw", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553418544", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:26:39Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553418531, + "node_id": "RDE_lADOGvRYu86NDn_GzwAAAANjc4sj", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553418531", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_dismissed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:26:39Z", + "dismissed_review": { + "state": "approved", + "review_id": 2353839077, + "dismissal_message": null, + "dismissal_commit_id": "fb9a445653a77d6e0664a0844021bb134a1f5606" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553182583, + "node_id": "PVTISC_lADOGvRYu86ZVp2rzwAAAANjb_F3", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553182583", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:11:08Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553182805, + "node_id": "ATPVTE_lADOGvRYu86ZVp2rzwAAAANjb_JV", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553182805", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "added_to_project_v2", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:11:07Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14551012232, + "node_id": "SE_lADOGvRYu86YjcEdzwAAAANjTtOI", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14551012232", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T06:40:22Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14551012220, + "node_id": "MEE_lADOGvRYu86YjcEdzwAAAANjTtN8", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14551012220", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T06:40:22Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547344046, + "node_id": "SE_lADOGvRYu86Y2ihPzwAAAANjFtqu", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547344046", + "actor": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T22:00:47Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "id": 2564433999, + "node_id": "PR_kwDOGvRYu859hX16", + "number": 1924, + "title": "fix: PageNumberPaginator not reset when iterating through multiple pa…", + "user": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 4, + "created_at": "2024-10-03T16:09:04Z", + "updated_at": "2024-10-07T22:00:46Z", + "closed_at": "2024-10-05T10:55:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1924", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1924.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1924.patch", + "merged_at": "2024-10-05T10:55:43Z" + }, + "body": "…rent ressources\r\n\r\n\r\n### Description\r\n\r\n\r\n\r\n### Related Issues\r\n\r\n- Fixes #1921 1921\r\n- Closes #...\r\n- Resolves #...\r\n\r\n\r\n### Additional Context\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547344034, + "node_id": "MEE_lADOGvRYu86Y2ihPzwAAAANjFtqi", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547344034", + "actor": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T22:00:47Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "id": 2564433999, + "node_id": "PR_kwDOGvRYu859hX16", + "number": 1924, + "title": "fix: PageNumberPaginator not reset when iterating through multiple pa…", + "user": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 4, + "created_at": "2024-10-03T16:09:04Z", + "updated_at": "2024-10-07T22:00:46Z", + "closed_at": "2024-10-05T10:55:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1924", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1924.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1924.patch", + "merged_at": "2024-10-05T10:55:43Z" + }, + "body": "…rent ressources\r\n\r\n\r\n### Description\r\n\r\n\r\n\r\n### Related Issues\r\n\r\n- Fixes #1921 1921\r\n- Closes #...\r\n- Resolves #...\r\n\r\n\r\n### Additional Context\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547333543, + "node_id": "SE_lADOGvRYu86Y2L7wzwAAAANjFrGn", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547333543", + "actor": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T21:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "id": 2564341488, + "node_id": "PR_kwDOGvRYu859hDjp", + "number": 1923, + "title": "Feat/1922 rest api source add mulitple path parameters", + "user": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923858, + "node_id": "LA_kwDOGvRYu87glfSS", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/enhancement", + "name": "enhancement", + "color": "a2eeef", + "default": true, + "description": "New feature or request" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 3, + "created_at": "2024-10-03T15:22:13Z", + "updated_at": "2024-10-07T21:59:31Z", + "closed_at": "2024-10-05T16:58:10Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1923", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1923.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1923.patch", + "merged_at": "2024-10-05T16:58:10Z" + }, + "body": "\r\n### Description\r\nAdds support for multiple path parameters to child resources in a rest_api_source.\r\n\r\n\r\n### Related Issues\r\n- Closes #1922\r\n\r\n\r\n### Additional Context\r\nAdded some tests which seemed reasonable and ensured that it doesn't allow multiple parents. \r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547333535, + "node_id": "MEE_lADOGvRYu86Y2L7wzwAAAANjFrGf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547333535", + "actor": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T21:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "id": 2564341488, + "node_id": "PR_kwDOGvRYu859hDjp", + "number": 1923, + "title": "Feat/1922 rest api source add mulitple path parameters", + "user": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923858, + "node_id": "LA_kwDOGvRYu87glfSS", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/enhancement", + "name": "enhancement", + "color": "a2eeef", + "default": true, + "description": "New feature or request" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 3, + "created_at": "2024-10-03T15:22:13Z", + "updated_at": "2024-10-07T21:59:31Z", + "closed_at": "2024-10-05T16:58:10Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1923", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1923.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1923.patch", + "merged_at": "2024-10-05T16:58:10Z" + }, + "body": "\r\n### Description\r\nAdds support for multiple path parameters to child resources in a rest_api_source.\r\n\r\n\r\n### Related Issues\r\n- Closes #1922\r\n\r\n\r\n### Additional Context\r\nAdded some tests which seemed reasonable and ensured that it doesn't allow multiple parents. \r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14546778649, + "node_id": "REFE_lADOGvRYu86ZQmUgzwAAAANjDjoZ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14546778649", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "8798c1753a529db1d78d3fd4189e552c3705d250", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/8798c1753a529db1d78d3fd4189e552c3705d250", + "created_at": "2024-10-07T20:59:43Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "id": 2571265312, + "node_id": "PR_kwDOGvRYu85926oH", + "number": 1934, + "title": "master merge for 1.2.0 release", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T19:20:05Z", + "updated_at": "2024-10-07T20:59:42Z", + "closed_at": "2024-10-07T20:59:42Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1934", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1934.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1934.patch", + "merged_at": "2024-10-07T20:59:42Z" + }, + "body": "\r\n### Description\r\nmaster merge for 1.2.0 release\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14546778497, + "node_id": "CE_lADOGvRYu86ZQmUgzwAAAANjDjmB", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14546778497", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T20:59:42Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "id": 2571265312, + "node_id": "PR_kwDOGvRYu85926oH", + "number": 1934, + "title": "master merge for 1.2.0 release", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T19:20:05Z", + "updated_at": "2024-10-07T20:59:42Z", + "closed_at": "2024-10-07T20:59:42Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1934", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1934.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1934.patch", + "merged_at": "2024-10-07T20:59:42Z" + }, + "body": "\r\n### Description\r\nmaster merge for 1.2.0 release\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14546778481, + "node_id": "ME_lADOGvRYu86ZQmUgzwAAAANjDjlx", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14546778481", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "8798c1753a529db1d78d3fd4189e552c3705d250", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/8798c1753a529db1d78d3fd4189e552c3705d250", + "created_at": "2024-10-07T20:59:42Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "id": 2571265312, + "node_id": "PR_kwDOGvRYu85926oH", + "number": 1934, + "title": "master merge for 1.2.0 release", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T19:20:05Z", + "updated_at": "2024-10-07T20:59:42Z", + "closed_at": "2024-10-07T20:59:42Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1934", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1934.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1934.patch", + "merged_at": "2024-10-07T20:59:42Z" + }, + "body": "\r\n### Description\r\nmaster merge for 1.2.0 release\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545639476, + "node_id": "HRDE_lADOGvRYu86ZNk3czwAAAANi_Ng0", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639476", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:30Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545639561, + "node_id": "PVTISC_lADOGvRYu86YeMHLzwAAAANi_NiJ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639561", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:29Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1902", + "id": 2558050763, + "node_id": "I_kwDOGvRYu86YeMHL", + "number": 1902, + "title": "Allow GCS staging buckets for Databricks destination", + "user": { + "login": "16bzwiener", + "id": 18385834, + "node_id": "MDQ6VXNlcjE4Mzg1ODM0", + "avatar_url": "https://avatars.githubusercontent.com/u/18385834?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/16bzwiener", + "html_url": "https://github.com/16bzwiener", + "followers_url": "https://api.github.com/users/16bzwiener/followers", + "following_url": "https://api.github.com/users/16bzwiener/following{/other_user}", + "gists_url": "https://api.github.com/users/16bzwiener/gists{/gist_id}", + "starred_url": "https://api.github.com/users/16bzwiener/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/16bzwiener/subscriptions", + "organizations_url": "https://api.github.com/users/16bzwiener/orgs", + "repos_url": "https://api.github.com/users/16bzwiener/repos", + "events_url": "https://api.github.com/users/16bzwiener/events{/privacy}", + "received_events_url": "https://api.github.com/users/16bzwiener/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-01T02:49:20Z", + "updated_at": "2024-10-07T19:15:28Z", + "closed_at": "2024-10-07T19:15:28Z", + "author_association": "NONE", + "active_lock_reason": null, + "body": "### Feature description\n\nThe Databricks destination stages data in an external bucket before copying the data into Delta Lake. I'm working on GCP and need to use a GCS bucket for staging the data. I was able to get this working with a forked version of the repo.\n\n### Are you a dlt user?\n\nYes, I run dlt in production.\n\n### Use case\n\nMy company is migrating to Databricks on GCP. We have a dozen dlt pipelines in production that will need to be pointed to Databricks. \n\n### Proposed solution\n\nModify `dlt/destinations/impl/databricks/databricks.py` to allow a `gs` bucket_scheme makes its way to https://github.com/dlt-hub/dlt/blob/854905fb56576bc608b01b6b047208df888160a7/dlt/destinations/impl/databricks/databricks.py#L83\r\nIf there isn't a storage credential, still throw an exception saying GCS buckets do not work with temporary credentials.\n\n### Related issues\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14545639135, + "node_id": "REFE_lADOGvRYu86ZNk3czwAAAANi_Nbf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639135", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "created_at": "2024-10-07T19:15:28Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545639053, + "node_id": "CE_lADOGvRYu86YeMHLzwAAAANi_NaN", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639053", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:28Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1902", + "id": 2558050763, + "node_id": "I_kwDOGvRYu86YeMHL", + "number": 1902, + "title": "Allow GCS staging buckets for Databricks destination", + "user": { + "login": "16bzwiener", + "id": 18385834, + "node_id": "MDQ6VXNlcjE4Mzg1ODM0", + "avatar_url": "https://avatars.githubusercontent.com/u/18385834?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/16bzwiener", + "html_url": "https://github.com/16bzwiener", + "followers_url": "https://api.github.com/users/16bzwiener/followers", + "following_url": "https://api.github.com/users/16bzwiener/following{/other_user}", + "gists_url": "https://api.github.com/users/16bzwiener/gists{/gist_id}", + "starred_url": "https://api.github.com/users/16bzwiener/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/16bzwiener/subscriptions", + "organizations_url": "https://api.github.com/users/16bzwiener/orgs", + "repos_url": "https://api.github.com/users/16bzwiener/repos", + "events_url": "https://api.github.com/users/16bzwiener/events{/privacy}", + "received_events_url": "https://api.github.com/users/16bzwiener/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-01T02:49:20Z", + "updated_at": "2024-10-07T19:15:28Z", + "closed_at": "2024-10-07T19:15:28Z", + "author_association": "NONE", + "active_lock_reason": null, + "body": "### Feature description\n\nThe Databricks destination stages data in an external bucket before copying the data into Delta Lake. I'm working on GCP and need to use a GCS bucket for staging the data. I was able to get this working with a forked version of the repo.\n\n### Are you a dlt user?\n\nYes, I run dlt in production.\n\n### Use case\n\nMy company is migrating to Databricks on GCP. We have a dozen dlt pipelines in production that will need to be pointed to Databricks. \n\n### Proposed solution\n\nModify `dlt/destinations/impl/databricks/databricks.py` to allow a `gs` bucket_scheme makes its way to https://github.com/dlt-hub/dlt/blob/854905fb56576bc608b01b6b047208df888160a7/dlt/destinations/impl/databricks/databricks.py#L83\r\nIf there isn't a storage credential, still throw an exception saying GCS buckets do not work with temporary credentials.\n\n### Related issues\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14545638875, + "node_id": "CE_lADOGvRYu86ZNk3czwAAAANi_NXb", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545638875", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:27Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545638850, + "node_id": "ME_lADOGvRYu86ZNk3czwAAAANi_NXC", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545638850", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "created_at": "2024-10-07T19:15:27Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + } + ] \ No newline at end of file diff --git a/tests/pipeline/test_pipeline_extra.py b/tests/pipeline/test_pipeline_extra.py index 821bec8e08..a51052d247 100644 --- a/tests/pipeline/test_pipeline_extra.py +++ b/tests/pipeline/test_pipeline_extra.py @@ -41,7 +41,12 @@ class BaseModel: # type: ignore[no-redef] from tests.utils import TEST_STORAGE_ROOT from tests.extract.utils import expect_extracted_file from tests.load.utils import DestinationTestConfiguration, destinations_configs -from tests.pipeline.utils import assert_load_info, load_data_table_counts, many_delayed +from tests.pipeline.utils import ( + assert_load_info, + load_data_table_counts, + load_json_case, + many_delayed, +) DUMMY_COMPLETE = dummy(completed_prob=1) # factory set up to complete jobs @@ -502,6 +507,51 @@ def test_empty_parquet(test_storage: FileStorage) -> None: assert set(table.schema.names) == {"id", "name", "_dlt_load_id", "_dlt_id"} +def test_parquet_with_flattened_columns() -> None: + # normalize json, write parquet file to filesystem + pipeline = dlt.pipeline( + "test_parquet_with_flattened_columns", destination=dlt.destinations.filesystem("_storage") + ) + info = pipeline.run( + [load_json_case("github_events")], table_name="events", loader_file_format="parquet" + ) + assert_load_info(info) + + # make sure flattened columns exist + assert "issue__reactions__url" in pipeline.default_schema.tables["events"]["columns"] + assert "issue_reactions_url" not in pipeline.default_schema.tables["events"]["columns"] + + events_table = pipeline._dataset().events.arrow() + assert "issue__reactions__url" in events_table.schema.names + assert "issue_reactions_url" not in events_table.schema.names + + # load table back into filesystem + info = pipeline.run(events_table, table_name="events2", loader_file_format="parquet") + assert_load_info(info) + + assert "issue__reactions__url" in pipeline.default_schema.tables["events2"]["columns"] + assert "issue_reactions_url" not in pipeline.default_schema.tables["events2"]["columns"] + + # load back into original table + info = pipeline.run(events_table, table_name="events", loader_file_format="parquet") + assert_load_info(info) + + events_table_new = pipeline._dataset().events.arrow() + assert events_table.schema == events_table_new.schema + # double row count + assert events_table.num_rows * 2 == events_table_new.num_rows + + # now add a column that clearly needs normalization + updated_events_table = events_table_new.append_column( + "Clearly!Normalize", events_table_new["issue__reactions__url"] + ) + info = pipeline.run(updated_events_table, table_name="events", loader_file_format="parquet") + assert_load_info(info) + + assert "clearly_normalize" in pipeline.default_schema.tables["events"]["columns"] + assert "Clearly!Normalize" not in pipeline.default_schema.tables["events"]["columns"] + + def test_resource_file_format() -> None: os.environ["RESTORE_FROM_DESTINATION"] = "False"