Skip to content

Commit

Permalink
Merge pull request #8 from bmsuisse/dev
Browse files Browse the repository at this point in the history
expose get polars schema
  • Loading branch information
aersam authored Jun 17, 2024
2 parents 91977e9 + 4386185 commit b75f17d
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 83 deletions.
2 changes: 1 addition & 1 deletion deltalake2db/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
from .duckdb import get_sql_for_delta_expr
from .duckdb import create_view_for_delta as duckdb_create_view_for_delta
from .duckdb import apply_storage_options as duckdb_apply_storage_options
from .polars import scan_delta_union as polars_scan_delta
from .polars import scan_delta_union as polars_scan_delta, get_polars_schema
from .protocol_check import is_protocol_supported
15 changes: 15 additions & 0 deletions deltalake2db/polars.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import polars as pl
from deltalake import DeltaTable, Field, DataType
from deltalake.schema import StructType, ArrayType, MapType
from collections import OrderedDict
import os


Expand Down Expand Up @@ -136,6 +137,20 @@ def _filter_cond(f: "pl.LazyFrame", conditions: dict) -> "pl.LazyFrame":
return f.filter(**conditions)


def get_polars_schema(
delta_table: Union[DeltaTable, Path, str],
) -> "OrderedDict[str, pl.PolarsDataType]":
from .protocol_check import check_is_supported

if isinstance(delta_table, Path) or isinstance(delta_table, str):
delta_table = DeltaTable(delta_table)
check_is_supported(delta_table)
res_dict = OrderedDict()
for f in delta_table.schema().fields:
res_dict[f.name] = _get_type(f.type)
return res_dict


def scan_delta_union(
delta_table: Union[DeltaTable, Path, str],
conditions: Optional[dict] = None,
Expand Down
Loading

0 comments on commit b75f17d

Please sign in to comment.