Skip to content

Commit

Permalink
Fix snippet with filter
Browse files Browse the repository at this point in the history
  • Loading branch information
VioletM committed Sep 11, 2024
1 parent ded43a6 commit 4dfce16
Showing 1 changed file with 2 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ from dlt.sources.filesystem import filesystem, read_csv
# Filter files accessing file_name field
filtered_files = filesystem(bucket_url="s3://bucket_name", file_glob="directory/*.csv")
filtered_files.add_filter(lambda item: ("London" in item.file_name) or ("Berlin" in item.file_name))
filtered_files.add_filter(lambda item: ("London" in item["file_name"]) or ("Berlin" in item["file_name"]))
filesystem_pipe = (filtered_files | read_csv())
pipeline = dlt.pipeline(pipeline_name="my_pipeline", destination="duckdb")
Expand Down Expand Up @@ -447,7 +447,7 @@ MAX_SIZE_IN_BYTES = 10
# Filter files accessing size_in_bytes field
filtered_files = filesystem(bucket_url="s3://bucket_name", file_glob="directory/*.csv")
filtered_files.add_filter(lambda item: item.size_in_bytes < MAX_SIZE_IN_BYTES)
filtered_files.add_filter(lambda item: item["size_in_bytes"] < MAX_SIZE_IN_BYTES)
filesystem_pipe = (filtered_files | read_csv())
pipeline = dlt.pipeline(pipeline_name="my_pipeline", destination="duckdb")
Expand Down

0 comments on commit 4dfce16

Please sign in to comment.