Skip to content

Commit

Permalink
fixes staging tests for athena
Browse files Browse the repository at this point in the history
  • Loading branch information
rudolfix authored and willi-mueller committed Sep 2, 2024
1 parent efe5e0d commit d848f1d
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions tests/load/pipeline/test_stage_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,14 @@ def test_staging_load(destination_config: DestinationTestConfiguration) -> None:

assert len(package_info.jobs["failed_jobs"]) == 0
# we have 4 parquet and 4 reference jobs plus one merge job
num_jobs = 4 + 4 + 1 if destination_config.supports_merge else 4 + 4
assert len(package_info.jobs["completed_jobs"]) == num_jobs
num_jobs = 4 + 4
num_sql_jobs = 0
if destination_config.supports_merge:
num_sql_jobs += 1
# sql job is used to copy parquet to Athena Iceberg table (_dlt_pipeline_state)
if destination_config.destination == "athena" and destination_config.table_format == "iceberg":
num_sql_jobs += 1
assert len(package_info.jobs["completed_jobs"]) == num_jobs + num_sql_jobs
assert (
len(
[
Expand Down Expand Up @@ -110,7 +116,7 @@ def test_staging_load(destination_config: DestinationTestConfiguration) -> None:
if x.job_file_info.file_format == "sql"
]
)
== 1
== num_sql_jobs
)

initial_counts = load_table_counts(
Expand Down

0 comments on commit d848f1d

Please sign in to comment.