Skip to content

Commit

Permalink
unify filesystem schema behavior with other destinations
Browse files Browse the repository at this point in the history
  • Loading branch information
sh-rp committed Oct 11, 2024
1 parent 213b89c commit 50a5d47
Showing 1 changed file with 21 additions and 17 deletions.
38 changes: 21 additions & 17 deletions dlt/destinations/impl/filesystem/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -655,24 +655,28 @@ def _get_stored_schema_by_hash_or_newest(
"""Get the schema by supplied hash, falls back to getting the newest version matching the existing schema name"""
version_hash = self._to_path_safe_string(version_hash)
# find newest schema for pipeline or by version hash
selected_path = None
newest_load_id = "0"
for filepath, fileparts in self._iter_stored_schema_files():
if (
not version_hash
and (fileparts[0] == self.schema.name or any_schema_name)
and fileparts[1] > newest_load_id
):
newest_load_id = fileparts[1]
selected_path = filepath
elif fileparts[2] == version_hash:
selected_path = filepath
break
try:
selected_path = None
newest_load_id = "0"
for filepath, fileparts in self._iter_stored_schema_files():
if (
not version_hash
and (fileparts[0] == self.schema.name or any_schema_name)
and fileparts[1] > newest_load_id
):
newest_load_id = fileparts[1]
selected_path = filepath
elif fileparts[2] == version_hash:
selected_path = filepath
break

if selected_path:
return StorageSchemaInfo(
**json.loads(self.fs_client.read_text(selected_path, encoding="utf-8"))
)
if selected_path:
return StorageSchemaInfo(
**json.loads(self.fs_client.read_text(selected_path, encoding="utf-8"))
)
except DestinationUndefinedEntity:
# ignore missing table
pass

return None

Expand Down

0 comments on commit 50a5d47

Please sign in to comment.