Skip to content

Commit

Permalink
avoid lambda for mp
Browse files Browse the repository at this point in the history
  • Loading branch information
kai-tub committed Jul 1, 2024
1 parent 4fb89e6 commit 61f44aa
Showing 1 changed file with 20 additions and 13 deletions.
33 changes: 20 additions & 13 deletions rico_hdl/rico_hdl.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,18 @@ def safetensor_generator_s1(patch_path: str) -> bytes:
return save(data, metadata=None)


def optional_compressed_safetensor_generator(
safetensor_generator, compress: bool = False
):
if not compress:
return safetensor_generator

def wrapper(x):
return zstd_compressor(safetensor_generator(x))

return wrapper


@app.command()
def hyspecnet_11k(
target_dir: TargetDir,
Expand Down Expand Up @@ -266,27 +278,22 @@ def bigearthnet(
# Otherwise an error in the latter CLI argument could produce an incomplete LMDB
env = open_lmdb(target_dir)

safetensor_gen_s1 = (
(lambda x: zstd_compressor(safetensor_generator_s1(x)))
if compress
else safetensor_generator_s1
)
safetensor_gen_s2 = (
(lambda x: zstd_compressor(safetensor_generator_s2(x)))
if compress
else safetensor_generator_s2
)

if bigearthnet_s1_dir is not None:
log.debug("Writing BigEarthNet-S1 data into LMDB")
lmdb_writer(
env, s1_patch_paths, bigearthnet_lmdb_key_extractor, safetensor_gen_s1
env,
s1_patch_paths,
bigearthnet_lmdb_key_extractor,
optional_compressed_safetensor_generator(safetensor_generator_s1, compress),
)

if bigearthnet_s2_dir is not None:
log.debug("Writing BigEarthNet-S2 data into LMDB")
lmdb_writer(
env, s2_patch_paths, bigearthnet_lmdb_key_extractor, safetensor_gen_s2
env,
s2_patch_paths,
bigearthnet_lmdb_key_extractor,
optional_compressed_safetensor_generator(safetensor_generator_s2, compress),
)


Expand Down

0 comments on commit 61f44aa

Please sign in to comment.