From a03a1caa6c3f3bcd4dd4979eb6299c4b1ce733be Mon Sep 17 00:00:00 2001 From: Sjoerd de Vries Date: Wed, 13 Sep 2023 23:09:50 +0200 Subject: [PATCH] add bashdocker support to unbashify --- seamless/metalevel/unbashify.py | 72 +++++++++++++++++++ tests/lowlevel/test-list.txt | 1 + .../transformation-raw-bashdocker.out | 0 .../lowlevel/transformation-raw-bashdocker.py | 53 ++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 tests/lowlevel/test-outputs/transformation-raw-bashdocker.out create mode 100644 tests/lowlevel/transformation-raw-bashdocker.py diff --git a/seamless/metalevel/unbashify.py b/seamless/metalevel/unbashify.py index 69790a17f..17ca5a8bc 100644 --- a/seamless/metalevel/unbashify.py +++ b/seamless/metalevel/unbashify.py @@ -21,14 +21,86 @@ def get_bash_checksums(): _bash_checksums["executor_code_buffer"] = executor_code_buffer _bash_checksums["semantic_code_checksum"] = semantic_code_checksum + sctx = load_stdgraph("bashdocker_transformer") + executor_code_checksum = sctx.executor_code.checksum + executor_code_buffer = sctx.executor_code.buffer + executor_code = sctx.executor_code.value + semantic_code_checksum = _get_semantic( + executor_code, bytes.fromhex(executor_code_checksum) + ) + _bash_checksums["docker_executor_code_checksum"] = executor_code_checksum + _bash_checksums["docker_executor_code_buffer"] = executor_code_buffer + _bash_checksums["docker_semantic_code_checksum"] = semantic_code_checksum + return _bash_checksums.copy() +def unbashify_docker(transformation_dict, semantic_cache, env: dict): + from seamless.core.direct.run import prepare_code, prepare_transformation_pin_value + tdict = deepcopy(transformation_dict) + tdict["__language__"] = "python" + # TODO: will this work directly with different return types / hash patterns? + tdict["__output__"] = ("result", "bytes", None) + bash_checksums = get_bash_checksums() + + pins = [p for p in sorted(tdict.keys()) if p != "code" and not p.startswith("__")] + + docker = env.pop("docker") + docker_image = docker["name"] + docker_options = docker.get("options", {}) + # TODO: pass on version and checksum as well? + if "powers" not in env: + env["powers"] = [] + env["powers"].append("docker") + env2_checksum = prepare_transformation_pin_value(env, "plain").hex() + tdict["__env__"] = env2_checksum + + new_pins = { + "pins_": (pins, "plain"), + "docker_image_": (docker_image, "str"), + "docker_options": (docker_options, "plain"), + } + for pinname, (value, celltype) in new_pins.items(): + p_checksum = prepare_transformation_pin_value(value, celltype).hex() + tdict[pinname] = (celltype, None, p_checksum) + + code_pin = tdict["code"] + tdict["docker_command"] = code_pin + semantic_code_checksum = prepare_code( + bash_checksums["docker_semantic_code_checksum"], + bash_checksums["docker_executor_code_buffer"], + bash_checksums["docker_executor_code_checksum"] + ) + tdict["code"] = ("python", "transformer", semantic_code_checksum.hex()) + semkey = (semantic_code_checksum.bytes(), "python", "transformer") + semantic_cache[semkey] = [bytes.fromhex(bash_checksums["docker_executor_code_checksum"])] + return tdict + def unbashify(transformation_dict, semantic_cache): from seamless.core.direct.run import prepare_code, prepare_transformation_pin_value + from seamless.core.manager import Manager + from ..core.environment import ( + validate_capabilities, + validate_conda_environment, + validate_docker + ) + assert transformation_dict["__language__"] == "bash" assert "bashcode" not in transformation_dict assert "pins_" not in transformation_dict + env_checksum = transformation_dict.get("__env__") + env = None + if env_checksum is not None: + manager = Manager() + env = manager.resolve(bytes.fromhex(env_checksum), celltype="plain", copy=True) + + if env is not None and env.get("docker") is not None: + ok1 = validate_capabilities(env)[0] + ok2 = validate_conda_environment(env)[0] + ok3 = validate_docker(env)[0] + if not (ok1 or ok2 or ok3): + return unbashify_docker(transformation_dict, semantic_cache, env) + tdict = deepcopy(transformation_dict) tdict["__language__"] = "python" # TODO: will this work directly with different return types / hash patterns? diff --git a/tests/lowlevel/test-list.txt b/tests/lowlevel/test-list.txt index ea7630740..900827504 100644 --- a/tests/lowlevel/test-list.txt +++ b/tests/lowlevel/test-list.txt @@ -58,6 +58,7 @@ transformation-checksum.sh: run with Seamless database. If not in the Docker ima transformation-raw.py transformation-raw-ipython.py transformation-raw-bash.py +transformation-raw-bashdocker.py meta.py simple-duplex.py simple-remote.py: run together with seamless-jobslave diff --git a/tests/lowlevel/test-outputs/transformation-raw-bashdocker.out b/tests/lowlevel/test-outputs/transformation-raw-bashdocker.out new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lowlevel/transformation-raw-bashdocker.py b/tests/lowlevel/transformation-raw-bashdocker.py new file mode 100644 index 000000000..98654ae00 --- /dev/null +++ b/tests/lowlevel/transformation-raw-bashdocker.py @@ -0,0 +1,53 @@ +import inspect, textwrap + +import seamless +seamless.delegate(False) + +from seamless import calculate_checksum +from seamless.core.cache.buffer_cache import buffer_cache +from seamless.core.cache.transformation_cache import ( + transformation_cache, DummyTransformer, tf_get_buffer, + syntactic_is_semantic, syntactic_to_semantic, + transformation_cache +) + +from seamless.core.protocol.serialize import serialize + +async def build_transformation(): + bash_code = "nginx -v" + inp = { + "code": ("text", bash_code), + "__env__": ("plain", {"docker": {"name": "nginx:1.25.2"}}), + } + tf_dunder = {} + transformation = { + "__language__": "bash", + "__output__": ("result", "bytes", None) + } + for k,v in inp.items(): + celltype, value = v + buf = await serialize(value, celltype) + checksum = calculate_checksum(buf) + buffer_cache.cache_buffer(checksum, buf) + if k == "__env__": + tf_dunder[k] = checksum.hex() + else: + transformation[k] = celltype, None, checksum.hex() + + tf_buf = tf_get_buffer(transformation) + tf_checksum = calculate_checksum(tf_buf) + buffer_cache.cache_buffer(tf_checksum, tf_buf) + + tf = DummyTransformer(tf_checksum) + result = await transformation_cache.run_transformation_async(tf_checksum, tf_dunder=tf_dunder, fingertip=False) + if result is not None: + result = buffer_cache.get_buffer(result, remote=False) + if result is not None: + try: + result = result.decode() + except UnicodeDecodeError: + pass + print(result) + +import asyncio +asyncio.get_event_loop().run_until_complete(build_transformation())