TESTING: all automatic workflow bash tests are working, except pin-fi…

…lesystem
sjdv1982 · Sep 13, 2024 · 07a4a57 · 07a4a57
1 parent 39aafec
commit 07a4a57
Show file tree

Hide file tree

Showing 15 changed files with 94 additions and 72 deletions.
diff --git a/seamless/workflow/midlevel/translate.py b/seamless/workflow/midlevel/translate.py
@@ -64,20 +64,6 @@ def print_error(*args):
 
 def set_structured_cell_from_checksum(cell, checksum, is_deepcell=False):
     trigger = False
-    """
-    if "temp" in checksum:
-        assert len(checksum) == 1, checksum.keys()
-        temp_checksum = checksum["temp"]
-        if cell.hash_pattern is not None:
-            temp_cs = Checksum(temp_checksum)
-            temp_cs2 = apply_hash_pattern_sync(
-                temp_cs, cell.hash_pattern
-            )
-            temp_checksum = temp_cs2.hex()
-        cell.auth._set_checksum(temp_checksum, initial=True, from_structured_cell=False)
-        trigger = True
-    else:
-    """
     if "value" in checksum:
         # not done! value calculated anew...
         """
@@ -101,22 +87,22 @@ def set_structured_cell_from_checksum(cell, checksum, is_deepcell=False):
         """
 
     k = "origin" if is_deepcell else "auth"
-    if k in checksum and checksum[k] is not None:
+    if k in checksum and Checksum(checksum[k]):
         if cell.auth is None:
             if not is_deepcell:
                 msg = "Warning: {} has no independence, but an {} checksum is present"
                 print(msg.format(cell, k))
         else:
             cell.auth._set_checksum(
-                checksum[k], from_structured_cell=True, initial=True
+                Checksum(checksum[k]), from_structured_cell=True, initial=True
             )
             cell._data._void = False
             cell._data._status_reason = None
             trigger = True
 
     schema_checksum = empty_dict_checksum
     if "schema" in checksum:
-        schema_checksum = checksum["schema"]
+        schema_checksum = Checksum(checksum["schema"])
     cell.schema._set_checksum(schema_checksum, from_structured_cell=True, initial=True)
     trigger = True
 

diff --git a/tests/workflow/load-vault.py b/tests/workflow/load-vault.py
@@ -1,6 +1,6 @@
 import json
 import seamless
-from seamless import load_graph, Context
+from seamless.workflow import load_graph, Context
 
 seamless.delegate(False)
 

diff --git a/tests/workflow/macro-elision-database.py b/tests/workflow/macro-elision-database.py
@@ -7,7 +7,7 @@
 else:
     print("Database found")
 
-from seamless.workflow import Context, Macro
+from seamless.workflow.highlevel import Context, Macro
 
 ctx = Context()
 m = ctx.m = Macro()

diff --git a/tests/workflow/macro-elision-database.sh b/tests/workflow/macro-elision-database.sh
@@ -9,7 +9,7 @@ echo 'Run 1'
 python3 -u macro-elision-database.py
 echo 'Start database'
 seamless-delegate-stop >& /dev/null
-seamless-delegate none >& /dev/null
+seamless-delegate none-devel >& /dev/null
 sleep 1
 echo
 echo 'Run 2'

diff --git a/tests/workflow/pin-filesystem.py b/tests/workflow/pin-filesystem.py
@@ -18,8 +18,7 @@
     print("Buffer read folder not found")
     seamless.delegate(False)
 
-from seamless.Cell import FolderCell
-from seamless.workflow import Context, DeepFolderCell, Transformer
+from seamless.workflow import Context, DeepFolderCell, Transformer, FolderCell
 
 try:
     deepfolder_checksum = seamless.parse_checksum(

diff --git a/tests/workflow/save-vault.py b/tests/workflow/save-vault.py
@@ -1,6 +1,6 @@
 import json
 import seamless
-from seamless import load_graph, Context
+from seamless.workflow import Context
 
 seamless.delegate(False)
 

diff --git a/tests/workflow/test-list.txt b/tests/workflow/test-list.txt
@@ -51,6 +51,7 @@ macro-simple.py
 macro.py
 macro-elision.py
 macro-elision-database.sh: 
+  run outside Docker image.
   run as i=macro-elision-database; seamless-run ./$i.sh > test-outputs/$i.out 2>&1
 reactor-edit-macro.py
 delay.py

diff --git a/tests/workflow/test-outputs/docker-singularity.out b/tests/workflow/test-outputs/docker-singularity.out
@@ -2,6 +2,7 @@ a
 b 
 c 
 None
+Waiting for: Seamless transformer: .tf.tf
 
 <Silk: {'test.npy': array([ 0,  3,  6,  9, 12, 15, 18, 21, 24, 27, 30, 33]), 'test.txt': 'hello\n'} >
 <Silk: [ 0  3  6  9 12 15 18 21 24 27 30 33] >

diff --git a/tests/workflow/test-outputs/get-file-and-directory-IN-DOCKER.out b/tests/workflow/test-outputs/get-file-and-directory-IN-DOCKER.out
@@ -1,12 +1,12 @@
-seamless: Verbosity set to 1
-seamless: Upload 1 files, total 5.00 B
-seamless: Verbosity set to 1
-seamless: Upload 3 files, total 169.00 B
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload 1 files, total 5.00 B
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload 3 files, total 169.00 B
 Repeat, no files will be uploaded...
-seamless: Verbosity set to 1
-seamless: Upload no files
-seamless: Verbosity set to 1
-seamless: Upload no files
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload no files
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload no files
 Stage 1
 
 None
@@ -52,7 +52,7 @@ The Docker containers will run side by side under the host, symbolic links will
 
 The following pins access their buffer through symbolic links:
 
-    inputfolder -> /cwd/tests/highlevel/dummy-bufferfolder/deployed/069a577ce64ff98c9730e55b8f13f94ba3eaba9f3066be0df90942ed0f7096e5
+    inputfolder -> /cwd/tests/workflow/dummy-bufferfolder/deployed/069a577ce64ff98c9730e55b8f13f94ba3eaba9f3066be0df90942ed0f7096e5
 
 
 

diff --git a/tests/workflow/test-outputs/get-file-and-directory.out b/tests/workflow/test-outputs/get-file-and-directory.out
@@ -1,12 +1,12 @@
-seamless: Verbosity set to 1
-seamless: Upload 1 files, total 5.00 B
-seamless: Verbosity set to 1
-seamless: Upload 3 files, total 169.00 B
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload 1 files, total 5.00 B
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload 3 files, total 169.00 B
 Repeat, no files will be uploaded...
-seamless: Verbosity set to 1
-seamless: Upload no files
-seamless: Verbosity set to 1
-seamless: Upload no files
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload no files
+seamless-upload: Verbosity set to 1
+seamless-upload: Upload no files
 Stage 1
 
 None

diff --git a/tests/workflow/test-outputs/imperative-async-parallel-jupyter.out b/tests/workflow/test-outputs/imperative-async-parallel-jupyter.out
@@ -1,4 +1,6 @@
 START
+In [3]:         
+In [3]: 
 In [3]: await main()                    
 In [3]: await main()
 10

diff --git a/tests/workflow/test-outputs/imperative-async.out b/tests/workflow/test-outputs/imperative-async.out
@@ -2,38 +2,47 @@
    ...: 
    ...: from seamless import transformer
    ...: 
+   ...: 
    ...: @transformer(return_transformation=True)
    ...: def func(a, b):
    ...:     import time
+   ...: 
    ...:     time.sleep(0.5)
    ...:     return 100 * a + b
    ...: 
-   ...: result = await func(88, 17).task() # takes 0.5 sec
+   ...: 
+   ...: result = await func(88, 17).task()  # takes 0.5 sec
    ...: print(result)
-   ...: result = await func(88, 17).task() # immediate
+   ...: result = await func(88, 17).task()  # immediate
    ...: print(result)
-   ...: result = await func(21, 17).task() # immediate
-   ...: print(result)                     
+   ...: result = await func(21, 17).task()  # immediate
+   ...: print(result)
+   ...:         
 In [2]: # %load imperative-async.py
    ...: import seamless
+   ...: 
    ...: seamless.delegate(False)
    ...: 
    ...: print("START")
    ...: 
    ...: from seamless import transformer
    ...: 
+   ...: 
    ...: @transformer(return_transformation=True)
    ...: def func(a, b):
    ...:     import time
+   ...: 
    ...:     time.sleep(0.5)
    ...:     return 100 * a + b
    ...: 
-   ...: result = await func(88, 17).task() # takes 0.5 sec
+   ...: 
+   ...: result = await func(88, 17).task()  # takes 0.5 sec
    ...: print(result)
-   ...: result = await func(88, 17).task() # immediate
+   ...: result = await func(88, 17).task()  # immediate
    ...: print(result)
-   ...: result = await func(21, 17).task() # immediate
+   ...: result = await func(21, 17).task()  # immediate
    ...: print(result)
+   ...: 
 START
 8817
 8817

diff --git a/tests/workflow/test-outputs/imperative-jupyter.out b/tests/workflow/test-outputs/imperative-jupyter.out
@@ -1,47 +1,49 @@
    ...:     print("START")
-   ...:     raise Exception # deliberately
+   ...:     raise Exception  # deliberately
+   ...: 
+   ...: 
    ...: try:
-   ...:     func0(0,0)
+   ...:     func0(0, 0)
    ...: except SeamlessTransformationError:
    ...:     traceback.print_exc(1)
    ...: print("/exception")
    ...: print()
    ...:         
-   ...: transformation_checksum = ctx.tf.get_transformation_checksum()
-   ...: transformation_dict = ctx.resolve(transformation_checksum, "plain")
-   ...: 
-   ...: from seamless.core.direct.run import run_transformation_dict_async
-   ...: from seamless.core.cache.buffer_cache import buffer_cache
-   ...: from seamless.core.protocol.deserialize import deserialize_sync as dese                                                                               r
-   ...: ialize
-   ...: 
-   ...: result_checksum = await run_transformation_dict_async(transformation_di                                                                               c
-   ...: t, fingertip=False)
-   ...: result = deserialize(buffer_cache.get_buffer(result_checksum), result_c                                                                               h
-   ...: ecksum, "mixed", copy=True)
+   ...: result_checksum = await run_transformation_dict_async(
+   ...:     transformation_dict, fingertip=False
+   ...: )
+   ...: result = Checksum(result_checksum).resolve("mixed")
    ...: print(result)
    ...: 
+   ...: 
    ...: @transformer
    ...: def func(a, b):
    ...:     import time
+   ...: 
    ...:     time.sleep(0.5)
    ...:     return 100 * a + b
+   ...: 
+   ...: 
    ...: func = transformer(func)
    ...: 
-   ...: result = func(88, 17) # takes 0.5 sec
+   ...: result = func(88, 17)  # takes 0.5 sec
    ...: print(result)
-   ...: result = func(88, 17) # immediate
+   ...: result = func(88, 17)  # immediate
    ...: print(result)
-   ...: result = func(21, 17) # immediate
+   ...: result = func(21, 17)  # immediate
    ...: print(result)
    ...: 
    ...: print("The following will give an exception:")
+   ...: 
+   ...: 
    ...: @transformer
    ...: def func0(a, b):
    ...:     print("START")
-   ...:     raise Exception # deliberately
+   ...:     raise Exception  # deliberately
+   ...: 
+   ...: 
    ...: try:
-   ...:     func0(0,0)
+   ...:     func0(0, 0)
    ...: except SeamlessTransformationError:
    ...:     traceback.print_exc(1)
    ...: print("/exception")
@@ -60,13 +62,13 @@ The following will give an exception:
 /exception
 
 Traceback (most recent call last):
-  File "/tmp/ipykernel_1541/3721671167.py", line 47, in <module>
-    func0(0,0)
-seamless.core.transformation.SeamlessTransformationError: Traceback (most recent call last):
+  File "/tmp/ipykernel_1047/3750347486.py", line 60, in <module>
+    func0(0, 0)
+seamless.workflow.core.transformation.SeamlessTransformationError: Traceback (most recent call last):
   File "transformer", line 6, in <module>
     result = func0(a=a,b=b)
   File "transformer", line 4, in func0
-    raise Exception # deliberately
+    raise Exception  # deliberately
 Exception
 
 *************************************************

diff --git a/tests/workflow/test-outputs/simpler-ipython.out b/tests/workflow/test-outputs/simpler-ipython.out
@@ -1,9 +1,11 @@
 Cannot install event loop hook for "asyncio" when running with `--simple-prompt`.
 NOTE: Tk is supported natively; use Tk apps and Tk backends with `--simple-prompt`.
-31.4 ns +- 1.49 ns per loop (mean +- std. dev. of 7 runs, 10,000,000 loops each)
+71.5 ns +- 8 ns per loop (mean +- std. dev. of 7 runs, 10,000,000 loops each)
+Waiting for: Seamless transformer: .transform.tf
 Waiting for: Seamless transformer: .transform.tf
 <Silk: None >
-28.7 ns +- 0.94 ns per loop (mean +- std. dev. of 7 runs, 10,000,000 loops each)
+70.1 ns +- 3.91 ns per loop (mean +- std. dev. of 7 runs, 10,000,000 loops each)
+Waiting for: Seamless transformer: .transform.tf
 Waiting for: Seamless transformer: .transform.tf
 <Silk: None >
 Status: OK
diff --git a/wip/TODO b/wip/TODO
@@ -0,0 +1,20 @@
+Before 0.13 release:
+  - high level tests, bash automatic, DONE except pin-filesystem
+  - high level tests, delegation
+  - high level tests, non-automatic
+  - regenerate webgen, run webgen/test
+  - cleanup new-project, run tests/workflow/webinterface.sh (also interactively)
+  - cleanup /seamless-tools/scripts
+  - cleanup /seamless-tools/scripts/utils
+  - cleanup /seamless/bin
+  - cleanup /seamless-tools/seamless-cli
+  - direct tests
+  - fairserver tests
+  - examples
+  - build and publish seamless-framework conda package
+  - build and publish seamless-cli-bin + seamless-cli-complement conda packages
+  - build and test conda environments
+  - build and publish seamless-deps and seamless Docker images
+  - build, test and publish seamless-cli conda package
+  - update binderhub demo
+  - update Cloudless (big)