Merge pull request #34 from score-p/hotfix

Hotfix

src/jumper/kernel.py

@@ -5,6 +5,7 @@
 import subprocess
 import sys
 import time
+import shutil
 
 from enum import Enum
 from textwrap import dedent
@@ -94,6 +95,8 @@ def __init__(self, **kwargs):
         self.perfdata_handler = PerformanceDataHandler()
         self.nodelist = self.perfdata_handler.get_nodelist()
 
+        self.scorep_available_ = shutil.which("scorep")
+
     def cell_output(self, string, stream="stdout"):
         """
         Display string as cell output.
@@ -109,6 +112,13 @@ def standard_reply(self):
             "payload": [],
             "user_expressions": {},
         }
+
+    def scorep_not_available(self):
+        if not self.scorep_available_:
+            self.cell_output("Score-P not available, cell ignored.", "stderr")
+            return self.standard_reply()
+        else:
+            return None
 
     def marshaller_settings(self, code):
         """
@@ -119,9 +129,9 @@ def marshaller_settings(self, code):
             self.pershelper.postprocess()
 
             marshaller_match = re.search(
-                r"MARSHALLER=(\w+)", code.split("\n", 1)[1]
+                r"MARSHALLER=([\w-]+)", code.split("\n", 1)[1]
             )
-            mode_match = re.search(r"MODE=(\w+)", code.split("\n", 1)[1])
+            mode_match = re.search(r"MODE=([\w-]+)", code.split("\n", 1)[1])
             marshaller = (
                 marshaller_match.group(1) if marshaller_match else None
             )
@@ -130,11 +140,10 @@ def marshaller_settings(self, code):
             if marshaller:
                 if not self.pershelper.set_marshaller(marshaller):
                     self.cell_output(
-                        f"Marshaller '{marshaller}' is not recognized, "
+                        f"Marshaller '{marshaller}' is not available or compatible, "
                         f"kernel will use '{self.pershelper.marshaller}'.",
                         "stderr",
                     )
-                    return self.standard_reply()
             if mode:
                 if not self.pershelper.set_mode(mode):
                     self.cell_output(
@@ -196,13 +205,13 @@ def set_scorep_pythonargs(self, code):
         Read and record Score-P Python binding arguments from the cell.
         """
         if self.mode == KernelMode.DEFAULT:
-            self.scorep_binding_args += code.split("\n")[1:]
+            self.scorep_binding_args = code.split("\n")[1].replace(' ', '\n').split("\n")
             self.cell_output(
                 "Score-P Python binding arguments set successfully: "
                 + str(self.scorep_binding_args)
             )
         elif self.mode == KernelMode.WRITEFILE:
-            self.writefile_scorep_binding_args += code.split("\n")[1:]
+            self.writefile_scorep_binding_args = code.split("\n")[1].replace(' ', '\n').split("\n")
             self.cell_output("Score-P bindings arguments recorded.")
         else:
             self.cell_output(
@@ -388,7 +397,6 @@ def end_writefile(self):
                     f"{' '.join(self.writefile_scorep_binding_args)} "
                     f"{self.writefile_python_name}"
                 )
-            #self.cell_output(f"{''.join(self.writefile_scorep_env)}\n")
             self.cell_output("Finished converting to Python script.")
         else:
             self.cell_output(
@@ -397,6 +405,31 @@ def end_writefile(self):
             )
         return self.standard_reply()
 
+    def abort_writefile(self):
+        """
+        Cancel writefile mode.
+        """
+        if self.mode == KernelMode.WRITEFILE:
+            self.mode = KernelMode.DEFAULT
+
+            if os.path.exists(self.writefile_bash_name):
+                os.remove(self.writefile_bash_name)
+            if os.path.exists(self.writefile_python_name):
+                os.remove(self.writefile_python_name)
+
+            self.writefile_base_name = "jupyter_to_script"
+            self.writefile_bash_name = ""
+            self.writefile_python_name = ""
+            self.writefile_scorep_binding_args = []
+            self.writefile_multicell = False
+            self.cell_output("Writefile mode aborted.")
+        else:
+            self.cell_output(
+                f"KernelWarning: Currently in {self.mode}, command ignored.",
+                "stderr",
+            )
+        return self.standard_reply()
+
     def ghost_cell_error(self, reply_status, error_message):
         self.shell.execution_count += 1
         reply_status["execution_count"] = self.shell.execution_count - 1
@@ -967,75 +1000,85 @@ async def do_execute(
         elif code.startswith("%%set_perfmonitor"):
             return self.set_perfmonitor(code)
         elif code.startswith("%%scorep_python_binding_arguments"):
-            return self.set_scorep_pythonargs(code)
+            return self.scorep_not_available() or self.set_scorep_pythonargs(code)
         elif code.startswith("%%serializer_settings"):
             self.cell_output(
                 "Deprecated. Use: %%marshalling_settings\n[MARSHALLER=]\n[MODE=]",
                 "stdout",
             )
             return self.standard_reply()
         elif code.startswith("%%marshalling_settings"):
-            return self.marshaller_settings(code)
+            return self.scorep_not_available() or self.marshaller_settings(code)
         elif code.startswith("%%enable_multicellmode"):
-            return self.enable_multicellmode()
+            return self.scorep_not_available() or self.enable_multicellmode()
         elif code.startswith("%%abort_multicellmode"):
-            return self.abort_multicellmode()
+            return self.scorep_not_available() or self.abort_multicellmode()
         elif code.startswith("%%finalize_multicellmode"):
             # Cannot be put into a separate function due to tight coupling
             # between do_execute and scorep_execute
-            if self.mode == KernelMode.MULTICELL:
-                self.mode = KernelMode.DEFAULT
-                try:
-                    reply_status = await self.scorep_execute(
-                        self.multicell_code,
+            if not self.scorep_available_:
+                self.cell_output("Score-P not available, cell ignored.", "stderr")
+                return self.standard_reply()
+            else:
+                if self.mode == KernelMode.MULTICELL:
+                    self.mode = KernelMode.DEFAULT
+                    try:
+                        reply_status = await self.scorep_execute(
+                            self.multicell_code,
+                            silent,
+                            store_history,
+                            user_expressions,
+                            allow_stdin,
+                            cell_id=cell_id,
+                        )
+                    except Exception:
+                        self.cell_output(
+                            "KernelError: Multicell execution failed.", "stderr"
+                        )
+                        return self.standard_reply()
+                    self.multicell_code = ""
+                    self.multicell_cellcount = 0
+                    return reply_status
+                elif self.mode == KernelMode.WRITEFILE:
+                    self.writefile_multicell = False
+                    return self.standard_reply()
+                else:
+                    self.cell_output(
+                        f"KernelWarning: Currently in {self.mode}, ignore command",
+                        "stderr",
+                    )
+                    return self.standard_reply()
+        elif code.startswith("%%start_writefile"):
+            return self.scorep_not_available() or self.start_writefile(code)
+        elif code.startswith("%%abort_writefile"):
+            return self.scorep_not_available() or self.abort_writefile()
+        elif code.startswith("%%end_writefile"):
+            return self.scorep_not_available() or self.end_writefile()
+        elif code.startswith("%%execute_with_scorep"):
+            if not self.scorep_available_:
+                self.cell_output("Score-P not available, cell ignored.", "stderr")
+                return self.standard_reply()
+            else:
+                if self.mode == KernelMode.DEFAULT:
+                    return await self.scorep_execute(
+                        code.split("\n", 1)[1],
                         silent,
                         store_history,
                         user_expressions,
                         allow_stdin,
                         cell_id=cell_id,
                     )
-                except Exception:
-                    self.cell_output(
-                        "KernelError: Multicell execution failed.", "stderr"
+                elif self.mode == KernelMode.MULTICELL:
+                    return self.append_multicellmode(
+                        magics_cleanup(code.split("\n", 1)[1])[1]
+                    )
+                elif self.mode == KernelMode.WRITEFILE:
+                    scorep_env, nomagic_code = magics_cleanup(code.split("\n", 1)[1])
+                    self.writefile_scorep_env.extend(scorep_env)
+                    return self.append_writefile(
+                        nomagic_code,
+                        explicit_scorep=True,
                     )
-                    return self.standard_reply()
-                self.multicell_code = ""
-                self.multicell_cellcount = 0
-                return reply_status
-            elif self.mode == KernelMode.WRITEFILE:
-                self.writefile_multicell = False
-                return self.standard_reply()
-            else:
-                self.cell_output(
-                    f"KernelWarning: Currently in {self.mode}, ignore command",
-                    "stderr",
-                )
-                return self.standard_reply()
-        elif code.startswith("%%start_writefile"):
-            return self.start_writefile(code)
-        elif code.startswith("%%end_writefile"):
-            return self.end_writefile()
-        elif code.startswith("%%execute_with_scorep"):
-            if self.mode == KernelMode.DEFAULT:
-                return await self.scorep_execute(
-                    code.split("\n", 1)[1],
-                    silent,
-                    store_history,
-                    user_expressions,
-                    allow_stdin,
-                    cell_id=cell_id,
-                )
-            elif self.mode == KernelMode.MULTICELL:
-                return self.append_multicellmode(
-                    magics_cleanup(code.split("\n", 1)[1])[1]
-                )
-            elif self.mode == KernelMode.WRITEFILE:
-                scorep_env, nomagic_code = magics_cleanup(code.split("\n", 1)[1])
-                self.writefile_scorep_env.extend(scorep_env)
-                return self.append_writefile(
-                    nomagic_code,
-                    explicit_scorep=True,
-                )
         else:
             if self.mode == KernelMode.DEFAULT:
                 self.pershelper.parse(magics_cleanup(code)[1], "jupyter")

src/jumper/userpersistence.py

@@ -4,8 +4,7 @@
 import astunparse
 from pathlib import Path
 import uuid
-
-import dill
+import importlib
 
 scorep_script_name = "scorep_script.py"
 
@@ -83,12 +82,13 @@ def postprocess(self):
             os.remove(scorep_script_name)
 
     def set_marshaller(self, marshaller):
-        # TODO: valid marshallers should not be configured in code but via an
-        # environment variable
-        valid_marshallers = {"dill", "cloudpickle", "parallel_marshall"}
-        return marshaller in valid_marshallers and (
-            setattr(self, "marshaller", marshaller) or True
-        )
+        try:
+            marshaller_module = importlib.import_module(marshaller)
+        except ImportError:
+            return False
+        if not hasattr(marshaller_module, 'dump') or not hasattr(marshaller_module, 'load'):
+            return False
+        return (setattr(self, "marshaller", marshaller) or True)
 
     def set_mode(self, mode):
         valid_modes = {"disk", "memory"}
@@ -213,10 +213,10 @@ def dump_runtime(
     }
 
     with os.fdopen(os.open(os_environ_dump_, os.O_WRONLY | os.O_CREAT), 'wb') as file:
-        dill.dump(filtered_os_environ_, file)
+        marshaller.dump(filtered_os_environ_, file)
 
     with os.fdopen(os.open(sys_path_dump_, os.O_WRONLY | os.O_CREAT), 'wb') as file:
-        dill.dump(sys_path_, file)
+        marshaller.dump(sys_path_, file)
 
 
 def dump_variables(variables_names, globals_, var_dump_, marshaller):
@@ -243,10 +243,10 @@ def load_runtime(
     loaded_sys_path_ = []
 
     with os.fdopen(os.open(os_environ_dump_, os.O_RDONLY), 'rb') as file:
-        loaded_os_environ_ = dill.load(file)
+        loaded_os_environ_ = marshaller.load(file)
 
     with os.fdopen(os.open(sys_path_dump_, os.O_RDONLY), 'rb') as file:
-        loaded_sys_path_ = dill.load(file)
+        loaded_sys_path_ = marshaller.load(file)
 
     # os_environ_.clear()
     os_environ_.update(loaded_os_environ_)

tests/kernel/writemode.yaml

@@ -1,5 +1,5 @@
 -
-  - "%%start_writefile test_kernel_tmp/my_jupyter_to_script"
+  - "%%start_writefile"
   - - |
       Started converting to Python script. See files:
       /home/runner/work/scorep_jupyter_kernel_python/scorep_jupyter_kernel_python/test_kernel_tmp/my_jupyter_to_script_run.sh
@@ -11,6 +11,22 @@
     %env SCOREP_TOTAL_MEMORY=3g
     %env SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces
   - - "Python commands without instrumentation recorded."
+-
+  - "%%abort_writefile"
+  - - "Writefile mode aborted."
+-
+  - "%%start_writefile test_kernel_tmp/my_jupyter_to_script"
+  - - |
+      Started converting to Python script. See files:
+      /home/carthage/Documents/scorep_jupyter_kernel_python/test_kernel_tmp/my_jupyter_to_script_run.sh
+      /home/carthage/Documents/scorep_jupyter_kernel_python/test_kernel_tmp/my_jupyter_to_script.py
+-
+  - |-
+    %env SCOREP_ENABLE_TRACING=1
+    %env SCOREP_ENABLE_PROFILING=0
+    %env SCOREP_TOTAL_MEMORY=3g
+    %env SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces
+  - - "Python commands without instrumentation recorded."
 -
   - |-
     %%scorep_python_binding_arguments