Refactor kernel tests: migrate from YAML-based to notebook-based test cases (#46)

* Migrated kernel tests from YAML-driven definitions to Jupyter notebooks (`.ipynb`).

Summary of changes:
- All YAML test inputs replaced with `.ipynb` notebooks.
- Each notebook-based test can now be run independently of the others.
- Replaced `check_stream_output()` with `check_from_notebook()` to parse and execute notebook cells.
- Added environment variable setup (`SCORP_ENABLE_TRACING`, `SCORP_TOTAL_MEMORY`, etc.) for consistent test behavior.

* fix capital letters in env variables, smaller clarifications

* Update README.md

* Update README.md

* Update README.md

---------

Co-authored-by: OutlyingWest <alexeybuv7@amail.com>
Co-authored-by: Elias Werner <elias.werner@tu-dresden.de>
Co-authored-by: Elias Werner <eliwerner3@googlemail.com>

Author: 4 people

README.md

@@ -46,7 +46,7 @@ For binding to Score-P, the kernel uses the [Score-P Python bindings](https://gi
 To install the kernel and required dependencies:
 
 ```
-pip install scorep_jupyter
+pip install scorep-jupyter
 python -m scorep_jupyter.install
 ```
 
@@ -67,7 +67,7 @@ From the Score-P Python bindings:
 > Please make sure that `scorep-config` is in your `PATH` variable.
 > For Ubuntu LTS systems there is a non-official ppa of Score-P available: https://launchpad.net/~andreasgocht/+archive/ubuntu/scorep .
 
-To use the coarse-grained performance measurements, simply install the JUmPER extension via:
+To use the coarse-grained performance measurements, simply install the jumper ipython extension via:
 
 ```
 pip install jumper_extension
@@ -110,17 +110,17 @@ MARSHALLER=[dill,cloudpickle]
 MODE=[disk,memory]
 ```
 
-When using persistence in `disk` mode, user can also define directory to which serializer output will be saved with `SCOREP_KERNEL_PERSISTENCE_DIR` environment variable.
+When using persistence in `disk` mode, user can also define directory to which serializer output will be saved with `SCOREP_JUPYTER_PERSISTENCE_DIR` environment variable.
 ```
-%env SCOREP_KERNEL_PERSISTENCE_DIR=path/to/dir
+%env SCOREP_JUPYTER_PERSISTENCE_DIR=path/to/dir
 ```
-To see the detailed report for marshalling steps - `JUMPER_MARSHALLING_DETAILED_REPORT` environment variable can be set.
+To see the detailed report for marshalling steps - `SCOREP_JUPYTER_MARSHALLING_DETAILED_REPORT` environment variable can be set.
 ```
-%env JUMPER_MARSHALLING_DETAILED_REPORT=1
+%env SCOREP_JUPYTER_MARSHALLING_DETAILED_REPORT=1
 ```
-You can disable visual animations shown during long-running tasks by setting the `JUMPER_DISABLE_PROCESSING_ANIMATIONS` environment variable.
+You can disable visual animations shown during long-running tasks by setting the `SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS` environment variable.
 ```
-%env JUMPER_DISABLE_PROCESSING_ANIMATIONS=1
+%env SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS=1
 ```
 
 `%%execute_with_scorep`
@@ -204,7 +204,7 @@ Similar yields for cloudpickle. Use the `%%marshalling_settings` magic command t
 When dealing with big data structures, there might be a big runtime overhead at the beginning and the end of a Score-P cell. This is due to additional data saving and loading processes for persistency in the background. However this does not affect the actual user code and the Score-P measurements.
 
 ## Logging Configuration
-To adjust logging and obtain more detailed output about the behavior of the JUmPER kernel, refer to the `src/logging_config.py` file.
+To adjust logging and obtain more detailed output about the behavior of the scorep_jupyter kernel, refer to the `src/logging_config.py` file.
 
 This file contains configuration options for controlling the verbosity, format, and destination of log messages. You can customize it to suit your debugging needs.
 

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
   "jupyter-client",
   "astunparse",
   "dill",
+  "nbformat",
   "scorep"
 ]
 

src/parallel_marshall/parallel_marshall.py

@@ -10,16 +10,18 @@
 # mode is automatically determined by the file object that is passed for
 # dumping
 mode = ""
-backend = str(os.environ.get("scorep_jupyter_PARALLEL_MARSHALL_BACKEND", "dill"))
-if os.environ.get("scorep_jupyter_PARALLEL_MARSHALL_NWORKERS"):
+backend = str(
+    os.environ.get("SCOREP_JUPYTER_PARALLEL_MARSHALL_BACKEND", "dill")
+)
+if os.environ.get("SCOREP_JUPYTER_PARALLEL_MARSHALL_NWORKERS"):
     workers = min(
-        int(os.environ.get("scorep_jupyter_PARALLEL_MARSHALL_NWORKERS")),
+        int(os.environ.get("SCOREP_JUPYTER_PARALLEL_MARSHALL_NWORKERS")),
         multiprocessing.cpu_count(),
         multiprocessing.cpu_count(),
     )
 else:
     workers = multiprocessing.cpu_count()
-debug = int(os.environ.get("scorep_jupyter_PARALLEL_MARSHALL_DEBUG", 20))
+debug = int(os.environ.get("SCOREP_JUPYTER_PARALLEL_MARSHALL_DEBUG", 20))
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(filename="parallel_marshall.log", level=logging.INFO)

src/scorep_jupyter/install.py

@@ -6,7 +6,13 @@
 from scorep_jupyter.logo import logo_image
 
 kernel_spec = {
-    "argv": [sys.executable, "-m", "scorep_jupyter.kernel", "-f", "{connection_file}"],
+    "argv": [
+        sys.executable,
+        "-m",
+        "scorep_jupyter.kernel",
+        "-f",
+        "{connection_file}",
+    ],
     "name": "scorep_jupyter",
     "display_name": "Score-P_Python",
     "language": "python",

src/scorep_jupyter/kernel.py

@@ -1,5 +1,4 @@
 import datetime
-import json
 import os
 import re
 import selectors
@@ -16,7 +15,10 @@
 from scorep_jupyter.userpersistence import PersHelper, scorep_script_name
 from scorep_jupyter.userpersistence import magics_cleanup, create_busy_spinner
 import importlib
-from scorep_jupyter.kernel_messages import KernelErrorCode, KERNEL_ERROR_MESSAGES
+from scorep_jupyter.kernel_messages import (
+    KernelErrorCode,
+    KERNEL_ERROR_MESSAGES,
+)
 
 # import scorep_jupyter.multinode_monitor.slurm_monitor as slurm_monitor
 
@@ -75,7 +77,7 @@ def __init__(self, **kwargs):
 
         self.scorep_binding_args = []
 
-        os.environ["SCOREP_KERNEL_PERSISTENCE_DIR"] = "./"
+        os.environ["SCOREP_JUPYTER_PERSISTENCE_DIR"] = "./"
         self.pershelper = PersHelper("dill", "memory")
 
         self.mode = KernelMode.DEFAULT
@@ -99,7 +101,7 @@ def __init__(self, **kwargs):
             self.scorep_python_available_ = False
 
         logging.config.dictConfig(LOGGING)
-        self.log = logging.getLogger('kernel')
+        self.log = logging.getLogger("kernel")
 
     def cell_output(self, string, stream="stdout"):
         """
@@ -143,9 +145,7 @@ def marshaller_settings(self, code):
             code_parts = code.split("\n", 1)
             content = code_parts[1] if len(code_parts) > 1 else ""
 
-            marshaller_match = re.search(
-                r"MARSHALLER=([\w-]+)", content
-            )
+            marshaller_match = re.search(r"MARSHALLER=([\w-]+)", content)
             mode_match = re.search(r"MODE=([\w-]+)", content)
             marshaller = (
                 marshaller_match.group(1) if marshaller_match else None
@@ -322,7 +322,8 @@ def start_writefile(self, code):
                     dedent(
                         f"""
                         # This is the automatic conversion of
-                        # Jupyter Notebook -> Python script by scorep_jupyter kernel.
+                        # Jupyter Notebook -> Python script by scorep_jupyter
+                        # kernel.
                         # Code corresponding to the cells not marked for
                         # Score-P instrumentation is framed by
                         # "with scorep.instrumenter.disable()
@@ -568,17 +569,22 @@ async def scorep_execute(
 
         stdout_lock = threading.Lock()
         process_busy_spinner = create_busy_spinner(stdout_lock)
-        process_busy_spinner.start('Process is running...')
+        process_busy_spinner.start("Process is running...")
 
-        multicellmode_timestamps = []
+        # Due to splitting into scorep-kernel and ipython extension,
+        # multicell mode is not supported for coarse-grained measurements
+        # anymore (in the extension) and we do not show the single cells in
+        # the ipython extension visualizations after executing them with scorep
+        # however, since we are using scorep anyway, the ipython extension is
+        # not useful, since we can count hardware counters anyway
+        # multicellmode_timestamps = []
 
         try:
-            multicellmode_timestamps = self.read_scorep_process_pipe(
-                proc, stdout_lock
-            )
-            process_busy_spinner.stop('Done.')
+            # multicellmode_timestamps =
+            self.read_scorep_process_pipe(proc, stdout_lock)
+            process_busy_spinner.stop("Done.")
         except KeyboardInterrupt:
-            process_busy_spinner.stop('Kernel interrupted.')
+            process_busy_spinner.stop("Kernel interrupted.")
 
         # In disk mode, subprocess already terminated
         # after dumping persistence to file
@@ -703,12 +709,14 @@ def read_scorep_process_pipe(
                     sel.unregister(key.fileobj)
                     continue
 
-                decoded_line = line.decode(sys.getdefaultencoding(), errors='ignore')
+                decoded_line = line.decode(
+                    sys.getdefaultencoding(), errors="ignore"
+                )
 
                 if key.fileobj is proc.stderr:
                     with stdout_lock:
-                        self.log.warning(f'{decoded_line.strip()}')
-                elif 'MCM_TS' in decoded_line:
+                        self.log.warning(f"{decoded_line.strip()}")
+                elif "MCM_TS" in decoded_line:
                     multicellmode_timestamps.append(decoded_line)
                 else:
                     with stdout_lock:
@@ -866,12 +874,10 @@ def log_error(self, code: KernelErrorCode, **kwargs):
         mode = self.pershelper.mode
         marshaller = self.pershelper.marshaller
 
-        template = KERNEL_ERROR_MESSAGES.get(code, "Unknown error. Mode: {mode}, Marshaller: {marshaller}")
-        message = template.format(
-            mode=mode,
-            marshaller=marshaller,
-            **kwargs
+        template = KERNEL_ERROR_MESSAGES.get(
+            code, "Unknown error. Mode: {mode}, Marshaller: {marshaller}"
         )
+        message = template.format(mode=mode, marshaller=marshaller, **kwargs)
 
         self.log.error(message)
         self.cell_output("KernelError: " + message, "stderr")

src/scorep_jupyter/kernel_messages.py

@@ -24,12 +24,15 @@ class KernelErrorCode(Enum):
         "Failed to set up persistence communication files/pipes "
     ),
     KernelErrorCode.PERSISTENCE_DUMP_FAIL: (
-        "[mode: {mode}] Failed to serialize notebook persistence ({direction}, marshaller: {marshaller})."
+        "[mode: {mode}] Failed to serialize notebook persistence "
+        "({direction}, marshaller: {marshaller})."
     ),
     KernelErrorCode.PERSISTENCE_LOAD_FAIL: (
-        "[mode: {mode}] Failed to load persistence ({direction}, marshaller: {marshaller})."
+        "[mode: {mode}] Failed to load persistence "
+        "({direction}, marshaller: {marshaller})."
     ),
     KernelErrorCode.SCOREP_SUBPROCESS_FAIL: (
-        "[mode: {mode}] Subprocess terminated unexpectedly. Persistence not recorded (marshaller: {marshaller})."
+        "[mode: {mode}] Subprocess terminated unexpectedly. "
+        "Persistence not recorded (marshaller: {marshaller})."
     ),
 }

src/scorep_jupyter/logging_config.py

@@ -64,7 +64,9 @@ def filter(self, record):
     "filters": {
         "jupyter_filter": {"()": JupyterLogFilter},
         "ignore_error_filter": {"()": IgnoreErrorFilter},
-        "scorep_jupyter_kernel_only_filter": {"()": scorep_jupyterKernelOnlyFilter},
+        "scorep_jupyter_kernel_only_filter": {
+            "()": scorep_jupyterKernelOnlyFilter
+        },
     },
     "root": {
         "handlers": [],

src/scorep_jupyter/userpersistence.py

@@ -11,7 +11,6 @@
 import uuid
 import importlib
 
-
 scorep_script_name = "scorep_script.py"
 
 
@@ -24,7 +23,7 @@ def __init__(self, marshaller="dill", mode="memory"):
         self.subprocess_definitions = ""
         self.subprocess_variables = []
         self.base_path = Path(
-            os.environ["SCOREP_KERNEL_PERSISTENCE_DIR"]
+            os.environ["SCOREP_JUPYTER_PERSISTENCE_DIR"]
         ) / Path("./kernel_persistence/")
         self.paths = {
             "jupyter": {"os_environ": "", "sys_path": "", "var": ""},
@@ -195,7 +194,8 @@ def jupyter_update(self, code):
         jupyter_update = (
             "import sys\n"
             "import os\n"
-            "from scorep_jupyter.userpersistence import load_runtime, load_variables\n"
+            "from scorep_jupyter.userpersistence import load_runtime, "
+            "load_variables\n"
             f"load_runtime(os.environ, sys.path,"
             f"'{self.paths['subprocess']['os_environ']}',"
             f"'{self.paths['subprocess']['sys_path']}',{self.marshaller})\n"
@@ -231,7 +231,7 @@ def parse(self, code, mode):
 
     def set_dump_report_level(self):
         self.is_dump_detailed_report = int(
-            os.getenv("scorep_jupyter_MARSHALLING_DETAILED_REPORT", "0")
+            os.getenv("SCOREP_JUPYTER_MARSHALLING_DETAILED_REPORT", "0")
         )
 
 
@@ -241,7 +241,9 @@ def dump_runtime(
     # Don't dump environment variables set by Score-P bindings.
     # Will force it to re-initialize instead of calling reset_preload()
     filtered_os_environ_ = {
-        k: v for k, v in os_environ_.items() if not k.startswith("SCOREP_")
+        k: v
+        for k, v in os_environ_.items()
+        if not k.startswith("SCOREP_") or "SCOREP_JUPYTER" in k
     }
 
     with os.fdopen(
@@ -271,8 +273,7 @@ def dump_variables(variables_names, globals_, var_dump_, marshaller):
         if non_persistent_class in globals().keys():
             user_variables[el].__class__ = globals()[non_persistent_class]
 
-    with (os.fdopen(os.open(var_dump_, os.O_WRONLY | os.O_CREAT), "wb")
-          as file):
+    with os.fdopen(os.open(var_dump_, os.O_WRONLY | os.O_CREAT), "wb") as file:
         marshaller.dump(user_variables, file)
 
 
@@ -369,36 +370,41 @@ def magics_cleanup(code):
     """
     lines = code.splitlines(True)
     scorep_env = []
-    
+
     # Cell magics that should skip entire cell content for persistence
     non_persistent_cell_magics = ["%%bash"]  # Non-Python content
     # Cell magics that should keep Python content but skip magic line
     python_cell_magics = ["%%prun", "%%capture"]
     whitelist_prefixes_line = ["%prun", "%time"]
-    
+
     # Check if this is a cell magic
     if lines and lines[0].strip().startswith("%%"):
         first_line = lines[0].strip()
-        if any(first_line.startswith(prefix) for prefix in non_persistent_cell_magics):
+        if any(
+            first_line.startswith(prefix)
+            for prefix in non_persistent_cell_magics
+        ):
             # For non-Python cell magics like %%bash
             # Skip the entire cell content for persistence
             return scorep_env, ""
-        elif any(first_line.startswith(prefix) for prefix in python_cell_magics):
+        elif any(
+            first_line.startswith(prefix) for prefix in python_cell_magics
+        ):
             # For Python cell magics like %%prun, %%capture
             # Skip only the magic line, keep the Python content for persistence
             filtered_lines = lines[1:]  # Skip first line (the magic)
             return scorep_env, "".join(filtered_lines)
-    
+
     # Process line by line for non-cell magics or non-whitelisted cell magics
     filtered_lines = []
-    
+
     for line in lines:
         stripped_line = line.strip()
-        
+
         # Keep empty lines and comments
         if not stripped_line or stripped_line.startswith("#"):
             filtered_lines.append(line)
-            
+
         # Handle %env specially
         elif stripped_line.startswith("%env"):
             env_var = stripped_line.split(" ", 1)[1]
@@ -410,22 +416,27 @@ def magics_cleanup(code):
                     filtered_lines.append(f'os.environ["{key}"]="{val}"\n')
             else:
                 key = env_var
-                filtered_lines.append(f"print(\"env: {key}=os.environ['{key}']\")\n")
-                
+                filtered_lines.append(
+                    f"print(\"env: {key}=os.environ['{key}']\")\n"
+                )
+
         # Handle whitelisted line magics - keep the command part
-        elif any(stripped_line.startswith(prefix) for prefix in whitelist_prefixes_line):
+        elif any(
+            stripped_line.startswith(prefix)
+            for prefix in whitelist_prefixes_line
+        ):
             parts = line.split(" ", 1)
             if len(parts) > 1:
                 filtered_lines.append(parts[1])
-                
+
         # Remove all other magic commands and shell commands
         elif stripped_line.startswith("%") or stripped_line.startswith("!"):
             continue
-            
+
         # Keep regular Python code
         else:
             filtered_lines.append(line)
-    
+
     nomagic_code = "".join(filtered_lines)
     return scorep_env, nomagic_code
 
@@ -488,7 +499,9 @@ def stop(self, done_message="Done."):
 
 
 def create_busy_spinner(lock=None):
-    is_enabled = os.getenv("scorep_jupyter_DISABLE_PROCESSING_ANIMATIONS") != "1"
+    is_enabled = (
+        os.getenv("SCOREP_JUPYTER_DISABLE_PROCESSING_ANIMATIONS") != "1"
+    )
     if is_enabled:
         return BusySpinner(lock)
     else: