Run black on python files with 120 char line length

Author: ocaisa

scripts/calculate_hashes.py

@@ -9,6 +9,7 @@
 # Output file
 output_file = "hashes.json"
 
+
 # Function to compute SHA256 hash of a file
 def compute_hash(file_path):
     sha256 = hashlib.sha256()
@@ -17,23 +18,24 @@ def compute_hash(file_path):
             sha256.update(chunk)
     return sha256.hexdigest()
 
+
 # Collect hashes
 hashes = {}
 for root, dirs, files in os.walk(directory):
     for file in files:
         if file.endswith((".json", ".yaml", ".yml")):
             file_path = os.path.join(root, file)
             relative_path = os.path.relpath(file_path, directory)
-            timestamp = ''
+            timestamp = ""
             if file.endswith(".json"):
                 with open(file, "r") as f:
                     data = json.load(f)
                     timestamp = data["timestamp"]
-            elif file.endswith(( ".yaml", ".yml")):
+            elif file.endswith((".yaml", ".yml")):
                 with open(file, "r") as f:
                     data = yaml.load(f, Loader=yaml.FullLoader)
                     timestamp = data["timestamp"]
-            hashes[relative_path] = {'hash': compute_hash(file_path), 'timestamp': timestamp}
+            hashes[relative_path] = {"hash": compute_hash(file_path), "timestamp": timestamp}
 
 # Write hashes to JSON file
 with open(output_file, "w") as f:

scripts/generate_data_files.py

@@ -20,17 +20,20 @@
 EESSI_REFERENCE_ARCHITECTURE = "x86_64/intel/icelake"
 
 # Give order to my toolchains so I can easily figure out what "latest" means
-EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = OrderedDict({
-    '2025.06': [
-        {'name': 'foss', 'version': '2025a'},
-        {'name': 'foss', 'version': '2024a'},
-    ],
-    '2023.06': [
-        {'name': 'foss', 'version': '2023b'},
-        {'name': 'foss', 'version': '2023a'},
-        {'name': 'foss', 'version': '2022b'},
-    ],
-})
+EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = OrderedDict(
+    {
+        "2025.06": [
+            {"name": "foss", "version": "2025a"},
+            {"name": "foss", "version": "2024a"},
+        ],
+        "2023.06": [
+            {"name": "foss", "version": "2023b"},
+            {"name": "foss", "version": "2023a"},
+            {"name": "foss", "version": "2022b"},
+        ],
+    }
+)
+
 
 @contextmanager
 def suppress_stdout():
@@ -56,26 +59,17 @@ def load_and_list_modules(module_name):
         module --terse list 2>&1
     """
 
-    result = subprocess.run(
-        ["bash", "-c", cmd],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True
-    )
+    result = subprocess.run(["bash", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
 
     if result.returncode != 0:
         raise RuntimeError(f"Failed to load module '{module_name}':\n{result.stdout}")
 
     # Parse module list output
-    modules = [
-        line
-        for line in result.stdout.splitlines()
-        if "/" in line
-    ]
-    
+    modules = [line for line in result.stdout.splitlines() if "/" in line]
+
     # Filter out the modules we expect to be loaded
-    eessi_extend_module_stub = 'EESSI-extend/'
-    eb_module_stub = 'EasyBuild/'
+    eessi_extend_module_stub = "EESSI-extend/"
+    eb_module_stub = "EasyBuild/"
     if module_name.startswith(eessi_extend_module_stub):
         # Don't filter anything
         pass
@@ -84,7 +78,11 @@ def load_and_list_modules(module_name):
         modules = [module for module in modules if not module.startswith(eessi_extend_module_stub)]
     else:
         # Filter EESSI-extend and EasyBuild
-        modules = [module for module in modules if not module.startswith(eessi_extend_module_stub) and not module.startswith(eb_module_stub)]
+        modules = [
+            module
+            for module in modules
+            if not module.startswith(eessi_extend_module_stub) and not module.startswith(eb_module_stub)
+        ]
 
     return modules
 
@@ -96,23 +94,23 @@ def use_timestamped_reprod_if_exists(original_path):
     """
     # Default to returning the original path
     returned_path = original_path
-    
+
     # Split path
     parts = original_path.strip(os.sep).split(os.sep)
 
     # Find the last occurrence of 'software'
-    idx = len(parts) - 1 - parts[::-1].index('software')
+    idx = len(parts) - 1 - parts[::-1].index("software")
 
     # Replace 'software' by 'reprod'
-    parts[idx] = 'reprod'
+    parts[idx] = "reprod"
 
     # Path up to version directory (software/software/version)
-    pre_timestamp = os.sep.join([''] + parts[:idx+3])
+    pre_timestamp = os.sep.join([""] + parts[: idx + 3])
     # Path after version directory (easybuild/reprod/easyblocks)
-    post_version = parts[idx+3:]
+    post_version = parts[idx + 3 :]
 
     # Look for timestamp directories under pre_timestamp
-    timestamp_dirs = [d for d in glob.glob(os.path.join(pre_timestamp, '*')) if os.path.isdir(d)]
+    timestamp_dirs = [d for d in glob.glob(os.path.join(pre_timestamp, "*")) if os.path.isdir(d)]
     if timestamp_dirs:
         latest_timestamp = max(timestamp_dirs)  # lexicographic order
         # Reconstruct path: reprod/.../version/<latest_timestamp>/easybuild/reprod/easyblocks
@@ -122,48 +120,49 @@ def use_timestamped_reprod_if_exists(original_path):
 
     return returned_path
 
+
 def collect_eb_files(base_path):
     """
     Scan for .eb files and their corresponding *-easybuild-devel files,
     extract the major EasyBuild version from devel files, and group .eb files by major version.
     For folders containing 'EasyBuild' or 'EESSI-extend', assume the loaded EasyBuild version if extraction fails.
-    
+
     Parameters:
         base_path (str): Root folder to scan for .eb files.
 
     Returns:
         dict: {major_version: [list of .eb file paths]}
     """
     eb_files_by_version = defaultdict(list)
-    version_pattern = re.compile(r'software/EasyBuild/(\d+)\.(\d+)\.(\d+)/bin')
+    version_pattern = re.compile(r"software/EasyBuild/(\d+)\.(\d+)\.(\d+)/bin")
 
     # Get major version from loaded EasyBuild installation for exceptions
     easybuild_major_version = str(EASYBUILD_VERSION.version[0])
 
     # Find all .eb files recursively
-    eb_files = glob.glob(os.path.join(base_path, '*/*/easybuild/*.eb'))
+    eb_files = glob.glob(os.path.join(base_path, "*/*/easybuild/*.eb"))
 
     for eb_file in eb_files:
         folder = os.path.dirname(eb_file)
 
         # Look for the -easybuild-devel file in the same folder
-        devel_files = glob.glob(os.path.join(folder, '*-easybuild-devel'))
+        devel_files = glob.glob(os.path.join(folder, "*-easybuild-devel"))
         if not devel_files:
             raise FileNotFoundError(f"No *-easybuild-devel file found in folder: {folder}")
 
         # Pick the latest devel file if multiple exist
         latest_devel = max(devel_files, key=os.path.getmtime)
 
         # Extract the EasyBuild version
-        with open(latest_devel, 'r') as f:
+        with open(latest_devel, "r") as f:
             content = f.read()
             match = version_pattern.search(content)
 
             # Handle exception folders
-            if 'EasyBuild' in folder or 'EESSI-extend' in folder:
+            if "EasyBuild" in folder or "EESSI-extend" in folder:
                 major_version = match.group(1) if match else easybuild_major_version
                 # Don't add EESSI-extend to EB4 or the same file will appear twice
-                if 'EESSI-extend' in folder and major_version == '4':
+                if "EESSI-extend" in folder and major_version == "4":
                     continue
             else:
                 if not match:
@@ -183,7 +182,7 @@ def collect_eb_files(base_path):
         "-e",
         required=True,
         choices=VALID_EESSI_VERSIONS,
-        help=f"Allowed versions: {', '.join(VALID_EESSI_VERSIONS)}"
+        help=f"Allowed versions: {', '.join(VALID_EESSI_VERSIONS)}",
     )
 
     args = parser.parse_args()
@@ -192,7 +191,9 @@ def collect_eb_files(base_path):
     print(f"Using EESSI version: {eessi_version}")
 
     # We use a single architecture path to gather information about the software versions
-    base_path = f'/cvmfs/software.eessi.io/versions/{eessi_version}/software/linux/{EESSI_REFERENCE_ARCHITECTURE}/software/'
+    base_path = (
+        f"/cvmfs/software.eessi.io/versions/{eessi_version}/software/linux/{EESSI_REFERENCE_ARCHITECTURE}/software/"
+    )
     result = collect_eb_files(base_path)
 
     set_up_configuration(args="")
@@ -203,56 +204,66 @@ def collect_eb_files(base_path):
     eessi_software["eessi_version"][eessi_version] = {}
     # Add a timestamp
     eessi_software["timestamp"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
-    
+
     # Store the toolchain hierarchies supported by the EESSI version
     eessi_software["eessi_version"][eessi_version]["toolchain_hierarchy"] = {}
     for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version]:
         toolchain_family = f"{top_level_toolchain['name']}_{top_level_toolchain['version']}"
         # Get the hierarchy and always add the system toolchain
-        eessi_software["eessi_version"][eessi_version]["toolchain_hierarchy"][toolchain_family] = [{'name': 'system', 'version': 'system'}] + get_toolchain_hierarchy(top_level_toolchain)
-    
+        eessi_software["eessi_version"][eessi_version]["toolchain_hierarchy"][toolchain_family] = [
+            {"name": "system", "version": "system"}
+        ] + get_toolchain_hierarchy(top_level_toolchain)
+
     for eb_version_of_install, files in sorted(result.items()):
         print(f"Major version {eb_version_of_install}:")
         if eb_version_of_install == str(EASYBUILD_VERSION.version[0]):
             total_files = len(files)
             for i, file in enumerate(files, start=1):
                 percent = (i / total_files) * 100
                 print(f"{percent:.1f}% - {file}")
-    
+
                 # Don't try to parse an EasyBuild easyconfig that is not the same major release
-                if '/software/EasyBuild/' in file and f'/EasyBuild/{eb_version_of_install}' not in file:
+                if "/software/EasyBuild/" in file and f"/EasyBuild/{eb_version_of_install}" not in file:
                     continue
                 # print(process_easyconfig(path)[0]['ec'].asdict())
-                
+
                 eb_hooks_path = use_timestamped_reprod_if_exists(f"{os.path.dirname(file)}/reprod/easyblocks")
-                easyblocks_dir = include_easyblocks(tmpdir, [eb_hooks_path+"/*.py"])
+                easyblocks_dir = include_easyblocks(tmpdir, [eb_hooks_path + "/*.py"])
                 with suppress_stdout():
-                    parsed_ec=process_easyconfig(file)[0]
+                    parsed_ec = process_easyconfig(file)[0]
                 # included easyblocks are the first entry in sys.path, so just pop them but keep a list of what was used
                 sys.path.pop(0)
-                easyblocks_used = [os.path.basename(f) for f in glob.glob(f"{easyblocks_dir}/**/*.py", recursive=True) if os.path.basename(f) != '__init__.py']
+                easyblocks_used = [
+                    os.path.basename(f)
+                    for f in glob.glob(f"{easyblocks_dir}/**/*.py", recursive=True)
+                    if os.path.basename(f) != "__init__.py"
+                ]
                 shutil.rmtree(easyblocks_dir)
-                
+
                 # Use the path as the key since we know it is unique
-                eessi_software["eessi_version"][eessi_version][file] = parsed_ec['ec'].asdict()
-                eessi_software["eessi_version"][eessi_version][file]['mtime'] = os.path.getmtime(file)
-                
+                eessi_software["eessi_version"][eessi_version][file] = parsed_ec["ec"].asdict()
+                eessi_software["eessi_version"][eessi_version][file]["mtime"] = os.path.getmtime(file)
+
                 # Make sure we can load the module before adding it's information to the main dict
                 try:
-                    eessi_software["eessi_version"][eessi_version][file]['required_modules'] = load_and_list_modules(parsed_ec['full_mod_name'])
+                    eessi_software["eessi_version"][eessi_version][file]["required_modules"] = load_and_list_modules(
+                        parsed_ec["full_mod_name"]
+                    )
                 except RuntimeError as e:
                     print(f"Ignoring {file} due to error processing module: {e}")
                     eessi_software["eessi_version"][eessi_version].pop(file)
                     continue
 
                 # Store everything we now know about the installation as a dict
                 # Add important data that is related to the module environment
-                eessi_software["eessi_version"][eessi_version][file]['full_mod_name'] = parsed_ec['full_mod_name']
-                eessi_software["eessi_version"][eessi_version][file]['short_mod_name'] = parsed_ec['short_mod_name']
-                eessi_software["eessi_version"][eessi_version][file]['required_modules'] = load_and_list_modules(parsed_ec['full_mod_name'])
+                eessi_software["eessi_version"][eessi_version][file]["full_mod_name"] = parsed_ec["full_mod_name"]
+                eessi_software["eessi_version"][eessi_version][file]["short_mod_name"] = parsed_ec["short_mod_name"]
+                eessi_software["eessi_version"][eessi_version][file]["required_modules"] = load_and_list_modules(
+                    parsed_ec["full_mod_name"]
+                )
                 # Retain the easyblocks used so we can use a heuristic to figure out the type of extensions (R, Python, Perl)
-                eessi_software["eessi_version"][eessi_version][file]['easyblocks'] = easyblocks_used
-    
+                eessi_software["eessi_version"][eessi_version][file]["easyblocks"] = easyblocks_used
+
     # Store the result
     with open(f"eessi_software_{eessi_version}-eb{str(EASYBUILD_VERSION.version[0])}.yaml", "w") as f:
         yaml.dump(eessi_software, f)

scripts/generate_schema_md.py

@@ -2,6 +2,7 @@
 import sys
 from pathlib import Path
 
+
 def explain_json_schema(obj, indent=0):
     spacing = "  " * indent
     explanation = ""
@@ -22,6 +23,7 @@ def explain_json_schema(obj, indent=0):
         explanation += type(obj).__name__
     return explanation
 
+
 def main():
     if len(sys.argv) < 2:
         print("Usage: python generate_schema_md.py <path_to_json_file>")
@@ -48,5 +50,6 @@ def main():
 
     print(md_content)
 
+
 if __name__ == "__main__":
     main()

scripts/merge_data_files.py

@@ -3,15 +3,13 @@
 import yaml
 from datetime import datetime, timezone
 
+
 def earliest_if_within_two_hours(timestamps):
     if not timestamps:
         raise ValueError("No timestamps provided")
 
     # Parse timestamps into aware datetime objects
-    times = [
-        datetime.fromisoformat(t.replace("Z", "+00:00"))
-        for t in timestamps
-    ]
+    times = [datetime.fromisoformat(t.replace("Z", "+00:00")) for t in timestamps]
 
     times.sort()
     earliest = times[0]
@@ -25,6 +23,7 @@ def earliest_if_within_two_hours(timestamps):
     # Return earliest *string*, matching input format
     return earliest.isoformat().replace("+00:00", "Z")
 
+
 def strict_merge(a, b, path=""):
     """Recursively merge dictionary b into a, erroring on mismatched values."""
     if not isinstance(a, dict) or not isinstance(b, dict):