Merge branch 'main' into white_matter

Author: alejoe91

doc/development/development.rst

@@ -397,3 +397,52 @@ After this you need to add a block in `Install Sorters <https://github.com/Spike
 to describe your sorter.
 
 Finally, make a pull request so we can review the code and incorporate into the sorters module of SpikeInterface!
+
+
+
+How to make a release
+---------------------
+
+Checklist
+^^^^^^^^^
+* pyproject.toml: check that the version is ahead of current release. Also, comment out the @ (git dependencies)
+* In the top level ``__init__`` (located at ``src/spikeinterface/__init__.py``) set ``DEV_MODE`` to ``False`` (this is used for the docker installations)
+* Create a new release note for the appropriate version on doc/releases/new_version_tag.
+
+There can be large releases like:
+
+``doc/releases/0.101.0.rst``
+
+Which contain a section called "Main Changes" and minor releases which include only bug fixes like:
+
+``doc/releases/0.101.2.rst``
+
+To collect all the PRs and bug fixes we have a script in:
+``doc/scripts/``
+called ``auto-release-notes.sh``. Run it with ``bash auto-release-notes.sh`` and it will create the release notes for the module specific changes.
+
+The first time you run the script, GitHub will guide you through an authorization process if you've not already done so.
+
+The signature of the script is:
+
+.. code-block:: bash
+
+    bash auto-release-notes.sh <start_date> <end_date>
+
+Where the start date is the date of the last release and the end date is the current date. Dates are in YYYY-MM-DD format
+
+The date of the last release can be found on `PyPI <https://pypi.org/project/spikeinterface/>`_.
+
+
+As a specific example:
+.. code-block:: bash
+
+    bash auto-release-notes.sh 2025-02-19 2025-03-24
+
+* Finish the release notes and merge
+* Locally tag the main branch with the newly merged release notes with the new version
+* Push the tag to the remote repository which will trigger the release action (.github/workflows/publish-to-pypi.yml)
+* Do an after-release `PR <https://github.com/SpikeInterface/spikeinterface/pull/3828/files>`_:
+    - Uncomment the git installs in pyproject
+    - Set ``DEV_MODE`` to ``True`` in the top level ``__init__`` (located at ``src/spikeinterface/__init__.py``)
+    - Update `pyproject.toml` version one patch ahead or one minor if it is larger one.

pyproject.toml

@@ -151,8 +151,6 @@ test = [
     "pytest-cov",
     "psutil",
 
-    "huggingface_hub",
-
     # preprocessing
     "ibllib", # for IBL
 
@@ -195,8 +193,8 @@ docs = [
     "hdbscan>=0.8.33",   # For sorters spykingcircus2 + tridesclous
     "numba", # For many postprocessing functions
     "networkx",
-    "skops", # For auotmated curation
-    "scikit-learn", # For auotmated curation
+    "skops", # For automated curation
+    "scikit-learn", # For automated curation
     # Download data
     "pooch>=1.8.2",
     "datalad>=1.0.2",

src/spikeinterface/core/analyzer_extension_core.py

@@ -709,7 +709,7 @@ class ComputeNoiseLevels(AnalyzerExtension):
     depend_on = []
     need_recording = True
     use_nodepipeline = False
-    need_job_kwargs = False
+    need_job_kwargs = True
     need_backward_compatibility_on_load = True
 
     def __init__(self, sorting_analyzer):
@@ -729,9 +729,12 @@ def _merge_extension_data(
         # this does not depend on units
         return self.data.copy()
 
-    def _run(self, verbose=False):
+    def _run(self, verbose=False, **job_kwargs):
         self.data["noise_levels"] = get_noise_levels(
-            self.sorting_analyzer.recording, return_scaled=self.sorting_analyzer.return_scaled, **self.params
+            self.sorting_analyzer.recording,
+            return_scaled=self.sorting_analyzer.return_scaled,
+            **self.params,
+            **job_kwargs,
         )
 
     def _get_data(self):

src/spikeinterface/core/binaryrecordingextractor.py

@@ -148,6 +148,18 @@ def get_binary_description(self):
         )
         return d
 
+    def __del__(self):
+        """
+        Ensures that all segment resources are properly cleaned up when this recording extractor is deleted.
+        Closes any open file handles in the recording segments.
+        """
+        # Close all recording segments
+        if hasattr(self, "_recording_segments"):
+            for segment in self._recording_segments:
+                # This will trigger the __del__ method of the BinaryRecordingSegment
+                # which will close the file handle
+                del segment
+
 
 BinaryRecordingExtractor.write_recording.__doc__ = BinaryRecordingExtractor.write_recording.__doc__.format(
     _shared_job_kwargs_doc
@@ -223,6 +235,15 @@ def get_traces(
 
         return traces
 
+    def __del__(self):
+        # Ensure that the file handle is closed when the segment is garbage-collected
+        try:
+            if hasattr(self, "file") and self.file and not self.file.closed:
+                self.file.close()
+        except Exception as e:
+            warnings.warn(f"Error closing file handle in BinaryRecordingSegment: {e}")
+            pass
+
 
 # For backward compatibility (old good time)
 BinDatRecordingExtractor = BinaryRecordingExtractor

src/spikeinterface/generation/tests/test_hybrid_tools.py

@@ -1,3 +1,5 @@
+import pytest
+
 import numpy as np
 
 from spikeinterface.core import Templates
@@ -69,7 +71,8 @@ def test_generate_hybrid_from_templates():
     assert sorting_hybrid.get_num_units() == num_units
 
 
-def test_estimate_templates(create_cache_folder):
+@pytest.mark.skip("Spykingcircus2 is not stable enought for estimating templates from recording")
+def test_estimate_templates_from_recording(create_cache_folder):
     cache_folder = create_cache_folder
     rec, _ = generate_ground_truth_recording(num_units=10, sampling_frequency=20000, seed=0)
     templates = estimate_templates_from_recording(
@@ -79,7 +82,11 @@ def test_estimate_templates(create_cache_folder):
 
 
 if __name__ == "__main__":
+    from pathlib import Path
+
+    cache_folder = Path(__file__).resolve().parents[4] / "cache_folder" / "generation"
+
     # test_generate_hybrid_no_motion()
-    test_generate_hybrid_motion()
-    # test_estimate_templates()
+    # test_generate_hybrid_motion()
+    test_estimate_templates_from_recording(cache_folder)
     # test_generate_hybrid_with_sorting()

src/spikeinterface/preprocessing/preprocessinglist.py

@@ -25,6 +25,7 @@
     CenterRecording,
     center,
 )
+
 from .scale import scale_to_uV
 
 from .whiten import WhitenRecording, whiten, compute_whitening_matrix

src/spikeinterface/sorters/external/kilosort4.py

@@ -138,11 +138,7 @@ def check_sorter_version(cls):
 
     @classmethod
     def _setup_recording(cls, recording, sorter_output_folder, params, verbose):
-        from probeinterface import write_prb
-
-        pg = recording.get_probegroup()
-        probe_filename = sorter_output_folder / "probe.prb"
-        write_prb(probe_filename, pg)
+        cls._setup_json_probe_map(recording, sorter_output_folder)
 
         if params["use_binary_file"]:
             if not recording.binary_compatible_with(time_axis=0, file_paths_length=1):
@@ -189,7 +185,7 @@ def _run_from_folder(cls, sorter_output_folder, params, verbose):
 
         sorter_output_folder = sorter_output_folder.absolute()
 
-        probe_filename = sorter_output_folder / "probe.prb"
+        probe_filename = sorter_output_folder / "chanMap.json"
 
         torch_device = params["torch_device"]
         if torch_device == "auto":
@@ -358,7 +354,10 @@ def _run_from_folder(cls, sorter_output_folder, params, verbose):
         )
         if version.parse(ks_version) >= version.parse("4.0.28"):
             cluster_spikes_kwargs.update(dict(verbose=verbose))
-        clu, Wall = cluster_spikes(**cluster_spikes_kwargs)
+        if version.parse(ks_version) <= version.parse("4.0.30"):
+            clu, Wall = cluster_spikes(**cluster_spikes_kwargs)
+        else:
+            clu, Wall, st, tF = cluster_spikes(**cluster_spikes_kwargs)
 
         if params["skip_kilosort_preprocessing"]:
             ops["preprocessing"] = dict(
@@ -386,3 +385,29 @@ def _run_from_folder(cls, sorter_output_folder, params, verbose):
     @classmethod
     def _get_result_from_folder(cls, sorter_output_folder):
         return KilosortBase._get_result_from_folder(sorter_output_folder)
+
+    @classmethod
+    def _setup_json_probe_map(cls, recording, sorter_output_folder):
+        """Create a JSON probe map file for Kilosort4."""
+        from kilosort.io import save_probe
+        import numpy as np
+
+        groups = recording.get_channel_groups()
+        positions = np.array(recording.get_channel_locations())
+        if positions.shape[1] != 2:
+            raise RuntimeError("3D 'location' are not supported. Set 2D locations instead.")
+
+        n_chan = recording.get_num_channels()
+        chanMap = np.arange(n_chan)
+        xc = positions[:, 0]
+        yc = positions[:, 1]
+        kcoords = groups.astype(float)
+
+        probe = {
+            "chanMap": chanMap,
+            "xc": xc,
+            "yc": yc,
+            "kcoords": kcoords,
+            "n_chan": n_chan,
+        }
+        save_probe(probe, str(sorter_output_folder / "chanMap.json"))