Merge pull request #11 from dgoeries/stable-release

feat: Create stable release, fix memory leak in ltd

Author: dgoeries

.github/workflows/python-app.yml

@@ -8,27 +8,31 @@ on:
 
 jobs:
   build:
-
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13", "3.13"]
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Check out repository
+      uses: actions/checkout@v4
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install setuptools
-        python -m pip install flake8 pytest
+        python -m pip install setuptools flake8 pytest
+
     - name: Build package
       run: |
         python -m pip install -e . -v
+
     - name: Test with pytest
       run: |
-        pytest
+        pytest

.pre-commit-config.yaml

@@ -27,4 +27,4 @@ repos:
     rev: v3.19.0
     hooks:
       - id: pyupgrade
-        args: [ --py310-plus ]
+        args: [ --py311-plus ]

pyproject.toml

@@ -8,10 +8,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "downsample"
-version = "0.0.5"
+version = "0.1.0"
 readme = {file = "README.txt", content-type = "text/markdown"}
 description = "Downsample algorithms module for Python written in C"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 authors = [
     { name = "Dennis Göries", email = "dennis@goeries.de" }
 ]
@@ -24,7 +24,7 @@ dependencies = [
 
 license = { text = "MIT" }
 classifiers = [
-    "Development Status :: 4 - Beta",
+    "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",

src/downsample/_ltd.c

@@ -39,13 +39,13 @@ static PyObject *split_bucket_at(PyObject *buckets_list, int index) {
     // Get the bucket of interest and index
     PyArrayObject *bucket =
         (PyArrayObject *)PyList_GetItem(buckets_list, index);
+
     npy_intp bucket_size = PyArray_DIM(bucket, 0);
     npy_intp dim = PyArray_DIM(bucket, 1);
 
     // calculate split sizes and split the bucket into half
     npy_intp bucket_a_length = (npy_intp)ceil((double)bucket_size / 2.0);
     npy_intp bucket_b_length = bucket_size - bucket_a_length;
-
     npy_intp dims_a[2] = {bucket_a_length, dim};
     npy_intp dims_b[2] = {bucket_b_length, dim};
 
@@ -61,13 +61,19 @@ static PyObject *split_bucket_at(PyObject *buckets_list, int index) {
 
     // resulting bucket size increases by 1
     PyObject *result_buckets = PyList_New(bucket_count + 1);
-    // Repopulate result_buckets with the items from buckets_list
+
     for (Py_ssize_t i = 0; i < bucket_count; i++) {
+        // Note: Don't copy bucket at 'index', it is split later.
+        if (i == index) {
+            continue;
+        }
         PyObject *item = PyList_GetItem(buckets_list, i);
         Py_INCREF(item);
         PyList_SET_ITEM(result_buckets, (i < index) ? i : i + 1, item);
     }
+
     // Insert to the new list the split bucket at index and index +1
+    // Note: PyList_SET_ITEM steals the references of bucket_a and bucket_b
     PyList_SET_ITEM(result_buckets, index, bucket_a);
     PyList_SET_ITEM(result_buckets, index + 1, bucket_b);
 
@@ -109,6 +115,7 @@ static PyObject *merge_bucket_at(PyObject *buckets_list, int index) {
     }
     // place the merged bucket at index
     PyList_SET_ITEM(result_buckets, index, merged_bucket);
+
     // And finally copy buckets after index + 1 from buckets_list to
     // result_buckets, shifting each index by one (due to the removal of one
     // bucket).
@@ -117,11 +124,10 @@ static PyObject *merge_bucket_at(PyObject *buckets_list, int index) {
         Py_INCREF(item);
         PyList_SET_ITEM(result_buckets, i - 1, item);
     }
-
+    Py_DECREF(buckets_list);
     return result_buckets;
 }
 
-
 static PyObject *LTTB_for_buckets(PyObject *buckets_list) {
     Py_ssize_t bucket_count = PyList_Size(buckets_list);
     npy_intp num_points = bucket_count;
@@ -131,17 +137,14 @@ static PyObject *LTTB_for_buckets(PyObject *buckets_list) {
     double *x_data = (double *)PyArray_DATA((PyArrayObject *)x_array);
     double *y_data = (double *)PyArray_DATA((PyArrayObject *)y_array);
 
-    // Get the first point of the first bucket and initialize sampled data
     PyArrayObject *first_bucket =
         (PyArrayObject *)PyList_GetItem(buckets_list, 0);
 
     double *first_point_data = (double *)PyArray_GETPTR2(first_bucket, 0, 0);
     x_data[0] = first_point_data[0];
     y_data[0] = first_point_data[1];
-    // Store the last selected data point
     double *last_selected_data = first_point_data;
 
-    // Main LTTB loop
     for (Py_ssize_t i = 1; i < bucket_count - 1; i++) {
         PyArrayObject *bucket =
             (PyArrayObject *)PyList_GetItem(buckets_list, i);
@@ -154,8 +157,8 @@ static PyObject *LTTB_for_buckets(PyObject *buckets_list) {
 
         double max_area = -1.0;
         npy_intp max_area_index = -1;
-
         npy_intp bucket_size = PyArray_DIM(bucket, 0);
+
         for (npy_intp j = 0; j < bucket_size; j++) {
             double *point_data = (double *)PyArray_GETPTR2(bucket, j, 0);
             double area = calculate_triangle_area(
@@ -173,17 +176,16 @@ static PyObject *LTTB_for_buckets(PyObject *buckets_list) {
         Py_DECREF(average_point);
     }
 
-    // Append the first point of the last bucket
     PyArrayObject *last_bucket =
         (PyArrayObject *)PyList_GetItem(buckets_list, bucket_count - 1);
     double *last_point_data = (double *)PyArray_GETPTR2(last_bucket, 0, 0);
     x_data[bucket_count - 1] = last_point_data[0];
     y_data[bucket_count - 1] = last_point_data[1];
-    // Return x and y arrays as a tuple
+
     PyObject *result = PyTuple_Pack(2, x_array, y_array);
     Py_DECREF(x_array);
     Py_DECREF(y_array);
-    Py_DECREF(buckets_list);
+    // This function borrows the list, hence it should not destroy it (DECREF).
 
     return result;
 }
@@ -345,75 +347,59 @@ static PyObject *calculate_sse_for_buckets(PyObject *buckets_list) {
     return sse_array;
 }
 
-
 static PyObject *ltd_for_buckets(PyObject *buckets_list) {
-    // 1: The data has been split into an almost equal number of buckets as the
-    // threshold
-    //   - first bucket only containing the first data point
-    //   - last bucket containing only the last data point .
-    // First and last buckets will then excluded in the bucket resizing
-
-    // 2: Calculate the SSE for the buckets with one point in
-    // adjacent buckets overlapping
-    // 3: while halting condition is not met continue
-    // 4: Find the bucket F with the highest SSE
-    // 5: Find the pair of adjacent buckets A and B with the lowest SSE sum.
-    //    The pair should not contain F
-    // 6: Split bucket F into roughly two equal buckets.
-    // 7: Merge the buckets A and B
-    // 8: Calculate the SSE of the newly split up and merged buckets
-    // 9: end
-    // 10: Use the Largest-Triangle-Three-Buckets algorithm on the resulting
-    // buckets for point selection
-
-    // 1.
-    Py_ssize_t num_buckets = PyList_Size(buckets_list);
+    // We modify the local 'buckets_list' variable (swap it),
+    // so we must own a reference to it initially.
+    Py_INCREF(buckets_list);
 
+    Py_ssize_t num_buckets = PyList_Size(buckets_list);
     int threshold = (int)num_buckets;
     int num_iterations = ((int)num_buckets * 10) / threshold;
 
     for (int i = 0; i < num_iterations; i++) {
-        // 2. + 3.
         PyObject *sse_for_buckets = calculate_sse_for_buckets(buckets_list);
-        // 4.
+
         npy_intp highest_sse_bucket_index = find_highest_sse_bucket_index(
             buckets_list, (PyArrayObject *)sse_for_buckets);
+
         if (highest_sse_bucket_index < 0) {
             Py_DECREF(sse_for_buckets);
             break;
         }
-        // 5.
+
         npy_intp lowest_sse_adjacent_bucket_index =
             find_lowest_sse_adjacent_buckets_index(
                 (PyArrayObject *)sse_for_buckets, highest_sse_bucket_index);
+
         if (lowest_sse_adjacent_bucket_index < 0) {
             Py_DECREF(sse_for_buckets);
             break;
         }
 
-        // 6.
+        // Split
         PyObject *updated_buckets =
             split_bucket_at(buckets_list, highest_sse_bucket_index);
+
         Py_DECREF(buckets_list);
         buckets_list = updated_buckets;
 
         if (lowest_sse_adjacent_bucket_index > highest_sse_bucket_index) {
             lowest_sse_adjacent_bucket_index += 1;
         }
-        // 7.
+
         PyObject *merged_buckets =
             merge_bucket_at(buckets_list, lowest_sse_adjacent_bucket_index);
-        // 8.
+
         Py_DECREF(buckets_list);
         buckets_list = merged_buckets;
 
-        Py_DECREF(
-            sse_for_buckets); // Release SSE array for the current iteration
+        Py_DECREF(sse_for_buckets);
     }
-    // end 9.
-    // 10.
+
     PyObject *lttb_result = LTTB_for_buckets(buckets_list);
-    // Don't forget to release the final reference of buckets_list
+
+    // Finally, release the final buckets_list (balances the initial INCREF
+    // or creation in loop
     Py_DECREF(buckets_list);
 
     return lttb_result;

src/downsample/tests/test_memory_leak.py

@@ -1,12 +1,13 @@
+import gc
 import tracemalloc
 
 import numpy as np
 import pytest
 
-from downsample import ltob, lttb
+from downsample import ltd, lttb
 
 
-@pytest.mark.parametrize("func", [lttb, ltob])
+@pytest.mark.parametrize("func", [lttb, ltd])
 def test_memory_leak(func):
     """
     Test memory leak for different LTTB functions.
@@ -18,8 +19,8 @@ def test_memory_leak(func):
 
     # Test parameters (shared for all functions)
     size = 1_000_000
-    threshold = 100
-    iterations = 1_000
+    threshold = 1000
+    iterations = 1000
 
     # Generate test data
     x = np.linspace(0, 10, size)
@@ -31,7 +32,6 @@ def test_memory_leak(func):
     for _ in range(iterations):
         result = func(x, y, threshold)
         del result
-        import gc
         gc.collect()
 
     # Snapshot after function execution