Merge pull request #1308 from Sage-Bionetworks/v4.11.0-rc-dev

[SYNPY-1735] Release python client v4.11.0

Author: BryanFauble

.github/workflows/build.yml

@@ -34,15 +34,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.13'
-    - uses: pre-commit/action@v3.0.1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - uses: pre-commit/action@v3.0.1
 
   # run unit (and integration tests if account secrets available) on our build matrix
   test:
@@ -51,11 +50,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macos-13, windows-2022]
+        os: [ubuntu-22.04, macos-15-intel, windows-2022]
 
         # if changing the below change the run-integration-tests versions and the check-deploy versions
         # Make sure that we are running the integration tests on the first and last versions of the matrix
-        python: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
     runs-on: ${{ matrix.os }}
 
@@ -84,15 +83,15 @@ jobs:
           path: |
             ${{ steps.get-dependencies.outputs.site_packages_loc }}
             ${{ steps.get-dependencies.outputs.site_bin_dir }}
-          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v27
+          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v31
 
       - name: Install py-dependencies
         if: steps.cache-dependencies.outputs.cache-hit != 'true'
         shell: bash
         run: |
           python -m pip install --upgrade pip
 
-          pip install -e ".[boto3,pandas,pysftp,tests]"
+          pip install -e ".[boto3,pandas,pysftp,tests,curator]"
 
           # ensure that numpy c extensions are installed on windows
           # https://stackoverflow.com/a/59346525
@@ -109,7 +108,7 @@ jobs:
           pytest -sv --cov-append --cov=. --cov-report xml tests/unit
       - name: Check for Secret availability
         id: secret-check
-        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) || contains(fromJSON('["3.13"]'), matrix.python) }}
+        if: ${{ contains(fromJSON('["3.10"]'), matrix.python) || contains(fromJSON('["3.14"]'), matrix.python) }}
         # perform secret check & put boolean result as an output
         shell: bash
         run: |
@@ -125,34 +124,42 @@ jobs:
             echo "synapse_pat_available=true" >> $GITHUB_OUTPUT;
           fi
 
+      - name: Download Failed Tests from Previous Attempt
+        if: ${{ github.run_attempt > 1 && (contains(fromJSON('["3.10"]'), matrix.python) || contains(fromJSON('["3.14"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true'}}
+        uses: actions/download-artifact@v4
+        with:
+          name: failed-tests-${{ matrix.os }}-${{ matrix.python }}
+        continue-on-error: true
+
       # run integration tests iff the decryption keys for the test configuration are available.
       # they will not be available in pull requests from forks.
       # run integration tests on the oldest and newest supported versions of python.
       # we don't run on the entire matrix to avoid a 3xN set of concurrent tests against
       # the target server where N is the number of supported python versions.
       - name: run-integration-tests
+        id: integration_tests
         shell: bash
 
         # keep versions consistent with the first and last from the strategy matrix
-        if: ${{ (contains(fromJSON('["3.9"]'), matrix.python) || contains(fromJSON('["3.13"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true'}}
+        if: ${{ (contains(fromJSON('["3.10"]'), matrix.python) || contains(fromJSON('["3.14"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true' && github.event_name != 'release'}}
         run: |
           # Set SYNAPSE_PROFILE based on OS and Python version
           if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
-            if [ "${{ matrix.python }}" == "3.9" ]; then
+            if [ "${{ matrix.python }}" == "3.10" ]; then
               export SYNAPSE_PROFILE="TestUbuntuMinimumPython"
-            elif [ "${{ matrix.python }}" == "3.13" ]; then
+            elif [ "${{ matrix.python }}" == "3.14" ]; then
               export SYNAPSE_PROFILE="TestUbuntuMaximumPython"
             fi
           elif [ "${{ startsWith(matrix.os, 'windows') }}" == "true" ]; then
-            if [ "${{ matrix.python }}" == "3.9" ]; then
+            if [ "${{ matrix.python }}" == "3.10" ]; then
               export SYNAPSE_PROFILE="TestWindowsMinimumPython"
-            elif [ "${{ matrix.python }}" == "3.13" ]; then
+            elif [ "${{ matrix.python }}" == "3.14" ]; then
               export SYNAPSE_PROFILE="TestWindowsMaximumPython"
             fi
           elif [ "${{ startsWith(matrix.os, 'macos') }}" == "true" ]; then
-            if [ "${{ matrix.python }}" == "3.9" ]; then
+            if [ "${{ matrix.python }}" == "3.10" ]; then
               export SYNAPSE_PROFILE="TestMacosMinimumPython"
-            elif [ "${{ matrix.python }}" == "3.13" ]; then
+            elif [ "${{ matrix.python }}" == "3.14" ]; then
               export SYNAPSE_PROFILE="TestMacosMaximumPython"
             fi
           fi
@@ -194,21 +201,96 @@ jobs:
           # Setup ignore patterns based on Python version
           IGNORE_FLAGS="--ignore=tests/integration/synapseclient/test_command_line_client.py"
 
-          if [ "${{ matrix.python }}" == "3.9" ]; then
-            # For min Python version, ignore async tests
-            IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/async/"
-            echo "Running integration tests for Min Python version (3.9) - ignoring async tests"
-          elif [ "${{ matrix.python }}" == "3.13" ]; then
-            # For max Python version, ignore synchronous tests
-            IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/synchronous/"
-            echo "Running integration tests for Max Python version (3.13) - ignoring synchronous tests"
-          fi
+          # Check if we should run only failed tests from previous attempt
+          if [[ -f failed_tests.txt && -s failed_tests.txt ]]; then
+            echo "::notice::Retry attempt ${{ github.run_attempt }} detected - running only previously failed tests"
+            cat failed_tests.txt
 
-          # use loadscope to avoid issues running tests concurrently that share scoped fixtures
-          pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration -n 8 $IGNORE_FLAGS --dist loadscope
+            # Run only the failed tests
+            pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml \
+              --junit-xml=test-results.xml \
+              -n 8 --dist loadscope \
+              $(cat failed_tests.txt | tr '\n' ' ')
+          else
+            echo "::notice::First attempt or no previous failures - running full integration test suite"
+
+            # use loadscope to avoid issues running tests concurrently that share scoped fixtures
+            pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml \
+              --junit-xml=test-results.xml \
+              tests/integration -n 8 $IGNORE_FLAGS --dist loadscope
+          fi
 
           # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
           pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration/synapseclient/test_command_line_client.py
+
+      - name: Extract Failed Tests
+        if: always() && steps.integration_tests.outcome == 'failure'
+        shell: bash
+        run: |
+          python -c "
+          import xml.etree.ElementTree as ET
+          import os
+          tree = ET.parse('test-results.xml')
+          root = tree.getroot()
+          failed = []
+          for testcase in root.iter('testcase'):
+              if testcase.find('failure') is not None or testcase.find('error') is not None:
+                  classname = testcase.get('classname')
+                  name = testcase.get('name')
+                  file_attr = testcase.get('file')
+
+                  # Use the file attribute if available, otherwise convert classname
+                  if file_attr:
+                      # file attribute is already in the correct format
+                      test_path = f'{file_attr}::{classname.split(\".\")[-1]}::{name}'
+                  else:
+                      # Convert classname from dot notation to file path
+                      # e.g., 'tests.integration.foo.test_bar.TestClass' -> 'tests/integration/foo/test_bar.py::TestClass'
+                      parts = classname.split('.')
+                      # Find the test file (usually starts with 'test_')
+                      module_parts = []
+                      class_parts = []
+                      found_test_file = False
+                      for part in parts:
+                          if not found_test_file:
+                              module_parts.append(part)
+                              if part.startswith('test_'):
+                                  found_test_file = True
+                          else:
+                              class_parts.append(part)
+
+                      file_path = '/'.join(module_parts) + '.py'
+                      if class_parts:
+                          test_path = f'{file_path}::{\"::\".join(class_parts)}::{name}'
+                      else:
+                          test_path = f'{file_path}::{name}'
+
+                  failed.append(test_path)
+
+          with open('failed_tests.txt', 'w') as f:
+              f.write('\n'.join(failed))
+          print(f'Found {len(failed)} failed tests')
+          for test in failed:
+              print(f'  - {test}')
+          print(f'Current attempt: ${{ github.run_attempt }}')
+          "
+
+      - name: Upload Failed Tests for Next Attempt
+        if: always() && steps.integration_tests.outcome == 'failure' && github.run_attempt < 3
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-tests-${{ matrix.os }}-${{ matrix.python }}
+          path: failed_tests.txt
+          retention-days: 2
+          overwrite: true
+
+      - name: Fail job if integration tests failed after all retries
+        if: always() && steps.integration_tests.outcome == 'failure'
+        shell: bash
+        run: |
+          echo "::error::Integration tests failed after ${{ github.run_attempt }} attempt(s)"
+          exit 1
+
       - name: Upload coverage report
         id: upload_coverage_report
         uses: actions/upload-artifact@v4
@@ -225,12 +307,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
       - name: Check coverage-report artifact existence
         id: check_coverage_report
         uses: LIT-Protocol/artifact-exists-action@v0
         with:
-          name: "coverage-report"
+          name: 'coverage-report'
       - name: Download coverage report
         uses: actions/download-artifact@v4
         if: steps.check_coverage_report.outputs.exists == 'true'
@@ -240,21 +322,21 @@ jobs:
         id: check_coverage_xml
         uses: andstor/file-existence-action@v3
         with:
-          files: "coverage.xml"
+          files: 'coverage.xml'
         # This is a workaround described in https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057
       - name: Override Coverage Source Path for Sonar
         if: steps.check_coverage_xml.outputs.files_exists == 'true'
         run: sed -i "s/<source>\/home\/runner\/work\/synapsePythonClient<\/source>/<source>\/github\/workspace<\/source>/g" coverage.xml
       - name: SonarCloud Scan
-        uses: SonarSource/sonarqube-scan-action@v5.3.1
+        uses: SonarSource/sonarqube-scan-action@v6.0.0
         if: ${{ always() }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
   # on a GitHub release, build the pip package and upload it as a GitHub release asset
   package:
-    needs: [test,pre-commit]
+    needs: [test, pre-commit]
 
     runs-on: ubuntu-22.04
 
@@ -269,7 +351,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: '3.10'
 
       - name: set-release-env
         shell: bash
@@ -366,7 +448,6 @@ jobs:
       #     asset_path: dist/${{ steps.build-package.outputs.bdist-package-name }}
       #     asset_content_type: application/zip
 
-
   # re-download the built package to the appropriate pypi server.
   # we upload prereleases to test.pypi.org and releases to pypi.org.
   deploy:
@@ -404,10 +485,10 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-24.04, macos-13, windows-2022]
+        os: [ubuntu-24.04, macos-15-intel, windows-2022]
 
         # python versions should be consistent with the strategy matrix and the runs-integration-tests versions
-        python: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
     runs-on: ${{ matrix.os }}
 

.gitignore

@@ -6,6 +6,7 @@ virtualenvs
 examples_temp
 .DS_Store
 deploy.sh
+.venv/
 
 junk/
 nose.cfg

CONTRIBUTING.md

@@ -93,7 +93,7 @@ copied to forks).
 #### Installing the Python Client in a virtual environment with pipenv
 Perform the following one-time steps to set up your local environment.
 
-1. This package uses Python, if you have not already, please install [pyenv](https://github.com/pyenv/pyenv#installation) to manage your Python versions. Versions supported by this package are all versions >=3.9 and <=3.13. If you do not install `pyenv` make sure that Python and `pip` are installed correctly and have been added to your PATH by running `python3 --version` and `pip3 --version`. If your installation was successful, your terminal will return the versions of Python and `pip` that you installed.  **Note**: If you have `pyenv` it will install a specific version of Python for you.
+1. This package uses Python, if you have not already, please install [pyenv](https://github.com/pyenv/pyenv#installation) to manage your Python versions. Versions supported by this package are all versions >=3.10 and <=3.14. If you do not install `pyenv` make sure that Python and `pip` are installed correctly and have been added to your PATH by running `python3 --version` and `pip3 --version`. If your installation was successful, your terminal will return the versions of Python and `pip` that you installed.  **Note**: If you have `pyenv` it will install a specific version of Python for you.
 
 2. Install `pipenv` by running `pip install pipenv`.
     - If you already have `pipenv` installed, ensure that the version is >=2023.9.8 to avoid compatibility issues.
@@ -223,6 +223,64 @@ When integration tests are ran in the Github CI/CD pipeline it will upload the t
 #### Integration testing for external collaborators
 As an external collaborator you will not have access to a development account and environment to run the integration tests against. Either request that a Sage Bionetworks staff member run your integration tests via a pull request, or, contact us via the [Service Desk](https://sagebionetworks.jira.com/servicedesk/customer/portal/9) to requisition a development account for integration testing only.
 
+### Managing Python version changes
+
+When adding support for a new Python version or dropping support for an old version, several files across the codebase and CI/CD pipelines must be updated to ensure consistency and proper testing coverage.
+
+#### Adding a new Python version
+
+When adding support for a new Python version (e.g., adding Python 3.15), update the following:
+
+**Code configuration files:**
+1. **`setup.cfg`**:
+   - Add the new version to the `classifiers` list under `[metadata]` (e.g., `Programming Language :: Python :: 3.15`)
+   - Update the `python_requires` constraint under `[options]` to include the new version (e.g., `>=3.10, <3.16`)
+
+2. **`pyproject.toml`**:
+   - Update the `target-version` list in the `[tool.black]` section to include the new version if needed
+
+3. **`Dockerfile`**:
+   - Update the base image to use the new Python version (e.g., `FROM python:3.15-slim`)
+
+**CI/CD configuration files:**
+1. **`.github/workflows/build.yml`**:
+   - Add the new version to the `python` matrix under the `test` job strategy
+   - Ensure the new version is included in integration test runs (typically the latest version should be tested)
+   - Update any Python version comments or documentation within the workflow
+
+**Testing:**
+- Run the full test suite (both unit and integration tests) on the new Python version locally before submitting a PR
+- Verify that all CI/CD pipelines pass with the new version included
+
+#### Dropping an old Python version
+
+When dropping support for an old Python version (e.g., removing Python 3.10), update the following:
+
+**Code configuration files:**
+1. **`setup.cfg`**:
+   - Remove the old version from the `classifiers` list under `[metadata]`
+   - Update the `python_requires` constraint under `[options]` to reflect the new minimum version (e.g., `>=3.11, <3.15`)
+
+2. **`pyproject.toml`**:
+   - Update the `target-version` list in the `[tool.black]` section to remove the old version
+
+3. **`Dockerfile`**:
+   - Ensure the base image uses a supported Python version
+
+**CI/CD configuration files:**
+1. **`.github/workflows/build.yml`**:
+   - Remove the old version from the `python` matrix under the `test` job strategy
+   - Update the cache key version (e.g., increment `v28` to `v29`) to invalidate old caches
+
+**Documentation:**
+- Update the README.md and any getting started documentation to reflect the new supported Python version range
+- Update CONTRIBUTING.md (this file) if it mentions specific Python versions in examples
+
+**Important considerations:**
+- Python version changes should be coordinated with a release and clearly communicated in release notes
+- Breaking compatibility with a Python version is a significant change and should typically coincide with a major or minor version bump
+- Always test thoroughly on the minimum and maximum supported Python versions before release
+
 ### Asynchronous methods
 [Asyncio](https://docs.python.org/3/library/asyncio.html) is the future of the Synapse
 Python Client. As such, the expectation is that all future methods that rely on async

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.14-slim
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
 
 RUN apt-get update \

Pipfile

@@ -10,4 +10,4 @@ synapseclient = {file = ".", path = "."}
 python_version = "3.12.6"
 
 [dev-packages]
-synapseclient = {file = ".", editable = true, path = ".", extras = ["dev", "tests", "pandas", "pysftp", "boto3", "docs"]}
+synapseclient = {file = ".", editable = true, path = ".", extras = ["dev", "tests", "pandas", "pysftp", "boto3", "docs", "curator"]}