Merge branch 'graphnet-team:main' into fix_workflow

Author: sevmag

.github/actions/install/action.yml

@@ -13,30 +13,61 @@ inputs:
   extras:
     description: "Extras flags for pip installation of graphnet"
     required: true
-    default: "[develop,torch]"
+    default: "develop"
+
+  torch_version:
+    description: "Version of PyTorch"
+    required: true
+    default: "2.6.0"
+    options: ["2.7.0", "2.6.0", "2.5.0", "2.4.0"]
+
   hardware:
     description: "The requirements file depends on the hardware, i.e., CPU, GPU, or macOS"
     required: true
     default: "cpu"
-    options: ["cpu", "cu118", "cu121", "macos"]
+    options: ["cpu", "cu118", "cu121", "cu124", "cu126", "macos"]
+
+  use_vm:
+    description: "Should the installation happen in a virtual environment named venv?"
+    required: true
+    default: false
+    options: [true, false]
 
 runs:
   using: "composite"
   steps:
-    - name: Infer installation flags
+    - name: Prepare pip flags
+      id: flags
+      run: |
+        [[ "${{ inputs.editable }}" == "true" ]] && echo "editable_flag=-e" >> $GITHUB_OUTPUT || echo "editable_flag=" >> $GITHUB_OUTPUT
+        [[ "${{ inputs.user }}" == "true" ]] && echo "user_flag=--user" >> $GITHUB_OUTPUT || echo "user_flag=" >> $GITHUB_OUTPUT
+
+        version="${{ inputs.torch_version }}"
+        IFS='.' read -r major minor _ <<< "$version"
+        echo "torch_flag=${major}${minor}" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Install dependencies (pip & setuptools)
+      shell: bash
       run: |
-        PIP_FLAG_EDITABLE=`[[ ${{ inputs.editable }} =~ (T|t)rue ]] && printf "%s\n" "-e " || echo " "`
-        PIP_FLAG_USER=`[[ ${{ inputs.user }} =~ (T|t)rue ]] && printf "%s\n" "--user " || echo " "`
-        echo "PIP_FLAGS=${PIP_FLAG_USER}${PIP_FLAG_EDITABLE}" >> $GITHUB_ENV
+        if [[ "${{ inputs.use_vm }}" == "true" ]]; then source ~/venv/bin/activate; fi
+        pip install --upgrade 'pip>=20'
+        pip install --upgrade 'setuptools>=68.2.2'
+
+    - name: Print pip and setuptools versions
       shell: bash
-    - name: Install dependencies
       run: |
-        pip install --upgrade pip>=20
-        pip install wheel setuptools==59.5.0
+        if [[ "${{ inputs.use_vm }}" == "true" ]]; then source ~/venv/bin/activate; fi
+        pip --version
+        pip show setuptools
+
+    - name: Install GraphNeT
       shell: bash
-    - name: Install package
       run: |
-        echo requirements/torch_${{ inputs.hardware }}.txt ${{ env.PIP_FLAGS }} .${{ inputs.extras }}
-        pip install -r requirements/torch_${{ inputs.hardware }}.txt ${{ env.PIP_FLAGS }} .${{ inputs.extras }}
+        if [[ "${{ inputs.use_vm }}" == "true" ]]; then source ~/venv/bin/activate; fi
+
+        echo "Installing graphnet with flags: ${{ steps.flags.outputs.user_flag }} ${{ steps.flags.outputs.editable_flag }}"
+        pip install ${{ steps.flags.outputs.user_flag }} ${{ steps.flags.outputs.editable_flag }} .[torch-${{ steps.flags.outputs.torch_flag }}-${{ inputs.hardware }},${{ inputs.extras }}] \
+          -f https://data.pyg.org/whl/torch-${{ inputs.torch_version }}+${{ inputs.hardware }}.html
+
         pip install git+https://github.com/thoglu/jammy_flows.git
-      shell: bash

.github/workflows/build.yml

@@ -33,81 +33,141 @@ jobs:
          fi
 
   build-icetray:
-    name: Unit tests - IceTray
+    name: Unit tests - IceTray (v1.13.0 - 3.10)  
     needs: [ check-codeclimate-credentials ]
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        torch_version: ['2.7.0']
+        hardware: ["cpu"]
     container:
-      image: icecube/icetray:icetray-prod-v1.8.1-ubuntu20.04-X64
-      options: --user root
+      # GitHub Actions overwrite the docker container entrypoint
+      # inspect <image> --format '{{.Config.Entrypoint}}'
+      image: icecube/icetray:icetray-devel-v1.13.0-ubuntu22.04-2025-02-12
+    
     steps:
-      - name: install git
+      - name: Mimmick Docker Entrypoint
+        # The entrypoint of the container sets python paths
+        # Because the entrypoint is skipped by GitHub, we set it here
+        run: |
+          echo "I3_SRC=/opt/icetray/share/icetray" >> $GITHUB_ENV
+          echo "I3_BUILD=/opt/icetray/share/icetray" >> $GITHUB_ENV
+          echo "I3_TESTDATA=/root/icetray/build/test-data" >> $GITHUB_ENV
+          echo "I3_PRODDATA=/root/icetray/build/prod-data" >> $GITHUB_ENV
+          echo "ROOTSYS=/opt/icetray/cernroot" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/icetray/lib:/opt/icetray/cernroot/lib:/opt/icetray/lib/tools:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "DYLD_LIBRARY_PATH=/opt/icetray/lib:/opt/icetray/cernroot/lib:/opt/icetray/lib/tools:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "PYTHONPATH=/opt/icetray/lib:/opt/icetray/cernroot/lib:$PYTHONPATH" >> $GITHUB_ENV
+          echo "PATH=/opt/icetray/bin:$PATH" >> $GITHUB_ENV
+          echo "HDF5_USE_FILE_LOCKING=FALSE" >> $GITHUB_ENV
+      - name: Verify IceCube is importable
+        run: python3 -c "import icecube; print('Import Successful')"
+      - name: Show python version
+        run: | 
+          python3 --version
+          pip --version
+          pip3 list
+          python3 -c "import icecube; print(icecube.__path__)"
+          pip show setuptools
+
+      - name: Install git
         run: |
           apt-get --yes install sudo
           sudo apt update --fix-missing --yes
           sudo apt upgrade --yes
-          sudo apt-get install --yes git-all
-      - name: Set environment variables
+          sudo apt-get install --yes git
+
+      - uses: actions/checkout@v4
+
+      - name: Create virtual environment
+        shell: bash
         run: |
-          echo "PATH=/usr/local/icetray/bin:$PATH" >> $GITHUB_ENV
-          echo "PYTHONPATH=/usr/local/icetray/lib:$PYTHONPATH" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=/usr/local/icetray/lib:/usr/local/icetray/cernroot/lib:/usr/local/icetray/lib/tools:$LD_LIBRARY_PATH" >> $GITHUB_ENV
-      - uses: actions/checkout@v3
+          python3 -m venv ~/venv --upgrade-deps
+          source ~/venv/bin/activate
+          pip --version
+          python --version
+          pip show setuptools
+
       - name: Print available disk space before graphnet install
         run: df -h
-      - name: Upgrade packages already installed on icecube/icetray
+
+      - name: Upgrade packages in virtual environment
+        shell: bash
         run: |
-          pip install --upgrade astropy  # Installed version incompatible with numpy 1.23.0 [https://github.com/astropy/astropy/issues/12534]
-          pip install --ignore-installed PyYAML  # Distutils installed [https://github.com/pypa/pip/issues/5247]
-          pip install --upgrade psutil # Original version from IceTray Environment incompatible
+          source ~/venv/bin/activate
+          pip install --upgrade astropy
+          pip install --upgrade PyYAML
+          pip install --upgrade psutil
+          pip install --upgrade setuptools
+          pip install --upgrade versioneer
+          pip show setuptools
+          pip show versioneer
       - name: Install package
         uses: ./.github/actions/install
         with:
           editable: true
+          use_vm: true
+          torch_version: ${{ matrix.torch_version }}
+          hardware: ${{ matrix.hardware }}
       - name: Print packages in pip
+        shell: bash
         run: |
+          source ~/venv/bin/activate
           pip show torch
           pip show torch-geometric
           pip show torch-cluster
           pip show torch-sparse
           pip show torch-scatter
           pip show jammy_flows
       - name: Run unit tests and generate coverage report
+        shell: bash
         run: |
+          source ~/venv/bin/activate
           coverage run --source=graphnet -m pytest tests/ --ignore=tests/examples/04_training --ignore=tests/utilities 
           coverage run -a --source=graphnet -m pytest tests/examples/04_training
           coverage run -a --source=graphnet -m pytest tests/utilities 
           coverage xml -o coverage.xml
-    
+          coverage report -m
       - name: Work around permission issue
         run: |
           git config --global --add safe.directory /__w/graphnet/graphnet
       - name: Publish code coverage
-        uses: paambaati/codeclimate-action@v3.0.0
+        uses: paambaati/codeclimate-action@v9.0.0
         if: needs.check-codeclimate-credentials.outputs.has_credentials == 'true'
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CODECLIMATE_TEST_REPORTER_ID }}
-        with:
-          coverageCommand: coverage report
+        with: 
           coverageLocations: coverage.xml:coverage.py
+          
 
-  build-matrix:
-    name: Unit tests - Python versions
+  build-matrix-examples:
+    name: Examples - Ubuntu 22.04
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: ['3.11']
+        torch_version: ['2.7.0', '2.6.0', '2.5.0']
+        hardware: ["cpu"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Print available disk space before graphnet install
         run: df -h
+      - name: Show existing environment
+        run: |
+          echo "PIP_FLAGS=${PIP_FLAGS}"
+          python --version
+          pip --version
+          pip debug --verbose
       - name: Install package
         uses: ./.github/actions/install
         with:
           editable: true
+          torch_version: ${{ matrix.torch_version }}
+          hardware: ${{ matrix.hardware }}
       - name: Print available disk space after graphnet install
         run: df -h
       - name: Print packages in pip
@@ -117,32 +177,45 @@ jobs:
           pip show torch-cluster
           pip show torch-sparse
           pip show torch-scatter
+          pip show dill
           pip show numpy
-
-
       - name: Run unit tests and generate coverage report
         run: |
           set -o pipefail  # To propagate exit code from pytest
-          coverage run --source=graphnet -m pytest tests/ --ignore=tests/utilities --ignore=tests/data/ --ignore=tests/deployment/ --ignore=tests/examples/01_icetray/
-          coverage run -a --source=graphnet -m pytest tests/utilities
-          coverage report -m
+          pytest tests/examples --ignore=tests/examples/01_icetray/
       - name: Print available disk space after unit tests
         run: df -h
 
-  build-macos:
-    name: Unit tests - macOS
-    runs-on: macos-13
+  build-macos-examples:
+    # Runtime on macOS is multiplied with a factor 10 by github. Minimize!
+    name: Examples - macOS 15
+    runs-on: macos-15
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.11']
+        torch_version: ['2.7.0']
+        hardware: ["cpu"]
+
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
-      - name: Install package
+          python-version: ${{ matrix.python-version }}
+
+      - name: Show Python version
+        run: |
+          python --version
+          which python
+          pip --version
+
+      - name: Install dependencies
         uses: ./.github/actions/install
         with:
-          editable: true
-          hardware: "cpu"
+          torch_version: ${{ matrix.torch_version }}
+          hardware: ${{ matrix.hardware }}
       - name: Print packages in pip
         run: |
             pip show torch
@@ -153,15 +226,17 @@ jobs:
       - name: Run unit tests and generate coverage report
         run: |
           set -o pipefail  # To propagate exit code from pytest
-          coverage run --source=graphnet -m pytest tests/  --ignore=tests/data/ --ignore=tests/deployment/ --ignore=tests/examples/ --ignore=tests/utilities
-          coverage run -a --source=graphnet -m pytest tests/utilities
-          coverage report -m
+          pytest tests/examples --ignore=tests/examples/01_icetray/
 
   docs:
     name: Documentation Compilation
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
       - name: Install package
         uses: ./.github/actions/install
       - name: Build documentation

.github/workflows/code-quality.yml

@@ -22,9 +22,16 @@ jobs:
       matrix:
         hook: ["black", "flake8", "docformatter", "pydocstyle", "mypy"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
       - name: Install package
         uses: ./.github/actions/install
+        with:
+          
       - name: Run ${{ matrix.hook }}
         run: |
+          pip show ${{ matrix.hook }}
           pre-commit run ${{ matrix.hook }} --all-files

.github/workflows/docs.yml

@@ -16,9 +16,17 @@ jobs:
     name: Build and publish Documentation
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
       - name: Install package
         uses: ./.github/actions/install
+        with:
+          editable: true
+          use_vm: false
+          torch_version: "2.6.0"
+          hardware: "cpu"
       - name: Build documentation
         run: |
           cd docs

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
     - id: flake8
       language_version: python3
   - repo: https://github.com/pycqa/docformatter
-    rev: v1.7.5
+    rev: v1.7.7
     hooks:
     - id: docformatter
       language_version: python3

docs/source/installation/install.rst

@@ -3,7 +3,7 @@
 Installation
 ============
 
-|graphnet|\ GraphNeT is available for Python 3.8 to Python 3.11.
+|graphnet|\ GraphNeT is available for Python 3.9 to Python 3.11.
 
 .. note::
    We recommend installing |graphnet|\ GraphNeT in a separate environment, e.g. using a Python virtual environment or Anaconda (see details on installation `here <https://www.anaconda.com/products/individual>`_).