[DEVOPS-584] Merge remote-tracking branch 'upstream/main' into DEVOPS-584

# Conflicts:
#	.github/workflows/reusable-python-pytest.yml
#	.github/workflows/reusable-python-static_analysis.yml

Author: sebhmg

.azure-pipelines/template-test-minimal-conda-env-install.yml

@@ -1,5 +1,8 @@
 # File: template-test-minimal-conda-env-install.yml
 parameters:
+  - name: pythonVersion
+    type: string
+    default: '3.10'
   # Determines the type of package repository to use for testing installation (prod or dev).
   - name: environment
     type: string
@@ -8,19 +11,27 @@ parameters:
       - prod
       - dev
   # Maps environment names to the repositories required for each environment.
-  - name: envToRepo
+  - name: envToVirtualRepos
     type: object
     default:
-      all:
-        - geology
-        - geophysics
-      all_deps_only:
-        - geology
-        - geophysics
+      all: geoplus
       geology: geology
       geophysics: geophysics
       interop: public
-      simpeg: geophysics
+      simpeg: public
+  - name: envToRemoteRepos
+    type: object
+    default:
+      all:
+        - conda-forge
+      geology:
+        - conda-forge
+      geophysics:
+        - conda-forge
+      interop:
+        - conda-forge
+      simpeg:
+        - conda-forge
 
 stages:
   - stage: TestInstallationOnMinimalCondaEnv
@@ -37,30 +48,31 @@ stages:
             inputs:
               targetType: 'inline'
               script: |
-                envFolder="environments"
+                buildEnvFolder="build"
                 artifactFolder="artifact_envFiles"
 
                 # Create test directories and files
-                mkdir -p "$envFolder" "$artifactFolder"
+                mkdir -p "environments" "$artifactFolder"
 
                 # DiscoverFiles
-                files=$(find $envFolder -name "py-3.10-win-64-*.yml")
+                files=$(find $buildEnvFolder -name "py-${{parameters.pythonVersion}}-win-64-*.yml")
                 if [[ -z "$files" ]]; then
-                    echo "No win-64 environment files found in $envFolder"
+                    echo "No win-64 environment files found in $buildEnvFolder"
                     exit 1
                 fi
 
-                envToRepoJson='${{ convertToJson(parameters.envToRepo) }}'
+                envToVirtualReposJson='${{ convertToJson(parameters.envToVirtualRepos) }}'
+                envToRemoteReposJson='${{ convertToJson(parameters.envToRemoteRepos) }}'
 
                 # Initialize the matrix JSON
                 matrix="{}"
 
-                # Iterate through each key-value pair in envToRepo
+                # Iterate through each key-value pair in envToVirtualRepos
                 while read -r entry; do
                   key=$(echo "$entry" | jq -r '.key')
                   value=$(echo "$entry" | jq -r '.value')
                   # Find the corresponding file
-                  matchingFile=$(echo "$files" | grep -m 1 "$key")
+                  matchingFile=$(echo "$files" | grep -m 1 "$key\.conda\.lock")
 
                   if [ -z "$matchingFile" ]; then
                     echo "No matching file found for $key, skipping..."
@@ -75,19 +87,23 @@ stages:
                     repos=$value
                   fi
 
+                  # also add repos from envToRemoteRepos for the current key
+                  remoteRepos=$(echo "$envToRemoteReposJson" | jq -r ".$key | join(\" \")")
+
                   # Create object
                   object_string=$( jq -n \
                                     --arg envType "$key" \
                                     --arg filename "$filename" \
                                     --arg repositories "$repos" \
-                                    '{ ($envType): {name: $envType, filename: $filename, repositories: $repositories }}' )
+                                    --arg remotes "$remoteRepos" \
+                                    '{ ($envType): {name: $envType, filename: $filename, repositories: $repositories, remotes: $remotes }}' )
 
                   # Merge with the main json_obj
                   matrix=$(echo "$matrix" | jq ". + $object_string")
 
                   # Copy matching files to the artifact folder
                   cp "$matchingFile" "$artifactFolder/"
-                done < <(echo "$envToRepoJson" | jq -c 'to_entries[]')
+                done < <(echo "$envToVirtualReposJson" | jq -c 'to_entries[]')
 
                 json_matrix=$(echo $matrix | jq -c .)
 
@@ -101,8 +117,8 @@ stages:
         pool:
           vmImage: 'windows-2022'
         container:
-          image: mirageoscienceltd.jfrog.io/mira-docker-local/windows-servercore-ltsc2022-micromamba:latest
-          endpoint: Docker-Artifactory
+          image: ghcr.io/mirageoscience/windows-servercore-ltsc2022-micromamba:latest
+          endpoint: mira-github-docker
         dependsOn: DiscoverFiles
         strategy:
           matrix: $[ dependencies.DiscoverFiles.outputs['DiscoverFiles.matrix'] ]
@@ -118,58 +134,44 @@ stages:
             inputs:
               targetType: 'inline'
               script: |
-                $string = "$(repositories)"
-                $array = $string.Split(" ")
                 $conda_channels = ""
-                $i = 0
-                foreach ($repo in $array) {
-                  echo "Adding repository ${repo}-conda-${{ parameters.environment }}"
-                  $url = "https://$(JFROG_ARTIFACTORY_USER):$(JFROG_ARTIFACTORY_TOKEN)@$(JFROG_ARTIFACTORY_URL)/conda/${repo}-conda-${{ parameters.environment }}"
-                  micromamba config append channels $url
-                  $conda_channels += $url
-                  if (($array.Length - $i) -gt 1) {
-                    $conda_channels += ","
-                  }
-                    $i++
-                }
-                echo "##vso[task.setvariable variable=conda_channels;issecret=true]$conda_channels"
-          - task: PowerShell@2
-            name: ConfigurePyPISources
-            displayName: Configure PyPI sources
-            inputs:
-              targetType: 'inline'
-              script: |
-                $string = "$(repositories)"
-                $array = $string.Split(" ")
-                $index_url = ""
-                $extra_index_url = ""
-                $i = 0
-                foreach ($repo in $array) {
-                  echo "Adding repository ${repo}-pypi-${{ parameters.environment }}"
-                  $url = "https://$(JFROG_ARTIFACTORY_USER):$(JFROG_ARTIFACTORY_TOKEN)@$(JFROG_ARTIFACTORY_URL)/pypi/${repo}-pypi-${{ parameters.environment }}/simple"
-                  if ($i -eq 0) {
-                    $index_url = $url
-                  } else {
-                    $extra_index_url += $url
-                    if (($array.Length - $i) -gt 1) {
-                      $extra_index_url += " "
+                $conda_virtual_repos = "$(repositories)".Split(" ")
+                if ($conda_virtual_repos -ne "") {
+                  foreach ($repo in $conda_virtual_repos) {
+                    $repo_name = "${repo}-noremote-conda-${{ parameters.environment }}"
+                    echo "Adding Conda channel for '${repo_name}'"
+                    $url = "https://$(JFROG_ARTIFACTORY_USER):$(JFROG_ARTIFACTORY_TOKEN)@$(JFROG_ARTIFACTORY_URL)/conda/${repo_name}"
+                    if ($conda_channels -ne "") {
+                      $conda_channels += ","
                     }
+                    $conda_channels += $url
                   }
-                  $i++
                 }
-                echo "##vso[task.setvariable variable=extra_index_url;issecret=true]$extra_index_url"
-                echo "##vso[task.setvariable variable=index_url;issecret=true]$index_url"
+
+                $remote_repos = "$(remotes)".Split(" ") -join ","
+                echo "Adding Conda channels: '${remote_repos}'"
+                if ($conda_channels -and $remote_repos) {
+                  $conda_channels += ","
+                }
+                $conda_channels += "${remote_repos}"
+                echo "##vso[task.setvariable variable=conda_channels;issecret=true]$conda_channels"
           - task: PowerShell@2
             name: InstallEnvironment
             displayName: Install environment
             inputs:
               targetType: 'inline'
               script: |
                 $Env:CONDA_CHANNELS = "$(conda_channels)"
-                $Env:CONDARC = "$(Pipeline.Workspace)/envFiles"
-                $Env:PIP_INDEX_URL = "$(index_url)"
-                $Env:PIP_EXTRA_INDEX_URL = "$(extra_index_url)"
+                $Env:MAMBA_CHANNEL_ALIAS = "https://repo.prefix.dev"
+                $Env:MAMBA_CHANNEL_PRIORITY = "strict"  # make sure Artifactory takes precedence over conda-forge
                 $Env:PIP_NO_DEPS = 1
                 cd $(Pipeline.Workspace)/envFiles
-                micromamba env create -y -f $(filename) -n $(name)
-            
+                micromamba --no-rc env create -y -f $(filename) -n $(name)
+          - task: PowerShell@2
+            name: CheckEnvironment
+            displayName: Check environment
+            inputs:
+              targetType: 'inline'
+              script: |
+                micromamba --no-rc run -n $(name) python -c "import geoh5py; print(f'geoh5py version: {geoh5py.__version__}')"
+                micromamba --no-rc run -n $(name) python -c "import numpy; print(f'numpy version: {numpy.__version__}')"

.docker-images/windows-servercore-ltsc2022-micromamba/Dockerfile

@@ -2,7 +2,7 @@ FROM mcr.microsoft.com/windows/servercore:ltsc2022
 USER ContainerAdministrator
 RUN powershell -Command Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://vcredist.com/install.ps1'))
 USER ContainerUser
-ADD https://github.com/mamba-org/micromamba-releases/releases/download/1.5.10-0/micromamba-win-64 micromamba.exe
+ADD https://github.com/mamba-org/micromamba-releases/releases/download/2.1.0-0/micromamba-win-64 micromamba.exe
 RUN mkdir %USERPROFILE%\micromamba && \
     copy micromamba.exe %USERPROFILE%\micromamba && \
     del micromamba.exe && \

.github/actions/reusable-python-setup_conda/action.yml

@@ -5,20 +5,44 @@ inputs:
     description: 'Python version to use'
     required: true
     type: string
+  JFROG_ARTIFACTORY_URL:
+    description: 'JFrog Artifactory URL, needed if packages are on Artifactory'
+    required: false
+    type: string
+  JFROG_ARTIFACTORY_TOKEN:
+    description: 'JFrog Artifactory Token, needed if packages are on Artifactory'
+    required: false
+    type: string
+
 
 runs:
   using: "composite"
   steps:
     - uses: actions/checkout@v4
-    - name: Setup CONDA_LOCK_ENV_FILE
+    - name: Setup env vars
       shell: bash
       run: |
         os_conda=$( [[ $RUNNER_OS == "Linux" ]] && echo "linux" || echo "win")
-        echo "os_conda=$os_conda" >> $GITHUB_ENV
-    - name: Setup conda env
-      uses: mamba-org/setup-micromamba@v1
+        echo "os_conda=$os_conda" >> "$GITHUB_ENV"
+        echo "micromamba_version=2.1.0-0" >> "$GITHUB_ENV"
+        echo "MAMBA_CHANNEL_ALIAS=https://repo.prefix.dev" >> "$GITHUB_ENV"
+    - name: Setup micromamba
+      uses: mamba-org/setup-micromamba@v2
+      with:
+        micromamba-version: ${{ env.micromamba_version }}
+        init-shell: bash
+        cache-downloads: true
+        environment-name: ""
+        post-cleanup: none
+    - name: Authenticate to Artifactory
+      if: ${{ inputs.JFROG_ARTIFACTORY_URL && inputs.JFROG_ARTIFACTORY_TOKEN }}
+      shell: 'bash -l {0}'
+      run: |
+        micromamba auth login ${{ inputs.JFROG_ARTIFACTORY_URL }} -u github -p ${{ inputs.JFROG_ARTIFACTORY_TOKEN }}
+    - name: Create conda env
+      uses: mamba-org/setup-micromamba@v2
       with:
-        micromamba-version: 1.5.8-0
+        micromamba-version: ${{ env.micromamba_version }}
         environment-file: ${{ env.CONDA_LOCK_ENV_FILE }}
         environment-name: test_env
         init-shell: bash
@@ -27,5 +51,6 @@ runs:
       env:
         PYTHONUTF8: 1
         CONDA_CHANNEL_PRIORITY: strict
+        MAMBA_CHANNEL_PRIORITY: strict
         PIP_NO_DEPS: 1 # all dependencies are installed from conda
         CONDA_LOCK_ENV_FILE: 'environments/py-${{ inputs.python-version }}-${{ env.os_conda }}-64-dev.conda.lock.yml'

.github/actions/reusable-python-setup_poetry/action.yml

@@ -9,6 +9,10 @@ inputs:
     description: 'Runner OS'
     required: true
     type: string
+  install-extras:
+    description: 'Extra dependencies to install (eg. ["torch", "cuda"])'
+    required: false
+    type: string
   virtual-repo-names:
     description: 'Matrix of virtual repository names to resolve (eg. ["public-dev-pypi"])'
     required: false
@@ -25,17 +29,17 @@ runs:
       id: full-python-version
       shell: bash
       run: echo "version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")" >> $GITHUB_OUTPUT
-    - name: Install poetry
-      shell: bash
-      run: |
-        curl -sSL https://install.python-poetry.org | POETRY_HOME=$HOME/.poetry python -
-        echo "$HOME/.poetry/bin" >> $GITHUB_PATH
-    - name: Config poetry
+    - name: Install and configure Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 2.1.3
+        plugins: poetry-dynamic-versioning
+        virtualenvs-in-project: true
+        virtualenvs-path: .venv
+        installer-parallel: true
+    - name: Config repos for Poetry
       shell: bash
       run: |
-        poetry config virtualenvs.create true
-        poetry config virtualenvs.in-project true
-
         repository_list=$(echo '${{ inputs.virtual-repo-names }}' | jq -r '.[]')
         for repo_name in ${repository_list}; do
           first_part=$(echo "$repo_name" | cut -d'-' -f1 | tr '[:upper:]' '[:lower:]')
@@ -45,8 +49,6 @@ runs:
     - name: Set up cache
       uses: actions/cache@v4
       id: cache
-      env:
-        cache-number: ${{ inputs.cache-number }}
       with:
         path: .venv
         key: venv-${{ inputs.runner-os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}-{{ inputs.cache-number }}
@@ -56,4 +58,6 @@ runs:
       run: timeout 10s poetry run pip --version || rm -rf .venv
     - name: Install dependencies
       shell: bash
-      run: poetry install -vvv
+      run: |
+        extra_args=$(echo '${{ inputs.install-extras }}' | jq -r 'map("-E " + .) | join(" ")')
+        poetry install -vvv $(echo "$extra_args")

.github/dependabot.yml

@@ -9,4 +9,4 @@ updates:
     directory: "/"
     schedule:
       interval: "monthly"
-    target-branch: "develop"
+    target-branch: "main"

.github/workflows/jira-issue_to_jira.yml

@@ -1,5 +1,8 @@
 name: Create JIRA issue
 
+permissions:
+  issues: write
+
 on:
   issues:
     types: [opened]

.github/workflows/jira-pr_add_jira_summary.yml

@@ -1,5 +1,9 @@
 name: Add JIRA issue summary
 
+permissions:
+  contents: read
+  pull-requests: write
+
 on:
   pull_request_target:
     types: [opened]

.github/workflows/pre-commit.yml

@@ -1,5 +1,9 @@
 name: pre-commit
 
+permissions:
+  contents: read
+  statuses: write
+
 on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]

.github/workflows/reusable-jira-issue_to_jira.yml

@@ -20,7 +20,7 @@ on:
         required: true
         type: string
       components:
-        description: 'JIRA components (eg. [{"name": "geoh5py"}])'
+        description: 'JIRA components (e.g. [{"name": "geoh5py"}])'
         required: false
         type: string
 
@@ -30,6 +30,14 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
+    - name: Set Jira fields
+      id: set_jira_fields
+      run: |
+        if [[ -z "${{ inputs.components }}" ]]; then
+          echo 'jira_fields={}' >> $GITHUB_OUTPUT
+        else
+          echo 'jira_fields={"components": ${{ inputs.components }} }' >> $GITHUB_OUTPUT
+        fi
     - name: JIRA Login
       uses: atlassian/gajira-login@v3.0.1
       env:
@@ -45,7 +53,7 @@ jobs:
         summary: ${{ github.event.issue.title }}
         description: "_from [GitHub issue #${{ github.event.issue.number }}|${{ github.event.issue.html_url }}]_"
         # Additional fields in JSON format
-        fields: '{"components": ${{ inputs.components }} }'
+        fields: "${{ steps.set_jira_fields.outputs.jira_fields }}"
     - name: Post JIRA link
       uses: peter-evans/create-or-update-comment@v4
       with: