Merge pull request #282 from clamsproject/develop

releasing 1.5.0

Author: keighrim

.github/workflows/container.yml

@@ -11,6 +11,10 @@ on:
         required: true
         type: string
         description: 'name of the container build file to use'
+      ref:
+        required: false
+        type: string
+        description: 'git ref to checkout (defaults to version)'
   workflow_dispatch:
     inputs:
       version:
@@ -21,30 +25,45 @@ on:
         required: true
         type: string
         description: 'name of the container build file to use'
+      ref:
+        required: false
+        type: string
+        description: 'git ref to checkout (defaults to version)'
 
 env:
   REGISTRY: ghcr.io
 
 jobs:
-  build-and-push-image:
-    name: "🐳 Build and push image"
-    runs-on: ubuntu-latest
+  build:
+    name: "🐳 Build (${{ matrix.platform }})"
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
     permissions:
       contents: read
       packages: write
 
     steps:
+    - name: "🔧 Prepare"
+      id: prepare
+      run: |
+        platform=${{ matrix.platform }}
+        echo "pair=${platform//\//-}" >> $GITHUB_OUTPUT
+        slug=$(echo "${{ inputs.buildfilename }}" | sed 's|[^a-zA-Z0-9]|-|g; s/^-*//; s/-*$//')
+        echo "slug=${slug}" >> $GITHUB_OUTPUT
+
     - name: "🛍️ Checkout repository"
       uses: actions/checkout@v4
       with:
-        ref: ${{ inputs.version }}
-        fetch-depth: 0
-
-    - name: "🎛 Set up QEMU"
-      uses: docker/setup-qemu-action@v3
+        ref: ${{ inputs.ref || inputs.version }}
 
     - name: "👷 Set up Docker Buildx"
-      id: buildx
       uses: docker/setup-buildx-action@v3
 
     - name: "🏷 Prepare OCI annotations"
@@ -68,23 +87,11 @@ jobs:
         export nameonly="${filename%.*}"
         if [ ${nameonly} == ${filename} ]; then echo "SUFFIX=" >> $GITHUB_ENV ; else echo "SUFFIX=-${nameonly}" >> $GITHUB_ENV; fi
 
-    - name: "🏷 Get SDK version from latest tag"
-      id: sdkversion
-      run: |
-        export version=$(git describe --tags --abbrev=0)
-        echo "CLAMS_VERSION=${version}" >> $GITHUB_OUTPUT
-
-    - name: "🏷 Prepare docker tags, labels"
+    - name: "🏷 Prepare docker labels"
       id: meta
       uses: docker/metadata-action@v5
-      env:
-        CLAMS_VERSION: ${{ steps.sdkversion.outputs.CLAMS_VERSION }}
       with:
         images: ${{ env.REGISTRY }}/${{ github.repository }}${{ env.SUFFIX }}
-        tags: |
-          type=pep440,pattern={{version}},value=${{ env.CLAMS_VERSION }}
-          type=ref,event=tag
-          type=ref,event=pr
         labels: |
           ${{ env.EXISTING_LABELS }}
 
@@ -95,18 +102,84 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: "🏗 Build and push image"
-      uses: docker/build-push-action@v5
-      env:
-        CLAMS_VERSION: ${{ steps.sdkversion.outputs.CLAMS_VERSION }}
+    - name: "🏗 Build and push by digest"
+      id: build
+      uses: docker/build-push-action@v6
       with:
         context: ${{ env.CONTEXT }}
-        platforms: linux/amd64,linux/arm64
+        platforms: ${{ matrix.platform }}
         file: ${{ inputs.buildfilename }}
-        tags: ${{ steps.meta.outputs.tags }}
-        # using {{ steps.meta.outputs.labels }} doesn't work with multi-line variable ($EXISTING_LABLES)
         labels: ${{ env.DOCKER_METADATA_OUTPUT_LABELS }}
         build-args: |
-          clams_version=${{ env.CLAMS_VERSION }}
-        push: true
+          clams_version=${{ inputs.version }}
+        outputs: type=image,"name=${{ env.REGISTRY }}/${{ github.repository }}${{ env.SUFFIX }}",push-by-digest=true,name-canonical=true,push=true
+
+    - name: "📤 Export digest"
+      run: |
+        mkdir -p /tmp/digests
+        digest="${{ steps.build.outputs.digest }}"
+        touch "/tmp/digests/${digest#sha256:}"
 
+    - name: "📦 Upload digest"
+      uses: actions/upload-artifact@v4
+      with:
+        name: digests-${{ steps.prepare.outputs.slug }}-${{ steps.prepare.outputs.pair }}
+        path: /tmp/digests/*
+        if-no-files-found: error
+        retention-days: 1
+
+  merge:
+    name: "🔗 Create multi-platform manifest"
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: "🔧 Prepare"
+      id: prepare
+      run: |
+        slug=$(echo "${{ inputs.buildfilename }}" | sed 's|[^a-zA-Z0-9]|-|g; s/^-*//; s/-*$//')
+        echo "slug=${slug}" >> $GITHUB_OUTPUT
+
+    - name: "🏷 Get image name suffix"
+      id: getsuffix
+      run: |
+        export filename=$(basename ${{ inputs.buildfilename }})
+        export nameonly="${filename%.*}"
+        if [ ${nameonly} == ${filename} ]; then echo "SUFFIX=" >> $GITHUB_ENV ; else echo "SUFFIX=-${nameonly}" >> $GITHUB_ENV; fi
+
+    - name: "📥 Download digests"
+      uses: actions/download-artifact@v4
+      with:
+        path: /tmp/digests
+        pattern: digests-${{ steps.prepare.outputs.slug }}-*
+        merge-multiple: true
+
+    - name: "👷 Set up Docker Buildx"
+      uses: docker/setup-buildx-action@v3
+
+    - name: "🏷 Prepare docker tags"
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ github.repository }}${{ env.SUFFIX }}
+        tags: |
+          type=pep440,pattern={{version}},value=${{ inputs.version }}
+          type=ref,event=tag
+          type=ref,event=pr
+
+    - name: "🔏 Log in to registry"
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: "🔗 Create manifest list and push"
+      working-directory: /tmp/digests
+      run: |
+        docker buildx imagetools create \
+          $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          $(printf '${{ env.REGISTRY }}/${{ github.repository }}${{ env.SUFFIX }}@sha256:%s ' *)

.github/workflows/containers.yml

@@ -116,6 +116,15 @@ jobs:
       buildfilename: './container/opencv4.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
+  call-build-hf:
+    name: "🤙 Call container workflow with `hf`"
+    needs: ['set-version', 'call-build-hf5']
+    uses: ./.github/workflows/container.yml
+    secrets: inherit
+    with:
+      buildfilename: './container/hf.containerfile'
+      version: ${{ needs.set-version.outputs.version }}
+
   call-build-opencv4-tf2:
     name: "🤙 Call container workflow with `opencv4-tf2`"
     needs: ['set-version', 'call-build-opencv4']
@@ -134,56 +143,74 @@ jobs:
       buildfilename: './container/opencv4-torch2.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
-  call-build-tf2-hf:
-    name: "🤙 Call container workflow with `tf2-hf`"
-    needs: ['set-version', 'call-build-tf2']
+  call-build-hf4:
+    name: "🤙 Call container workflow with `hf4`"
+    needs: ['set-version', 'call-build-torch2']
     uses: ./.github/workflows/container.yml
     secrets: inherit
     with:
-      buildfilename: './container/tf2-hf.containerfile'
+      buildfilename: './container/hf4.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
-  call-build-hf:
-    name: "🤙 Call container workflow with `hf`"
+  call-build-hf5:
+    name: "🤙 Call container workflow with `hf5`"
     needs: ['set-version', 'call-build-torch2']
     uses: ./.github/workflows/container.yml
     secrets: inherit
     with:
-      buildfilename: './container/hf.containerfile'
+      buildfilename: './container/hf5.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
-  call-build-ffmpeg-tf2-hf:
-    name: "🤙 Call container workflow with `ffmpeg-tf2-hf`"
-    needs: ['set-version', 'call-build-ffmpeg-tf2']
+  call-build-ffmpeg-hf:
+    name: "🤙 Call container workflow with `ffmpeg-hf`"
+    needs: ['set-version', 'call-build-ffmpeg-hf5']
     uses: ./.github/workflows/container.yml
     secrets: inherit
     with:
-      buildfilename: './container/ffmpeg-tf2-hf.containerfile'
+      buildfilename: './container/ffmpeg-hf.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
-  call-build-ffmpeg-hf:
-    name: "🤙 Call container workflow with `ffmpeg-hf`"
+  call-build-ffmpeg-hf4:
+    name: "🤙 Call container workflow with `ffmpeg-hf4`"
     needs: ['set-version', 'call-build-ffmpeg-torch2']
     uses: ./.github/workflows/container.yml
     secrets: inherit
     with:
-      buildfilename: './container/ffmpeg-hf.containerfile'
+      buildfilename: './container/ffmpeg-hf4.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
-  call-build-opencv4-tf2-hf:
-    name: "🤙 Call container workflow with `opencv4-tf2-hf`"
-    needs: ['set-version', 'call-build-opencv4-tf2']
+  call-build-ffmpeg-hf5:
+    name: "🤙 Call container workflow with `ffmpeg-hf5`"
+    needs: ['set-version', 'call-build-ffmpeg-torch2']
     uses: ./.github/workflows/container.yml
     secrets: inherit
     with:
-      buildfilename: './container/opencv4-tf2-hf.containerfile'
+      buildfilename: './container/ffmpeg-hf5.containerfile'
       version: ${{ needs.set-version.outputs.version }}
 
   call-build-opencv4-hf:
     name: "🤙 Call container workflow with `opencv4-hf`"
-    needs: ['set-version', 'call-build-opencv4-torch2']
+    needs: ['set-version', 'call-build-opencv4-hf5']
     uses: ./.github/workflows/container.yml
     secrets: inherit
     with:
       buildfilename: './container/opencv4-hf.containerfile'
       version: ${{ needs.set-version.outputs.version }}
+
+  call-build-opencv4-hf4:
+    name: "🤙 Call container workflow with `opencv4-hf4`"
+    needs: ['set-version', 'call-build-opencv4-torch2']
+    uses: ./.github/workflows/container.yml
+    secrets: inherit
+    with:
+      buildfilename: './container/opencv4-hf4.containerfile'
+      version: ${{ needs.set-version.outputs.version }}
+
+  call-build-opencv4-hf5:
+    name: "🤙 Call container workflow with `opencv4-hf5`"
+    needs: ['set-version', 'call-build-opencv4-torch2']
+    uses: ./.github/workflows/container.yml
+    secrets: inherit
+    with:
+      buildfilename: './container/opencv4-hf5.containerfile'
+      version: ${{ needs.set-version.outputs.version }}

.github/workflows/publish.yml

@@ -1,13 +1,54 @@
-name: "📦 Publish (docs, PyPI)"
+name: "📦 Publish (PyPI + docs)"
 
-on: 
-  push: 
-    tags: 
+on:
+  push:
+    tags:
       - '[0-9]+.[0-9]+.[0-9]+'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to publish (e.g., 3.2.0)'
+        required: true
 
 jobs:
-  packge-and-upload:
-    name: "🤙 Call SDK publish workflow"
+  check-pypi:
+    name: "🔍 Check PyPI for existing package"
+    runs-on: ubuntu-latest
+    outputs:
+      exists: ${{ steps.check.outputs.exists }}
+      version: ${{ steps.check.outputs.version }}
+    steps:
+      - id: check
+        run: |
+          PACKAGE=$(echo "${{ github.repository }}" | cut -d/ -f2 | tr '-' '_')
+          VERSION=${{ inputs.version || github.ref_name }}
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/$PACKAGE/$VERSION/json")
+          if [ "$STATUS" = "200" ]; then
+            echo "Package $PACKAGE $VERSION already exists on PyPI, skipping upload"
+            echo "exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "Package $PACKAGE $VERSION not found on PyPI, proceeding with upload"
+            echo "exists=false" >> $GITHUB_OUTPUT
+          fi
+
+  publish-pypi:
+    name: "📦 Build and upload to PyPI"
+    needs: check-pypi
+    if: needs.check-pypi.outputs.exists == 'false'
     uses: clamsproject/.github/.github/workflows/sdk-publish.yml@main
     secrets: inherit
 
+  publish-docs:
+    name: "📖 Build and publish docs"
+    needs: [check-pypi, publish-pypi]
+    if: always() && needs.check-pypi.result == 'success' && needs.publish-pypi.result != 'failure'
+    uses: clamsproject/clamsproject.github.io/.github/workflows/sdk-docs.yml@main
+    with:
+      source_repo: clamsproject/clams-python
+      source_ref: ${{ needs.check-pypi.outputs.version }}
+      project_name: clams-python
+      build_command: 'python3 build-tools/docs.py --output-dir docs'
+      docs_output_dir: 'docs'
+      python_version: '3.11'
+    secrets: inherit

build-tools/docs.py

@@ -14,11 +14,13 @@ def run_command(command, cwd=None, check=True, env=None):
         sys.exit(result.returncode)
     return result
 
-def build_docs_local(source_dir: Path):
+def build_docs_local(source_dir: Path, output_dir: Path = None):
     """
     Builds documentation for the provided source directory.
     Assumes it's running in an environment with necessary tools.
     """
+    if output_dir is None:
+        output_dir = source_dir / "docs-test"
     print("--- Running in Local Build Mode ---")
 
     # 1. Generate source code and install in editable mode.
@@ -49,7 +51,7 @@ def build_docs_local(source_dir: Path):
     # 3. Build the documentation using Sphinx.
     print("\n--- Step 3: Building Sphinx documentation ---")
     docs_source_dir = source_dir / "documentation"
-    docs_build_dir = source_dir / "docs-test"
+    docs_build_dir = output_dir
 
     # Schema generation is now handled in conf.py
     # schema_src = source_dir / "clams" / "appmetadata.jsonschema"
@@ -74,9 +76,12 @@ def main():
     parser = argparse.ArgumentParser(
         description="Build documentation for the clams-python project."
     )
+    parser.add_argument(
+        '--output-dir', type=Path, default=None,
+        help='Output directory for built docs (default: docs-test)')
     args = parser.parse_args()
-    
-    build_docs_local(Path.cwd())
+
+    build_docs_local(Path.cwd(), output_dir=args.output_dir)
 
 if __name__ == "__main__":
     main()