DataflowTemplates/.github/workflows/upload-python-package.yml at main · GoogleCloudPlatform/DataflowTemplates · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# ====================================================================================
# Purpose:
#   This workflow builds and uploads the job-builder-util-transforms Python package to Google Cloud Storage (GCS).
#   It is designed to distribute the utility transforms package for use in Dataflow jobs.
#   The workflow:
#     1. Sets up Python and installs build dependencies.
#     2. Builds the Python package (wheel and sdist).
#     3. Uploads the built artifacts and Beam YAML provider listing to a specified GCS bucket.
#
# How to Run:
#   This workflow is triggered manually via GitHub Actions `workflow_dispatch`.
#   Required Inputs:
#     - bucket_path: The GCS bucket destination (e.g., gs://my-bucket/path).
# ====================================================================================

name: Build and Upload Python Package for Job Builder

on:
  workflow_dispatch:
    inputs:
      bucket_path:
        description: 'GCS bucket path (e.g., gs://my-bucket/path)'
        required: true
      version:
        description: 'Version number for the package (e.g., 0.1.0)'
        required: true
      only_upload_listing:
        description: 'If true, skips uploading the Python package to GCS and only uploads the provider listing'
        required: false
        type: boolean
        default: false
      date_override:
        description: 'Optional date string to use instead of generating one'
        required: false
        type: string

permissions:
  contents: read

jobs:
  build-and-upload:
    runs-on: [self-hosted, release]

    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: '3.11'

      - name: Install build dependencies
        run: |
          python -m pip install --upgrade pip
          pip install poetry

      - name: Install project dependencies
        working-directory: python/src/main/python/job-builder-util-transforms
        run: poetry install

      - name: Run Tests
        working-directory: python/src/main/python/job-builder-util-transforms
        run: poetry run pytest ../../../test/python/job-builder-util-transforms/

      - name: Build Package
        working-directory: python/src/main/python/job-builder-util-transforms
        run: poetry build

      - name: Upload to GCS
        env:
          BUCKET_PATH: ${{ github.event.inputs.bucket_path }}
          VERSION: ${{ github.event.inputs.version }}
          ONLY_UPLOAD_LISTING: ${{ github.event.inputs.only_upload_listing }}
          DATE_OVERRIDE: ${{ github.event.inputs.date_override }}
        run: |
          DATE="${DATE_OVERRIDE:-$(date +'%Y-%m-%d')}"
          GCS_PACKAGE_PATH="$BUCKET_PATH/$DATE/job_builder_util_transforms-$VERSION.tar.gz"
          if [ "$ONLY_UPLOAD_LISTING" != "true" ]; then
            gcloud storage cp python/src/main/python/job-builder-util-transforms/dist/job_builder_util_transforms-*.tar.gz "$GCS_PACKAGE_PATH"
          fi
          PACKAGE_PATH_NO_GS="${GCS_PACKAGE_PATH#gs://}"
          sed "s|<package_path>|$PACKAGE_PATH_NO_GS|g" python/src/main/python/job-builder-util-transforms/provider_listing_template.yaml > "python/src/main/python/job-builder-util-transforms/yaml_provider_listing_$VERSION.yaml"
          gcloud storage cp "python/src/main/python/job-builder-util-transforms/yaml_provider_listing_$VERSION.yaml" "$BUCKET_PATH/$DATE/yaml_provider_listing_$VERSION.yaml"