diff --git a/.github/workflows/bot_pr_approval.yaml b/.github/workflows/bot_pr_approval.yaml index e38c5841d..b1157d331 100644 --- a/.github/workflows/bot_pr_approval.yaml +++ b/.github/workflows/bot_pr_approval.yaml @@ -3,6 +3,9 @@ name: Provide approval for bot PRs on: pull_request: +permissions: + pull-requests: write + jobs: bot_pr_approval: uses: canonical/operator-workflows/.github/workflows/bot_pr_approval.yaml@main diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c88ff8b1f..66c656897 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,6 +3,9 @@ name: Tests on: pull_request: +permissions: + pull-requests: write + jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@main diff --git a/README.md b/README.md index 17e5a4993..f831c743d 100644 --- a/README.md +++ b/README.md @@ -87,49 +87,60 @@ charms and their interactions: * [GitHub Runner](https://charmhub.io/github-runner): The central component that manages self-hosted GitHub runners. It interacts with OpenStack to spawn runner VMs and communicates with GitHub to register and manage runners. * [Image Builder](https://charmhub.io/github-runner-image-builder): Responsible for generating images. It builds images on the builder OpenStack project and uploads them to the GitHub Runner OpenStack project. -* [MongoDB](https://charmhub.io/mongodb): Acts as a message queue to handle reactive runner requests. The [github-runner-webhook-router](https://charmhub.io/github-runner-webhook-router) charm will put events in MongoDB that will be consumed by the `github-runner` charm. Only for reactive runners. +* [Planner](https://charmhub.io/github-runner-planner): Consumes webhook events from an AMQP broker, tracks job state in PostgreSQL, and streams runner demand ("pressure") to the `github-runner` charm. Optional — without the Planner, the `github-runner` charm pre-spawns a fixed pool of runners configured via `base-virtual-machines`. The Planner integration requires: + * [Webhook Gateway](https://charmhub.io/github-runner-webhook-gateway): Receives and validates incoming GitHub webhook events and forwards them to the AMQP broker. + * [RabbitMQ](https://charmhub.io/rabbitmq-k8s): AMQP broker that carries webhook events from the Webhook Gateway to the Planner. + * [PostgreSQL](https://charmhub.io/postgresql): Persistent storage for the Planner's job state, flavor definitions, and auth token metadata. * [tmate-ssh-server](https://charmhub.io/tmate-ssh-server): Provides terminal-sharing capabilities to enable debugging of GitHub runners. Optional. * [COS lite stack](https://charmhub.io/topics/canonical-observability-stack/editions/lite): Provides observability to the GitHub runners ecosystem. Optional. Below is a diagram representing these components and their relationships, excluding the [COS lite stack](https://charmhub.io/topics/canonical-observability-stack/editions/lite): ```mermaid -C4Container -title Container diagram for the github-runner Charm System - Container_Boundary(c1, "Image Builder") { - Container(imagebuilder, "Image Builder", "", "Provides images to all related charms") - } - System_Ext(osbuilding, "OpenStack", "OpenStack deployment used for building images") -Container_Boundary(c2, "GitHub Runner"){ - Container(githubrunner, "GitHub Runner Charm", "", "Manages self-hosted runners") -} -Container_Boundary(c3, "monbodb"){ - Container(mongodb, "MongoDB", "", "Used as a message queue for reactive runner requests") -} -Container_Boundary(c4, "tmate-ssh-server"){ - Container(tmate_ssh, "tmate-ssh-server", "", "Terminal sharing capabilities to debug GitHub runners") -} - -Container_Boundary(c5, "github-runner-webhook-router"){ - Container(router, "github-runner-webhook-router", "", "Listens to GitHub webhooks") -} - - Rel(imagebuilder, osbuilding, "builds images") - UpdateRelStyle(imagebuilder, osbuilding, $offsetY="-30", $offsetX="10") - Rel(imagebuilder, osgithubrunner, "uploads images") - UpdateRelStyle(imagebuilder, osgithubrunner, $offsetY="-30", $offsetX="-90") - Rel(imagebuilder, githubrunner, "image ids") - UpdateRelStyle(imagebuilder, githubrunner, $offsetY="-10", $offsetX="-30") - System_Ext(osgithubrunner, "OpenStack", "OpenStack deployment used for spawning runner VMs") - System_Ext(github, "GitHub", "GitHub API") - Rel(githubrunner, osgithubrunner, "spawns VMs") - UpdateRelStyle(githubrunner, osgithubrunner, $offsetY="-30", $offsetX="10") - Rel(githubrunner, github, "Manage runners") - Rel(githubrunner, imagebuilder, "OpenStack credentials") - UpdateRelStyle(githubrunner, imagebuilder, $offsetY="10", $offsetX="-60") - Rel(mongodb, githubrunner, "database credentials") - Rel(tmate_ssh, githubrunner, "debug-ssh credentials") - Rel(router, mongodb, "new runner requests") +flowchart TD + GH(["GitHub"]) + OS_BUILD(["OpenStack\n(image building)"]) + OS_RUNNERS(["OpenStack\n(runner VMs)"]) + subgraph MQSG["RabbitMQ"] + MQ(["RabbitMQ Charm"]) + end + + subgraph PGSG["PostgreSQL"] + PG[(PostgreSQL)] + end + + subgraph IB["Image Builder"] + imagebuilder["Image Builder Charm"] + end + + subgraph GRC["GitHub Runner"] + githubrunner["GitHub Runner Charm"] + end + + subgraph TMATE["tmate-ssh-server"] + tmate["tmate-ssh-server"] + end + + subgraph WG["Webhook Gateway"] + webhookgateway["Webhook Gateway Charm"] + end + + subgraph PL["Planner"] + planner["Planner Charm"] + end + + imagebuilder -->|"builds images"| OS_BUILD + imagebuilder -->|"uploads images"| OS_RUNNERS + imagebuilder -->|"image ids"| githubrunner + githubrunner -->|"provides OpenStack credentials"| imagebuilder + githubrunner -->|"spawns VMs"| OS_RUNNERS + githubrunner <-->|"manage runners"| GH + tmate -->|"debug-ssh credentials"| githubrunner + GH -->|"workflow job webhooks"| webhookgateway + webhookgateway -->|"validates webhooks"| MQ + MQ -->|"webhook events"| planner + planner -->|"job state"| PG + planner -->|"pressure info (HTTP streaming)"| githubrunner ``` diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4ced2a443..e56fd149d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -527,8 +527,6 @@ def image_builder_config_fixture( "build-interval": "12", "revision-history-limit": "2", "openstack-auth-url": openstack_config.auth_url, - # Bandit thinks this is a hardcoded password - "openstack-password": openstack_config.password, # nosec: B105 "openstack-project-domain-name": openstack_config.project_domain_name, "openstack-project-name": openstack_config.project_name, "openstack-user-domain-name": openstack_config.user_domain_name, @@ -561,11 +559,20 @@ def image_builder_fixture( if not openstack_config.test_image_id: logging.info("Deploying image builder %s", image_builder_app_name) + password_secret_name = f"{image_builder_app_name}-openstack-password" + password_secret_id = juju.add_secret( + name=password_secret_name, + content={"password": openstack_config.password}, + ) + config = { + **image_builder_config, + "openstack-password-secret": str(password_secret_id), + } juju.deploy( "github-runner-image-builder", app=image_builder_app_name, channel="latest/edge", - config=image_builder_config, + config=config, constraints={ "root-disk": "20480M", "mem": "2048M", @@ -575,6 +582,7 @@ def image_builder_fixture( "cores": "2", }, ) + juju.grant_secret(password_secret_name, image_builder_app_name) yield image_builder_app_name