From e6e04d5071f9cd0e5b69083e4196b1e76de5f0fb Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 26 Feb 2026 14:01:07 -0800 Subject: [PATCH 1/9] Wire up Polaris to CI --- .github/workflows/python-ci-polaris.yml | 57 ++++++++++ Makefile | 34 ++++++ dev/docker-compose-polaris.yml | 76 +++++++++++++ dev/polaris_creds.env | 2 + dev/provision_polaris.py | 137 ++++++++++++++++++++++++ 5 files changed, 306 insertions(+) create mode 100644 .github/workflows/python-ci-polaris.yml create mode 100644 dev/docker-compose-polaris.yml create mode 100644 dev/polaris_creds.env create mode 100644 dev/provision_polaris.py diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml new file mode 100644 index 0000000000..68de3c99fd --- /dev/null +++ b/.github/workflows/python-ci-polaris.yml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Python CI - Polaris" + +on: + push: + branches: + - 'main' + pull_request: + paths: + - 'pyiceberg/**' + - 'tests/**' + - 'dev/docker-compose-polaris.yml' + - 'dev/provision_polaris.py' + - '.github/workflows/python-ci-polaris.yml' + - 'Makefile' + - 'pyproject.toml' + - 'uv.lock' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + polaris-integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Install + run: make install + - name: Run Polaris integration tests + run: make test-polaris + - name: Show debug logs + if: ${{ failure() }} + run: docker compose -f dev/docker-compose-polaris.yml logs diff --git a/Makefile b/Makefile index d262de45a9..397c1ae185 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,8 @@ test: ## Run all unit tests (excluding integration) test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests +test-polaris: test-polaris-setup test-polaris-exec test-polaris-cleanup ## Run Polaris integration tests + test-integration-setup: install ## Start Docker services for integration tests docker compose -f dev/docker-compose-integration.yml kill docker compose -f dev/docker-compose-integration.yml rm -f @@ -123,6 +125,38 @@ test-integration-cleanup: ## Clean up integration test environment fi $(CLEANUP_COMMAND) +test-polaris-setup: install ## Start Docker services for Polaris integration tests + docker compose -f dev/docker-compose-polaris.yml kill + docker compose -f dev/docker-compose-polaris.yml rm -f + docker compose -f dev/docker-compose-polaris.yml up -d --build --wait + uv run $(PYTHON_ARG) python dev/provision_polaris.py > dev/polaris_creds.env + +test-polaris-exec: ## Run Polaris integration tests + @eval $$(cat dev/polaris_creds.env) && \ + PYICEBERG_TEST_CATALOG="polaris" \ + PYICEBERG_CATALOG__POLARIS__TYPE="rest" \ + PYICEBERG_CATALOG__POLARIS__URI="http://localhost:8181/api/catalog" \ + PYICEBERG_CATALOG__POLARIS__OAUTH2_SERVER_URI="http://localhost:8181/api/catalog/v1/oauth/tokens" \ + PYICEBERG_CATALOG__POLARIS__CREDENTIAL="$$CLIENT_ID:$$CLIENT_SECRET" \ + PYICEBERG_CATALOG__POLARIS__SCOPE="PRINCIPAL_ROLE:ALL" \ + PYICEBERG_CATALOG__POLARIS__WAREHOUSE="polaris" \ + PYICEBERG_CATALOG__POLARIS__HEADER__X_ICEBERG_ACCESS_DELEGATION="vended-credentials" \ + PYICEBERG_CATALOG__POLARIS__HEADER__REALM="POLARIS" \ + PYICEBERG_CATALOG__POLARIS__S3__ENDPOINT="http://localhost:9000" \ + PYICEBERG_CATALOG__POLARIS__S3__ACCESS_KEY_ID="admin" \ + PYICEBERG_CATALOG__POLARIS__S3__SECRET_ACCESS_KEY="password" \ + PYICEBERG_CATALOG__POLARIS__S3__REGION="us-east-1" \ + $(TEST_RUNNER) pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" $(PYTEST_ARGS) + # Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request. + + +test-polaris-cleanup: ## Clean up Polaris integration test environment + @if [ "${KEEP_COMPOSE}" != "1" ]; then \ + echo "Cleaning up Polaris Docker containers..."; \ + docker compose -f dev/docker-compose-polaris.yml down -v --remove-orphans --timeout 0 2>/dev/null || true; \ + rm -f dev/polaris_creds.env; \ + fi + test-integration-rebuild: ## Rebuild integration Docker services from scratch docker compose -f dev/docker-compose-integration.yml kill docker compose -f dev/docker-compose-integration.yml rm -f diff --git a/dev/docker-compose-polaris.yml b/dev/docker-compose-polaris.yml new file mode 100644 index 0000000000..260dbf2024 --- /dev/null +++ b/dev/docker-compose-polaris.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + polaris: + image: apache/polaris:latest + container_name: pyiceberg-polaris + networks: + iceberg_net: + ports: + - 8181:8181 + - 8182:8182 + environment: + - POLARIS_BOOTSTRAP_CREDENTIALS=POLARIS,root,s3cr3t + - polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true + - polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3"] + - polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true + - polaris.readiness.ignore-severe-issues=true + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + healthcheck: + test: ["CMD", "curl", "http://localhost:8182/q/health"] + interval: 10s + timeout: 10s + retries: 5 + minio: + image: minio/minio + container_name: pyiceberg-polaris-minio + networks: + iceberg_net: + aliases: + - warehouse.minio + ports: + - 9001:9001 + - 9000:9000 + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio + command: ["server", "/data", "--console-address", ":9001"] + mc: + image: minio/mc + container_name: pyiceberg-polaris-mc + networks: + iceberg_net: + depends_on: + - minio + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " + +networks: + iceberg_net: diff --git a/dev/polaris_creds.env b/dev/polaris_creds.env new file mode 100644 index 0000000000..da037c8603 --- /dev/null +++ b/dev/polaris_creds.env @@ -0,0 +1,2 @@ +CLIENT_ID=dae157528c18583c +CLIENT_SECRET=e056dc0ab9f23c2ab87d9da4ad93cdaa diff --git a/dev/provision_polaris.py b/dev/provision_polaris.py new file mode 100644 index 0000000000..ed2ddf2e08 --- /dev/null +++ b/dev/provision_polaris.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import requests + +POLARIS_URL = "http://localhost:8181/api/management/v1" +POLARIS_TOKEN_URL = "http://localhost:8181/api/catalog/v1/oauth/tokens" + + +def get_token(client_id: str, client_secret: str) -> str: + response = requests.post( + POLARIS_TOKEN_URL, + data={ + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + "scope": "PRINCIPAL_ROLE:ALL", + }, + headers={"realm": "POLARIS"}, + ) + response.raise_for_status() + return response.json()["access_token"] + + +def provision() -> None: + # Initial authentication with root credentials + token = get_token("root", "s3cr3t") + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json", "realm": "POLARIS"} + + # 1. Create Principal + principal_name = "pyiceberg_principal" + principal_resp = requests.post( + f"{POLARIS_URL}/principals", + headers=headers, + json={"name": principal_name, "type": "PRINCIPAL"}, + ) + if principal_resp.status_code == 409: + principal_resp = requests.post( + f"{POLARIS_URL}/principals/{principal_name}/rotate-credentials", + headers=headers, + ) + principal_resp.raise_for_status() + principal_data = principal_resp.json() + client_id = principal_data["credentials"]["clientId"] + client_secret = principal_data["credentials"]["clientSecret"] + + # 2. Assign service_admin role to our principal + requests.put( + f"{POLARIS_URL}/principals/{principal_name}/principal-roles", + headers=headers, + json={"principalRole": {"name": "service_admin"}}, + ).raise_for_status() + + # 3. Create Principal Role for catalog access + role_name = "pyiceberg_role" + requests.post( + f"{POLARIS_URL}/principal-roles", + headers=headers, + json={"principalRole": {"name": role_name}}, + ) # Ignore error if exists + + # 4. Link Principal to Principal Role + requests.put( + f"{POLARIS_URL}/principals/{principal_name}/principal-roles", + headers=headers, + json={"principalRole": {"name": role_name}}, + ).raise_for_status() + + # 5. Create Catalog + catalog_name = "polaris" + requests.post( + f"{POLARIS_URL}/catalogs", + headers=headers, + json={ + "catalog": { + "name": catalog_name, + "type": "INTERNAL", + "readOnly": False, + "properties": { + "default-base-location": "s3://warehouse/polaris/", + "polaris.config.drop-with-purge.enabled": "true", + }, + "storageConfigInfo": { + "storageType": "S3", + "allowedLocations": ["s3://warehouse/polaris/"], + "region": "us-east-1", + "endpoint": "http://minio:9000", + }, + } + }, + ) # Ignore error if exists + + # 6. Link catalog_admin role to our principal role + requests.put( + f"{POLARIS_URL}/principal-roles/{role_name}/catalog-roles/{catalog_name}", + headers=headers, + json={"catalogRole": {"name": "catalog_admin"}}, + ).raise_for_status() + + # 7. Grant explicit privileges to catalog_admin role for this catalog + for privilege in [ + "CATALOG_MANAGE_CONTENT", + "CATALOG_MANAGE_METADATA", + "TABLE_CREATE", + "TABLE_WRITE_DATA", + "TABLE_LIST", + "NAMESPACE_CREATE", + "NAMESPACE_LIST", + ]: + requests.put( + f"{POLARIS_URL}/catalogs/{catalog_name}/catalog-roles/catalog_admin/grants", + headers=headers, + json={"grant": {"type": "catalog", "privilege": privilege}}, + ).raise_for_status() + + # Print credentials for use in CI + print(f"CLIENT_ID={client_id}") + print(f"CLIENT_SECRET={client_secret}") + + +if __name__ == "__main__": + provision() From 8f1962bc71c5045a57486a9eda7b6081677996f1 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 26 Feb 2026 14:03:46 -0800 Subject: [PATCH 2/9] don't commit polaris temp creds --- .gitignore | 2 ++ dev/polaris_creds.env | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 dev/polaris_creds.env diff --git a/.gitignore b/.gitignore index ef8c522482..e32278d18b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ htmlcov pyiceberg/avro/decoder_fast.c pyiceberg/avro/*.html pyiceberg/avro/*.so + +dev/polaris_creds.env diff --git a/dev/polaris_creds.env b/dev/polaris_creds.env deleted file mode 100644 index da037c8603..0000000000 --- a/dev/polaris_creds.env +++ /dev/null @@ -1,2 +0,0 @@ -CLIENT_ID=dae157528c18583c -CLIENT_SECRET=e056dc0ab9f23c2ab87d9da4ad93cdaa From 55cda5753f97742b0e2a8aa0e4cabadbd48d7f81 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 26 Feb 2026 14:15:24 -0800 Subject: [PATCH 3/9] CI fix --- .github/workflows/python-ci-polaris.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml index 68de3c99fd..3de8b0aace 100644 --- a/.github/workflows/python-ci-polaris.yml +++ b/.github/workflows/python-ci-polaris.yml @@ -48,6 +48,8 @@ jobs: uses: astral-sh/setup-uv@v7 with: enable-cache: true + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install run: make install - name: Run Polaris integration tests From 92856de1e491436f137bd8f5aede3b5e8cf139c2 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 26 Feb 2026 14:22:11 -0800 Subject: [PATCH 4/9] permissions --- .github/workflows/python-ci-polaris.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml index 3de8b0aace..06989539eb 100644 --- a/.github/workflows/python-ci-polaris.yml +++ b/.github/workflows/python-ci-polaris.yml @@ -32,6 +32,9 @@ on: - 'pyproject.toml' - 'uv.lock' +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} From 65fa58640d481f9ebed903eb727e4aee0589b18b Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Tue, 17 Mar 2026 16:07:25 -0700 Subject: [PATCH 5/9] review --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 397c1ae185..891d1a1bfc 100644 --- a/Makefile +++ b/Makefile @@ -148,7 +148,7 @@ test-polaris-exec: ## Run Polaris integration tests PYICEBERG_CATALOG__POLARIS__S3__REGION="us-east-1" \ $(TEST_RUNNER) pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" $(PYTEST_ARGS) # Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request. - + # TODO: Remove test exception after https://github.com/apache/polaris/pull/4013 is merged. test-polaris-cleanup: ## Clean up Polaris integration test environment @if [ "${KEEP_COMPOSE}" != "1" ]; then \ From 5dc703c20690040d4c98e8bdedc8f4866a0463ad Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 1 Apr 2026 16:24:03 -0700 Subject: [PATCH 6/9] Make docker-compose-polaris match the quick start --- Makefile | 20 +-- dev/docker-compose-polaris.yml | 263 ++++++++++++++++++++++++++++----- dev/provision_polaris.py | 137 ----------------- dev/test-polaris.sh | 54 +++++++ 4 files changed, 280 insertions(+), 194 deletions(-) delete mode 100644 dev/provision_polaris.py create mode 100644 dev/test-polaris.sh diff --git a/Makefile b/Makefile index 891d1a1bfc..28bb0ff6b9 100644 --- a/Makefile +++ b/Makefile @@ -129,26 +129,10 @@ test-polaris-setup: install ## Start Docker services for Polaris integration tes docker compose -f dev/docker-compose-polaris.yml kill docker compose -f dev/docker-compose-polaris.yml rm -f docker compose -f dev/docker-compose-polaris.yml up -d --build --wait - uv run $(PYTHON_ARG) python dev/provision_polaris.py > dev/polaris_creds.env + docker compose -f dev/docker-compose-polaris.yml exec -T polaris-setup cat /tmp/polaris_creds.env > dev/polaris_creds.env test-polaris-exec: ## Run Polaris integration tests - @eval $$(cat dev/polaris_creds.env) && \ - PYICEBERG_TEST_CATALOG="polaris" \ - PYICEBERG_CATALOG__POLARIS__TYPE="rest" \ - PYICEBERG_CATALOG__POLARIS__URI="http://localhost:8181/api/catalog" \ - PYICEBERG_CATALOG__POLARIS__OAUTH2_SERVER_URI="http://localhost:8181/api/catalog/v1/oauth/tokens" \ - PYICEBERG_CATALOG__POLARIS__CREDENTIAL="$$CLIENT_ID:$$CLIENT_SECRET" \ - PYICEBERG_CATALOG__POLARIS__SCOPE="PRINCIPAL_ROLE:ALL" \ - PYICEBERG_CATALOG__POLARIS__WAREHOUSE="polaris" \ - PYICEBERG_CATALOG__POLARIS__HEADER__X_ICEBERG_ACCESS_DELEGATION="vended-credentials" \ - PYICEBERG_CATALOG__POLARIS__HEADER__REALM="POLARIS" \ - PYICEBERG_CATALOG__POLARIS__S3__ENDPOINT="http://localhost:9000" \ - PYICEBERG_CATALOG__POLARIS__S3__ACCESS_KEY_ID="admin" \ - PYICEBERG_CATALOG__POLARIS__S3__SECRET_ACCESS_KEY="password" \ - PYICEBERG_CATALOG__POLARIS__S3__REGION="us-east-1" \ - $(TEST_RUNNER) pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" $(PYTEST_ARGS) - # Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request. - # TODO: Remove test exception after https://github.com/apache/polaris/pull/4013 is merged. + TEST_RUNNER="$(TEST_RUNNER)" sh dev/test-polaris.sh $(PYTEST_ARGS) test-polaris-cleanup: ## Clean up Polaris integration test environment @if [ "${KEEP_COMPOSE}" != "1" ]; then \ diff --git a/dev/docker-compose-polaris.yml b/dev/docker-compose-polaris.yml index 260dbf2024..3820e90cd4 100644 --- a/dev/docker-compose-polaris.yml +++ b/dev/docker-compose-polaris.yml @@ -16,61 +16,246 @@ # under the License. services: + + rustfs: + image: rustfs/rustfs:1.0.0-alpha.81 + networks: + iceberg_net: + ports: + # API port + - "9000:9000" + # UI port + - "9001:9001" + environment: + RUSTFS_ACCESS_KEY: polaris_root + RUSTFS_SECRET_KEY: polaris_pass + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "true" + RUSTFS_CONSOLE_ADDRESS: ":9001" + healthcheck: + test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health && curl --fail http://127.0.0.1:9001/rustfs/console/health"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 40s + polaris: image: apache/polaris:latest container_name: pyiceberg-polaris networks: iceberg_net: ports: - - 8181:8181 - - 8182:8182 + # API port + - "8181:8181" + # Management port (metrics and health checks) + - "8182:8182" + # Optional, allows attaching a debugger to the Polaris JVM + - "5005:5005" + depends_on: + rustfs: + condition: service_healthy + bucket-setup: + condition: service_completed_successfully environment: - - POLARIS_BOOTSTRAP_CREDENTIALS=POLARIS,root,s3cr3t - - polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true - - polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3"] - - polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true - - polaris.readiness.ignore-severe-issues=true - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 + JAVA_DEBUG: true + JAVA_DEBUG_PORT: "*:5005" + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: polaris_root + AWS_SECRET_ACCESS_KEY: polaris_pass + POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t + polaris.realm-context.realms: POLARIS + quarkus.otel.sdk.disabled: "true" + polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true" + polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES": '["FILE","S3"]' + polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS": "true" + polaris.readiness.ignore-severe-issues: "true" healthcheck: - test: ["CMD", "curl", "http://localhost:8182/q/health"] - interval: 10s + test: ["CMD", "curl", "--fail", "http://localhost:8182/q/health"] + interval: 2s timeout: 10s - retries: 5 - minio: - image: minio/minio - container_name: pyiceberg-polaris-minio + retries: 10 + start_period: 10s + + bucket-setup: + image: amazon/aws-cli:2.34.19 networks: iceberg_net: - aliases: - - warehouse.minio - ports: - - 9001:9001 - - 9000:9000 + depends_on: + rustfs: + condition: service_healthy environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - command: ["server", "/data", "--console-address", ":9001"] - mc: - image: minio/mc - container_name: pyiceberg-polaris-mc + AWS_ACCESS_KEY_ID: polaris_root + AWS_SECRET_ACCESS_KEY: polaris_pass + AWS_ENDPOINT_URL: http://rustfs:9000 + entrypoint: "/bin/sh" + command: + - "-c" + - >- + echo Creating RustFS bucket... && + aws s3 mb s3://bucket123 && + aws s3 ls && + echo Bucket setup complete. + + polaris-setup: + image: alpine/curl:8.17.0 networks: iceberg_net: depends_on: - - minio + polaris: + condition: service_healthy environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " - until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc mb minio/warehouse; - /usr/bin/mc policy set public minio/warehouse; - tail -f /dev/null - " + - CLIENT_ID=root + - CLIENT_SECRET=s3cr3t + - CATALOG_NAME=polaris + - REALM=POLARIS + entrypoint: /bin/sh + command: + - -c + - | + set -e + apk add --no-cache jq + + echo "Obtaining root access token..." + TOKEN_RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/catalog/v1/oauth/tokens \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=client_credentials&client_id=$${CLIENT_ID}&client_secret=$${CLIENT_SECRET}&scope=PRINCIPAL_ROLE:ALL" 2>&1) || { + echo "❌ Failed to obtain access token" + echo "$$TOKEN_RESPONSE" >&2 + exit 1 + } + + TOKEN=$$(echo $$TOKEN_RESPONSE | jq -r '.access_token') + if [ -z "$$TOKEN" ] || [ "$$TOKEN" = "null" ]; then + echo "❌ Failed to parse access token from response" + echo "$$TOKEN_RESPONSE" + exit 1 + fi + echo "✅ Obtained access token" + + echo "Creating catalog '$$CATALOG_NAME' in realm $$REALM..." + PAYLOAD='{ + "catalog": { + "name": "'$$CATALOG_NAME'", + "type": "INTERNAL", + "readOnly": false, + "properties": { + "default-base-location": "s3://bucket123", + "polaris.config.drop-with-purge.enabled": "true" + }, + "storageConfigInfo": { + "storageType": "S3", + "allowedLocations": ["s3://bucket123"], + "endpoint": "http://localhost:9000", + "endpointInternal": "http://rustfs:9000", + "pathStyleAccess": true + } + } + }' + + RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/catalogs \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "Polaris-Realm: $$REALM" \ + -d "$$PAYLOAD" 2>&1) && echo -n "" || { + echo "❌ Failed to create catalog" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Catalog created" + + echo "" + echo "Creating principal 'quickstart_user'..." + PRINCIPAL_RESPONSE=$$(curl --fail-with-body -s -X POST http://polaris:8181/api/management/v1/principals \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"principal": {"name": "quickstart_user", "properties": {}}}' 2>&1) || { + echo "❌ Failed to create principal" + echo "$$PRINCIPAL_RESPONSE" >&2 + exit 1 + } + + USER_CLIENT_ID=$$(echo $$PRINCIPAL_RESPONSE | jq -r '.credentials.clientId') + USER_CLIENT_SECRET=$$(echo $$PRINCIPAL_RESPONSE | jq -r '.credentials.clientSecret') + if [ -z "$$USER_CLIENT_ID" ] || [ "$$USER_CLIENT_ID" = "null" ] || [ -z "$$USER_CLIENT_SECRET" ] || [ "$$USER_CLIENT_SECRET" = "null" ]; then + echo "❌ Failed to parse user credentials from response" + echo "$$PRINCIPAL_RESPONSE" + exit 1 + fi + echo "✅ Principal created with clientId: $$USER_CLIENT_ID" + + echo "Creating principal role 'quickstart_user_role'..." + RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/principal-roles \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"principalRole": {"name": "quickstart_user_role", "properties": {}}}' 2>&1) && echo -n "" || { + echo "❌ Failed to create principal role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Principal role created" + + echo "Creating catalog role 'quickstart_catalog_role'..." + RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/catalogs/$$CATALOG_NAME/catalog-roles \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"catalogRole": {"name": "quickstart_catalog_role", "properties": {}}}' 2>&1) && echo -n "" || { + echo "❌ Failed to create catalog role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Catalog role created" + + echo "Assigning principal role to principal..." + RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/principals/quickstart_user/principal-roles \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"principalRole": {"name": "quickstart_user_role"}}' 2>&1) && echo -n "" || { + echo "❌ Failed to assign principal role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Principal role assigned" + + echo "Assigning catalog role to principal role..." + RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/principal-roles/quickstart_user_role/catalog-roles/$$CATALOG_NAME \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"catalogRole": {"name": "quickstart_catalog_role"}}' 2>&1) && echo -n "" || { + echo "❌ Failed to assign catalog role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Catalog role assigned" + + echo "Granting CATALOG_MANAGE_CONTENT privilege..." + RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/catalogs/$$CATALOG_NAME/catalog-roles/quickstart_catalog_role/grants \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"type": "catalog", "privilege": "CATALOG_MANAGE_CONTENT"}' 2>&1) && echo -n "" || { + echo "❌ Failed to grant privileges" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Privileges granted" + + echo "CLIENT_ID=$$USER_CLIENT_ID" > /tmp/polaris_creds.env + echo "CLIENT_SECRET=$$USER_CLIENT_SECRET" >> /tmp/polaris_creds.env + + touch /tmp/polaris-setup-done + tail -f /dev/null + healthcheck: + interval: 1s + timeout: 10s + retries: 240 + test: ["CMD", "test", "-f", "/tmp/polaris-setup-done"] networks: iceberg_net: diff --git a/dev/provision_polaris.py b/dev/provision_polaris.py deleted file mode 100644 index ed2ddf2e08..0000000000 --- a/dev/provision_polaris.py +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import requests - -POLARIS_URL = "http://localhost:8181/api/management/v1" -POLARIS_TOKEN_URL = "http://localhost:8181/api/catalog/v1/oauth/tokens" - - -def get_token(client_id: str, client_secret: str) -> str: - response = requests.post( - POLARIS_TOKEN_URL, - data={ - "grant_type": "client_credentials", - "client_id": client_id, - "client_secret": client_secret, - "scope": "PRINCIPAL_ROLE:ALL", - }, - headers={"realm": "POLARIS"}, - ) - response.raise_for_status() - return response.json()["access_token"] - - -def provision() -> None: - # Initial authentication with root credentials - token = get_token("root", "s3cr3t") - headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json", "realm": "POLARIS"} - - # 1. Create Principal - principal_name = "pyiceberg_principal" - principal_resp = requests.post( - f"{POLARIS_URL}/principals", - headers=headers, - json={"name": principal_name, "type": "PRINCIPAL"}, - ) - if principal_resp.status_code == 409: - principal_resp = requests.post( - f"{POLARIS_URL}/principals/{principal_name}/rotate-credentials", - headers=headers, - ) - principal_resp.raise_for_status() - principal_data = principal_resp.json() - client_id = principal_data["credentials"]["clientId"] - client_secret = principal_data["credentials"]["clientSecret"] - - # 2. Assign service_admin role to our principal - requests.put( - f"{POLARIS_URL}/principals/{principal_name}/principal-roles", - headers=headers, - json={"principalRole": {"name": "service_admin"}}, - ).raise_for_status() - - # 3. Create Principal Role for catalog access - role_name = "pyiceberg_role" - requests.post( - f"{POLARIS_URL}/principal-roles", - headers=headers, - json={"principalRole": {"name": role_name}}, - ) # Ignore error if exists - - # 4. Link Principal to Principal Role - requests.put( - f"{POLARIS_URL}/principals/{principal_name}/principal-roles", - headers=headers, - json={"principalRole": {"name": role_name}}, - ).raise_for_status() - - # 5. Create Catalog - catalog_name = "polaris" - requests.post( - f"{POLARIS_URL}/catalogs", - headers=headers, - json={ - "catalog": { - "name": catalog_name, - "type": "INTERNAL", - "readOnly": False, - "properties": { - "default-base-location": "s3://warehouse/polaris/", - "polaris.config.drop-with-purge.enabled": "true", - }, - "storageConfigInfo": { - "storageType": "S3", - "allowedLocations": ["s3://warehouse/polaris/"], - "region": "us-east-1", - "endpoint": "http://minio:9000", - }, - } - }, - ) # Ignore error if exists - - # 6. Link catalog_admin role to our principal role - requests.put( - f"{POLARIS_URL}/principal-roles/{role_name}/catalog-roles/{catalog_name}", - headers=headers, - json={"catalogRole": {"name": "catalog_admin"}}, - ).raise_for_status() - - # 7. Grant explicit privileges to catalog_admin role for this catalog - for privilege in [ - "CATALOG_MANAGE_CONTENT", - "CATALOG_MANAGE_METADATA", - "TABLE_CREATE", - "TABLE_WRITE_DATA", - "TABLE_LIST", - "NAMESPACE_CREATE", - "NAMESPACE_LIST", - ]: - requests.put( - f"{POLARIS_URL}/catalogs/{catalog_name}/catalog-roles/catalog_admin/grants", - headers=headers, - json={"grant": {"type": "catalog", "privilege": privilege}}, - ).raise_for_status() - - # Print credentials for use in CI - print(f"CLIENT_ID={client_id}") - print(f"CLIENT_SECRET={client_secret}") - - -if __name__ == "__main__": - provision() diff --git a/dev/test-polaris.sh b/dev/test-polaris.sh new file mode 100644 index 0000000000..f14647825a --- /dev/null +++ b/dev/test-polaris.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# Load credentials +if [ -f dev/polaris_creds.env ]; then + # We use a subshell or export to ensure they are available + export $(cat dev/polaris_creds.env | xargs) +else + echo "dev/polaris_creds.env not found. Please run 'make test-polaris-setup' first." + exit 1 +fi + +# Set environment variables for PyIceberg +export PYICEBERG_TEST_CATALOG="polaris" +export PYICEBERG_CATALOG__POLARIS__TYPE="rest" +export PYICEBERG_CATALOG__POLARIS__URI="http://localhost:8181/api/catalog" +export PYICEBERG_CATALOG__POLARIS__OAUTH2_SERVER_URI="http://localhost:8181/api/catalog/v1/oauth/tokens" +export PYICEBERG_CATALOG__POLARIS__CREDENTIAL="$CLIENT_ID:$CLIENT_SECRET" +export PYICEBERG_CATALOG__POLARIS__SCOPE="PRINCIPAL_ROLE:ALL" +export PYICEBERG_CATALOG__POLARIS__WAREHOUSE="polaris" +export PYICEBERG_CATALOG__POLARIS__HEADER__X_ICEBERG_ACCESS_DELEGATION="vended-credentials" +export PYICEBERG_CATALOG__POLARIS__HEADER__REALM="POLARIS" + +# S3 Configuration for RustFS +export PYICEBERG_CATALOG__POLARIS__S3__ENDPOINT="http://localhost:9000" +export PYICEBERG_CATALOG__POLARIS__S3__ACCESS_KEY_ID="polaris_root" +export PYICEBERG_CATALOG__POLARIS__S3__SECRET_ACCESS_KEY="polaris_pass" +export PYICEBERG_CATALOG__POLARIS__S3__REGION="us-west-2" +export PYICEBERG_CATALOG__POLARIS__S3__PATH_STYLE_ACCESS="true" + +# Default test runner if not provided +RUNNER=${TEST_RUNNER:-uv run python -m} + +# Run pytest +# Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request. +# TODO: Remove test exception after https://github.com/apache/polaris/pull/4013 is merged. +$RUNNER pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" "$@" From 5dbd5d5d4ce0d2eff558c657913dcd6d8e7d570f Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 1 Apr 2026 16:38:15 -0700 Subject: [PATCH 7/9] Pin version --- .github/workflows/python-ci-polaris.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml index 06989539eb..a7ad64e586 100644 --- a/.github/workflows/python-ci-polaris.yml +++ b/.github/workflows/python-ci-polaris.yml @@ -48,7 +48,7 @@ jobs: with: python-version: '3.12' - name: Install UV - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: enable-cache: true - name: Install system dependencies From 318ef215bf42e7871117331753f2ebfcc759ce57 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 1 Apr 2026 16:39:35 -0700 Subject: [PATCH 8/9] pin rest --- .github/workflows/python-ci-polaris.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml index a7ad64e586..967abc9076 100644 --- a/.github/workflows/python-ci-polaris.yml +++ b/.github/workflows/python-ci-polaris.yml @@ -43,8 +43,8 @@ jobs: polaris-integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: '3.12' - name: Install UV From b935c93d4477acc51c3283e9e33b0e0804a6f5f9 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 1 Apr 2026 16:41:31 -0700 Subject: [PATCH 9/9] persist credentials --- .github/workflows/python-ci-polaris.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml index 967abc9076..386f6a5002 100644 --- a/.github/workflows/python-ci-polaris.yml +++ b/.github/workflows/python-ci-polaris.yml @@ -44,6 +44,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: '3.12'