diff --git a/.github/workflows/python-ci-polaris.yml b/.github/workflows/python-ci-polaris.yml new file mode 100644 index 0000000000..386f6a5002 --- /dev/null +++ b/.github/workflows/python-ci-polaris.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Python CI - Polaris" + +on: + push: + branches: + - 'main' + pull_request: + paths: + - 'pyiceberg/**' + - 'tests/**' + - 'dev/docker-compose-polaris.yml' + - 'dev/provision_polaris.py' + - '.github/workflows/python-ci-polaris.yml' + - 'Makefile' + - 'pyproject.toml' + - 'uv.lock' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + polaris-integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 + with: + enable-cache: true + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos + - name: Install + run: make install + - name: Run Polaris integration tests + run: make test-polaris + - name: Show debug logs + if: ${{ failure() }} + run: docker compose -f dev/docker-compose-polaris.yml logs diff --git a/.gitignore b/.gitignore index ef8c522482..e32278d18b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ htmlcov pyiceberg/avro/decoder_fast.c pyiceberg/avro/*.html pyiceberg/avro/*.so + +dev/polaris_creds.env diff --git a/Makefile b/Makefile index d262de45a9..28bb0ff6b9 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,8 @@ test: ## Run all unit tests (excluding integration) test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests +test-polaris: test-polaris-setup test-polaris-exec test-polaris-cleanup ## Run Polaris integration tests + test-integration-setup: install ## Start Docker services for integration tests docker compose -f dev/docker-compose-integration.yml kill docker compose -f dev/docker-compose-integration.yml rm -f @@ -123,6 +125,22 @@ test-integration-cleanup: ## Clean up integration test environment fi $(CLEANUP_COMMAND) +test-polaris-setup: install ## Start Docker services for Polaris integration tests + docker compose -f dev/docker-compose-polaris.yml kill + docker compose -f dev/docker-compose-polaris.yml rm -f + docker compose -f dev/docker-compose-polaris.yml up -d --build --wait + docker compose -f dev/docker-compose-polaris.yml exec -T polaris-setup cat /tmp/polaris_creds.env > dev/polaris_creds.env + +test-polaris-exec: ## Run Polaris integration tests + TEST_RUNNER="$(TEST_RUNNER)" sh dev/test-polaris.sh $(PYTEST_ARGS) + +test-polaris-cleanup: ## Clean up Polaris integration test environment + @if [ "${KEEP_COMPOSE}" != "1" ]; then \ + echo "Cleaning up Polaris Docker containers..."; \ + docker compose -f dev/docker-compose-polaris.yml down -v --remove-orphans --timeout 0 2>/dev/null || true; \ + rm -f dev/polaris_creds.env; \ + fi + test-integration-rebuild: ## Rebuild integration Docker services from scratch docker compose -f dev/docker-compose-integration.yml kill docker compose -f dev/docker-compose-integration.yml rm -f diff --git a/dev/docker-compose-polaris.yml b/dev/docker-compose-polaris.yml new file mode 100644 index 0000000000..3820e90cd4 --- /dev/null +++ b/dev/docker-compose-polaris.yml @@ -0,0 +1,261 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + + rustfs: + image: rustfs/rustfs:1.0.0-alpha.81 + networks: + iceberg_net: + ports: + # API port + - "9000:9000" + # UI port + - "9001:9001" + environment: + RUSTFS_ACCESS_KEY: polaris_root + RUSTFS_SECRET_KEY: polaris_pass + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "true" + RUSTFS_CONSOLE_ADDRESS: ":9001" + healthcheck: + test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health && curl --fail http://127.0.0.1:9001/rustfs/console/health"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 40s + + polaris: + image: apache/polaris:latest + container_name: pyiceberg-polaris + networks: + iceberg_net: + ports: + # API port + - "8181:8181" + # Management port (metrics and health checks) + - "8182:8182" + # Optional, allows attaching a debugger to the Polaris JVM + - "5005:5005" + depends_on: + rustfs: + condition: service_healthy + bucket-setup: + condition: service_completed_successfully + environment: + JAVA_DEBUG: true + JAVA_DEBUG_PORT: "*:5005" + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: polaris_root + AWS_SECRET_ACCESS_KEY: polaris_pass + POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t + polaris.realm-context.realms: POLARIS + quarkus.otel.sdk.disabled: "true" + polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true" + polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES": '["FILE","S3"]' + polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS": "true" + polaris.readiness.ignore-severe-issues: "true" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8182/q/health"] + interval: 2s + timeout: 10s + retries: 10 + start_period: 10s + + bucket-setup: + image: amazon/aws-cli:2.34.19 + networks: + iceberg_net: + depends_on: + rustfs: + condition: service_healthy + environment: + AWS_ACCESS_KEY_ID: polaris_root + AWS_SECRET_ACCESS_KEY: polaris_pass + AWS_ENDPOINT_URL: http://rustfs:9000 + entrypoint: "/bin/sh" + command: + - "-c" + - >- + echo Creating RustFS bucket... && + aws s3 mb s3://bucket123 && + aws s3 ls && + echo Bucket setup complete. + + polaris-setup: + image: alpine/curl:8.17.0 + networks: + iceberg_net: + depends_on: + polaris: + condition: service_healthy + environment: + - CLIENT_ID=root + - CLIENT_SECRET=s3cr3t + - CATALOG_NAME=polaris + - REALM=POLARIS + entrypoint: /bin/sh + command: + - -c + - | + set -e + apk add --no-cache jq + + echo "Obtaining root access token..." + TOKEN_RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/catalog/v1/oauth/tokens \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=client_credentials&client_id=$${CLIENT_ID}&client_secret=$${CLIENT_SECRET}&scope=PRINCIPAL_ROLE:ALL" 2>&1) || { + echo "❌ Failed to obtain access token" + echo "$$TOKEN_RESPONSE" >&2 + exit 1 + } + + TOKEN=$$(echo $$TOKEN_RESPONSE | jq -r '.access_token') + if [ -z "$$TOKEN" ] || [ "$$TOKEN" = "null" ]; then + echo "❌ Failed to parse access token from response" + echo "$$TOKEN_RESPONSE" + exit 1 + fi + echo "✅ Obtained access token" + + echo "Creating catalog '$$CATALOG_NAME' in realm $$REALM..." + PAYLOAD='{ + "catalog": { + "name": "'$$CATALOG_NAME'", + "type": "INTERNAL", + "readOnly": false, + "properties": { + "default-base-location": "s3://bucket123", + "polaris.config.drop-with-purge.enabled": "true" + }, + "storageConfigInfo": { + "storageType": "S3", + "allowedLocations": ["s3://bucket123"], + "endpoint": "http://localhost:9000", + "endpointInternal": "http://rustfs:9000", + "pathStyleAccess": true + } + } + }' + + RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/catalogs \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "Polaris-Realm: $$REALM" \ + -d "$$PAYLOAD" 2>&1) && echo -n "" || { + echo "❌ Failed to create catalog" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Catalog created" + + echo "" + echo "Creating principal 'quickstart_user'..." + PRINCIPAL_RESPONSE=$$(curl --fail-with-body -s -X POST http://polaris:8181/api/management/v1/principals \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"principal": {"name": "quickstart_user", "properties": {}}}' 2>&1) || { + echo "❌ Failed to create principal" + echo "$$PRINCIPAL_RESPONSE" >&2 + exit 1 + } + + USER_CLIENT_ID=$$(echo $$PRINCIPAL_RESPONSE | jq -r '.credentials.clientId') + USER_CLIENT_SECRET=$$(echo $$PRINCIPAL_RESPONSE | jq -r '.credentials.clientSecret') + if [ -z "$$USER_CLIENT_ID" ] || [ "$$USER_CLIENT_ID" = "null" ] || [ -z "$$USER_CLIENT_SECRET" ] || [ "$$USER_CLIENT_SECRET" = "null" ]; then + echo "❌ Failed to parse user credentials from response" + echo "$$PRINCIPAL_RESPONSE" + exit 1 + fi + echo "✅ Principal created with clientId: $$USER_CLIENT_ID" + + echo "Creating principal role 'quickstart_user_role'..." + RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/principal-roles \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"principalRole": {"name": "quickstart_user_role", "properties": {}}}' 2>&1) && echo -n "" || { + echo "❌ Failed to create principal role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Principal role created" + + echo "Creating catalog role 'quickstart_catalog_role'..." + RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/catalogs/$$CATALOG_NAME/catalog-roles \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"catalogRole": {"name": "quickstart_catalog_role", "properties": {}}}' 2>&1) && echo -n "" || { + echo "❌ Failed to create catalog role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Catalog role created" + + echo "Assigning principal role to principal..." + RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/principals/quickstart_user/principal-roles \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"principalRole": {"name": "quickstart_user_role"}}' 2>&1) && echo -n "" || { + echo "❌ Failed to assign principal role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Principal role assigned" + + echo "Assigning catalog role to principal role..." + RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/principal-roles/quickstart_user_role/catalog-roles/$$CATALOG_NAME \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"catalogRole": {"name": "quickstart_catalog_role"}}' 2>&1) && echo -n "" || { + echo "❌ Failed to assign catalog role" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Catalog role assigned" + + echo "Granting CATALOG_MANAGE_CONTENT privilege..." + RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/catalogs/$$CATALOG_NAME/catalog-roles/quickstart_catalog_role/grants \ + -H "Authorization: Bearer $$TOKEN" \ + -H "Polaris-Realm: $$REALM" \ + -H "Content-Type: application/json" \ + -d '{"type": "catalog", "privilege": "CATALOG_MANAGE_CONTENT"}' 2>&1) && echo -n "" || { + echo "❌ Failed to grant privileges" + echo "$$RESPONSE" >&2 + exit 1 + } + echo "✅ Privileges granted" + + echo "CLIENT_ID=$$USER_CLIENT_ID" > /tmp/polaris_creds.env + echo "CLIENT_SECRET=$$USER_CLIENT_SECRET" >> /tmp/polaris_creds.env + + touch /tmp/polaris-setup-done + tail -f /dev/null + healthcheck: + interval: 1s + timeout: 10s + retries: 240 + test: ["CMD", "test", "-f", "/tmp/polaris-setup-done"] + +networks: + iceberg_net: diff --git a/dev/test-polaris.sh b/dev/test-polaris.sh new file mode 100644 index 0000000000..f14647825a --- /dev/null +++ b/dev/test-polaris.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# Load credentials +if [ -f dev/polaris_creds.env ]; then + # We use a subshell or export to ensure they are available + export $(cat dev/polaris_creds.env | xargs) +else + echo "dev/polaris_creds.env not found. Please run 'make test-polaris-setup' first." + exit 1 +fi + +# Set environment variables for PyIceberg +export PYICEBERG_TEST_CATALOG="polaris" +export PYICEBERG_CATALOG__POLARIS__TYPE="rest" +export PYICEBERG_CATALOG__POLARIS__URI="http://localhost:8181/api/catalog" +export PYICEBERG_CATALOG__POLARIS__OAUTH2_SERVER_URI="http://localhost:8181/api/catalog/v1/oauth/tokens" +export PYICEBERG_CATALOG__POLARIS__CREDENTIAL="$CLIENT_ID:$CLIENT_SECRET" +export PYICEBERG_CATALOG__POLARIS__SCOPE="PRINCIPAL_ROLE:ALL" +export PYICEBERG_CATALOG__POLARIS__WAREHOUSE="polaris" +export PYICEBERG_CATALOG__POLARIS__HEADER__X_ICEBERG_ACCESS_DELEGATION="vended-credentials" +export PYICEBERG_CATALOG__POLARIS__HEADER__REALM="POLARIS" + +# S3 Configuration for RustFS +export PYICEBERG_CATALOG__POLARIS__S3__ENDPOINT="http://localhost:9000" +export PYICEBERG_CATALOG__POLARIS__S3__ACCESS_KEY_ID="polaris_root" +export PYICEBERG_CATALOG__POLARIS__S3__SECRET_ACCESS_KEY="polaris_pass" +export PYICEBERG_CATALOG__POLARIS__S3__REGION="us-west-2" +export PYICEBERG_CATALOG__POLARIS__S3__PATH_STYLE_ACCESS="true" + +# Default test runner if not provided +RUNNER=${TEST_RUNNER:-uv run python -m} + +# Run pytest +# Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request. +# TODO: Remove test exception after https://github.com/apache/polaris/pull/4013 is merged. +$RUNNER pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" "$@"