From 05cb4eab3ec893e0b9012eae1be225c521660168 Mon Sep 17 00:00:00 2001 From: Kevin Klaes Date: Thu, 15 Jan 2026 12:22:17 -0700 Subject: [PATCH 1/5] Add relaxPermChecks and encryption enabling flags to ps-entry --- build/ps-entry.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/ps-entry.sh b/build/ps-entry.sh index 27a5db2ce5..1a55a1b262 100755 --- a/build/ps-entry.sh +++ b/build/ps-entry.sh @@ -490,6 +490,10 @@ if [[ $originalArgOne == mongo* ]]; then _mongod_hack_ensure_arg --logappend "${mongodHackedArgs[@]}" fi + _mongod_hack_ensure_no_arg --relaxPermChecks "${mongodHackedArgs[@]}" + _mongod_hack_ensure_no_arg --enableEncryption "${mongodHackedArgs[@]}" + _mongod_hack_ensure_no_arg_val --encryptionKeyFile "${mongodHackedArgs[@]}" + set -- "${mongodHackedArgs[@]}" # MongoDB 3.6+ defaults to localhost-only binding From 8cd82526504675a243baccbf5c5da7dc85e87ffb Mon Sep 17 00:00:00 2001 From: Kevin Klaes Date: Tue, 20 Jan 2026 16:52:50 -0700 Subject: [PATCH 2/5] Trigger actions --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ba64edbd70..d93a2a1dd4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Percona Operator for MongoDB + ![Percona Kubernetes Operators](kubernetes.svg) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) From 4ac3ed6867318f80aadc69e2be1cacc64e3d4118 Mon Sep 17 00:00:00 2001 From: Kevin Klaes Date: Tue, 20 Jan 2026 17:01:22 -0700 Subject: [PATCH 3/5] Build and push image to ghcr --- .github/workflows/build-operator.yml | 253 +++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 .github/workflows/build-operator.yml diff --git a/.github/workflows/build-operator.yml b/.github/workflows/build-operator.yml new file mode 100644 index 0000000000..cb1d69e62a --- /dev/null +++ b/.github/workflows/build-operator.yml @@ -0,0 +1,253 @@ +name: Build and Publish Operator + +on: + push: + branches: + - main + paths: + - 'pkg/**' + - 'cmd/**' + - 'config/**' + - 'deploy/**' + - 'build/**' + - 'go.mod' + - 'go.sum' + - 'Makefile' + - '.github/workflows/build-operator.yml' + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: 'Tag to build (e.g., v1.21.2-objectrocket)' + required: false + type: string + +env: + REGISTRY: ghcr.io + IMAGE_NAME: objectrocket/percona-server-mongodb-operator + GO_VERSION: '1.25' + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Run unit tests + run: make test + + - name: Upload coverage report + uses: codecov/codecov-action@v4 + if: always() + with: + files: ./cover.out + flags: unittests + name: operator-coverage + continue-on-error: true + + build-and-push: + name: Build and Push Operator Image + runs-on: ubuntu-latest + needs: test + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract version and metadata + id: meta + run: | + # Determine version based on trigger type + if [ "${{ github.event_name }}" = "release" ]; then + VERSION="${{ github.event.release.tag_name }}" + VERSION="${VERSION#v}" # Remove 'v' prefix if present + elif [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.tag }}" ]; then + VERSION="${{ inputs.tag }}" + VERSION="${VERSION#v}" + else + # For main branch, use version from pkg/version/version.txt or git describe + if [ -f pkg/version/version.txt ]; then + VERSION=$(cat pkg/version/version.txt) + else + VERSION=$(git describe --tags --always --dirty) + fi + VERSION="${VERSION}-objectrocket" + fi + + # Extract major.minor version + MAJOR_MINOR=$(echo "$VERSION" | grep -oE '^[0-9]+\.[0-9]+' || echo "") + + # Get git metadata + GIT_COMMIT=$(git rev-parse HEAD) + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + BUILD_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "major_minor=$MAJOR_MINOR" >> $GITHUB_OUTPUT + echo "git_commit=$GIT_COMMIT" >> $GITHUB_OUTPUT + echo "git_branch=$GIT_BRANCH" >> $GITHUB_OUTPUT + echo "build_time=$BUILD_TIME" >> $GITHUB_OUTPUT + + # Construct image tags + TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${VERSION}" + if [ -n "$MAJOR_MINOR" ]; then + TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${MAJOR_MINOR}" + fi + if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/main" ]; then + TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + fi + + echo "tags=$TAGS" >> $GITHUB_OUTPUT + + - name: Extract Docker metadata + id: docker_meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ steps.meta.outputs.version }} + type=raw,value=${{ steps.meta.outputs.major_minor }},enable=${{ steps.meta.outputs.major_minor != '' }} + type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + labels: | + org.opencontainers.image.title=Percona Server MongoDB Operator + org.opencontainers.image.description=Kubernetes operator for managing Percona Server for MongoDB clusters + org.opencontainers.image.vendor=ObjectRocket + org.opencontainers.image.version=${{ steps.meta.outputs.version }} + org.opencontainers.image.revision=${{ steps.meta.outputs.git_commit }} + org.opencontainers.image.source=${{ github.repositoryUrl }} + org.opencontainers.image.created=${{ steps.meta.outputs.build_time }} + + - name: Build and push operator image + uses: docker/build-push-action@v6 + with: + context: . + file: ./build/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + build-args: | + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + GIT_BRANCH=${{ steps.meta.outputs.git_branch }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Generate image digest + id: digest + if: github.event_name != 'pull_request' + run: | + DIGEST=$(docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} --format '{{.Manifest.Digest}}') + echo "digest=$DIGEST" >> $GITHUB_OUTPUT + echo "Image digest: $DIGEST" + + build-artifacts: + name: Build and Package Artifacts + runs-on: ubuntu-latest + needs: test + permissions: + contents: write + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Generate CRDs and manifests + run: | + make generate + make manifests + + - name: Create artifact directory + run: | + mkdir -p artifacts + cp deploy/crd.yaml artifacts/ + cp deploy/rbac.yaml artifacts/ + cp deploy/cw-rbac.yaml artifacts/ + cp deploy/operator.yaml artifacts/ + cp deploy/cw-operator.yaml artifacts/ + cp deploy/bundle.yaml artifacts/ + cp deploy/cw-bundle.yaml artifacts/ + + - name: Upload CRDs and RBAC artifacts + uses: actions/upload-artifact@v4 + with: + name: operator-manifests + path: artifacts/ + retention-days: 30 + + - name: Create release artifacts (on release) + if: github.event_name == 'release' + run: | + tar -czf operator-manifests-${{ github.event.release.tag_name }}.tar.gz -C artifacts . + + - name: Upload release artifacts + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./operator-manifests-${{ github.event.release.tag_name }}.tar.gz + asset_name: operator-manifests-${{ github.event.release.tag_name }}.tar.gz + asset_content_type: application/gzip + continue-on-error: true + + summary: + name: Build Summary + runs-on: ubuntu-latest + needs: [test, build-and-push, build-artifacts] + if: always() + steps: + - name: Generate summary + run: | + echo "## Operator Build Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Workflow:** ${{ github.workflow }}" >> $GITHUB_STEP_SUMMARY + echo "**Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "**Branch:** ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY + echo "**Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Job Status" >> $GITHUB_STEP_SUMMARY + echo "- Tests: ${{ needs.test.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Build and Push: ${{ needs.build-and-push.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Build Artifacts: ${{ needs.build-artifacts.result }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ "${{ needs.build-and-push.result }}" = "success" ]; then + echo "### Published Images" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Next Steps" >> $GITHUB_STEP_SUMMARY + echo "The manifest update workflow will automatically update deployment manifests in liberty-infrastructure." >> $GITHUB_STEP_SUMMARY + fi From d93f3d34f1860a9ee41b669e5e7089181e4489b0 Mon Sep 17 00:00:00 2001 From: joshWretlind Date: Wed, 18 Feb 2026 08:36:20 -0700 Subject: [PATCH 4/5] Fix keyfile permissions (#3) * Fix keyfile permissions * Remove extra debug line * Update build/ps-entry.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- build/ps-entry.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/build/ps-entry.sh b/build/ps-entry.sh index 1a55a1b262..fce992158f 100755 --- a/build/ps-entry.sh +++ b/build/ps-entry.sh @@ -462,6 +462,28 @@ if [[ $originalArgOne == mongo* ]]; then fi fi + keyFilePath="" + if _mongod_hack_have_arg --keyFile "${mongodHackedArgs[@]}"; then + keyFilePath="$(_mongod_hack_get_arg_val --keyFile "${mongodHackedArgs[@]}")" + elif _parse_config "${mongodHackedArgs[@]}"; then + keyFilePath="$(jq -r '.security.keyFile // empty' "$jsonConfigFile")" + fi + + if [ -n "$keyFilePath" ] && [ -f "$keyFilePath" ]; then + keyFileTmp="${TMPDIR:-/tmp}/mongodb-keyfile" + if [ "$keyFilePath" != "$keyFileTmp" ]; then + if ! cp -f "$keyFilePath" "$keyFileTmp"; then + echo >&2 "error: failed to copy keyFile from $keyFilePath to $keyFileTmp" + exit 1 + fi + fi + if ! chmod 0400 "$keyFileTmp"; then + echo >&2 "error: failed to chmod keyFile $keyFileTmp" + exit 1 + fi + _mongod_hack_ensure_arg_val --keyFile "$keyFileTmp" "${mongodHackedArgs[@]}" + fi + if [ "$MONGODB_VERSION" != 'v4.0' ]; then _mongod_hack_rename_arg '--sslAllowInvalidCertificates' '--tlsAllowInvalidCertificates' "${mongodHackedArgs[@]}" _mongod_hack_rename_arg '--sslAllowInvalidHostnames' '--tlsAllowInvalidHostnames' "${mongodHackedArgs[@]}" From d60affb60695b037ae7ce9f7519baeab458e49bc Mon Sep 17 00:00:00 2001 From: Kevin Klaes Date: Thu, 19 Feb 2026 09:55:28 -0700 Subject: [PATCH 5/5] Initial steering docs --- .kiro/steering/operator-details.md | 525 +++++++++++++++++++++++++++++ 1 file changed, 525 insertions(+) create mode 100644 .kiro/steering/operator-details.md diff --git a/.kiro/steering/operator-details.md b/.kiro/steering/operator-details.md new file mode 100644 index 0000000000..8ff9eca7ba --- /dev/null +++ b/.kiro/steering/operator-details.md @@ -0,0 +1,525 @@ +--- +inclusion: fileMatch +fileMatchPattern: 'percona-server-mongodb-operator/**' +--- + +# Percona Server for MongoDB Operator Development Guide + +This document provides detailed guidance for working with the Liberty Platform's custom fork of the Percona Server for MongoDB Operator. + +## Repository Purpose + +Custom fork of the [Percona Operator for MongoDB](https://github.com/percona/percona-server-mongodb-operator) that deploys and manages MongoDB instances on Kubernetes. The operator automates deployments, scaling, backups, and day-to-day operations for both replica sets and sharded clusters. + +## Key Features + +- **Automated workflows** - Simplified MongoDB management +- **High availability** - No single point of failure +- **Easy sharding and scaling** - Horizontal scaling support +- **Integrated backups** - Automated backup and restore +- **Monitoring integration** - PMM (Percona Monitoring and Management) +- **Automated updates** - Rolling updates with zero downtime +- **Password rotation** - Automated credential management +- **Private registries** - Support for custom container registries + +## Repository Structure + +``` +cmd/ + manager/ # Operator main entry point + mongodb-healthcheck/ # Health check binary +pkg/ + apis/ # Custom Resource Definitions (CRDs) + psmdb/v1/ # PerconaServerMongoDB CRD + controller/ # Reconciliation logic + perconaservermongodb/ + psmdb/ # MongoDB-specific logic + backup/ # Backup operations + cluster/ # Cluster management + secret/ # Secret management + k8s/ # Kubernetes client utilities + naming/ # Resource naming conventions + util/ # Shared utilities +config/ + crd/ # CRD manifests + rbac/ # RBAC configurations + manager/ # Operator deployment +deploy/ + bundle.yaml # Complete deployment bundle + operator.yaml # Operator deployment + rbac.yaml # RBAC resources + crd.yaml # CRD definitions + cr.yaml # Example custom resource + cr-minimal.yaml # Minimal CR example +build/ + Dockerfile # Operator container image + ps-entry.sh # MongoDB entrypoint + pbm-entry.sh # Backup agent entrypoint +e2e-tests/ # End-to-end test suites +vendor/ # Go dependencies +``` + +## Custom Resource Definition (CRD) + +The operator manages MongoDB through the `PerconaServerMongoDB` custom resource. + +### Basic CR Structure + +```yaml +apiVersion: psmdb.percona.com/v1 +kind: PerconaServerMongoDB +metadata: + name: my-cluster +spec: + crVersion: "1.21.0" + image: ghcr.io/objectrocket/percona-server-mongodb:6.0-k8s + + replsets: + - name: rs0 + size: 3 + volumeSpec: + persistentVolumeClaim: + resources: + requests: + storage: 10Gi + + backup: + enabled: true + image: percona/percona-backup-mongodb:2.0.0 + storages: + s3-us-west: + type: s3 + s3: + bucket: my-backups + region: us-west-2 +``` + +### Key Spec Fields + +- **crVersion**: Operator version compatibility +- **image**: MongoDB container image +- **replsets**: Replica set configurations +- **sharding**: Sharded cluster configuration +- **backup**: Backup configuration +- **secrets**: Secret references +- **pmm**: Monitoring configuration +- **updateStrategy**: Update behavior + +## Development Workflow + +### Prerequisites + +1. **Go** (version specified in go.mod) +2. **Docker** (for building images) +3. **kubectl** (for Kubernetes interaction) +4. **Kubernetes cluster** (for testing) +5. **make** (for build automation) + +### Building the Operator + +```bash +# Build operator binary +make build + +# Build operator container image +make docker-build + +# Build and push image +make docker-build docker-push IMG=ghcr.io/your-org/psmdb-operator:latest +``` + +### Running Locally + +```bash +# Install CRDs +make install + +# Run operator locally (outside cluster) +make run + +# Deploy to cluster +make deploy IMG=ghcr.io/your-org/psmdb-operator:latest +``` + +### Testing + +```bash +# Run unit tests +make test + +# Run e2e tests (requires cluster) +cd e2e-tests +./run + +# Run specific e2e test +./run default-cr +``` + +## Key Components + +### Controller + +The main reconciliation loop in `pkg/controller/perconaservermongodb/`. + +**Responsibilities:** +- Watch PerconaServerMongoDB resources +- Reconcile desired state with actual state +- Manage StatefulSets, Services, ConfigMaps +- Handle backup and restore operations +- Coordinate rolling updates + +**Key files:** +- `controller.go` - Main controller logic +- `psmdb_controller.go` - Reconciliation loop +- `status.go` - Status updates + +### PSMDB Package + +MongoDB-specific logic in `pkg/psmdb/`. + +**Modules:** +- `backup/` - Backup and restore operations +- `cluster/` - Cluster configuration +- `secret/` - Secret management +- `tls/` - TLS certificate handling +- `mongo/` - MongoDB client operations + +### APIs + +Custom resource definitions in `pkg/apis/psmdb/v1/`. + +**Key files:** +- `perconaservermongodb_types.go` - CR type definitions +- `psmdb_defaults.go` - Default values +- `psmdb_validation.go` - Validation logic + +## Common Tasks + +### Adding a New Feature + +1. **Update CRD types** in `pkg/apis/psmdb/v1/` +2. **Add validation** in `psmdb_validation.go` +3. **Implement reconciliation logic** in controller +4. **Update CRD manifests**: `make manifests` +5. **Generate code**: `make generate` +6. **Add tests** in appropriate test suite +7. **Update documentation** + +### Modifying Reconciliation Logic + +1. **Locate relevant controller code** in `pkg/controller/` +2. **Make changes** to reconciliation logic +3. **Test locally**: `make run` +4. **Run unit tests**: `make test` +5. **Run e2e tests** for affected functionality + +### Updating CRD + +1. **Modify types** in `pkg/apis/psmdb/v1/perconaservermongodb_types.go` +2. **Update validation** if needed +3. **Regenerate manifests**: `make manifests` +4. **Regenerate code**: `make generate` +5. **Update deploy files**: `make bundle` +6. **Test with new CR fields** + +### Adding E2E Test + +1. **Create test directory** in `e2e-tests/` +2. **Add test script** (usually `run.sh`) +3. **Create test CR** (YAML manifest) +4. **Add assertions** for expected behavior +5. **Update test CSV** files if needed +6. **Run test**: `./run your-test-name` + +## Integration with Liberty Platform + +### Deployment via FluxCD + +The operator is deployed to database clusters via liberty-infrastructure: + +```yaml +# In liberty-infrastructure/platform/percona-mongodb-operator/ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: percona-mongodb-operator +spec: + chart: + spec: + chart: psmdb-operator + version: 1.21.0 +``` + +### Instance Creation via Temporal + +liberty-temporal-platform creates MongoDB instances: + +```go +// In temporal-platform/internal/psmdb/ +func (a *Activities) CreateInstance(ctx context.Context, req *protos.CreateInstanceRequest) (*protos.CreateInstanceResponse, error) { + // Create PerconaServerMongoDB CR + psmdb := &psmdbv1.PerconaServerMongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.InstanceId, + Namespace: req.Namespace, + }, + Spec: psmdbv1.PerconaServerMongoDBSpec{ + Image: req.MongodbImage, + Replsets: []psmdbv1.ReplsetSpec{ + { + Name: "rs0", + Size: req.ReplicaCount, + }, + }, + }, + } + + // Apply to cluster + return k8sClient.Create(ctx, psmdb) +} +``` + +### Image Usage + +The operator uses MongoDB images from liberty-docker-images: + +```yaml +spec: + image: ghcr.io/objectrocket/percona-server-mongodb:6.0-k8s + backup: + image: percona/percona-backup-mongodb:2.0.0 +``` + +## Makefile Targets + +Common make targets: + +```bash +# Development +make build # Build operator binary +make run # Run locally +make install # Install CRDs +make uninstall # Remove CRDs + +# Code generation +make generate # Generate code +make manifests # Generate CRD manifests +make bundle # Generate deployment bundle + +# Docker +make docker-build # Build container image +make docker-push # Push container image + +# Testing +make test # Run unit tests +make test-e2e # Run e2e tests + +# Deployment +make deploy # Deploy to cluster +make undeploy # Remove from cluster + +# Utilities +make fmt # Format code +make vet # Run go vet +make lint # Run linter +``` + +## E2E Testing + +### Test Structure + +Each e2e test is a directory containing: +- `run.sh` - Test execution script +- `conf/*.yaml` - Test CR manifests +- `compare/*.yaml` - Expected state files + +### Running Tests + +```bash +# Run all tests +cd e2e-tests +./run + +# Run specific test +./run default-cr + +# Run test suite +./run --suite=run-pr.csv +``` + +### Test Categories + +- **Basic**: `default-cr`, `one-pod`, `limits` +- **Scaling**: `scaling`, `smart-update` +- **Backup**: `demand-backup`, `scheduled-backup`, `pitr` +- **Sharding**: `data-sharded`, `upgrade-sharded` +- **Security**: `custom-tls`, `ldap`, `users` +- **Monitoring**: `monitoring-2-0`, `monitoring-pmm3` +- **Chaos**: `self-healing-chaos`, `operator-self-healing-chaos` + +### Writing Tests + +1. **Create test directory**: `e2e-tests/my-test/` +2. **Add run.sh**: + ```bash + #!/bin/bash + set -o errexit + + test_dir=$(realpath $(dirname $0)) + . ${test_dir}/../functions + + create_infra $namespace + + desc 'create PSMDB cluster' + apply_cluster $test_dir/conf/my-cluster.yaml + + desc 'check if all pods are ready' + wait_for_running $cluster-rs0 3 + + desc 'verify functionality' + # Add test assertions + + destroy $namespace + ``` + +3. **Add CR manifest**: `conf/my-cluster.yaml` +4. **Add expected states**: `compare/` (if needed) + +## Best Practices + +### Code Organization + +1. **Keep controllers focused** - Single responsibility +2. **Use helper functions** - Reusable logic in pkg/ +3. **Validate early** - Check CR validity before reconciliation +4. **Handle errors gracefully** - Return errors for retry +5. **Update status** - Keep CR status current + +### CRD Design + +1. **Use clear field names** - Self-documenting +2. **Provide defaults** - Sensible default values +3. **Add validation** - Prevent invalid configurations +4. **Version carefully** - Plan for API evolution +5. **Document fields** - Add comments and examples + +### Testing + +1. **Write unit tests** - Test logic in isolation +2. **Add e2e tests** - Test real-world scenarios +3. **Test edge cases** - Failure scenarios +4. **Use test fixtures** - Reusable test data +5. **Clean up resources** - Proper test teardown + +### Reconciliation + +1. **Be idempotent** - Same input = same output +2. **Handle partial failures** - Graceful degradation +3. **Use finalizers** - Clean up external resources +4. **Requeue on errors** - Retry failed operations +5. **Update status last** - After successful reconciliation + +## Troubleshooting + +### Operator Not Starting + +**Check logs**: +```bash +kubectl logs -n psmdb-operator deployment/percona-server-mongodb-operator +``` + +**Common issues**: +- CRDs not installed: `make install` +- RBAC permissions: Check `deploy/rbac.yaml` +- Image pull errors: Verify image exists + +### CR Not Reconciling + +**Check CR status**: +```bash +kubectl get psmdb my-cluster -o yaml +``` + +**Check operator logs**: +```bash +kubectl logs -n psmdb-operator deployment/percona-server-mongodb-operator | grep my-cluster +``` + +**Common issues**: +- Invalid CR spec: Check validation errors +- Resource conflicts: Check for existing resources +- Insufficient resources: Check node capacity + +### Pods Not Starting + +**Check pod status**: +```bash +kubectl get pods -l app.kubernetes.io/instance=my-cluster +kubectl describe pod my-cluster-rs0-0 +``` + +**Common issues**: +- Image pull errors: Verify image and credentials +- PVC issues: Check storage class and capacity +- Init container failures: Check init logs + +### Backup Failures + +**Check backup status**: +```bash +kubectl get psmdb-backup +kubectl describe psmdb-backup my-backup +``` + +**Check PBM logs**: +```bash +kubectl logs my-cluster-rs0-0 -c backup-agent +``` + +**Common issues**: +- Storage credentials: Verify secret +- Network access: Check connectivity to storage +- Insufficient space: Check storage capacity + +## Upstream Sync + +### Syncing with Upstream Percona + +To sync with upstream Percona operator: + +```bash +# Add upstream remote (if not already added) +git remote add upstream https://github.com/percona/percona-server-mongodb-operator.git + +# Fetch upstream changes +git fetch upstream + +# View changes +git log HEAD..upstream/main + +# Merge or rebase +git merge upstream/main +# or +git rebase upstream/main + +# Resolve conflicts if any +# Test thoroughly +# Push to fork +``` + +### Maintaining Custom Changes + +1. **Document customizations** - Track Liberty-specific changes +2. **Use feature branches** - Isolate custom features +3. **Tag releases** - Mark stable versions +4. **Test after sync** - Run full e2e suite +5. **Update dependencies** - Keep go.mod current + +## Additional Resources + +- [Percona Operator Documentation](https://docs.percona.com/percona-operator-for-mongodb/) +- [Operator SDK Documentation](https://sdk.operatorframework.io/) +- [Kubernetes Operator Pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) +- [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) +- [Controller Runtime](https://github.com/kubernetes-sigs/controller-runtime) +- Upstream repository: https://github.com/percona/percona-server-mongodb-operator