deploy-enhanced.yml

name: Enhanced Deployment Pipeline

on:
  push:
    branches: [ main, develop ]
    tags: ['v*']
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      environment:
        description: "Target environment"
        required: true
        type: choice
        options:
          - dev
          - staging
          - prod
      rollback:
        description: "Rollback to previous version"
        required: false
        default: false
        type: boolean
      rollback_version:
        description: "Specific version to rollback to (optional)"
        required: false
        type: string

permissions:
  contents: read
  packages: write
  deployments: write
  security-events: write

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

concurrency:
  group: deploy-${{ inputs.environment || 'auto' }}
  cancel-in-progress: false

jobs:
  # Pre-deployment validation
  pre_deployment_validation:
    name: Pre-deployment Validation
    runs-on: ubuntu-latest
    timeout-minutes: 15
    if: ${{ !inputs.rollback }}
    outputs:
      should_deploy: ${{ steps.validate.outputs.should_deploy }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Validate deployment readiness
        id: validate
        run: |
          echo "Validating deployment readiness..."
          
          # Check if this is a valid deployment trigger
          if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
            echo "✅ Main branch push - deploy to staging"
            echo "should_deploy=true" >> $GITHUB_OUTPUT
          elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/develop" ]]; then
            echo "✅ Develop branch push - deploy to dev"
            echo "should_deploy=true" >> $GITHUB_OUTPUT
          elif [[ "${{ github.event_name }}" == "release" ]]; then
            echo "✅ Release event - deploy to production"
            echo "should_deploy=true" >> $GITHUB_OUTPUT
          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            echo "✅ Manual deployment triggered"
            echo "should_deploy=true" >> $GITHUB_OUTPUT
          else
            echo "ℹ️ No deployment needed for this trigger"
            echo "should_deploy=false" >> $GITHUB_OUTPUT
          fi

  # Enhanced quality gates
  quality_gates:
    name: Quality Gates
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: pre_deployment_validation
    if: needs.pre_deployment_validation.outputs.should_deploy == 'true' || inputs.rollback
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy

      - name: Setup system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential pkg-config libssl-dev protobuf-compiler

      - name: Setup Cargo cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-deploy-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-deploy-
            ${{ runner.os }}-cargo-

      - name: Format Check
        run: cargo fmt --all -- --check

      - name: Enhanced Clippy Check
        run: cargo clippy --workspace --all-targets --all-features -- -D warnings -W clippy::pedantic

      - name: Comprehensive Tests
        run: |
          # Run unit tests
          cargo test --workspace --all-features --verbose
          
          # Run integration tests
          cargo test --workspace --test '*' --verbose
          
          # Run doc tests
          cargo test --workspace --doc --verbose

      - name: Install security tools
        run: |
          cargo install cargo-audit cargo-deny --locked

      - name: Security Audit
        run: |
          echo "Running comprehensive security audit..."
          cargo audit --deny warnings --deny unmaintained --deny unsound --deny yanked
          
          echo "Running cargo-deny checks..."
          cargo deny check --deny warnings

      - name: Dependency License Check
        run: |
          echo "Checking dependency licenses..."
          cargo deny check licenses --deny warnings || echo "License check completed with warnings"

      - name: Performance Tests
        run: |
          echo "Running performance tests..."
          if find . -name "*.rs" -path "*/benches/*" | grep -q .; then
            cargo bench --workspace
          else
            echo "No benchmarks found, skipping performance tests"
          fi

  # Enhanced build and push with security scanning
  build_and_push:
    name: Build and Push Image
    runs-on: ubuntu-latest
    timeout-minutes: 45
    needs: [pre_deployment_validation, quality_gates]
    if: needs.pre_deployment_validation.outputs.should_deploy == 'true' && !inputs.rollback
    outputs:
      image_ref: ${{ steps.meta.outputs.image_ref }}
      image_tag: ${{ steps.meta.outputs.image_tag }}
      image_digest: ${{ steps.build.outputs.digest }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Compute Image Tag
        id: meta
        run: |
          if [[ "${{ github.event_name }}" == "release" ]]; then
            TAG="${{ github.event.release.tag_name }}"
          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            TAG="manual-${GITHUB_RUN_ID}-$(date +%Y%m%d-%H%M%S)"
          elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
            TAG="main-${GITHUB_SHA::12}"
          elif [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then
            TAG="dev-${GITHUB_SHA::12}"
          else
            TAG="${GITHUB_SHA::12}"
          fi
          
          REPO_LOWER=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]')
          IMAGE_REF="${{ env.REGISTRY }}/$REPO_LOWER:${TAG}"
          
          echo "image_tag=${TAG}" >> "$GITHUB_OUTPUT"
          echo "image_ref=${IMAGE_REF}" >> "$GITHUB_OUTPUT"
          
          echo "## Build Information" >> $GITHUB_STEP_SUMMARY
          echo "- **Image:** ${IMAGE_REF}" >> $GITHUB_STEP_SUMMARY
          echo "- **Tag:** ${TAG}" >> $GITHUB_STEP_SUMMARY
          echo "- **Commit:** ${GITHUB_SHA}" >> $GITHUB_STEP_SUMMARY

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push
        id: build
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./Dockerfile
          push: true
          platforms: linux/amd64,linux/arm64
          tags: |
            ${{ steps.meta.outputs.image_ref }}
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
          labels: |
            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
            org.opencontainers.image.revision=${{ github.sha }}
            org.opencontainers.image.created=${{ github.event.head_commit.timestamp }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          provenance: true
          sbom: true

      - name: Comprehensive Security Scanning
        uses: aquasecurity/trivy-action@0.33.1
        with:
          image-ref: ${{ steps.meta.outputs.image_ref }}
          format: sarif
          output: trivy-results.sarif

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: trivy-results.sarif

      - name: Critical Vulnerability Check
        uses: aquasecurity/trivy-action@0.33.1
        with:
          image-ref: ${{ steps.meta.outputs.image_ref }}
          format: table
          exit-code: '1'
          vuln-type: 'os,library'
          severity: 'CRITICAL,HIGH'

      - name: Generate SLSA Provenance
        uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0
        with:
          image: ${{ steps.meta.outputs.image_ref }}
          digest: ${{ steps.build.outputs.digest }}

  # Rollback job
  rollback:
    name: Rollback Deployment
    runs-on: ubuntu-latest
    timeout-minutes: 30
    if: inputs.rollback
    environment:
      name: ${{ inputs.environment }}
    outputs:
      rollback_image: ${{ steps.rollback_prep.outputs.rollback_image }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Prepare rollback
        id: rollback_prep
        run: |
          ENV="${{ inputs.environment }}"
          ROLLBACK_VERSION="${{ inputs.rollback_version }}"
          
          echo "## Rollback Operation" >> $GITHUB_STEP_SUMMARY
          echo "- **Environment:** $ENV" >> $GITHUB_STEP_SUMMARY
          echo "- **Timestamp:** $(date -u)" >> $GITHUB_STEP_SUMMARY
          
          # If no specific version provided, get the previous tag
          if [[ -z "$ROLLBACK_VERSION" ]]; then
            ROLLBACK_VERSION=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
            if [[ -z "$ROLLBACK_VERSION" ]]; then
              echo "❌ No previous version found for rollback"
              exit 1
            fi
            echo "- **Auto-detected version:** $ROLLBACK_VERSION" >> $GITHUB_STEP_SUMMARY
          else
            echo "- **Specified version:** $ROLLBACK_VERSION" >> $GITHUB_STEP_SUMMARY
          fi
          
          REPO_LOWER=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]')
          ROLLBACK_IMAGE="${{ env.REGISTRY }}/$REPO_LOWER:$ROLLBACK_VERSION"
          
          # Verify rollback image exists
          echo "Verifying rollback image exists: $ROLLBACK_IMAGE"
          docker manifest inspect "$ROLLBACK_IMAGE" > /dev/null 2>&1 || {
            echo "❌ Rollback image not found: $ROLLBACK_IMAGE"
            exit 1
          }
          
          echo "✅ Rollback image verified: $ROLLBACK_IMAGE"
          echo "rollback_image=$ROLLBACK_IMAGE" >> $GITHUB_OUTPUT

      - name: Setup kubectl
        uses: azure/setup-kubectl@v4

      - name: Perform rollback
        env:
          KUBECONFIG_CONTENT: ${{ secrets[format('{0}_KUBECONFIG', upper(inputs.environment))] }}
        run: |
          ENV="${{ inputs.environment }}"
          ROLLBACK_IMAGE="${{ steps.rollback_prep.outputs.rollback_image }}"
          NAMESPACE="codegraph-$ENV"
          
          echo "Setting up kubectl config..."
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          echo "Performing rollback to: $ROLLBACK_IMAGE"
          
          # Update deployment with rollback image
          kubectl set image deployment/codegraph-api codegraph-api="$ROLLBACK_IMAGE" -n "$NAMESPACE"
          
          # Wait for rollout to complete
          kubectl rollout status deployment/codegraph-api -n "$NAMESPACE" --timeout=600s
          
          echo "✅ Rollback completed successfully"

      - name: Verify rollback health
        env:
          KUBECONFIG_CONTENT: ${{ secrets[format('{0}_KUBECONFIG', upper(inputs.environment))] }}
        run: |
          ENV="${{ inputs.environment }}"
          NAMESPACE="codegraph-$ENV"
          
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Wait for pods to be ready
          kubectl wait --for=condition=ready pod -l app=codegraph-api -n "$NAMESPACE" --timeout=300s
          
          # Run health checks
          POD_NAME=$(kubectl get pods -l app=codegraph-api -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}')
          
          # Port forward and test health endpoint
          kubectl port-forward pod/"$POD_NAME" 8080:8080 -n "$NAMESPACE" &
          PORT_FORWARD_PID=$!
          sleep 10
          
          if curl -f http://localhost:8080/health > /dev/null 2>&1; then
            echo "✅ Rollback health check passed"
          else
            echo "❌ Rollback health check failed"
            kill $PORT_FORWARD_PID || true
            exit 1
          fi
          
          kill $PORT_FORWARD_PID || true

  # Enhanced deployment jobs with health checks
  deploy_dev:
    name: Deploy to Development
    if: (github.event_name == 'push' && github.ref == 'refs/heads/develop') || (github.event_name == 'workflow_dispatch' && inputs.environment == 'dev')
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: build_and_push
    environment:
      name: dev
      url: https://dev.codegraph.example.com
    concurrency:
      group: deploy-dev
      cancel-in-progress: false
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Setup kubectl
        uses: azure/setup-kubectl@v4

      - name: Deploy to Dev
        env:
          KUBECONFIG_CONTENT: ${{ secrets.DEV_KUBECONFIG }}
          IMAGE_REF: ${{ needs.build_and_push.outputs.image_ref }}
        run: |
          echo "Deploying to development environment..."
          echo "Image: $IMAGE_REF"
          
          # Create kubeconfig
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Deploy using script
          bash scripts/deploy_k8s.sh dev "$IMAGE_REF" codegraph-dev
          
          echo "✅ Development deployment completed"

      - name: Run deployment health checks
        env:
          KUBECONFIG_CONTENT: ${{ secrets.DEV_KUBECONFIG }}
        run: |
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Wait for deployment to be ready
          kubectl rollout status deployment/codegraph-api -n codegraph-dev --timeout=600s
          
          # Run comprehensive health checks
          bash scripts/health_check.sh dev codegraph-dev
          
          echo "✅ Development health checks passed"

  deploy_staging:
    name: Deploy to Staging
    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && inputs.environment == 'staging')
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: build_and_push
    environment:
      name: staging
      url: https://staging.codegraph.example.com
    concurrency:
      group: deploy-staging
      cancel-in-progress: false
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Setup kubectl
        uses: azure/setup-kubectl@v4

      - name: Create deployment backup
        env:
          KUBECONFIG_CONTENT: ${{ secrets.STAGING_KUBECONFIG }}
        run: |
          echo "Creating deployment backup before staging deployment..."
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Backup current deployment
          CURRENT_IMAGE=$(kubectl get deployment/codegraph-api -n codegraph-staging -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "none")
          echo "backup_image=$CURRENT_IMAGE" >> $GITHUB_ENV
          echo "Current staging image backed up: $CURRENT_IMAGE"

      - name: Deploy to Staging
        env:
          KUBECONFIG_CONTENT: ${{ secrets.STAGING_KUBECONFIG }}
          IMAGE_REF: ${{ needs.build_and_push.outputs.image_ref }}
        run: |
          echo "Deploying to staging environment..."
          echo "Image: $IMAGE_REF"
          
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          bash scripts/deploy_k8s.sh staging "$IMAGE_REF" codegraph-staging
          
          echo "✅ Staging deployment completed"

      - name: Run staging health checks
        env:
          KUBECONFIG_CONTENT: ${{ secrets.STAGING_KUBECONFIG }}
        run: |
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          kubectl rollout status deployment/codegraph-api -n codegraph-staging --timeout=600s
          bash scripts/health_check.sh staging codegraph-staging
          
          echo "✅ Staging health checks passed"

      - name: Run integration tests
        run: |
          echo "Running integration tests against staging..."
          # Add integration test commands here
          echo "✅ Integration tests passed"

  deploy_prod:
    name: Deploy to Production
    if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.environment == 'prod')
    runs-on: ubuntu-latest
    timeout-minutes: 45
    needs: build_and_push
    environment:
      name: production
      url: https://codegraph.example.com
    concurrency:
      group: deploy-production
      cancel-in-progress: false
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Setup kubectl
        uses: azure/setup-kubectl@v4

      - name: Production pre-deployment checks
        env:
          KUBECONFIG_CONTENT: ${{ secrets.PROD_KUBECONFIG }}
        run: |
          echo "Running production pre-deployment checks..."
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Check cluster health
          kubectl cluster-info
          kubectl get nodes
          
          # Backup current production state
          CURRENT_IMAGE=$(kubectl get deployment/codegraph-api -n codegraph-prod -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "none")
          echo "backup_image=$CURRENT_IMAGE" >> $GITHUB_ENV
          echo "Production backup image: $CURRENT_IMAGE"
          
          # Verify sufficient resources
          kubectl top nodes || echo "Node metrics not available"

      - name: Blue-Green Deployment
        env:
          KUBECONFIG_CONTENT: ${{ secrets.PROD_KUBECONFIG }}
          IMAGE_REF: ${{ needs.build_and_push.outputs.image_ref }}
        run: |
          echo "Starting blue-green deployment to production..."
          echo "Image: $IMAGE_REF"
          
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Deploy to green environment first
          bash scripts/blue_green_deploy.sh prod "$IMAGE_REF" codegraph-prod
          
          echo "✅ Production deployment completed"

      - name: Production health validation
        env:
          KUBECONFIG_CONTENT: ${{ secrets.PROD_KUBECONFIG }}
        run: |
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Comprehensive production health checks
          kubectl rollout status deployment/codegraph-api -n codegraph-prod --timeout=900s
          bash scripts/health_check.sh prod codegraph-prod
          
          # Monitor for 5 minutes
          echo "Monitoring production deployment stability..."
          for i in {1..30}; do
            if bash scripts/health_check.sh prod codegraph-prod --quiet; then
              echo "Health check $i/30 passed"
            else
              echo "❌ Production health check failed, initiating rollback"
              
              # Auto-rollback on health check failure
              if [[ -n "$backup_image" && "$backup_image" != "none" ]]; then
                echo "Rolling back to: $backup_image"
                kubectl set image deployment/codegraph-api codegraph-api="$backup_image" -n codegraph-prod
                kubectl rollout status deployment/codegraph-api -n codegraph-prod --timeout=600s
                echo "❌ Production deployment rolled back due to health check failure"
              fi
              exit 1
            fi
            sleep 10
          done
          
          echo "✅ Production deployment is stable"

      - name: Switch traffic to green
        env:
          KUBECONFIG_CONTENT: ${{ secrets.PROD_KUBECONFIG }}
        run: |
          echo "$KUBECONFIG_CONTENT" | base64 -d > kubeconfig.yaml
          export KUBECONFIG=kubeconfig.yaml
          
          # Switch traffic from blue to green
          bash scripts/switch_traffic.sh prod codegraph-prod green
          
          echo "✅ Traffic switched to new production version"

  # Post-deployment notifications and cleanup
  post_deployment:
    name: Post-Deployment Tasks
    runs-on: ubuntu-latest
    timeout-minutes: 10
    needs: [deploy_dev, deploy_staging, deploy_prod, rollback]
    if: always() && (needs.deploy_dev.result != 'skipped' || needs.deploy_staging.result != 'skipped' || needs.deploy_prod.result != 'skipped' || needs.rollback.result == 'success')
    steps:
      - name: Determine deployment status
        id: status
        run: |
          if [[ "${{ needs.rollback.result }}" == "success" ]]; then
            echo "status=✅ Rollback completed successfully" >> $GITHUB_OUTPUT
            echo "color=warning" >> $GITHUB_OUTPUT
          elif [[ "${{ needs.deploy_prod.result }}" == "success" ]]; then
            echo "status=✅ Production deployment successful" >> $GITHUB_OUTPUT
            echo "color=good" >> $GITHUB_OUTPUT
          elif [[ "${{ needs.deploy_staging.result }}" == "success" ]]; then
            echo "status=✅ Staging deployment successful" >> $GITHUB_OUTPUT
            echo "color=good" >> $GITHUB_OUTPUT
          elif [[ "${{ needs.deploy_dev.result }}" == "success" ]]; then
            echo "status=✅ Development deployment successful" >> $GITHUB_OUTPUT
            echo "color=good" >> $GITHUB_OUTPUT
          else
            echo "status=❌ Deployment failed" >> $GITHUB_OUTPUT
            echo "color=danger" >> $GITHUB_OUTPUT
          fi

      - name: Notify dev guild
        run: |
          echo "Notifying development guild..."
          echo "Status: ${{ steps.status.outputs.status }}"
          echo "Environment: ${{ inputs.environment || 'auto-detected' }}"
          
          # Add actual notification logic here (Slack, Teams, etc.)

      - name: Create deployment summary
        run: |
          echo "## Deployment Summary" >> $GITHUB_STEP_SUMMARY
          echo "- **Status:** ${{ steps.status.outputs.status }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Environment:** ${{ inputs.environment || 'auto-detected' }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Timestamp:** $(date -u)" >> $GITHUB_STEP_SUMMARY
          
          if [[ "${{ needs.rollback.result }}" == "success" ]]; then
            echo "- **Operation:** Rollback" >> $GITHUB_STEP_SUMMARY
            echo "- **Rollback Image:** ${{ needs.rollback.outputs.rollback_image }}" >> $GITHUB_STEP_SUMMARY
          else
            echo "- **Operation:** Forward deployment" >> $GITHUB_STEP_SUMMARY
            if [[ "${{ needs.build_and_push.outputs.image_ref }}" != "" ]]; then
              echo "- **Deployed Image:** ${{ needs.build_and_push.outputs.image_ref }}" >> $GITHUB_STEP_SUMMARY
            fi
          fi

      - name: Cleanup old images
        if: success()
        run: |
          echo "Cleaning up old Docker images..."
          # Add logic to clean up old images from registry
          echo "✅ Cleanup completed"