Merge branch 'datreeio:main' into develop

Author: r3nic1e

.github/pull_request_template.md

@@ -1,3 +1,5 @@
 ## PR Checklist
 
-- [ ] I have used the [CRD Extractor](https://github.com/datreeio/CRDs-catalog?tab=readme-ov-file#crd-extractor) tool to generate these CRDs
+- [ ] I generated these CRs using the [CRD Extractor tool](https://github.com/datreeio/CRDs-catalog?tab=readme-ov-file#crd-extractor). If I used a different method, I have described the method in this PR.
+- [ ] I am updating existing schemas and have specified the updated schema version.
+- [ ] I am adding new schemas and included a link to the GitHub repository that contains the source of these schemas.

README.md

@@ -53,7 +53,7 @@ This repository also contains a handy utility that extracts all CRDs from a clus
 
 ### What does this utility do?
 1. Checks that the prerequisites are installed.
-2. Extracts your CRDs from your cluster using kubectl.
+2. Extracts your CRDs from your cluster using kubectl (optionally from a specific [Kubernetes context](#using-a-specific-kubernetes-context) via `KUBECTL_CONTEXT`).
 3. Using the script from [openapi2jsonschema.py from kubeconform](https://github.com/yannh/kubeconform/blob/master/scripts/openapi2jsonschema.py) to convert your CRDs from openAPI to JSON schema.
 
 ### Supported Platforms
@@ -65,11 +65,45 @@ The following programs are required to be installed on the machine running this
 * [python3](https://www.python.org/downloads/)
 * [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
 
+Alternatively, you can use [Nix](https://nixos.org/) or [Devbox](https://www.jetify.com/devbox) to automatically set up all dependencies.
+
 ### Usage
-To use the CRD Extractor:  
+To use the CRD Extractor:
 1. Download the [latest release](https://github.com/datreeio/CRDs-catalog/releases/latest/download/crd-extractor.zip) from this repository.
 2. Extract, and run the utility:
 ```
+cd Utilities
+./crd-extractor.sh
+```
+
+#### Using a specific Kubernetes context
+To extract CRDs from a specific Kubernetes cluster context, set the `KUBECTL_CONTEXT` environment variable:
+```
+cd Utilities
+KUBECTL_CONTEXT=staging ./crd-extractor.sh
+```
+
+Or export it before running:
+```
+export KUBECTL_CONTEXT=staging
+cd Utilities
+./crd-extractor.sh
+```
+
+If `KUBECTL_CONTEXT` is not set, the utility will use your default kubectl context.
+
+#### Using Devbox
+If you have [Devbox](https://www.jetify.com/devbox) installed:
+```
+cd Utilities
+devbox run crd-extractor
+```
+
+#### Using Nix
+If you have [Nix](https://nixos.org/) with flakes enabled:
+```
+cd Utilities
+nix develop
 ./crd-extractor.sh
 ```
 

Utilities/crd-extractor.sh

@@ -1,10 +1,19 @@
 #!/usr/bin/env bash
+# shellcheck disable=SC2059
 
 set -euo pipefail
 
+# Support kubectl context via KUBECTL_CONTEXT environment variable
+KUBECTL_ARGS=(kubectl)
+if [ -n "${KUBECTL_CONTEXT:-}" ]; then
+    KUBECTL_ARGS=(kubectl --context="${KUBECTL_CONTEXT}")
+fi
+
 fetch_crd() {
     filename=${1%% *}
-    kubectl get crds "$filename" -o yaml >"$TMP/$filename.yaml" 2>&1
+    if ! "${KUBECTL_ARGS[@]}" get crds "$filename" -o yaml >"$TMP/$filename.yaml" 2>/dev/null; then
+        printf "Warning: Failed to extract CRD: %s\n" "$filename" >&2
+    fi
 }
 
 # Check if python3 is installed
@@ -52,19 +61,24 @@ fi
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-# Create temp folder for CRDs
+# Create temp folders for CRDs and conversion
 TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
+CONVERSION_TMP=$(mktemp -d)
+trap 'rm -rf "$TMP" "$CONVERSION_TMP"' EXIT
 
 # Create final schemas directory
 # Use current directory if OUTPUT_DIR is set, otherwise use default
 SCHEMAS_DIR=${OUTPUT_DIR:-$HOME/.datree/crdSchemas}
 mkdir -p "$SCHEMAS_DIR"
-cd "$SCHEMAS_DIR"
 
 # Get a list of all CRDs
-printf "Fetching list of CRDs...\n"
-IFS=$'\n' read -r -d '' -a CRD_LIST < <(kubectl get crds 2>&1 | sed -n '/NAME/,$p' | tail -n +2 && printf '\0')
+printf "Fetching list of CRDs..."
+if [ -n "${KUBECTL_CONTEXT:-}" ]; then
+    printf " (using context: %s)\n" "${KUBECTL_CONTEXT}"
+else
+    printf "\n"
+fi
+IFS=$'\n' read -r -d '' -a CRD_LIST < <("${KUBECTL_ARGS[@]}" get crds 2>&1 | sed -n '/NAME/,$p' | tail -n +2 && printf '\0')
 
 # If no CRDs exist in the cluster, exit
 if [ ${#CRD_LIST[@]} == 0 ]; then
@@ -89,40 +103,80 @@ for crd in "${CRD_LIST[@]}"; do
     fi
     ((++FETCHED_CRDS))
 done
+# shellcheck disable=SC2046
 wait $(jobs -p)
 
 
-# Convert crds to jsonSchema
+# Convert crds to jsonSchema - output to temp directory to avoid mixing with existing schemas
 CONVERTER_SCRIPT="$SCRIPT_DIR/openapi2jsonschema.py"
 export FILENAME_FORMAT="{fullgroup}_{kind}_{version}"
+cd "$CONVERSION_TMP"
 python3 "$CONVERTER_SCRIPT" "$TMP"/*.yaml
 conversionResult=$?
 
-# Copy and rename files to support kubeval
-rm -rf "$SCHEMAS_DIR/master-standalone"
-mkdir -p "$SCHEMAS_DIR/master-standalone"
-cp "$SCHEMAS_DIR"/*.json "$SCHEMAS_DIR/master-standalone"
-find "$SCHEMAS_DIR/master-standalone" -name '*json' -exec bash -c ' mv -f $0 ${0/\_/-stable-}' {} \;
-
-# Organize schemas by group
-for schema in "$SCHEMAS_DIR"/*.json; do
-    crdFileName=$(basename "$schema")
-    crdGroup=$(echo "$crdFileName" | cut -d"_" -f1)
-    outName=$(echo "$crdFileName" | cut -d"_" -f2-)
-    mkdir -p "$crdGroup"
-    mv "$schema" "./$crdGroup/$outName"
+# Track which groups we're processing in this run
+declare -A PROCESSED_GROUPS
+
+# Find newly generated schemas in the conversion temp directory
+NEW_SCHEMAS=()
+shopt -s nullglob
+for schema in "$CONVERSION_TMP"/*.json; do
+    [ -f "$schema" ] && NEW_SCHEMAS+=("$schema")
 done
+shopt -u nullglob
+
+# Only proceed if we have new schemas
+if [ ${#NEW_SCHEMAS[@]} -gt 0 ]; then
+    # Copy and rename files to support kubeval
+    rm -rf "$SCHEMAS_DIR/master-standalone"
+    mkdir -p "$SCHEMAS_DIR/master-standalone"
+    cp "$CONVERSION_TMP"/*.json "$SCHEMAS_DIR/master-standalone/" 2>/dev/null
+    find "$SCHEMAS_DIR/master-standalone" -name '*json' -exec bash -c ' mv -f $0 ${0/\_/-stable-}' {} \;
+
+    # Organize schemas by group
+    for schema in "${NEW_SCHEMAS[@]}"; do
+        crdFileName=$(basename "$schema")
+        crdGroup=$(echo "$crdFileName" | cut -d"_" -f1)
+        outName=$(echo "$crdFileName" | cut -d"_" -f2-)
+        mkdir -p "$SCHEMAS_DIR/$crdGroup"
+        cp "$schema" "$SCHEMAS_DIR/$crdGroup/$outName"
+        PROCESSED_GROUPS[$crdGroup]=1
+    done
+
+    # Copy organized schemas to git repository (when not using custom OUTPUT_DIR)
+    if [ -z "${OUTPUT_DIR:-}" ]; then
+        printf "\nCopying schemas to repository...\n"
+        for crdGroup in "${!PROCESSED_GROUPS[@]}"; do
+            groupDir="$SCHEMAS_DIR/$crdGroup"
+            if [ -d "$groupDir" ]; then
+                # Copy to parent directory of Utilities folder
+                repoDir="$(dirname "$SCRIPT_DIR")/$crdGroup"
+                mkdir -p "$repoDir"
+                cp -r "$groupDir"/*.json "$repoDir/" 2>/dev/null || true
+                printf "  Copied %s schemas\n" "$crdGroup"
+            fi
+        done
+    fi
+fi
 
 CYAN='\033[0;36m'
 GREEN='\033[0;32m'
 NC='\033[0m' # No Color
 
-if [ $conversionResult == 0 ]; then
-    printf "${GREEN}Successfully converted $FETCHED_CRDS CRDs to JSON schema${NC}\n"
-    printf "Schemas saved to: ${CYAN}$SCHEMAS_DIR${NC}\n"
+if [ $conversionResult == 0 ] && [ ${#NEW_SCHEMAS[@]} -gt 0 ]; then
+    printf "\n${GREEN}Successfully converted ${FETCHED_CRDS} CRDs to JSON schema${NC}\n"
+    printf "Schemas saved to: ${CYAN}${SCHEMAS_DIR}${NC}\n"
+
+    if [ -z "${OUTPUT_DIR:-}" ] && [ ${#PROCESSED_GROUPS[@]} -gt 0 ]; then
+        printf "\n${GREEN}Schemas organized by API group and copied to:${NC}\n"
+        for crdGroup in "${!PROCESSED_GROUPS[@]}"; do
+            repoDir="$(dirname "$SCRIPT_DIR")/$crdGroup"
+            printf "  - %s/\n" "$repoDir"
+        done | sort -u
+    fi
 
     printf "\nTo validate a CR using various tools, run the relevant command:\n"
     printf "\n- ${CYAN}datree:${NC}\n\$ datree test /path/to/file\n"
-    printf "\n- ${CYAN}kubeconform:${NC}\n\$ kubeconform -summary -output json -schema-location default -schema-location '$SCHEMAS_DIR/{{ .Group }}/{{ .ResourceKind }}_{{ .ResourceAPIVersion }}.json' /path/to/file\n"
-    printf "\n- ${CYAN}kubeval:${NC}\n\$ kubeval --additional-schema-locations file:\"$SCHEMAS_DIR\" /path/to/file\n\n"
+    printf "\n- ${CYAN}kubeconform:${NC}\n\$ kubeconform -summary -output json -schema-location default -schema-location '${SCHEMAS_DIR}/{{ .Group }}/{{ .ResourceKind }}_{{ .ResourceAPIVersion }}.json' /path/to/file\n"
+    printf "\n- ${CYAN}kubeval:${NC}\n\$ kubeval --additional-schema-locations file:\"${SCHEMAS_DIR}\" /path/to/file\n\n"
 fi

Utilities/devbox.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.14.2/.schema/devbox.schema.json",
+  "packages": [
+    "python313Packages.pyyaml@latest",
+    "kubectl@1.32",
+    "python313@latest"
+  ],
+  "shell": {
+    "init_hook": [
+      "echo \"$(python3 --version)\"",
+      "echo \"$(kubectl version --client)\"",
+      "echo \"context: $(kubectl config current-context 2>/dev/null || echo '<no context>')\""
+    ],
+    "scripts": {
+      "crd-extractor": [
+        "./crd-extractor.sh"
+      ]
+    }
+  }
+}