fix: replaced hard enforced local limits with kueue

Author: HardMax71

.github/actions/e2e-ready/action.yml

@@ -26,6 +26,44 @@ runs:
           /home/runner/.kube/config > backend/kubeconfig.yaml
         chmod 644 backend/kubeconfig.yaml
 
+    - name: Install Kueue
+      shell: bash
+      run: |
+        KUEUE_VERSION="${KUEUE_VERSION:-v0.16.1}"
+        kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml"
+        kubectl wait --for=condition=Available --timeout=120s \
+          deployment/kueue-controller-manager -n kueue-system
+        kubectl apply --server-side -f - <<'EOF'
+        apiVersion: kueue.x-k8s.io/v1beta1
+        kind: ResourceFlavor
+        metadata:
+          name: default-flavor
+        ---
+        apiVersion: kueue.x-k8s.io/v1beta1
+        kind: ClusterQueue
+        metadata:
+          name: executor-queue
+        spec:
+          namespaceSelector: {}
+          resourceGroups:
+          - coveredResources: ["cpu", "memory"]
+            flavors:
+            - name: default-flavor
+              resources:
+              - name: cpu
+                nominalQuota: "32"
+              - name: memory
+                nominalQuota: "4Gi"
+        ---
+        apiVersion: kueue.x-k8s.io/v1beta1
+        kind: LocalQueue
+        metadata:
+          name: executor-queue
+          namespace: integr8scode
+        spec:
+          clusterQueue: executor-queue
+        EOF
+
     - name: Use test environment config
       shell: bash
       run: |

.github/workflows/stack-tests.yml

@@ -31,6 +31,7 @@ env:
   KAFKA_IMAGE: confluentinc/cp-kafka:7.8.2
   K3S_VERSION: v1.32.11+k3s1
   K3S_INSTALL_SHA256: d75e014f2d2ab5d30a318efa5c326f3b0b7596f194afcff90fa7a7a91166d5f7
+  KUEUE_VERSION: v0.16.1
 
 jobs:
   # Fast unit tests (no infrastructure needed)

backend/app/services/k8s_worker/pod_builder.py

@@ -157,6 +157,7 @@ def _build_pod_metadata(
     ) -> k8s_client.V1ObjectMeta:
         """Build pod metadata with saga tracking"""
         labels = {"app": "integr8s", "component": "executor", "execution-id": execution_id, "language": language}
+        labels["kueue.x-k8s.io/queue-name"] = "executor-queue"
 
         labels["user-id"] = user_id[:63]  # K8s label value limit
 

backend/app/services/k8s_worker/worker.py

@@ -255,12 +255,10 @@ async def ensure_namespace_security(self) -> None:
 
         Creates:
         - Default-deny NetworkPolicy for executor pods (blocks lateral movement and exfiltration)
-        - ResourceQuota to cap aggregate CPU/memory consumption (no pod count limit)
         - Pod Security Admission labels (Restricted profile)
         """
         namespace = self._settings.K8S_NAMESPACE
         await self._ensure_executor_network_policy(namespace)
-        await self._ensure_executor_resource_quota(namespace)
         await self._apply_psa_labels(namespace)
 
     async def _ensure_executor_network_policy(self, namespace: str) -> None:
@@ -290,35 +288,6 @@ async def _ensure_executor_network_policy(self, namespace: str) -> None:
         )
         self.logger.info(f"NetworkPolicy '{policy_name}' applied in namespace {namespace}")
 
-    async def _ensure_executor_resource_quota(self, namespace: str) -> None:
-        """Create or update ResourceQuota to cap aggregate CPU/memory in the executor namespace."""
-        quota_name = "executor-quota"
-
-        quota = k8s_client.V1ResourceQuota(
-            api_version="v1",
-            kind="ResourceQuota",
-            metadata=k8s_client.V1ObjectMeta(
-                name=quota_name,
-                namespace=namespace,
-                labels={"app": "integr8s", "component": "security"},
-            ),
-            spec=k8s_client.V1ResourceQuotaSpec(
-                hard={
-                    "requests.cpu": self._settings.K8S_QUOTA_CPU,
-                    "requests.memory": self._settings.K8S_QUOTA_MEMORY,
-                    "limits.cpu": self._settings.K8S_QUOTA_CPU,
-                    "limits.memory": self._settings.K8S_QUOTA_MEMORY,
-                },
-            ),
-        )
-
-        await self.v1.patch_namespaced_resource_quota(  # type: ignore[call-arg]
-            name=quota_name, namespace=namespace, body=quota,
-            field_manager="integr8s", force=True,
-            _content_type="application/apply-patch+yaml",
-        )
-        self.logger.info(f"ResourceQuota '{quota_name}' applied in namespace {namespace}")
-
     async def _apply_psa_labels(self, namespace: str) -> None:
         """Apply Pod Security Admission labels to the executor namespace."""
         psa_labels = {

backend/app/settings.py

@@ -77,10 +77,6 @@ def __init__(
     K8S_POD_PRIORITY_CLASS_NAME: str | None = None
     K8S_POD_RUNTIME_CLASS_NAME: str | None = None  # e.g. "gvisor" for sandboxed execution
 
-    # Namespace-level ResourceQuota caps (total budget, not per-pod)
-    K8S_QUOTA_CPU: str = "32000m"
-    K8S_QUOTA_MEMORY: str = "4096Mi"
-
     SUPPORTED_RUNTIMES: dict[str, LanguageInfoDomain] = Field(default_factory=lambda: RUNTIME_MATRIX)
 
     EXAMPLE_SCRIPTS: dict[str, str] = Field(default_factory=lambda: EXEC_EXAMPLE_SCRIPTS)

backend/config.test.toml

@@ -16,8 +16,6 @@ K8S_POD_CPU_REQUEST = "200m"
 K8S_POD_MEMORY_REQUEST = "128Mi"
 K8S_POD_EXECUTION_TIMEOUT = 5
 K8S_NAMESPACE = "integr8scode"
-K8S_QUOTA_CPU = "32000m"
-K8S_QUOTA_MEMORY = "4096Mi"
 
 RATE_LIMITS = "99999/second"
 RATE_LIMIT_ENABLED = false

backend/config.toml

@@ -21,8 +21,6 @@ K8S_POD_CPU_REQUEST = "200m"
 K8S_POD_MEMORY_REQUEST = "128Mi"
 K8S_POD_EXECUTION_TIMEOUT = 5
 K8S_NAMESPACE = "integr8scode"
-K8S_QUOTA_CPU = "32000m"
-K8S_QUOTA_MEMORY = "4096Mi"
 
 RATE_LIMITS = "100/minute"
 

cert-generator/setup-k8s.sh

@@ -89,8 +89,45 @@ echo "Connected to Kubernetes"
 # Create namespace
 kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f -
 
-# Clean up stale executor pods from previous runs so they don't consume ResourceQuota
-kubectl delete pods -n integr8scode -l component=executor --field-selector=status.phase!=Running --ignore-not-found 2>/dev/null || true
+# Install Kueue (scheduling-gate based quota management)
+KUEUE_VERSION="${KUEUE_VERSION:-v0.16.1}"
+echo "Installing Kueue ${KUEUE_VERSION}..."
+kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml"
+kubectl wait --for=condition=Available --timeout=120s \
+  deployment/kueue-controller-manager -n kueue-system
+
+# Kueue resources: ResourceFlavor + ClusterQueue + LocalQueue
+kubectl apply --server-side -f - <<'KUEUE_EOF'
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ResourceFlavor
+metadata:
+  name: default-flavor
+---
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ClusterQueue
+metadata:
+  name: executor-queue
+spec:
+  namespaceSelector: {}
+  resourceGroups:
+  - coveredResources: ["cpu", "memory"]
+    flavors:
+    - name: default-flavor
+      resources:
+      - name: cpu
+        nominalQuota: "32"
+      - name: memory
+        nominalQuota: "4Gi"
+---
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: LocalQueue
+metadata:
+  name: executor-queue
+  namespace: integr8scode
+spec:
+  clusterQueue: executor-queue
+KUEUE_EOF
+echo "Kueue installed and configured"
 
 # Create ServiceAccount
 kubectl apply -f - <<EOF
@@ -146,9 +183,6 @@ rules:
 - apiGroups: ["networking.k8s.io"]
   resources: ["networkpolicies"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
-- apiGroups: [""]
-  resources: ["resourcequotas"]
-  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 EOF
 
 # Create RoleBinding

docs/security/policies.md

@@ -60,19 +60,27 @@ spec:
 
 This policy matches pods with the `component: executor` label, which the pod builder applies to all executor pods.
 
-### Resource Quota
+### Kueue (Resource Quota with Queuing)
 
-A ResourceQuota caps aggregate CPU and memory in the executor namespace. If the execution queue allows more pods than
-the namespace has resources for, Kubernetes keeps excess pods in Pending state rather than failing.
+[Kueue](https://kueue.sigs.k8s.io/) manages CPU/memory quota for executor pods. Unlike ResourceQuota (which rejects pod
+creation with 403 Forbidden when quota is full), Kueue adds a **scheduling gate** to pods — they exist in the API server
+but the scheduler ignores them (status: `SchedulingGated`). When quota frees up, the gate is removed and the pod is
+scheduled normally.
 
-| Resource          | Limit              | Source setting     |
-|-------------------|--------------------|--------------------|
-| `requests.cpu`    | `K8S_QUOTA_CPU`    | Namespace CPU cap  |
-| `requests.memory` | `K8S_QUOTA_MEMORY` | Namespace memory cap |
-| `limits.cpu`      | `K8S_QUOTA_CPU`    | Same as requests   |
-| `limits.memory`   | `K8S_QUOTA_MEMORY` | Same as requests   |
+Kueue resources (created by `setup-k8s.sh`):
 
-No `pods` count in the quota — concurrency is controlled by the execution queue (`max_concurrent_executions`).
+| Resource        | Name             | Purpose                                                      |
+|-----------------|------------------|--------------------------------------------------------------|
+| ResourceFlavor  | `default-flavor` | Represents the cluster's default resource pool               |
+| ClusterQueue    | `executor-queue` | Defines CPU/memory quota (32 CPU, 4Gi memory)               |
+| LocalQueue      | `executor-queue` | Namespace-scoped queue in `integr8scode`, bound to ClusterQueue |
+
+Executor pods carry the label `kueue.x-k8s.io/queue-name: executor-queue` (set by the pod builder). Kueue only manages
+pods with this label — DaemonSet pods (e.g., the image pre-puller) are invisible to Kueue.
+
+Key behavior:
+- `active_deadline_seconds` only counts from when the pod is scheduled, so gated time does not consume execution timeout
+- Concurrency is still controlled by the execution queue; Kueue acts as a safety net for aggregate resource consumption
 
 ### Pod Security Admission (PSA)