Fix dep conf vals and add meta-deps for nova scheduler (#169)

Author: PhilippMatthes

helm/cortex/values.yaml

@@ -251,18 +251,16 @@ conf:
                   - servers
       - name: vrops_hostsystem_contention_long_term_extractor
         dependencies:
-          features:
-            extractors:
-              - vrops_hostsystem_resolver
+          extractors:
+            - vrops_hostsystem_resolver
           sync:
             prometheus:
               metrics:
                 - alias: vrops_hostsystem_cpu_contention_long_term_percentage
       - name: vrops_hostsystem_contention_short_term_extractor
         dependencies:
-          features:
-            extractors:
-              - vrops_hostsystem_resolver
+          extractors:
+            - vrops_hostsystem_resolver
           sync:
             prometheus:
               metrics:
@@ -340,30 +338,25 @@ conf:
       # VMware-specific KPIs.
       - name: vmware_host_contention_kpi
         dependencies:
-          features:
-            extractors:
-              - vrops_hostsystem_contention_long_term_extractor
+          extractors:
+            - vrops_hostsystem_contention_long_term_extractor
       - name: vmware_project_noisiness_kpi
         dependencies:
-          features:
-            extractors:
-              - vrops_project_noisiness_extractor
+          extractors:
+            - vrops_project_noisiness_extractor
       # Shared KPIs.
       - name: host_utilization_kpi
         dependencies:
-          features:
-            extractors:
-              - host_utilization_extractor
+          extractors:
+            - host_utilization_extractor
       - name: vm_migration_statistics_kpi
         dependencies:
-          features:
-            extractors:
+          extractors:
               - vm_host_residency_extractor
       - name: vm_life_span_kpi
         dependencies:
-          features:
-            extractors:
-              - vm_life_span_histogram_extractor
+          extractors:
+            - vm_life_span_histogram_extractor
 
   scheduler:
     api:
@@ -390,11 +383,17 @@ conf:
             utilizedActivationLowerBound: 0.0
             utilizedActivationUpperBound: 1.0
           dependencies:
-            features:
-              extractors:
-                - storage_pool_utilization_extractor
+            extractors:
+              - storage_pool_utilization_extractor
 
     nova:
+      # Dependencies needed by all the Nova scheduler steps.
+      dependencies:
+        extractors:
+          # Needed for the step scoper which checks stuff like the hypervisor
+          # type and traits as well as the vm flavor in the request spec.
+          - host_capabilities_extractor
+
       # Configuration of scheduler steps that modify the Nova scheduler weights.
       # Each scheduler step can specify its own dependencies on extractors and
       # synced data.
@@ -413,9 +412,8 @@ conf:
             avgCPUUsageActivationLowerBound: 0.0
             avgCPUUsageActivationUpperBound: -1.0
           dependencies:
-            features:
-              extractors:
-                - vrops_project_noisiness_extractor
+            extractors:
+              - vrops_project_noisiness_extractor
         - name: vmware_avoid_long_term_contended_hosts
           options:
             # Min-max scaling for avg CPU contention on the host.
@@ -429,9 +427,8 @@ conf:
             maxCPUContentionActivationLowerBound: 0.0
             maxCPUContentionActivationUpperBound: -0.25
           dependencies:
-            features:
-              extractors:
-                - vrops_hostsystem_contention_long_term_extractor
+            extractors:
+              - vrops_hostsystem_contention_long_term_extractor
         - name: vmware_avoid_short_term_contended_hosts
           options:
             # Min-max scaling for avg CPU contention on the host.
@@ -445,9 +442,8 @@ conf:
             maxCPUContentionActivationLowerBound: 0.0
             maxCPUContentionActivationUpperBound: -0.25
           dependencies:
-            features:
-              extractors:
-                - vrops_hostsystem_contention_short_term_extractor
+            extractors:
+              - vrops_hostsystem_contention_short_term_extractor
 
         # KVM specific scheduler steps.
         - name: kvm_avoid_overloaded_hosts_cpu
@@ -463,9 +459,8 @@ conf:
             maxCPUUsageActivationLowerBound: 0.0
             maxCPUUsageActivationUpperBound: -0.25
           dependencies:
-            features:
-              extractors:
-                - node_exporter_host_cpu_usage_extractor
+            extractors:
+              - node_exporter_host_cpu_usage_extractor
         - name: kvm_avoid_overloaded_hosts_memory
           options:
             # Min-max scaling for avg memory usage on the host.
@@ -479,9 +474,8 @@ conf:
             maxMemoryUsageActivationLowerBound: 0.0
             maxMemoryUsageActivationUpperBound: -0.25
           dependencies:
-            features:
-              extractors:
-                - node_exporter_host_memory_active_extractor
+            extractors:
+              - node_exporter_host_memory_active_extractor
 
         # Balance general-purpose vms
         - name: shared_resource_balancing
@@ -516,10 +510,8 @@ conf:
             diskUtilizedActivationLowerBound: 1.0
             diskUtilizedActivationUpperBound: 0.0
           dependencies:
-            features:
-              extractors:
-                - host_utilization_extractor
-                - host_capabilities_extractor # Needed due to the host scope.
+            extractors:
+              - host_utilization_extractor
 
 # Generic modifiers added on the initial creation of this helm chart.
 

internal/conf/conf.go

@@ -165,6 +165,9 @@ type ManilaSchedulerConfig struct {
 type NovaSchedulerConfig struct {
 	// Scheduler step plugins by their name.
 	Plugins []SchedulerStepConfig `json:"plugins"`
+
+	// Dependencies needed by all the Nova scheduler steps.
+	DependencyConfig `json:"dependencies,omitempty"`
 }
 
 type SchedulerStepConfig struct {

internal/conf/validation.go

@@ -104,6 +104,10 @@ func (c *config) Validate() error {
 			return err
 		}
 	}
+	// Check general dependencies needed by all scheduler steps.
+	if err := c.Nova.validate(*c); err != nil {
+		return err
+	}
 	if c.API.LogRequestBodies {
 		slog.Warn("logging request bodies is enabled (debug feature)")
 	}

internal/conf/validation_test.go

@@ -222,3 +222,24 @@ func TestInvalidConf_MissingFeatureForKPI(t *testing.T) {
 		t.Fatalf("expected error, got nil")
 	}
 }
+
+func TestInvalidConf_NovaSchedulerDependency(t *testing.T) {
+	content := `
+{
+  "extractor": {
+    "plugins": [
+      {
+        "name": "extractor_1"
+      }
+    ]
+  },
+  "scheduler": {
+    "nova": { "dependencies": { "extractors": ["extractor_2"] } }
+  }
+}
+`
+	conf := newConfigFromBytes([]byte(content))
+	if err := conf.Validate(); err == nil {
+		t.Fatalf("expected error, got nil")
+	}
+}