fix: remove project label from projectstorage metrics

The project label on projectstorage_first_ready_seconds,
projectstorage_child_creations_total, and
projectstorage_reinitializing_errors_total creates a cardinality
explosion: 414 projects × 82 resource kinds × histogram buckets ×
9 pods = ~7.2M series, consuming 28% of all VictoriaMetrics
storage in production.

Drop the project label from all three metrics. The distribution
of storage init latency by resource_group and resource_kind is
the useful signal; per-project granularity is not needed and
causes the cardinality problem.

Reduces total series from ~4.7M to ~11K per pod.

Ref: datum-cloud/infra#2113

Author: zachsmith1

internal/apiserver/storage/project/mux.go

@@ -35,10 +35,10 @@ var (
 	childCreations = k8smetrics.NewCounterVec(
 		&k8smetrics.CounterOpts{
 			Name:           "projectstorage_child_creations_total",
-			Help:           "Per-project child storage creations",
+			Help:           "Child storage creations by resource type",
 			StabilityLevel: k8smetrics.ALPHA,
 		},
-		[]string{"project", "resource_group", "resource_kind"},
+		[]string{"resource_group", "resource_kind"},
 	)
 
 	firstReady = k8smetrics.NewHistogramVec(
@@ -48,7 +48,7 @@ var (
 			Buckets:        []float64{0.02, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10},
 			StabilityLevel: k8smetrics.ALPHA,
 		},
-		[]string{"project", "resource_group", "resource_kind"},
+		[]string{"resource_group", "resource_kind"},
 	)
 
 	reinitErrors = k8smetrics.NewCounterVec(
@@ -57,7 +57,7 @@ var (
 			Help:           "Ops that hit 'storage is (re)initializing'",
 			StabilityLevel: k8smetrics.ALPHA,
 		},
-		[]string{"project", "resource_group", "resource_kind", "verb"},
+		[]string{"resource_group", "resource_kind", "verb"},
 	)
 )
 
@@ -69,13 +69,13 @@ func isReinitErr(err error) bool {
 	return err != nil && strings.Contains(err.Error(), "storage is (re)initializing")
 }
 
-func incrReinit(project, group, kind, verb string) {
-	reinitErrors.WithLabelValues(project, group, kind, verb).Inc()
+func incrReinit(group, kind, verb string) {
+	reinitErrors.WithLabelValues(group, kind, verb).Inc()
 }
 
-func recordFirstReady(c *child, project, group, kind string) {
+func recordFirstReady(c *child, group, kind string) {
 	c.readyOnce.Do(func() {
-		firstReady.WithLabelValues(project, group, kind).
+		firstReady.WithLabelValues(group, kind).
 			Observe(time.Since(c.created).Seconds())
 	})
 }
@@ -107,21 +107,20 @@ type decoratorArgs struct {
 
 // instrumentedStorage wraps a storage.Interface to emit metrics once per child
 type instrumentedStorage struct {
-	inner   storage.Interface
-	child   *child
-	project string
+	inner storage.Interface
+	child *child
 
 	// normalized labels
 	group string // API group ("" => "core" when you query; we keep "" here)
 	kind  string // resource plural
 }
 
 func (i *instrumentedStorage) markSuccess() {
-	recordFirstReady(i.child, i.project, i.group, i.kind)
+	recordFirstReady(i.child, i.group, i.kind)
 }
 func (i *instrumentedStorage) markReinit(verb string, err error) error {
 	if isReinitErr(err) {
-		incrReinit(i.project, i.group, i.kind, verb)
+		incrReinit(i.group, i.kind, verb)
 	}
 	return err
 }
@@ -239,16 +238,15 @@ func (m *projectMux) childForProject(project string) (storage.Interface, error)
 	// Wrap the child once with instrumentation.
 	c := &child{s: s, destroy: destroy, created: time.Now()}
 	wrapped := &instrumentedStorage{
-		inner:   s,
-		child:   c,
-		project: project,
-		group:   m.args.resourceGroup,
-		kind:    m.args.resourceKind,
+		inner: s,
+		child: c,
+		group: m.args.resourceGroup,
+		kind:  m.args.resourceKind,
 	}
 	c.s = wrapped
 
 	m.children[project] = c
-	childCreations.WithLabelValues(project, m.args.resourceGroup, m.args.resourceKind).Inc()
+	childCreations.WithLabelValues(m.args.resourceGroup, m.args.resourceKind).Inc()
 
 	// Bootstrap system namespace synchronously to prevent resource creation failures
 	if project != "" && m.loopbackConfig != nil {