fix grafana setup + new dashboards

Signed-off-by: Rahul Shetty <rashetty@redhat.com>

Author: rh-rahulshetty

Makefile

@@ -286,15 +286,17 @@ deploy-observability: ## Deploy observability (OTel + OpenShift Prometheus)
 
 add-grafana: ## Add Grafana on top of observability stack
 	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Adding Grafana..."
+	@kubectl apply -f components/manifests/observability/overlays/with-grafana/grafana-pvc.yaml
 	@kubectl apply -k components/manifests/observability/overlays/with-grafana/
 	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) Grafana deployed"
 	@echo "  Create route: oc create route edge grafana --service=grafana -n $(NAMESPACE)"
 
-clean-observability: ## Remove observability components
+clean-observability: ## Remove observability components (preserves Grafana PVC)
 	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Removing observability..."
 	@kubectl delete -k components/manifests/observability/overlays/with-grafana/ 2>/dev/null || true
 	@kubectl delete -k components/manifests/observability/ 2>/dev/null || true
 	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) Observability removed"
+	@echo "  To also delete Grafana data: kubectl delete pvc grafana-storage -n $(NAMESPACE)"
 
 grafana-dashboard: ## Open Grafana (create route first)
 	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Opening Grafana..."

components/manifests/observability/README.md

@@ -40,11 +40,14 @@ Open **OpenShift Console → Observe → Metrics** and query:
 If you want custom dashboards:
 
 ```bash
-# Add Grafana overlay
+make add-grafana
+
+# Or manually
+kubectl apply -f components/manifests/observability/overlays/with-grafana/grafana-pvc.yaml
 kubectl apply -k components/manifests/observability/overlays/with-grafana/
 ```
 
-**Adds**: Grafana (additional 128MB) - still uses OpenShift Prometheus
+**Adds**: Grafana (additional 128MB) with pre-provisioned dashboards - still uses OpenShift Prometheus
 
 **Access Grafana**:
 ```bash
@@ -53,10 +56,10 @@ oc create route edge grafana --service=grafana -n ambient-code
 
 # Get URL
 oc get route grafana -n ambient-code -o jsonpath='{.spec.host}'
-# Login: admin/admin
+# Login: admin/admin (change on first login)
 ```
 
-**Import dashboard**: Upload `dashboards/ambient-operator-dashboard.json` in Grafana UI
+**Dashboards** are provisioned automatically from `overlays/with-grafana/dashboards/`. See [dashboards/README.md](./overlays/with-grafana/dashboards/README.md) for how to add new ones.
 
 ---
 
@@ -185,6 +188,6 @@ EOF
 ## Cleanup
 
 ```bash
-kubectl delete -k components/manifests/observability/overlays/with-grafana/  # If Grafana deployed
-kubectl delete -k components/manifests/observability/
+make clean-observability                    # Removes stack but preserves Grafana PVC
+kubectl delete pvc grafana-storage -n ambient-code  # Also delete Grafana data
 ```

components/manifests/observability/base/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: ambient-code
+
+resources:
+  - otel-collector.yaml
+  - servicemonitor.yaml