diff --git a/canonical-kubernetes/addons/prometheus/steps/01_install-prometheus/prometheus-scrape-k8s.yaml b/canonical-kubernetes/addons/prometheus/steps/01_install-prometheus/prometheus-scrape-k8s.yaml index f7f3438..38d22d4 100644 --- a/canonical-kubernetes/addons/prometheus/steps/01_install-prometheus/prometheus-scrape-k8s.yaml +++ b/canonical-kubernetes/addons/prometheus/steps/01_install-prometheus/prometheus-scrape-k8s.yaml @@ -1,4 +1,21 @@ -- job_name: 'k8s-api-endpoints' +# A scrape configuration for running Prometheus on a Kubernetes cluster. +# This uses separate scrape configs for cluster components (i.e. API server, node) +# and services to allow each to use different authentication configs. +# +# Kubernetes labels will be added as Prometheus labels on metrics via the +# `labelmap` relabeling action. +# +# If you are using Kubernetes 1.7.2 or earlier, please take note of the comments +# for the kubernetes-cadvisor job; you will need to edit or remove this job. + +# Scrape config for API servers. +# +# Kubernetes exposes API servers as endpoints to the default/kubernetes +# service so this uses `endpoints` role and uses relabelling to only keep +# the endpoints associated with the default/kubernetes service using the +# default named port `https`. This works for single API server deployments as +# well as HA API server deployments. +- job_name: 'kubernetes-apiservers' kubernetes_sd_configs: - api_server: K8S_API_ENDPOINT role: endpoints @@ -18,6 +35,13 @@ - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] action: keep regex: default;kubernetes;https + +# Scrape config for nodes (kubelet). +# +# Rather than connecting directly to the node, the scrape is proxied though the +# Kubernetes apiserver. This means it will work if Prometheus is running out of +# cluster, or can't connect to nodes for some other reason (e.g. because of +# firewalling). - job_name: 'kubernetes-nodes' kubernetes_sd_configs: - api_server: K8S_API_ENDPOINT @@ -43,6 +67,21 @@ regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics + +# Scrape config for Kubelet cAdvisor. +# +# This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics +# (those whose names begin with 'container_') have been removed from the +# Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to +# retrieve those metrics. +# +# In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor +# HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" +# in that case (and ensure cAdvisor's HTTP server hasn't been disabled with +# the --cadvisor-port=0 Kubelet flag). +# +# This job is not necessary and should be removed in Kubernetes 1.6 and +# earlier versions, or it will cause the metrics to be scraped twice. - job_name: 'kubernetes-cadvisor' kubernetes_sd_configs: - api_server: K8S_API_ENDPOINT @@ -68,3 +107,104 @@ regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + +# Example scrape config for service endpoints. +# +# The relabeling allows the actual service scrape endpoint to be configured +# for all or only some endpoints. +- job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + api_server: K8S_API_ENDPOINT + tls_config: + insecure_skip_verify: true + basic_auth: + username: admin + password: K8S_PASSWORD + scheme: https + scrape_interval: 30s + tls_config: + insecure_skip_verify: true + basic_auth: + username: admin + password: K8S_PASSWORD + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: (\d+) + target_label: __meta_kubernetes_pod_container_port_number + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + regex: () + target_label: __meta_kubernetes_service_annotation_prometheus_io_path + replacement: /metrics + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_pod_container_port_number, __meta_kubernetes_service_annotation_prometheus_io_path] + target_label: __metrics_path__ + regex: (.+);(.+);(.+);(.+) + replacement: /api/v1/namespaces/$1/services/$2:$3/proxy$4 + - target_label: __address__ + replacement: K8S_API_ENDPOINT:443 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: instance + + + +# Example scrape config for pods +# +# The relabeling allows the actual pod scrape to be configured +# for all the declared ports (or port-free target if none is declared) +# or only some ports. +- job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + api_server: K8S_API_ENDPOINT + tls_config: + insecure_skip_verify: true + basic_auth: + username: admin + password: K8S_PASSWORD + scheme: https + scrape_interval: 30s + tls_config: + insecure_skip_verify: true + basic_auth: + username: admin + password: K8S_PASSWORD + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + regex: () + target_label: __meta_kubernetes_pod_annotation_prometheus_io_path + replacement: /metrics + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_name, __meta_kubernetes_pod_container_port_number, __meta_kubernetes_pod_annotation_prometheus_io_path] + target_label: __metrics_path__ + regex: (.+);(.+);(.+);(.+) + replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4 + - target_label: __address__ + replacement: K8S_API_ENDPOINT:443 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: instance