feat: add OpenTelemetry metrics and traces to JAD

.github/workflows/e2e-tests.yaml

@@ -86,7 +86,7 @@ jobs:
           # deploying the infrastructure first, wait for it to finish and deploying the applications afterwards is 
           # the safest way to avoid race conditions between apps and infra, e.g. vault.
 
-          kubectl apply -f k8s/base
+          kubectl apply -k k8s/base
       - name: "Wait for JAD infrastructure to be ready"
         run: |-
           kubectl wait --namespace edc-v \
@@ -102,7 +102,7 @@ jobs:
           sed -i "s/imagePullPolicy:.*Always/imagePullPolicy: Never/g" k8s/apps/issuerservice.yaml
           sed -i "s/imagePullPolicy:.*Always/imagePullPolicy: Never/g" k8s/apps/identityhub.yaml
 
-          kubectl apply -f k8s/apps
+          kubectl apply -k k8s/apps
 
       - name: "Wait for JAD applications to be ready"
         run: |-

README.md

@@ -215,15 +215,15 @@ The recommended way is to deploy infrastructure services first, and application
 by running:
 
 ```shell
-kubectl apply -f k8s/base/
+kubectl apply -k k8s/base/
 
 # Wait for the infrastructure services to be ready:
 kubectl wait --namespace edc-v \
             --for=condition=ready pod \
             --selector=type=edcv-infra \
             --timeout=90s
 
-kubectl apply -f k8s/apps/
+kubectl apply -k k8s/apps/
 
 # Wait for seed jobs to be ready:
 kubectl wait --namespace edc-v \
@@ -235,12 +235,12 @@ Here's a copy-and-pasteable command to delete and redeploy everything:
 
 ```shell
 kubectl delete -k k8s/; \
-kubectl apply -f k8s/base && \
+kubectl apply -k k8s/base && \
 kubectl wait --namespace edc-v \
             --for=condition=ready pod \
             --selector=type=edcv-infra \
             --timeout=90s && \
-kubectl apply -f k8s/apps && \
+kubectl apply -k k8s/apps && \
 kubectl wait --namespace edc-v \
             --for=condition=complete job --all \
             --timeout=90s

build.gradle.kts

@@ -13,6 +13,7 @@
  */
 
 import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+import java.net.URI
 
 plugins {
     `java-library`
@@ -31,6 +32,21 @@ buildscript {
 val edcBuildId = libs.plugins.edc.build.get().pluginId
 val jadVersion: String by project
 
+val downloadOtelAgent by tasks.registering {
+    val outputFile = layout.buildDirectory.file("otel/opentelemetry-javaagent.jar")
+    outputs.file(outputFile)
+    doLast {
+        val url = URI("https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar").toURL()
+        val destFile = outputFile.get().asFile
+        destFile.parentFile.mkdirs()
+        url.openStream().use { input ->
+            destFile.outputStream().use { output ->
+                input.copyTo(output)
+            }
+        }
+    }
+}
+
 allprojects {
     apply(plugin = edcBuildId)
     apply(plugin = "org.eclipse.edc.autodoc")
@@ -48,6 +64,13 @@ subprojects {
 
             //actually apply the plugin to the (sub-)project
             apply(plugin = "com.bmuschko.docker-remote-api")
+
+            val copyOtelAgent = tasks.register<Copy>("copyOtelAgent") {
+                dependsOn(rootProject.tasks.named("downloadOtelAgent"))
+                from(rootProject.layout.buildDirectory.dir("otel"))
+                into(project.layout.buildDirectory.dir("otel"))
+            }
+
             // configure the "dockerize" task
             val dockerTask: DockerBuildImage = tasks.create("dockerize", DockerBuildImage::class) {
                 val dockerContextDir = project.projectDir
@@ -60,10 +83,11 @@ subprojects {
                 if (System.getProperty("platform") != null)
                     platform.set(System.getProperty("platform"))
                 buildArgs.put("JAR", "build/libs/${project.name}.jar")
+                buildArgs.put("OTEL_AGENT", "build/otel/opentelemetry-javaagent.jar")
                 inputDir.set(file(dockerContextDir))
             }
-            // make sure always runs after "dockerize" and after "copyOtel"
             dockerTask.dependsOn(tasks.named("shadowJar"))
+            dockerTask.dependsOn(copyOtelAgent)
         }
     }
 }

gradle/libs.versions.toml

@@ -53,6 +53,7 @@ edc-spi-http = { module = "org.eclipse.edc:http-spi", version.ref = "edc" }
 edc-spi-catalog = { module = "org.eclipse.edc:catalog-spi", version.ref = "edc" }
 edc-spi-transaction = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" }
 edc-spi-edrstore = { module = "org.eclipse.edc:edr-store-spi", version.ref = "edc" }
+edc-spi-jwt = { module = "org.eclipse.edc:jwt-spi", version.ref = "edc" }
 
 # identityhub SPI modules
 edc-ih-spi-credentials = { module = "org.eclipse.edc:verifiable-credential-spi", version.ref = "edc" }
@@ -86,6 +87,7 @@ restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssu
 jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" }
 jersey-multipart = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" }
 postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" }
+opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version = "1.61.0" }
 
 [plugins]
 shadow = { id = "com.gradleup.shadow", version = "9.4.1" }

k8s/apps/cfm-agents.yaml

@@ -36,27 +36,55 @@ spec:
         - name: keycloak-agent
           image: ghcr.io/eclipse-cfm/cfm/kcagent:latest
           imagePullPolicy: Always
+          command: [ "/kcagent" ]
+          args: [
+            "--mode=debug"
+          ]
+          envFrom:
+            - configMapRef:
+                name: telemetry-config
           volumeMounts:
             - name: keycloak-agent-config
               mountPath: /etc/appname
               readOnly: true
         - name: edcv-agent
           image: ghcr.io/eclipse-cfm/cfm/edcvagent:latest
           imagePullPolicy: Always
+          command: [ "/edcvagent" ]
+          args: [
+            "--mode=debug"
+          ]
+          envFrom:
+            - configMapRef:
+                name: telemetry-config
           volumeMounts:
             - name: edcv-agent-config
               mountPath: /etc/appname
               readOnly: true
         - name: registration-agent
           image: ghcr.io/eclipse-cfm/cfm/regagent:latest
           imagePullPolicy: Always
+          command: [ "/regagent" ]
+          args: [
+            "--mode=debug"
+          ]
+          envFrom:
+            - configMapRef:
+                name: telemetry-config
           volumeMounts:
             - name: registration-agent-config
               mountPath: /etc/appname
               readOnly: true
         - name: onboarding-agent
           image: ghcr.io/eclipse-cfm/cfm/obagent:latest
           imagePullPolicy: Always
+          command: [ "/obagent" ]
+          args: [
+            "--mode=debug"
+          ]
+          envFrom:
+            - configMapRef:
+                name: telemetry-config
           volumeMounts:
             - name: onboarding-agent-config
               mountPath: /etc/appname

k8s/apps/controlplane.yaml

@@ -38,6 +38,11 @@ spec:
           imagePullPolicy: Always
           envFrom:
             - configMapRef: { name: controlplane-config }
+            - configMapRef:
+                name: telemetry-config
+          env:
+            - name: OTEL_SERVICE_NAME
+              value: edc.controlplane
           ports:
             - containerPort: 8081
               name: management-port

k8s/apps/dataplane.yaml

@@ -39,6 +39,11 @@ spec:
           envFrom:
             - configMapRef:
                 name: dataplane-config
+            - configMapRef:
+                name: telemetry-config
+          env:
+            - name: OTEL_SERVICE_NAME
+              value: "edc.dataplane"
           ports:
             - containerPort: 11002
               name: public-port

k8s/apps/identityhub.yaml

@@ -39,6 +39,11 @@ spec:
           envFrom:
             - configMapRef:
                 name: ih-config
+            - configMapRef:
+                name: telemetry-config
+          env:
+            - name: OTEL_SERVICE_NAME
+              value: edc.identityhub
           ports:
             - containerPort: 7082
               name: creds-port

k8s/apps/issuerservice.yaml

@@ -41,6 +41,11 @@ spec:
               protocol: TCP
           envFrom:
             - configMapRef: { name: issuerservice-config }
+            - configMapRef:
+                name: telemetry-config
+          env:
+            - name: OTEL_SERVICE_NAME
+              value: edc.issuerservice
           livenessProbe:
             httpGet:
               path: /api/check/liveness

k8s/apps/kustomization.yaml

@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2025 Metaform Systems, Inc.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License, Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Contributors:
+#      Metaform Systems, Inc. - initial API and implementation
+#
+
+resources:
+  - controlplane-config.yaml
+  - controlplane.yaml
+  - dataplane-config.yaml
+  - dataplane.yaml
+  - issuerservice-config.yaml
+  - issuerservice.yaml
+  - issuerservice-seed-job.yaml
+  - identityhub-config.yaml
+  - identityhub.yaml
+  - edcv-agent-config.yaml
+  - keycloak-agent-config.yaml
+  - onboarding-agent-config.yaml
+  - registration-agent-config.yaml
+  - provision-manager.yaml
+  - provision-manager-config.yaml
+  - provision-manager-seed-job.yaml
+  - tenant-manager.yaml
+  - tenant-manager-config.yaml
+  - tenant-manager-seed-job.yaml
+  - telemetry-config.yaml
+  - cfm-agents.yaml
+  - redline.yaml
+  - redline-config.yaml
+  - redline-seed-job.yaml
+  - ui.yaml
+  - ui-config.yaml
+  - siglet-config.yaml
+  - siglet.yaml

k8s/apps/provision-manager.yaml

@@ -36,6 +36,13 @@ spec:
         - name: provision-manager
           image: ghcr.io/eclipse-cfm/cfm/pmanager:latest
           imagePullPolicy: Always
+          command: [ "/pmanager" ]
+          args: [
+            "--mode=debug"
+          ]
+          envFrom:
+            - configMapRef:
+                name: telemetry-config
           ports:
             - containerPort: 8080
           volumeMounts:

k8s/apps/redline.yaml

@@ -52,18 +52,18 @@ spec:
             httpGet:
               path: /api/public/health
               port: 8081
-            initialDelaySeconds: 60
-            periodSeconds: 10
-            timeoutSeconds: 5
-            failureThreshold: 3
+            initialDelaySeconds: 15
+            periodSeconds: 3
+            timeoutSeconds: 1
+            failureThreshold: 5
           readinessProbe:
             httpGet:
               path: /api/public/health
               port: 8081
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 3
+            initialDelaySeconds: 15
+            periodSeconds: 3
+            timeoutSeconds: 1
+            failureThreshold: 10
 
 ---
 apiVersion: v1

k8s/apps/telemetry-config.yaml

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 20256 Metaform Systems, Inc.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License, Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Contributors:
+#      Metaform Systems, Inc. - initial API and implementation
+#
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: telemetry-config
+  namespace: edc-v
+
+data:
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.edc-v.svc.cluster.local:4318"
+  OTEL_TRACES_EXPORTER: "otlp"
+  OTEL_METRICS_EXPORTER: "otlp"
+  OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://prometheus.edc-v.svc.cluster.local:9090/api/v1/otlp/v1/metrics"

k8s/apps/tenant-manager.yaml

@@ -36,6 +36,13 @@ spec:
         - name: tenant-manager
           image: ghcr.io/eclipse-cfm/cfm/tmanager:latest
           imagePullPolicy: Always
+          command: [ "/tmanager" ]
+          args: [
+            "--mode=debug"
+          ]
+          envFrom:
+            - configMapRef:
+                name: telemetry-config
           ports:
             - containerPort: 8080
           volumeMounts: