From 524b34b1a9c3db62a754aef9e92b3274629be3a8 Mon Sep 17 00:00:00 2001 From: Savita Manghnani Date: Tue, 12 May 2026 09:56:07 -0400 Subject: [PATCH 1/4] feat: make Ray dashboard URL configurable for HTTPS with TLS proxy Add --dashboard-domain-suffix and --dashboard-port operator flags so kuberay can construct HTTPS dashboard URLs for RayClusters that are already fronted by an envoy proxy with TLS certificates. When domainSuffix is set the URL becomes: https://..[:] e.g. raycluster-head-svc.my-namespace.svc.example.com:8266 When unset the operator falls back to the original http:// behaviour, preserving full backwards compatibility. Co-Authored-By: Claude Sonnet 4.6 --- .../ray_job_submission_service_server.go | 2 +- .../templates/deployment.yaml | 6 ++ helm-chart/kuberay-operator/values.yaml | 11 +++ .../config/v1alpha1/configuration_types.go | 13 ++- ray-operator/controllers/ray/utils/util.go | 46 +++++++--- .../controllers/ray/utils/util_test.go | 83 +++++++++++++++++++ ray-operator/main.go | 10 +++ ray-operator/test/sampleyaml/support.go | 2 +- 8 files changed, 160 insertions(+), 13 deletions(-) diff --git a/apiserver/pkg/server/ray_job_submission_service_server.go b/apiserver/pkg/server/ray_job_submission_service_server.go index 4fdead1e50f..83377dfcb01 100644 --- a/apiserver/pkg/server/ray_job_submission_service_server.go +++ b/apiserver/pkg/server/ray_job_submission_service_server.go @@ -41,7 +41,7 @@ type RayJobSubmissionServiceServer struct { // Create RayJobSubmissionServiceServer func NewRayJobSubmissionServiceServer(clusterServer *ClusterServer, options *RayJobSubmissionServiceServerOptions) *RayJobSubmissionServiceServer { zl := zerolog.New(os.Stdout).Level(zerolog.DebugLevel) - return &RayJobSubmissionServiceServer{clusterServer: clusterServer, options: options, log: zerologr.New(&zl).WithName("jobsubmissionservice"), dashboardClientFunc: utils.GetRayDashboardClientFunc(nil, false)} + return &RayJobSubmissionServiceServer{clusterServer: clusterServer, options: options, log: zerologr.New(&zl).WithName("jobsubmissionservice"), dashboardClientFunc: utils.GetRayDashboardClientFunc(nil, false, "", "")} } // Submit Ray job diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index 337dcc60bae..34de665a8df 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -123,6 +123,12 @@ spec: {{- if hasKey .Values "useKubernetesProxy" -}} {{- $argList = append $argList (printf "--use-kubernetes-proxy=%t" .Values.useKubernetesProxy) -}} {{- end -}} + {{- if and (hasKey .Values "dashboard") (.Values.dashboard.domainSuffix) -}} + {{- $argList = append $argList (printf "--dashboard-domain-suffix=%s" .Values.dashboard.domainSuffix) -}} + {{- end -}} + {{- if and (hasKey .Values "dashboard") (.Values.dashboard.port) -}} + {{- $argList = append $argList (printf "--dashboard-port=%s" (.Values.dashboard.port | toString)) -}} + {{- end -}} {{- if hasKey .Values "leaderElectionEnabled" -}} {{- $argList = append $argList (printf "--enable-leader-election=%t" .Values.leaderElectionEnabled) -}} {{- end -}} diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index c1892054323..d53b2118c90 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -149,6 +149,17 @@ operatorCommand: /manager # Using this option to configure kuberay-operator to comunitcate to Ray head pods by proxying through the Kubernetes API Server. # useKubernetesProxy: true +# dashboard configures how the operator constructs the Ray dashboard URL. +# When domainSuffix is set, the operator builds HTTPS URLs of the form: +# https://..[:] +# domainSuffix should include any Kubernetes subdomain prefix (e.g. "svc."): +# e.g. with the values below: raycluster-head-svc.my-namespace.svc.example.com:8266 +# This is useful when RayClusters are fronted by an envoy proxy with TLS already configured. +# When unset (default), the operator uses the original HTTP behaviour (backwards compatible). +# dashboard: +# domainSuffix: "svc.example.com" +# port: "8266" + # -- If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability. leaderElectionEnabled: true diff --git a/ray-operator/apis/config/v1alpha1/configuration_types.go b/ray-operator/apis/config/v1alpha1/configuration_types.go index 910ec0d11ab..ad713018611 100644 --- a/ray-operator/apis/config/v1alpha1/configuration_types.go +++ b/ray-operator/apis/config/v1alpha1/configuration_types.go @@ -86,10 +86,21 @@ type Configuration struct { // EnableMetrics indicates whether KubeRay operator should emit control plane metrics. EnableMetrics bool `json:"enableMetrics,omitempty"` + + // DashboardDomainSuffix is the domain suffix appended to the head service name when + // constructing HTTPS dashboard URLs. When set, the operator builds URLs of the form: + // https://..[:] + // domainSuffix should include any Kubernetes subdomain prefix (e.g. "svc."). + // When empty (default), the operator uses the original HTTP behaviour. + DashboardDomainSuffix string `json:"dashboardDomainSuffix,omitempty"` + + // DashboardPort is the port used when DashboardDomainSuffix is set. + // When empty, no port is appended to the dashboard URL. + DashboardPort string `json:"dashboardPort,omitempty"` } func (config Configuration) GetDashboardClient(mgr manager.Manager) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { - return utils.GetRayDashboardClientFunc(mgr, config.UseKubernetesProxy) + return utils.GetRayDashboardClientFunc(mgr, config.UseKubernetesProxy, config.DashboardDomainSuffix, config.DashboardPort) } func (config Configuration) GetHttpProxyClient(mgr manager.Manager) func(hostIp, podNamespace, podName string, port int) utils.RayHttpProxyClientInterface { diff --git a/ray-operator/controllers/ray/utils/util.go b/ray-operator/controllers/ray/utils/util.go index 70b1dadf3e1..89e96c971d8 100644 --- a/ray-operator/controllers/ray/utils/util.go +++ b/ray-operator/controllers/ray/utils/util.go @@ -73,6 +73,28 @@ func GetClusterDomainName() string { return DefaultDomainName } +// BuildDashboardURL constructs the dashboard URL for a given head service name and namespace. +// If domainSuffix is non-empty, it constructs an HTTPS URL using the custom domain with the format: +// +// https://..[:] +// +// domainSuffix should include any Kubernetes subdomain prefix, e.g.: +// +// "svc.example.com" +// → raycluster-head-svc.my-namespace.svc.example.com:8266 +// +// If domainSuffix is empty it falls back to the plain HTTP URL built from fallbackURL +// (backwards-compatible default behaviour). +func BuildDashboardURL(headSvcName, namespace, domainSuffix, port, fallbackURL string) string { + if domainSuffix == "" { + return "http://" + fallbackURL + } + if port != "" { + return fmt.Sprintf("https://%s.%s.%s:%s", headSvcName, namespace, domainSuffix, port) + } + return fmt.Sprintf("https://%s.%s.%s", headSvcName, namespace, domainSuffix) +} + // IsCreated returns true if pod has been created and is maintained by the API server func IsCreated(pod *corev1.Pod) bool { return pod.Status.Phase != "" @@ -943,7 +965,7 @@ func FetchHeadServiceURL(ctx context.Context, cli client.Client, rayCluster *ray return headServiceURL, nil } -func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { +func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool, dashboardDomainSuffix, dashboardPort string) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { return func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { dashboardClient := &dashboardclient.RayDashboardClient{} var authToken string @@ -969,14 +991,9 @@ func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) fun } if useKubernetesProxy { - var err error - headSvcName := rayCluster.Status.Head.ServiceName - if headSvcName == "" { - headSvcName, err = GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) - if err != nil { - err = fmt.Errorf("failed to construct Ray dashboard client: %w", err) - return nil, err - } + headSvcName, err := GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) + if err != nil { + return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) } dashboardClient.InitClient( @@ -989,11 +1006,20 @@ func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) fun return dashboardClient, nil } + // Build the dashboard URL. + // Priority (highest to lowest): + // 1. Custom HTTPS domain (dashboardDomainSuffix configured in operator) + // 2. Plain HTTP fallback (original behaviour) + headSvcName, err := GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) + if err != nil { + return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) + } + dashboardClient.InitClient( &http.Client{ Timeout: 2 * time.Second, }, - "http://"+url, + BuildDashboardURL(headSvcName, rayCluster.Namespace, dashboardDomainSuffix, dashboardPort, url), authToken, ) diff --git a/ray-operator/controllers/ray/utils/util_test.go b/ray-operator/controllers/ray/utils/util_test.go index 47afa55c85a..41a74e3e2fe 100644 --- a/ray-operator/controllers/ray/utils/util_test.go +++ b/ray-operator/controllers/ray/utils/util_test.go @@ -1792,3 +1792,86 @@ func TestGetWeightsFromHTTPRoute(t *testing.T) { }) } } + +func TestBuildDashboardURL(t *testing.T) { + tests := []struct { + name string + headSvcName string + namespace string + domainSuffix string + port string + fallbackURL string + want string + }{ + { + name: "no domain suffix — uses HTTP fallback URL (backwards compatible)", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "", + port: "", + fallbackURL: "10.0.0.1:8265", + want: "http://10.0.0.1:8265", + }, + { + name: "no domain suffix — port is ignored", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "", + port: "8266", + fallbackURL: "10.0.0.1:8265", + want: "http://10.0.0.1:8265", + }, + { + name: "suffix includes svc. — constructs HTTPS URL with namespace and port", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "svc.example.com", + port: "8266", + fallbackURL: "10.0.0.1:8265", + want: "https://raycluster-head-svc.my-namespace.svc.example.com:8266", + }, + { + name: "suffix includes svc. — constructs HTTPS URL with namespace, no port", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "svc.example.com", + port: "", + fallbackURL: "10.0.0.1:8265", + want: "https://raycluster-head-svc.my-namespace.svc.example.com", + }, + { + name: "custom head service name with dashes in different namespace", + headSvcName: "my-cluster-kuberay-head-svc", + namespace: "ml-team", + domainSuffix: "svc.prod.example.com", + port: "443", + fallbackURL: "10.0.0.2:8265", + want: "https://my-cluster-kuberay-head-svc.ml-team.svc.prod.example.com:443", + }, + { + name: "non-svc subdomain prefix", + headSvcName: "my-ray-cluster-head-svc", + namespace: "my-namespace", + domainSuffix: "mesh.example.com", + port: "", + fallbackURL: "10.0.0.1:8265", + want: "https://my-ray-cluster-head-svc.my-namespace.mesh.example.com", + }, + { + name: "empty fallback URL when no domain suffix", + headSvcName: "raycluster-head-svc", + namespace: "default", + domainSuffix: "", + port: "", + fallbackURL: "", + want: "http://", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildDashboardURL(tt.headSvcName, tt.namespace, tt.domainSuffix, tt.port, tt.fallbackURL) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/ray-operator/main.go b/ray-operator/main.go index ceba7d4772e..ee398ce1256 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -66,6 +66,8 @@ func main() { var logFileEncoder string var logStdoutEncoder string var useKubernetesProxy bool + var dashboardDomainSuffix string + var dashboardPort string var configFile string var featureGates string var enableBatchScheduler bool @@ -102,6 +104,12 @@ func main() { flag.StringVar(&configFile, "config", "", "Path to structured config file. Flags are ignored if config file is set.") flag.BoolVar(&useKubernetesProxy, "use-kubernetes-proxy", false, "Use Kubernetes proxy subresource when connecting to the Ray Head node.") + flag.StringVar(&dashboardDomainSuffix, "dashboard-domain-suffix", "", + "Domain suffix for constructing HTTPS dashboard URLs. When set, the operator uses "+ + "https://..[:] instead of the default HTTP URL. "+ + "The suffix should include any Kubernetes subdomain prefix, e.g. svc.example.com.") + flag.StringVar(&dashboardPort, "dashboard-port", "", + "Port appended to the dashboard URL when --dashboard-domain-suffix is set.") flag.StringVar(&featureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates. E.g. FeatureOne=true,FeatureTwo=false,...") flag.BoolVar(&enableMetrics, "enable-metrics", false, "Enable the emission of control plane metrics.") flag.Float64Var(&qps, "qps", float64(configapi.DefaultQPS), "The QPS value for the client communicating with the Kubernetes API server.") @@ -134,6 +142,8 @@ func main() { config.EnableBatchScheduler = enableBatchScheduler config.BatchScheduler = batchScheduler config.UseKubernetesProxy = useKubernetesProxy + config.DashboardDomainSuffix = dashboardDomainSuffix + config.DashboardPort = dashboardPort config.DeleteRayJobAfterJobFinishes = os.Getenv(utils.DELETE_RAYJOB_CR_AFTER_JOB_FINISHES) == "true" config.EnableMetrics = enableMetrics config.QPS = &qps diff --git a/ray-operator/test/sampleyaml/support.go b/ray-operator/test/sampleyaml/support.go index 42ffd19e0a2..ed7f4eddc5b 100644 --- a/ray-operator/test/sampleyaml/support.go +++ b/ray-operator/test/sampleyaml/support.go @@ -75,7 +75,7 @@ func QueryDashboardGetAppStatus(t Test, rayCluster *rayv1.RayCluster) func(Gomeg g.Expect(err).ToNot(HaveOccurred()) url := fmt.Sprintf("127.0.0.1:%d", localPort) - rayDashboardClientFunc := utils.GetRayDashboardClientFunc(nil, false) + rayDashboardClientFunc := utils.GetRayDashboardClientFunc(nil, false, "", "") rayDashboardClient, err := rayDashboardClientFunc(rayCluster, url) g.Expect(err).ToNot(HaveOccurred()) serveDetails, err := rayDashboardClient.GetServeDetails(t.Ctx()) From d568fb0122ef81c570683cc2a7d7bacd72e3a659 Mon Sep 17 00:00:00 2001 From: Savita Manghnani Date: Wed, 20 May 2026 19:46:40 -0400 Subject: [PATCH 2/4] feat: configure dashboard HTTPS and mTLS via environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace CLI flags --dashboard-domain-suffix and --dashboard-port with environment variables, keeping GetRayDashboardClientFunc signature unchanged from upstream (mgr, useKubernetesProxy only). Five new env vars (all optional, backwards compatible): KUBERAY_DASHBOARD_DOMAIN_SUFFIX – enables HTTPS dashboard URLs KUBERAY_DASHBOARD_PORT – port for HTTPS URLs KUBERAY_DASHBOARD_TLS_CA_CERT – CA cert for server verification KUBERAY_DASHBOARD_TLS_CLIENT_CERT – client cert for mTLS KUBERAY_DASHBOARD_TLS_CLIENT_KEY – client key for mTLS Also adds emissary injection annotation to operator pod template so mTLS client certs are provisioned automatically at the standard /etc/emissary/certs/client/ paths. Co-Authored-By: Claude Sonnet 4.6 --- .../ray_job_submission_service_server.go | 2 +- .../templates/deployment.yaml | 6 - helm-chart/kuberay-operator/values.yaml | 39 ++++-- .../config/v1alpha1/configuration_types.go | 13 +- .../controllers/ray/utils/constant.go | 21 ++++ ray-operator/controllers/ray/utils/util.go | 64 ++++++++-- .../controllers/ray/utils/util_test.go | 115 ++++++++++++++++++ ray-operator/main.go | 10 -- ray-operator/test/sampleyaml/support.go | 2 +- 9 files changed, 222 insertions(+), 50 deletions(-) diff --git a/apiserver/pkg/server/ray_job_submission_service_server.go b/apiserver/pkg/server/ray_job_submission_service_server.go index 83377dfcb01..4fdead1e50f 100644 --- a/apiserver/pkg/server/ray_job_submission_service_server.go +++ b/apiserver/pkg/server/ray_job_submission_service_server.go @@ -41,7 +41,7 @@ type RayJobSubmissionServiceServer struct { // Create RayJobSubmissionServiceServer func NewRayJobSubmissionServiceServer(clusterServer *ClusterServer, options *RayJobSubmissionServiceServerOptions) *RayJobSubmissionServiceServer { zl := zerolog.New(os.Stdout).Level(zerolog.DebugLevel) - return &RayJobSubmissionServiceServer{clusterServer: clusterServer, options: options, log: zerologr.New(&zl).WithName("jobsubmissionservice"), dashboardClientFunc: utils.GetRayDashboardClientFunc(nil, false, "", "")} + return &RayJobSubmissionServiceServer{clusterServer: clusterServer, options: options, log: zerologr.New(&zl).WithName("jobsubmissionservice"), dashboardClientFunc: utils.GetRayDashboardClientFunc(nil, false)} } // Submit Ray job diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index 34de665a8df..337dcc60bae 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -123,12 +123,6 @@ spec: {{- if hasKey .Values "useKubernetesProxy" -}} {{- $argList = append $argList (printf "--use-kubernetes-proxy=%t" .Values.useKubernetesProxy) -}} {{- end -}} - {{- if and (hasKey .Values "dashboard") (.Values.dashboard.domainSuffix) -}} - {{- $argList = append $argList (printf "--dashboard-domain-suffix=%s" .Values.dashboard.domainSuffix) -}} - {{- end -}} - {{- if and (hasKey .Values "dashboard") (.Values.dashboard.port) -}} - {{- $argList = append $argList (printf "--dashboard-port=%s" (.Values.dashboard.port | toString)) -}} - {{- end -}} {{- if hasKey .Values "leaderElectionEnabled" -}} {{- $argList = append $argList (printf "--enable-leader-election=%t" .Values.leaderElectionEnabled) -}} {{- end -}} diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index d53b2118c90..2729ecfea48 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -37,7 +37,10 @@ priorityClassName: "" labels: {} # -- Extra annotations. -annotations: {} +# Emissary injection provisions mTLS client certificates at /etc/emissary/certs/client/ +# which are consumed by the operator via the KUBERAY_DASHBOARD_TLS_* env vars below. +annotations: + emissary.datadoghq.com/inject-lifecycle: injected # -- Pod affinity affinity: {} @@ -149,16 +152,30 @@ operatorCommand: /manager # Using this option to configure kuberay-operator to comunitcate to Ray head pods by proxying through the Kubernetes API Server. # useKubernetesProxy: true -# dashboard configures how the operator constructs the Ray dashboard URL. -# When domainSuffix is set, the operator builds HTTPS URLs of the form: -# https://..[:] -# domainSuffix should include any Kubernetes subdomain prefix (e.g. "svc."): -# e.g. with the values below: raycluster-head-svc.my-namespace.svc.example.com:8266 -# This is useful when RayClusters are fronted by an envoy proxy with TLS already configured. -# When unset (default), the operator uses the original HTTP behaviour (backwards compatible). -# dashboard: -# domainSuffix: "svc.example.com" -# port: "8266" +# Dashboard URL and TLS configuration via environment variables. +# Set these under env: to configure HTTPS dashboard connections. +# +# KUBERAY_DASHBOARD_DOMAIN_SUFFIX – domain suffix for HTTPS dashboard URLs. +# When set, the operator builds: https://..[:] +# The suffix must include any Kubernetes subdomain prefix (e.g. "svc.cluster.local"). +# When unset (default), the operator uses the original plain-HTTP behaviour. +# KUBERAY_DASHBOARD_PORT – port appended to the URL when DOMAIN_SUFFIX is set. +# KUBERAY_DASHBOARD_TLS_CA_CERT – path to PEM CA cert for server cert verification. +# KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to PEM client cert for mTLS (emissary: /etc/emissary/certs/client/cert.pem). +# KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to PEM client key for mTLS (emissary: /etc/emissary/certs/client/key.pem). +# +# Example: +# env: +# - name: KUBERAY_DASHBOARD_DOMAIN_SUFFIX +# value: "svc.$(CLUSTER_DOMAIN)" +# - name: KUBERAY_DASHBOARD_PORT +# value: "8266" +# - name: KUBERAY_DASHBOARD_TLS_CA_CERT +# value: /etc/emissary/certs/client/ca.pem +# - name: KUBERAY_DASHBOARD_TLS_CLIENT_CERT +# value: /etc/emissary/certs/client/cert.pem +# - name: KUBERAY_DASHBOARD_TLS_CLIENT_KEY +# value: /etc/emissary/certs/client/key.pem # -- If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability. leaderElectionEnabled: true diff --git a/ray-operator/apis/config/v1alpha1/configuration_types.go b/ray-operator/apis/config/v1alpha1/configuration_types.go index ad713018611..910ec0d11ab 100644 --- a/ray-operator/apis/config/v1alpha1/configuration_types.go +++ b/ray-operator/apis/config/v1alpha1/configuration_types.go @@ -86,21 +86,10 @@ type Configuration struct { // EnableMetrics indicates whether KubeRay operator should emit control plane metrics. EnableMetrics bool `json:"enableMetrics,omitempty"` - - // DashboardDomainSuffix is the domain suffix appended to the head service name when - // constructing HTTPS dashboard URLs. When set, the operator builds URLs of the form: - // https://..[:] - // domainSuffix should include any Kubernetes subdomain prefix (e.g. "svc."). - // When empty (default), the operator uses the original HTTP behaviour. - DashboardDomainSuffix string `json:"dashboardDomainSuffix,omitempty"` - - // DashboardPort is the port used when DashboardDomainSuffix is set. - // When empty, no port is appended to the dashboard URL. - DashboardPort string `json:"dashboardPort,omitempty"` } func (config Configuration) GetDashboardClient(mgr manager.Manager) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { - return utils.GetRayDashboardClientFunc(mgr, config.UseKubernetesProxy, config.DashboardDomainSuffix, config.DashboardPort) + return utils.GetRayDashboardClientFunc(mgr, config.UseKubernetesProxy) } func (config Configuration) GetHttpProxyClient(mgr manager.Manager) func(hostIp, podNamespace, podName string, port int) utils.RayHttpProxyClientInterface { diff --git a/ray-operator/controllers/ray/utils/constant.go b/ray-operator/controllers/ray/utils/constant.go index 38b94c9b715..7537051f3c3 100644 --- a/ray-operator/controllers/ray/utils/constant.go +++ b/ray-operator/controllers/ray/utils/constant.go @@ -208,6 +208,27 @@ const ( // If set to true, we will use deterministic name for head pod. Otherwise, the non-deterministic name is used. ENABLE_DETERMINISTIC_HEAD_POD_NAME = "ENABLE_DETERMINISTIC_HEAD_POD_NAME" + // Dashboard environment variables configure the Ray dashboard URL and TLS settings. + // Set these via the operator Deployment's env section (deployment.yaml / Helm values). + // + // KUBERAY_DASHBOARD_DOMAIN_SUFFIX – domain suffix appended when constructing HTTPS dashboard URLs. + // When set, the operator builds: https://..[:] + // The suffix must include any Kubernetes subdomain prefix, e.g. "svc.cluster.local". + // When unset, the operator falls back to the original plain-HTTP behaviour. + // KUBERAY_DASHBOARD_PORT – port appended to the dashboard URL when KUBERAY_DASHBOARD_DOMAIN_SUFFIX is set. + // When unset, no port is appended. + // + // TLS certificate paths (only relevant when KUBERAY_DASHBOARD_DOMAIN_SUFFIX is set): + // KUBERAY_DASHBOARD_TLS_CA_CERT – path to the PEM CA cert used to verify the dashboard server cert. + // KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to the PEM client cert presented during mTLS handshake. + // KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to the PEM private key for the client cert. + // All three TLS vars must be set together for mTLS; only KUBERAY_DASHBOARD_TLS_CA_CERT is needed for one-way TLS. + KUBERAY_DASHBOARD_DOMAIN_SUFFIX = "KUBERAY_DASHBOARD_DOMAIN_SUFFIX" + KUBERAY_DASHBOARD_PORT = "KUBERAY_DASHBOARD_PORT" + KUBERAY_DASHBOARD_TLS_CA_CERT = "KUBERAY_DASHBOARD_TLS_CA_CERT" + KUBERAY_DASHBOARD_TLS_CLIENT_CERT = "KUBERAY_DASHBOARD_TLS_CLIENT_CERT" + KUBERAY_DASHBOARD_TLS_CLIENT_KEY = "KUBERAY_DASHBOARD_TLS_CLIENT_KEY" + // Ray core default configurations DefaultWorkerRayGcsReconnectTimeoutS = "600" diff --git a/ray-operator/controllers/ray/utils/util.go b/ray-operator/controllers/ray/utils/util.go index 89e96c971d8..b6b9eaf1bd7 100644 --- a/ray-operator/controllers/ray/utils/util.go +++ b/ray-operator/controllers/ray/utils/util.go @@ -3,6 +3,8 @@ package utils import ( "context" "crypto/sha1" //nolint:gosec // We are not using this for security purposes + "crypto/tls" + "crypto/x509" "encoding/base32" "fmt" "math" @@ -965,7 +967,52 @@ func FetchHeadServiceURL(ctx context.Context, cli client.Client, rayCluster *ray return headServiceURL, nil } -func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool, dashboardDomainSuffix, dashboardPort string) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { +// newDashboardHTTPClient builds an *http.Client for dashboard connections. +// When the KUBERAY_DASHBOARD_TLS_* environment variables are set it configures +// TLS (one-way) or mTLS (when client cert+key are also provided); otherwise it +// returns a plain client suitable for plain HTTP connections. +func newDashboardHTTPClient() (*http.Client, error) { + caFile := os.Getenv(KUBERAY_DASHBOARD_TLS_CA_CERT) + certFile := os.Getenv(KUBERAY_DASHBOARD_TLS_CLIENT_CERT) + keyFile := os.Getenv(KUBERAY_DASHBOARD_TLS_CLIENT_KEY) + + if caFile == "" && certFile == "" && keyFile == "" { + return &http.Client{Timeout: 2 * time.Second}, nil + } + + if (certFile == "") != (keyFile == "") { + return nil, fmt.Errorf("%s and %s must both be set for mTLS", KUBERAY_DASHBOARD_TLS_CLIENT_CERT, KUBERAY_DASHBOARD_TLS_CLIENT_KEY) + } + + tlsCfg := &tls.Config{} + + if caFile != "" { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("failed to read dashboard CA cert %q: %w", caFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("failed to parse dashboard CA cert %q", caFile) + } + tlsCfg.RootCAs = pool + } + + if certFile != "" { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("failed to load dashboard client cert/key (%q, %q): %w", certFile, keyFile, err) + } + tlsCfg.Certificates = []tls.Certificate{cert} + } + + return &http.Client{ + Timeout: 2 * time.Second, + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + }, nil +} + +func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { return func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { dashboardClient := &dashboardclient.RayDashboardClient{} var authToken string @@ -1006,20 +1053,19 @@ func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool, das return dashboardClient, nil } - // Build the dashboard URL. - // Priority (highest to lowest): - // 1. Custom HTTPS domain (dashboardDomainSuffix configured in operator) - // 2. Plain HTTP fallback (original behaviour) headSvcName, err := GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) if err != nil { return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) } + httpClient, err := newDashboardHTTPClient() + if err != nil { + return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) + } + dashboardClient.InitClient( - &http.Client{ - Timeout: 2 * time.Second, - }, - BuildDashboardURL(headSvcName, rayCluster.Namespace, dashboardDomainSuffix, dashboardPort, url), + httpClient, + BuildDashboardURL(headSvcName, rayCluster.Namespace, os.Getenv(KUBERAY_DASHBOARD_DOMAIN_SUFFIX), os.Getenv(KUBERAY_DASHBOARD_PORT), url), authToken, ) diff --git a/ray-operator/controllers/ray/utils/util_test.go b/ray-operator/controllers/ray/utils/util_test.go index 41a74e3e2fe..f29488154de 100644 --- a/ray-operator/controllers/ray/utils/util_test.go +++ b/ray-operator/controllers/ray/utils/util_test.go @@ -2,9 +2,18 @@ package utils import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" + "math/big" + "net/http" "os" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1875,3 +1884,109 @@ func TestBuildDashboardURL(t *testing.T) { }) } } + +func TestNewDashboardHTTPClient(t *testing.T) { + // Generate a minimal self-signed CA cert for tests that need a real PEM file. + caCertPEM := generateSelfSignedCACert(t) + caFile := writeTempPEM(t, caCertPEM) + + tests := []struct { + name string + envVars map[string]string + wantTLS bool + wantErr string + }{ + { + name: "no env vars — returns plain HTTP client", + envVars: map[string]string{}, + wantTLS: false, + }, + { + name: "only CA cert — one-way TLS, custom RootCAs set", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CA_CERT: caFile, + }, + wantTLS: true, + }, + { + name: "CA cert file missing — error", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CA_CERT: "/nonexistent/ca.pem", + }, + wantErr: "failed to read dashboard CA cert", + }, + { + name: "cert without key — error", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CLIENT_CERT: caFile, + }, + wantErr: KUBERAY_DASHBOARD_TLS_CLIENT_CERT, + }, + { + name: "key without cert — error", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CLIENT_KEY: caFile, + }, + wantErr: KUBERAY_DASHBOARD_TLS_CLIENT_CERT, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.envVars { + t.Setenv(k, v) + } + + client, err := newDashboardHTTPClient() + + if tt.wantErr != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, client) + return + } + + require.NoError(t, err) + require.NotNil(t, client) + + transport, hasTLSTransport := client.Transport.(*http.Transport) + if tt.wantTLS { + assert.True(t, hasTLSTransport, "expected TLS transport") + assert.NotNil(t, transport.TLSClientConfig) + assert.NotNil(t, transport.TLSClientConfig.RootCAs) + } else { + assert.False(t, hasTLSTransport, "expected plain client with no custom transport") + } + }) + } +} + +// generateSelfSignedCACert returns a PEM-encoded self-signed CA cert for testing. +func generateSelfSignedCACert(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func writeTempPEM(t *testing.T, data []byte) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "*.pem") + require.NoError(t, err) + _, err = f.Write(data) + require.NoError(t, err) + require.NoError(t, f.Close()) + return f.Name() +} diff --git a/ray-operator/main.go b/ray-operator/main.go index ee398ce1256..ceba7d4772e 100644 --- a/ray-operator/main.go +++ b/ray-operator/main.go @@ -66,8 +66,6 @@ func main() { var logFileEncoder string var logStdoutEncoder string var useKubernetesProxy bool - var dashboardDomainSuffix string - var dashboardPort string var configFile string var featureGates string var enableBatchScheduler bool @@ -104,12 +102,6 @@ func main() { flag.StringVar(&configFile, "config", "", "Path to structured config file. Flags are ignored if config file is set.") flag.BoolVar(&useKubernetesProxy, "use-kubernetes-proxy", false, "Use Kubernetes proxy subresource when connecting to the Ray Head node.") - flag.StringVar(&dashboardDomainSuffix, "dashboard-domain-suffix", "", - "Domain suffix for constructing HTTPS dashboard URLs. When set, the operator uses "+ - "https://..[:] instead of the default HTTP URL. "+ - "The suffix should include any Kubernetes subdomain prefix, e.g. svc.example.com.") - flag.StringVar(&dashboardPort, "dashboard-port", "", - "Port appended to the dashboard URL when --dashboard-domain-suffix is set.") flag.StringVar(&featureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates. E.g. FeatureOne=true,FeatureTwo=false,...") flag.BoolVar(&enableMetrics, "enable-metrics", false, "Enable the emission of control plane metrics.") flag.Float64Var(&qps, "qps", float64(configapi.DefaultQPS), "The QPS value for the client communicating with the Kubernetes API server.") @@ -142,8 +134,6 @@ func main() { config.EnableBatchScheduler = enableBatchScheduler config.BatchScheduler = batchScheduler config.UseKubernetesProxy = useKubernetesProxy - config.DashboardDomainSuffix = dashboardDomainSuffix - config.DashboardPort = dashboardPort config.DeleteRayJobAfterJobFinishes = os.Getenv(utils.DELETE_RAYJOB_CR_AFTER_JOB_FINISHES) == "true" config.EnableMetrics = enableMetrics config.QPS = &qps diff --git a/ray-operator/test/sampleyaml/support.go b/ray-operator/test/sampleyaml/support.go index ed7f4eddc5b..42ffd19e0a2 100644 --- a/ray-operator/test/sampleyaml/support.go +++ b/ray-operator/test/sampleyaml/support.go @@ -75,7 +75,7 @@ func QueryDashboardGetAppStatus(t Test, rayCluster *rayv1.RayCluster) func(Gomeg g.Expect(err).ToNot(HaveOccurred()) url := fmt.Sprintf("127.0.0.1:%d", localPort) - rayDashboardClientFunc := utils.GetRayDashboardClientFunc(nil, false, "", "") + rayDashboardClientFunc := utils.GetRayDashboardClientFunc(nil, false) rayDashboardClient, err := rayDashboardClientFunc(rayCluster, url) g.Expect(err).ToNot(HaveOccurred()) serveDetails, err := rayDashboardClient.GetServeDetails(t.Ctx()) From 905b4febdde025bc2ce6f4722a75f6dc5df3c467 Mon Sep 17 00:00:00 2001 From: Savita Manghnani Date: Wed, 20 May 2026 19:49:44 -0400 Subject: [PATCH 3/4] helm: move emissary annotation out of kuberay chart The emissary injection annotation is Datadog-specific infrastructure config and belongs in k8s-resources, not the upstream kuberay chart. Co-Authored-By: Claude Sonnet 4.6 --- helm-chart/kuberay-operator/values.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index 2729ecfea48..30ff0a02355 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -37,10 +37,7 @@ priorityClassName: "" labels: {} # -- Extra annotations. -# Emissary injection provisions mTLS client certificates at /etc/emissary/certs/client/ -# which are consumed by the operator via the KUBERAY_DASHBOARD_TLS_* env vars below. -annotations: - emissary.datadoghq.com/inject-lifecycle: injected +annotations: {} # -- Pod affinity affinity: {} From 9e0a23ef25fb3ed4fb819f1029b85cbc04a7fdd1 Mon Sep 17 00:00:00 2001 From: Savita Manghnani Date: Mon, 25 May 2026 00:13:19 -0700 Subject: [PATCH 4/4] feat: add KUBERAY_DASHBOARD_TLS_SERVER_NAME env var for TLS server name override Allows overriding the server name used for TLS certificate verification when the server cert is issued for a different hostname than the one in the dashboard URL. Co-Authored-By: Claude Sonnet 4.6 --- helm-chart/kuberay-operator/values.yaml | 8 +++++--- ray-operator/controllers/ray/utils/constant.go | 9 ++++++--- ray-operator/controllers/ray/utils/util.go | 4 ++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index 30ff0a02355..dba990cfa99 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -157,9 +157,11 @@ operatorCommand: /manager # The suffix must include any Kubernetes subdomain prefix (e.g. "svc.cluster.local"). # When unset (default), the operator uses the original plain-HTTP behaviour. # KUBERAY_DASHBOARD_PORT – port appended to the URL when DOMAIN_SUFFIX is set. -# KUBERAY_DASHBOARD_TLS_CA_CERT – path to PEM CA cert for server cert verification. -# KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to PEM client cert for mTLS (emissary: /etc/emissary/certs/client/cert.pem). -# KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to PEM client key for mTLS (emissary: /etc/emissary/certs/client/key.pem). +# KUBERAY_DASHBOARD_TLS_CA_CERT – path to PEM CA cert for server cert verification. +# KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to PEM client cert for mTLS. +# KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to PEM client key for mTLS. +# KUBERAY_DASHBOARD_TLS_SERVER_NAME – override the server name used for TLS cert verification. +# Set when the server cert is issued for a different name than the host in the dashboard URL. # # Example: # env: diff --git a/ray-operator/controllers/ray/utils/constant.go b/ray-operator/controllers/ray/utils/constant.go index 7537051f3c3..0a4a6e9eea4 100644 --- a/ray-operator/controllers/ray/utils/constant.go +++ b/ray-operator/controllers/ray/utils/constant.go @@ -219,15 +219,18 @@ const ( // When unset, no port is appended. // // TLS certificate paths (only relevant when KUBERAY_DASHBOARD_DOMAIN_SUFFIX is set): - // KUBERAY_DASHBOARD_TLS_CA_CERT – path to the PEM CA cert used to verify the dashboard server cert. - // KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to the PEM client cert presented during mTLS handshake. - // KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to the PEM private key for the client cert. + // KUBERAY_DASHBOARD_TLS_CA_CERT – path to the PEM CA cert used to verify the dashboard server cert. + // KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to the PEM client cert presented during mTLS handshake. + // KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to the PEM private key for the client cert. // All three TLS vars must be set together for mTLS; only KUBERAY_DASHBOARD_TLS_CA_CERT is needed for one-way TLS. + // KUBERAY_DASHBOARD_TLS_SERVER_NAME – overrides the server name used for TLS certificate verification. + // Useful when the server presents a cert issued for a different name than the host in the dashboard URL. KUBERAY_DASHBOARD_DOMAIN_SUFFIX = "KUBERAY_DASHBOARD_DOMAIN_SUFFIX" KUBERAY_DASHBOARD_PORT = "KUBERAY_DASHBOARD_PORT" KUBERAY_DASHBOARD_TLS_CA_CERT = "KUBERAY_DASHBOARD_TLS_CA_CERT" KUBERAY_DASHBOARD_TLS_CLIENT_CERT = "KUBERAY_DASHBOARD_TLS_CLIENT_CERT" KUBERAY_DASHBOARD_TLS_CLIENT_KEY = "KUBERAY_DASHBOARD_TLS_CLIENT_KEY" + KUBERAY_DASHBOARD_TLS_SERVER_NAME = "KUBERAY_DASHBOARD_TLS_SERVER_NAME" // Ray core default configurations DefaultWorkerRayGcsReconnectTimeoutS = "600" diff --git a/ray-operator/controllers/ray/utils/util.go b/ray-operator/controllers/ray/utils/util.go index b6b9eaf1bd7..e3c3a65f13a 100644 --- a/ray-operator/controllers/ray/utils/util.go +++ b/ray-operator/controllers/ray/utils/util.go @@ -986,6 +986,10 @@ func newDashboardHTTPClient() (*http.Client, error) { tlsCfg := &tls.Config{} + if sn := os.Getenv(KUBERAY_DASHBOARD_TLS_SERVER_NAME); sn != "" { + tlsCfg.ServerName = sn + } + if caFile != "" { caPEM, err := os.ReadFile(caFile) if err != nil {