diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index c1892054323..dba990cfa99 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -149,6 +149,33 @@ operatorCommand: /manager # Using this option to configure kuberay-operator to comunitcate to Ray head pods by proxying through the Kubernetes API Server. # useKubernetesProxy: true +# Dashboard URL and TLS configuration via environment variables. +# Set these under env: to configure HTTPS dashboard connections. +# +# KUBERAY_DASHBOARD_DOMAIN_SUFFIX – domain suffix for HTTPS dashboard URLs. +# When set, the operator builds: https://..[:] +# The suffix must include any Kubernetes subdomain prefix (e.g. "svc.cluster.local"). +# When unset (default), the operator uses the original plain-HTTP behaviour. +# KUBERAY_DASHBOARD_PORT – port appended to the URL when DOMAIN_SUFFIX is set. +# KUBERAY_DASHBOARD_TLS_CA_CERT – path to PEM CA cert for server cert verification. +# KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to PEM client cert for mTLS. +# KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to PEM client key for mTLS. +# KUBERAY_DASHBOARD_TLS_SERVER_NAME – override the server name used for TLS cert verification. +# Set when the server cert is issued for a different name than the host in the dashboard URL. +# +# Example: +# env: +# - name: KUBERAY_DASHBOARD_DOMAIN_SUFFIX +# value: "svc.$(CLUSTER_DOMAIN)" +# - name: KUBERAY_DASHBOARD_PORT +# value: "8266" +# - name: KUBERAY_DASHBOARD_TLS_CA_CERT +# value: /etc/emissary/certs/client/ca.pem +# - name: KUBERAY_DASHBOARD_TLS_CLIENT_CERT +# value: /etc/emissary/certs/client/cert.pem +# - name: KUBERAY_DASHBOARD_TLS_CLIENT_KEY +# value: /etc/emissary/certs/client/key.pem + # -- If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability. leaderElectionEnabled: true diff --git a/ray-operator/controllers/ray/utils/constant.go b/ray-operator/controllers/ray/utils/constant.go index 38b94c9b715..0a4a6e9eea4 100644 --- a/ray-operator/controllers/ray/utils/constant.go +++ b/ray-operator/controllers/ray/utils/constant.go @@ -208,6 +208,30 @@ const ( // If set to true, we will use deterministic name for head pod. Otherwise, the non-deterministic name is used. ENABLE_DETERMINISTIC_HEAD_POD_NAME = "ENABLE_DETERMINISTIC_HEAD_POD_NAME" + // Dashboard environment variables configure the Ray dashboard URL and TLS settings. + // Set these via the operator Deployment's env section (deployment.yaml / Helm values). + // + // KUBERAY_DASHBOARD_DOMAIN_SUFFIX – domain suffix appended when constructing HTTPS dashboard URLs. + // When set, the operator builds: https://..[:] + // The suffix must include any Kubernetes subdomain prefix, e.g. "svc.cluster.local". + // When unset, the operator falls back to the original plain-HTTP behaviour. + // KUBERAY_DASHBOARD_PORT – port appended to the dashboard URL when KUBERAY_DASHBOARD_DOMAIN_SUFFIX is set. + // When unset, no port is appended. + // + // TLS certificate paths (only relevant when KUBERAY_DASHBOARD_DOMAIN_SUFFIX is set): + // KUBERAY_DASHBOARD_TLS_CA_CERT – path to the PEM CA cert used to verify the dashboard server cert. + // KUBERAY_DASHBOARD_TLS_CLIENT_CERT – path to the PEM client cert presented during mTLS handshake. + // KUBERAY_DASHBOARD_TLS_CLIENT_KEY – path to the PEM private key for the client cert. + // All three TLS vars must be set together for mTLS; only KUBERAY_DASHBOARD_TLS_CA_CERT is needed for one-way TLS. + // KUBERAY_DASHBOARD_TLS_SERVER_NAME – overrides the server name used for TLS certificate verification. + // Useful when the server presents a cert issued for a different name than the host in the dashboard URL. + KUBERAY_DASHBOARD_DOMAIN_SUFFIX = "KUBERAY_DASHBOARD_DOMAIN_SUFFIX" + KUBERAY_DASHBOARD_PORT = "KUBERAY_DASHBOARD_PORT" + KUBERAY_DASHBOARD_TLS_CA_CERT = "KUBERAY_DASHBOARD_TLS_CA_CERT" + KUBERAY_DASHBOARD_TLS_CLIENT_CERT = "KUBERAY_DASHBOARD_TLS_CLIENT_CERT" + KUBERAY_DASHBOARD_TLS_CLIENT_KEY = "KUBERAY_DASHBOARD_TLS_CLIENT_KEY" + KUBERAY_DASHBOARD_TLS_SERVER_NAME = "KUBERAY_DASHBOARD_TLS_SERVER_NAME" + // Ray core default configurations DefaultWorkerRayGcsReconnectTimeoutS = "600" diff --git a/ray-operator/controllers/ray/utils/util.go b/ray-operator/controllers/ray/utils/util.go index 70b1dadf3e1..e3c3a65f13a 100644 --- a/ray-operator/controllers/ray/utils/util.go +++ b/ray-operator/controllers/ray/utils/util.go @@ -3,6 +3,8 @@ package utils import ( "context" "crypto/sha1" //nolint:gosec // We are not using this for security purposes + "crypto/tls" + "crypto/x509" "encoding/base32" "fmt" "math" @@ -73,6 +75,28 @@ func GetClusterDomainName() string { return DefaultDomainName } +// BuildDashboardURL constructs the dashboard URL for a given head service name and namespace. +// If domainSuffix is non-empty, it constructs an HTTPS URL using the custom domain with the format: +// +// https://..[:] +// +// domainSuffix should include any Kubernetes subdomain prefix, e.g.: +// +// "svc.example.com" +// → raycluster-head-svc.my-namespace.svc.example.com:8266 +// +// If domainSuffix is empty it falls back to the plain HTTP URL built from fallbackURL +// (backwards-compatible default behaviour). +func BuildDashboardURL(headSvcName, namespace, domainSuffix, port, fallbackURL string) string { + if domainSuffix == "" { + return "http://" + fallbackURL + } + if port != "" { + return fmt.Sprintf("https://%s.%s.%s:%s", headSvcName, namespace, domainSuffix, port) + } + return fmt.Sprintf("https://%s.%s.%s", headSvcName, namespace, domainSuffix) +} + // IsCreated returns true if pod has been created and is maintained by the API server func IsCreated(pod *corev1.Pod) bool { return pod.Status.Phase != "" @@ -943,6 +967,55 @@ func FetchHeadServiceURL(ctx context.Context, cli client.Client, rayCluster *ray return headServiceURL, nil } +// newDashboardHTTPClient builds an *http.Client for dashboard connections. +// When the KUBERAY_DASHBOARD_TLS_* environment variables are set it configures +// TLS (one-way) or mTLS (when client cert+key are also provided); otherwise it +// returns a plain client suitable for plain HTTP connections. +func newDashboardHTTPClient() (*http.Client, error) { + caFile := os.Getenv(KUBERAY_DASHBOARD_TLS_CA_CERT) + certFile := os.Getenv(KUBERAY_DASHBOARD_TLS_CLIENT_CERT) + keyFile := os.Getenv(KUBERAY_DASHBOARD_TLS_CLIENT_KEY) + + if caFile == "" && certFile == "" && keyFile == "" { + return &http.Client{Timeout: 2 * time.Second}, nil + } + + if (certFile == "") != (keyFile == "") { + return nil, fmt.Errorf("%s and %s must both be set for mTLS", KUBERAY_DASHBOARD_TLS_CLIENT_CERT, KUBERAY_DASHBOARD_TLS_CLIENT_KEY) + } + + tlsCfg := &tls.Config{} + + if sn := os.Getenv(KUBERAY_DASHBOARD_TLS_SERVER_NAME); sn != "" { + tlsCfg.ServerName = sn + } + + if caFile != "" { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("failed to read dashboard CA cert %q: %w", caFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("failed to parse dashboard CA cert %q", caFile) + } + tlsCfg.RootCAs = pool + } + + if certFile != "" { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("failed to load dashboard client cert/key (%q, %q): %w", certFile, keyFile, err) + } + tlsCfg.Certificates = []tls.Certificate{cert} + } + + return &http.Client{ + Timeout: 2 * time.Second, + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + }, nil +} + func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { return func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) { dashboardClient := &dashboardclient.RayDashboardClient{} @@ -969,14 +1042,9 @@ func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) fun } if useKubernetesProxy { - var err error - headSvcName := rayCluster.Status.Head.ServiceName - if headSvcName == "" { - headSvcName, err = GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) - if err != nil { - err = fmt.Errorf("failed to construct Ray dashboard client: %w", err) - return nil, err - } + headSvcName, err := GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) + if err != nil { + return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) } dashboardClient.InitClient( @@ -989,11 +1057,19 @@ func GetRayDashboardClientFunc(mgr manager.Manager, useKubernetesProxy bool) fun return dashboardClient, nil } + headSvcName, err := GenerateHeadServiceName(RayClusterCRD, rayCluster.Spec, rayCluster.Name) + if err != nil { + return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) + } + + httpClient, err := newDashboardHTTPClient() + if err != nil { + return nil, fmt.Errorf("failed to construct Ray dashboard client: %w", err) + } + dashboardClient.InitClient( - &http.Client{ - Timeout: 2 * time.Second, - }, - "http://"+url, + httpClient, + BuildDashboardURL(headSvcName, rayCluster.Namespace, os.Getenv(KUBERAY_DASHBOARD_DOMAIN_SUFFIX), os.Getenv(KUBERAY_DASHBOARD_PORT), url), authToken, ) diff --git a/ray-operator/controllers/ray/utils/util_test.go b/ray-operator/controllers/ray/utils/util_test.go index 47afa55c85a..f29488154de 100644 --- a/ray-operator/controllers/ray/utils/util_test.go +++ b/ray-operator/controllers/ray/utils/util_test.go @@ -2,9 +2,18 @@ package utils import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" + "math/big" + "net/http" "os" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1792,3 +1801,192 @@ func TestGetWeightsFromHTTPRoute(t *testing.T) { }) } } + +func TestBuildDashboardURL(t *testing.T) { + tests := []struct { + name string + headSvcName string + namespace string + domainSuffix string + port string + fallbackURL string + want string + }{ + { + name: "no domain suffix — uses HTTP fallback URL (backwards compatible)", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "", + port: "", + fallbackURL: "10.0.0.1:8265", + want: "http://10.0.0.1:8265", + }, + { + name: "no domain suffix — port is ignored", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "", + port: "8266", + fallbackURL: "10.0.0.1:8265", + want: "http://10.0.0.1:8265", + }, + { + name: "suffix includes svc. — constructs HTTPS URL with namespace and port", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "svc.example.com", + port: "8266", + fallbackURL: "10.0.0.1:8265", + want: "https://raycluster-head-svc.my-namespace.svc.example.com:8266", + }, + { + name: "suffix includes svc. — constructs HTTPS URL with namespace, no port", + headSvcName: "raycluster-head-svc", + namespace: "my-namespace", + domainSuffix: "svc.example.com", + port: "", + fallbackURL: "10.0.0.1:8265", + want: "https://raycluster-head-svc.my-namespace.svc.example.com", + }, + { + name: "custom head service name with dashes in different namespace", + headSvcName: "my-cluster-kuberay-head-svc", + namespace: "ml-team", + domainSuffix: "svc.prod.example.com", + port: "443", + fallbackURL: "10.0.0.2:8265", + want: "https://my-cluster-kuberay-head-svc.ml-team.svc.prod.example.com:443", + }, + { + name: "non-svc subdomain prefix", + headSvcName: "my-ray-cluster-head-svc", + namespace: "my-namespace", + domainSuffix: "mesh.example.com", + port: "", + fallbackURL: "10.0.0.1:8265", + want: "https://my-ray-cluster-head-svc.my-namespace.mesh.example.com", + }, + { + name: "empty fallback URL when no domain suffix", + headSvcName: "raycluster-head-svc", + namespace: "default", + domainSuffix: "", + port: "", + fallbackURL: "", + want: "http://", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildDashboardURL(tt.headSvcName, tt.namespace, tt.domainSuffix, tt.port, tt.fallbackURL) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestNewDashboardHTTPClient(t *testing.T) { + // Generate a minimal self-signed CA cert for tests that need a real PEM file. + caCertPEM := generateSelfSignedCACert(t) + caFile := writeTempPEM(t, caCertPEM) + + tests := []struct { + name string + envVars map[string]string + wantTLS bool + wantErr string + }{ + { + name: "no env vars — returns plain HTTP client", + envVars: map[string]string{}, + wantTLS: false, + }, + { + name: "only CA cert — one-way TLS, custom RootCAs set", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CA_CERT: caFile, + }, + wantTLS: true, + }, + { + name: "CA cert file missing — error", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CA_CERT: "/nonexistent/ca.pem", + }, + wantErr: "failed to read dashboard CA cert", + }, + { + name: "cert without key — error", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CLIENT_CERT: caFile, + }, + wantErr: KUBERAY_DASHBOARD_TLS_CLIENT_CERT, + }, + { + name: "key without cert — error", + envVars: map[string]string{ + KUBERAY_DASHBOARD_TLS_CLIENT_KEY: caFile, + }, + wantErr: KUBERAY_DASHBOARD_TLS_CLIENT_CERT, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.envVars { + t.Setenv(k, v) + } + + client, err := newDashboardHTTPClient() + + if tt.wantErr != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, client) + return + } + + require.NoError(t, err) + require.NotNil(t, client) + + transport, hasTLSTransport := client.Transport.(*http.Transport) + if tt.wantTLS { + assert.True(t, hasTLSTransport, "expected TLS transport") + assert.NotNil(t, transport.TLSClientConfig) + assert.NotNil(t, transport.TLSClientConfig.RootCAs) + } else { + assert.False(t, hasTLSTransport, "expected plain client with no custom transport") + } + }) + } +} + +// generateSelfSignedCACert returns a PEM-encoded self-signed CA cert for testing. +func generateSelfSignedCACert(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func writeTempPEM(t *testing.T, data []byte) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "*.pem") + require.NoError(t, err) + _, err = f.Write(data) + require.NoError(t, err) + require.NoError(t, f.Close()) + return f.Name() +}