Add NX-API PKCS#12 workaround for certificate provisioning

config/develop/manager_patch.yaml

@@ -6,3 +6,4 @@
     - --provider=openconfig
     - --requeue-interval=30s
     - --max-concurrent-reconciles=5
+    - --zap-log-level=3

go.mod

@@ -31,6 +31,7 @@ require (
 	rsc.io/script v0.0.2
 	sigs.k8s.io/controller-runtime v0.24.1
 	sigs.k8s.io/yaml v1.6.0
+	software.sslmate.com/src/go-pkcs12 v0.7.1
 )
 
 require (

internal/clientutil/clientutil.go

@@ -174,13 +174,20 @@ func (c *Client) TLSSecretPEM(ctx context.Context, ref *v1alpha1.SecretReference
 
 // Certificate loads a [tls.Certificate] from the referenced secret resource.
 // The secret must be of type 'kubernetes.io/tls' and contain the fields 'tls.crt' and 'tls.key'.
+// If a 'ca.crt' field is present, it is appended to the certificate chain.
 func (c *Client) Certificate(ctx context.Context, ref *v1alpha1.SecretReference) (*tls.Certificate, error) {
 	pem, err := c.TLSSecretPEM(ctx, ref)
 	if err != nil {
 		return nil, err
 	}
 
-	certificate, err := tls.X509KeyPair(pem.Certificate, pem.PrivateKey)
+	certPEMBlock := pem.Certificate
+	if len(pem.CA) > 0 {
+		certPEMBlock = append(certPEMBlock, '\n')
+		certPEMBlock = append(certPEMBlock, pem.CA...)
+	}
+
+	certificate, err := tls.X509KeyPair(certPEMBlock, pem.PrivateKey)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load x509 key pair: %w", err)
 	}

internal/provider/cisco/nxos/cert.go

@@ -6,23 +6,25 @@ package nxos
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
 	"crypto/rsa"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
 
-	"github.com/openconfig/gnoi/cert"
+	certpb "github.com/openconfig/gnoi/cert"
 	"google.golang.org/grpc"
+	pkcs12 "software.sslmate.com/src/go-pkcs12"
 
 	"github.com/ironcore-dev/network-operator/internal/transport/gnmiext"
 )
 
 // Certificate represents a X.509 certificate and its associated private key.
 // It can be used to load the certificate into a NX-OS device truspoint via gNOI.
 type Certificate struct {
-	Key  *rsa.PrivateKey
-	Cert *x509.Certificate
+	Key     *rsa.PrivateKey
+	Cert    *x509.Certificate
+	CACerts []*x509.Certificate
 }
 
 // Load loads the certificate into the specified trustpoint via the gNOI [cert service].
@@ -39,26 +41,31 @@ func (c *Certificate) Load(ctx context.Context, conn *grpc.ClientConn, trustpoin
 		return err
 	}
 
-	// Only the `LoadCertificate` method is currently supported on the Nexus 9000 series, despite the fact that the gNOI certificate service is deprecated in favor of the gNSI certz service.
-	// See: https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/programmability/cisco-nexus-9000-series-nx-os-programmability-guide-104x/gnoi---operation-interface.html
-	_, err = cert.NewCertificateManagementClient(conn).LoadCertificate(ctx, &cert.LoadCertificateRequest{ //nolint:staticcheck
-		Certificate:   &cert.Certificate{Type: cert.CertificateType_CT_X509, Certificate: b},
-		KeyPair:       &cert.KeyPair{PrivateKey: priv, PublicKey: pub},
-		CertificateId: trustpoint,
+	var chain []*certpb.Certificate
+	for _, ca := range c.CACerts {
+		var buf bytes.Buffer
+		if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: ca.Raw}); err != nil {
+			return fmt.Errorf("failed to encode CA certificate: %w", err)
+		}
+		chain = append(chain, &certpb.Certificate{Type: certpb.CertificateType_CT_X509, Certificate: buf.Bytes()})
+	}
+
+	// Nexus 9000 series only supports the gNOI certificate management service, despite the fact it is deprecated in favor of the gNSI certz service.
+	// See: https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/106x/programmability/cisco-nexus-9000-series-nx-os-programmability-guide-106x/gnoi---operation-interface.html
+	_, err = certpb.NewCertificateManagementClient(conn).LoadCertificate(ctx, &certpb.LoadCertificateRequest{ //nolint:staticcheck
+		Certificate:    &certpb.Certificate{Type: certpb.CertificateType_CT_X509, Certificate: b},
+		KeyPair:        &certpb.KeyPair{PrivateKey: priv, PublicKey: pub},
+		CertificateId:  trustpoint,
+		CaCertificates: chain,
 	}, grpc.WaitForReady(true))
 	return err
 }
 
 func (c *Certificate) Encode() ([]byte, error) {
-	// Self-sign the certificate as Cisco NX-OS does not support uploading a certificate chain via gNOI.
-	der, err := x509.CreateCertificate(rand.Reader, c.Cert, c.Cert, &c.Key.PublicKey, c.Key)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create certificate: %w", err)
-	}
 	var buf bytes.Buffer
-	err = pem.Encode(&buf, &pem.Block{
+	err := pem.Encode(&buf, &pem.Block{
 		Type:  "CERTIFICATE",
-		Bytes: der,
+		Bytes: c.Cert.Raw,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to encode certificate: %w", err)
@@ -115,3 +122,29 @@ func (*KeyPair) IsListItem() {}
 func (k *KeyPair) XPath() string {
 	return "System/userext-items/pkiext-items/keyring-items/KeyRing-list[name=" + k.Name + "]"
 }
+
+// EncodeCertificatePKCS12 encodes a tls.Certificate (leaf + chain + private key) into
+// PKCS#12 format protected by the given passphrase. LegacyRC2 encoding is used
+// for compatibility with NX-OS.
+func EncodeCertificatePKCS12(cert *tls.Certificate, passphrase string) ([]byte, error) {
+	key, ok := cert.PrivateKey.(*rsa.PrivateKey)
+	if !ok {
+		return nil, fmt.Errorf("unsupported private key type: expected *rsa.PrivateKey, got %T", cert.PrivateKey)
+	}
+
+	var chain []*x509.Certificate
+	for _, der := range cert.Certificate[1:] {
+		ca, err := x509.ParseCertificate(der)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse CA certificate: %w", err)
+		}
+		chain = append(chain, ca)
+	}
+
+	pfx, err := pkcs12.LegacyRC2.Encode(key, cert.Leaf, chain, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode PKCS#12: %w", err)
+	}
+
+	return pfx, nil
+}

internal/provider/cisco/nxos/provider.go

@@ -9,6 +9,10 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"maps"
@@ -91,7 +95,7 @@ func (p *Provider) Connect(ctx context.Context, conn *deviceutil.Connection) (er
 	}
 	// NXAPI only uses the address for URI construction.
 	c := *conn
-	c.Address = netip.MustParseAddrPort(conn.Address).String()
+	c.Address = netip.MustParseAddrPort(conn.Address).Addr().String()
 	p.nxapi, err = nxapi.NewClient(&c, timeout)
 	if err != nil {
 		return fmt.Errorf("failed to create nxapi client: %w", err)
@@ -641,27 +645,124 @@ func (p *Provider) GetPeerStatus(ctx context.Context, req *provider.BGPPeerStatu
 }
 
 func (p *Provider) EnsureCertificate(ctx context.Context, req *provider.EnsureCertificateRequest) error {
-	tp := new(Trustpoint)
-	tp.Name = req.ID
+	logger := logr.FromContextOrDiscard(ctx)
 
-	if err := p.Patch(ctx, tp); err != nil {
-		return err
+	if serial, err := p.installedCertSerial(ctx, req.ID); err == nil {
+		want := strings.ToUpper(req.Certificate.Leaf.SerialNumber.Text(16))
+		if strings.TrimLeft(serial, "0") == strings.TrimLeft(want, "0") {
+			logger.V(1).Info("Certificate already installed with matching serial", "serial", serial)
+			return nil
+		}
 	}
 
-	key, ok := req.Certificate.PrivateKey.(*rsa.PrivateKey)
-	if !ok {
-		return fmt.Errorf("unsupported private key type: expected *rsa.PrivateKey, got %T", req.Certificate.PrivateKey)
+	if NXVersion(p.client.Capabilities()) >= VersionNX10_7_1 {
+		tp := new(Trustpoint)
+		tp.Name = req.ID
+
+		if err := p.Patch(ctx, tp); err != nil {
+			return err
+		}
+
+		key, ok := req.Certificate.PrivateKey.(*rsa.PrivateKey)
+		if !ok {
+			return apistatus.NewInvalidArgumentError(apistatus.FieldViolation{
+				Field:       "spec.certificate.privateKey",
+				Description: fmt.Sprintf("unsupported private key type: expected *rsa.PrivateKey, got %T", req.Certificate.PrivateKey),
+			})
+		}
+
+		cert := &Certificate{Key: key, Cert: req.Certificate.Leaf}
+		for _, der := range req.Certificate.Certificate[1:] {
+			ca, err := x509.ParseCertificate(der)
+			if err != nil {
+				return fmt.Errorf("failed to parse CA certificate: %w", err)
+			}
+			cert.CACerts = append(cert.CACerts, ca)
+		}
+
+		err := cert.Load(ctx, p.conn, req.ID)
+		if err != nil {
+			return fmt.Errorf("failed to upload certificate via gNOI: %w", err)
+		}
+
+		logger.V(2).Info("Certificate upload completed")
+		return nil
 	}
 
-	kp := new(KeyPair)
-	kp.Name = req.ID
-	if err := p.client.GetConfig(ctx, kp); !errors.Is(err, gnmiext.ErrNil) {
-		// If the key pair already exists, we cannot update it, so we skip the rest of the process.
+	var pass [16]byte
+	_, _ = rand.Read(pass[:])
+	passphrase := hex.EncodeToString(pass[:])
+
+	pfx, err := EncodeCertificatePKCS12(req.Certificate, passphrase)
+	if err != nil {
+		return err
+	}
+
+	// Delete any existing trustpoint and RSA key before importing.
+	// gNMI delete is idempotent, so this is safe even on a fresh device.
+	if err := p.DeleteCertificate(ctx, &provider.DeleteCertificateRequest{ID: req.ID}); err != nil {
 		return err
 	}
 
-	cert := &Certificate{Key: key, Cert: req.Certificate.Leaf}
-	return cert.Load(ctx, p.conn, req.ID)
+	b64 := base64.StdEncoding.EncodeToString(pfx)
+	file := "/bootflash/" + req.ID + ".pfx"
+	b64File := file + ".b64"
+
+	cmds := []string{
+		"feature bash-shell",
+		"crypto ca trustpoint " + req.ID,
+	}
+	const chunkSize = 512
+	for i := 0; i < len(b64); i += chunkSize {
+		end := min(i+chunkSize, len(b64))
+		op := " >> "
+		if i == 0 {
+			op = " > "
+		}
+		cmds = append(cmds, "run bash echo -n '"+b64[i:end]+"'"+op+b64File)
+	}
+	cmds = append(
+		cmds,
+		"run bash base64 -d "+b64File+" > "+file,
+		"run bash rm "+b64File,
+		"crypto ca import "+req.ID+" pkcs12 bootflash:///"+req.ID+".pfx "+passphrase,
+		"run bash rm "+file,
+	)
+
+	_, err = p.nxapi.Do(ctx, nxapi.NewRequest(cmds...).WithRollback(nxapi.Stop))
+	if err != nil {
+		return fmt.Errorf("failed to upload certificate via NX-API: %w", err)
+	}
+
+	logger.V(2).Info("Certificate upload completed")
+	return nil
+}
+
+// installedCertSerial queries the device for the certificate installed under the
+// given trustpoint and returns its serial number as an uppercase hex string.
+// If the trustpoint does not exist or has no certificate, an error is returned.
+func (p *Provider) installedCertSerial(ctx context.Context, trustpoint string) (string, error) {
+	res, err := p.nxapi.Do(ctx, nxapi.NewRequest("show crypto ca certificates "+trustpoint))
+	if err != nil {
+		return "", err
+	}
+	if len(res) == 0 {
+		return "", errors.New("empty response")
+	}
+	var body struct {
+		Certificate struct {
+			Cert string `json:"certificate"`
+		} `json:"Certificate"`
+	}
+	if err := json.Unmarshal(res[0], &body); err != nil {
+		return "", err
+	}
+	for line := range strings.SplitSeq(body.Certificate.Cert, "\n") {
+		if after, ok := strings.CutPrefix(line, "serial="); ok {
+			return strings.ToUpper(strings.TrimSpace(after)), nil
+		}
+	}
+	return "", errors.New("serial not found in certificate output")
 }
 
 func (p *Provider) DeleteCertificate(ctx context.Context, req *provider.DeleteCertificateRequest) error {

internal/provider/cisco/nxos/version.go

@@ -6,22 +6,55 @@ package nxos
 import "github.com/ironcore-dev/network-operator/internal/transport/gnmiext"
 
 // Version represents the operating system version of the target device.
-type Version string
+// Versions are ordered so that comparison operators (>, >=, <, <=) reflect
+// the actual release ordering.
+type Version uint8
 
 const (
-	VersionUnknown  Version = "Unknown"
-	VersionNX10_4_3 Version = "10.4(3)"
-	VersionNX10_4_4 Version = "10.4(4)"
-	VersionNX10_4_5 Version = "10.4(5)"
-	VersionNX10_4_6 Version = "10.4(6)"
-	VersionNX10_5_1 Version = "10.5(1)"
-	VersionNX10_5_2 Version = "10.5(2)"
-	VersionNX10_5_3 Version = "10.5(3)"
-	VersionNX10_6_1 Version = "10.6(1)"
-	VersionNX10_6_2 Version = "10.6(2)"
-	VersionNX10_6_3 Version = "10.6(3)"
+	VersionUnknown  Version = iota
+	VersionNX10_4_3         // 10.4(3)
+	VersionNX10_4_4         // 10.4(4)
+	VersionNX10_4_5         // 10.4(5)
+	VersionNX10_4_6         // 10.4(6)
+	VersionNX10_5_1         // 10.5(1)
+	VersionNX10_5_2         // 10.5(2)
+	VersionNX10_5_3         // 10.5(3)
+	VersionNX10_6_1         // 10.6(1)
+	VersionNX10_6_2         // 10.6(2)
+	VersionNX10_6_3         // 10.6(3)
+
+	VersionNX10_7_1 // 10.7(1)
 )
 
+func (v Version) String() string {
+	switch v {
+	case VersionNX10_4_3:
+		return "10.4(3)"
+	case VersionNX10_4_4:
+		return "10.4(4)"
+	case VersionNX10_4_5:
+		return "10.4(5)"
+	case VersionNX10_4_6:
+		return "10.4(6)"
+	case VersionNX10_5_1:
+		return "10.5(1)"
+	case VersionNX10_5_2:
+		return "10.5(2)"
+	case VersionNX10_5_3:
+		return "10.5(3)"
+	case VersionNX10_6_1:
+		return "10.6(1)"
+	case VersionNX10_6_2:
+		return "10.6(2)"
+	case VersionNX10_6_3:
+		return "10.6(3)"
+	case VersionNX10_7_1:
+		return "10.7(1)"
+	default:
+		return "Unknown"
+	}
+}
+
 // nxosVersions maps the revision date of the Cisco-NX-OS-device yang model to the corresponding [Version].
 // It is used to determine the version of the target device based on the capabilities returned by the device.
 var nxosVersions = map[string]Version{
@@ -35,6 +68,10 @@ var nxosVersions = map[string]Version{
 	"2025-08-12": VersionNX10_6_1,
 	"2025-12-12": VersionNX10_6_2,
 	"2026-04-24": VersionNX10_6_3,
+
+	// VersionNX10_7_1 is the minimum version that supports CA chains via gNOI LoadCertificate.
+	// TODO: Update with the correct YANG model revision date once NX-OS 10.7(1) is released.
+	"2026-05-31": VersionNX10_7_1,
 }
 
 // NXVersion returns the NX-OS operating system version of the target device based on the supported models.

internal/transport/nxapi/nxapi.go

@@ -220,6 +220,12 @@ type RPCError struct {
 }
 
 func (e *RPCError) Error() string {
+	var detail struct {
+		Msg string `json:"msg"`
+	}
+	if json.Unmarshal(e.Data, &detail) == nil && detail.Msg != "" {
+		return fmt.Sprintf("nxapi: RPC error %d: %s: %s", e.Code, e.Message, detail.Msg)
+	}
 	return fmt.Sprintf("nxapi: RPC error %d: %s", e.Code, e.Message)
 }