diff --git a/internal/controller/core/suite_test.go b/internal/controller/core/suite_test.go index 81065d523..760555e02 100644 --- a/internal/controller/core/suite_test.go +++ b/internal/controller/core/suite_test.go @@ -6,6 +6,7 @@ package core import ( "context" "errors" + "fmt" "os" "path/filepath" "strconv" @@ -568,6 +569,10 @@ func (p *Provider) InterfaceNameEqual(_ context.Context, a, b string) (bool, err return a == b, nil } +func (p *Provider) LoopbackInterfaceName(id int) (string, error) { + return fmt.Sprintf("lo%d", id), nil +} + func (p *Provider) EnsureBanner(_ context.Context, req *provider.EnsureBannerRequest) error { p.Lock() defer p.Unlock() diff --git a/internal/controller/evpn/fabric_controller.go b/internal/controller/evpn/fabric_controller.go index 159018ddd..35b81b435 100644 --- a/internal/controller/evpn/fabric_controller.go +++ b/internal/controller/evpn/fabric_controller.go @@ -9,17 +9,23 @@ import ( "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/ironcore-dev/network-operator/api/core/v1alpha1" evpnv1alpha1 "github.com/ironcore-dev/network-operator/api/evpn/v1alpha1" + poolv1alpha1 "github.com/ironcore-dev/network-operator/api/pool/v1alpha1" "github.com/ironcore-dev/network-operator/internal/conditions" "github.com/ironcore-dev/network-operator/internal/provider" ) @@ -36,13 +42,17 @@ type FabricReconciler struct { // More info: https://book.kubebuilder.io/reference/raising-events Recorder events.EventRecorder - // Provider is the driver that will be used to create & delete the interface. + // Provider is the driver that will be used to create interfaces. Provider provider.ProviderFunc } // +kubebuilder:rbac:groups=evpn.networking.metal.ironcore.dev,resources=fabrics,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=evpn.networking.metal.ironcore.dev,resources=fabrics/status,verbs=get;update;patch // +kubebuilder:rbac:groups=evpn.networking.metal.ironcore.dev,resources=fabrics/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.metal.ironcore.dev,resources=devices,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.metal.ironcore.dev,resources=interfaces,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=pool.networking.metal.ironcore.dev,resources=claims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=pool.networking.metal.ironcore.dev,resources=ipaddresspools,verbs=get;list;watch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -64,6 +74,18 @@ func (r *FabricReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c return ctrl.Result{}, err } + if _, ok := r.Provider().(provider.InterfaceProvider); !ok { + if meta.SetStatusCondition(&fabric.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReadyCondition, + Status: metav1.ConditionFalse, + Reason: v1alpha1.NotImplementedReason, + Message: "Provider does not implement provider.InterfaceProvider", + }) { + return ctrl.Result{}, r.Status().Update(ctx, fabric) + } + return ctrl.Result{}, nil + } + if !fabric.DeletionTimestamp.IsZero() { if controllerutil.ContainsFinalizer(fabric, evpnv1alpha1.FinalizerName) { if err := r.finalize(ctx, fabric); err != nil { @@ -136,6 +158,15 @@ func (r *FabricReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&evpnv1alpha1.Fabric{}). + Owns(&poolv1alpha1.Claim{}). + Owns(&v1alpha1.Interface{}). + // Re-reconcile when a Device's labels change so that devices newly + // matching a deviceSelector are enrolled into the fabric. + Watches( + &v1alpha1.Device{}, + handler.EnqueueRequestsFromMapFunc(r.devicesToFabrics), + builder.WithPredicates(predicate.LabelChangedPredicate{}), + ). WithEventFilter(filter). Named("evpn-fabric"). Complete(r) @@ -147,7 +178,9 @@ type ReconcileFunc func(context.Context, *evpnv1alpha1.Fabric) (ctrl.Result, err func (r *FabricReconciler) reconcile(ctx context.Context, fabric *evpnv1alpha1.Fabric) (ctrl.Result, error) { phases := []ReconcileFunc{ - // r.reconcileNodes, + r.reconcileSystemLoopbacks, + r.reconcileVTEPLoopbacks, + r.reconcileAnycastRPLoopbacks, } for _, phase := range phases { res, err := phase(ctx, fabric) @@ -169,3 +202,207 @@ func (r *FabricReconciler) finalize(ctx context.Context, fabric *evpnv1alpha1.Fa _ = fabric return nil } + +const ( + LoopbackRouterID = 0 // Router-ID and BGP source address, present on all fabric devices + LoopbackVTEP = 1 // Primary VTEP address, present on VTEP devices + LoopbackVTEPAnycast = 2 // Anycast VTEP address, present on VTEP devices (deprecated in favour of ESI) + LoopbackAnycastRP = 100 // PIM anycast rendezvous point address, shared across RP devices +) + +// reconcileSystemLoopbacks ensures lo0 (Router-ID / BGP source) exists on every fabric device. +func (r *FabricReconciler) reconcileSystemLoopbacks(ctx context.Context, fabric *evpnv1alpha1.Fabric) (ctrl.Result, error) { + selector, err := metav1.LabelSelectorAsSelector(&fabric.Spec.DeviceSelector) + if err != nil { + return ctrl.Result{}, reconcile.TerminalError(fmt.Errorf("invalid deviceSelector: %w", err)) + } + devices := &v1alpha1.DeviceList{} + if err := r.List(ctx, devices, client.InNamespace(fabric.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return ctrl.Result{}, fmt.Errorf("listing devices: %w", err) + } + for i := range devices.Items { + claimName := fmt.Sprintf("%s-%s-lo%d", fabric.Name, devices.Items[i].Name, LoopbackRouterID) + claim, err := r.reconcileLoopbackClaim(ctx, fabric, claimName) + if err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileLoopbackInterface(ctx, fabric, &devices.Items[i], LoopbackRouterID, claim); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +// reconcileVTEPLoopbacks ensures lo1 (primary VTEP) and lo2 (anycast VTEP) exist on VTEP devices. +func (r *FabricReconciler) reconcileVTEPLoopbacks(ctx context.Context, fabric *evpnv1alpha1.Fabric) (ctrl.Result, error) { + selector, err := metav1.LabelSelectorAsSelector(&fabric.Spec.VTEP.DeviceSelector) + if err != nil { + return ctrl.Result{}, reconcile.TerminalError(fmt.Errorf("invalid vtep deviceSelector: %w", err)) + } + devices := &v1alpha1.DeviceList{} + if err := r.List(ctx, devices, client.InNamespace(fabric.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return ctrl.Result{}, fmt.Errorf("listing VTEP devices: %w", err) + } + for i := range devices.Items { + for _, id := range []int{LoopbackVTEP, LoopbackVTEPAnycast} { + claimName := fmt.Sprintf("%s-%s-lo%d", fabric.Name, devices.Items[i].Name, id) + claim, err := r.reconcileLoopbackClaim(ctx, fabric, claimName) + if err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileLoopbackInterface(ctx, fabric, &devices.Items[i], id, claim); err != nil { + return ctrl.Result{}, err + } + } + } + return ctrl.Result{}, nil +} + +// reconcileAnycastRPLoopbacks ensures lo100 (PIM anycast RP) exists on RP devices. +// One claim is allocated per AnycastRendezvousPoint group; all RP devices in the group +// share that single address. +func (r *FabricReconciler) reconcileAnycastRPLoopbacks(ctx context.Context, fabric *evpnv1alpha1.Fabric) (ctrl.Result, error) { + if fabric.Spec.BUM.PIM == nil { + return ctrl.Result{}, nil + } + for _, rp := range fabric.Spec.BUM.PIM.AnycastRendezvousPoints { + claimName := fmt.Sprintf("%s-%s-lo%d", fabric.Name, rp.Name, LoopbackAnycastRP) + claim, err := r.reconcileLoopbackClaim(ctx, fabric, claimName) + if err != nil { + return ctrl.Result{}, err + } + selector, err := metav1.LabelSelectorAsSelector(&rp.DeviceSelector) + if err != nil { + return ctrl.Result{}, reconcile.TerminalError(fmt.Errorf("invalid anycast rendezvous-point deviceSelector %q: %w", rp.Name, err)) + } + devices := &v1alpha1.DeviceList{} + if err := r.List(ctx, devices, client.InNamespace(fabric.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return ctrl.Result{}, fmt.Errorf("listing RP devices for %q: %w", rp.Name, err) + } + for i := range devices.Items { + if err := r.reconcileLoopbackInterface(ctx, fabric, &devices.Items[i], LoopbackAnycastRP, claim); err != nil { + return ctrl.Result{}, err + } + } + } + return ctrl.Result{}, nil +} + +// reconcileLoopbackClaim ensures a Claim with the given name exists and matches the desired spec. +// Returns the Claim object so callers can pass it directly to reconcileLoopbackInterface. +func (r *FabricReconciler) reconcileLoopbackClaim(ctx context.Context, fabric *evpnv1alpha1.Fabric, claimName string) (*poolv1alpha1.Claim, error) { + claim := &poolv1alpha1.Claim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: fabric.Namespace, + }, + } + res, err := controllerutil.CreateOrPatch(ctx, r.Client, claim, func() error { + claim.Spec = poolv1alpha1.ClaimSpec{ + PoolRef: v1alpha1.TypedLocalObjectReference{ + APIVersion: poolv1alpha1.GroupVersion.String(), + Kind: "IPAddressPool", + Name: fabric.Spec.Loopbacks.IPAddressPoolRef.Name, + }, + } + return controllerutil.SetControllerReference(fabric, claim, r.Scheme) + }) + if err != nil { + return nil, fmt.Errorf("reconciling claim %s: %w", claimName, err) + } + if res == controllerutil.OperationResultCreated { + r.Recorder.Eventf(fabric, nil, "Normal", "ClaimCreated", "Reconcile", "Created loopback address claim %s", claimName) + } + return claim, nil +} + +// reconcileLoopbackInterface creates or updates the Interface for a given device loopback +// once its Claim is allocated. A no-op if the claim is not yet allocated; the Owns() watch +// on Claim will re-enqueue this Fabric when the pool controller updates the claim status. +func (r *FabricReconciler) reconcileLoopbackInterface(ctx context.Context, fabric *evpnv1alpha1.Fabric, device *v1alpha1.Device, loopbackID int, claim *poolv1alpha1.Claim) error { + cond := conditions.Get(claim, poolv1alpha1.AllocatedCondition) + if cond == nil || cond.Status != metav1.ConditionTrue || claim.Status.Value == "" { + return nil + } + + prefix, err := v1alpha1.ParsePrefix(claim.Status.Value + "/32") + if err != nil { + return reconcile.TerminalError(fmt.Errorf("parsing allocated address %q: %w", claim.Status.Value, err)) + } + + handle, err := r.Provider().(provider.InterfaceProvider).LoopbackInterfaceName(loopbackID) + if err != nil { + return reconcile.TerminalError(fmt.Errorf("resolving loopback interface name for id %d: %w", loopbackID, err)) + } + + name := fmt.Sprintf("%s-%s-%s", fabric.Name, device.Name, handle) + intf := &v1alpha1.Interface{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: fabric.Namespace, + }, + } + res, err := controllerutil.CreateOrPatch(ctx, r.Client, intf, func() error { + intf.Spec.DeviceRef = v1alpha1.LocalObjectReference{Name: device.Name} + intf.Spec.Name = handle + intf.Spec.Type = v1alpha1.InterfaceTypeLoopback + intf.Spec.AdminState = v1alpha1.AdminStateUp + switch loopbackID { + case LoopbackRouterID: + intf.Spec.Description = "Router-ID, BGP Source" + case LoopbackVTEP: + intf.Spec.Description = "Primary VTEP" + case LoopbackVTEPAnycast: + intf.Spec.Description = "VTEP Anycast" + case LoopbackAnycastRP: + intf.Spec.Description = "Rendezvous Point" + } + if intf.Spec.IPv4 == nil { + intf.Spec.IPv4 = &v1alpha1.InterfaceIPv4{} + } + if len(intf.Spec.IPv4.Addresses) == 0 || intf.Spec.IPv4.Addresses[0] != prefix { + intf.Spec.IPv4.Addresses = []v1alpha1.IPPrefix{prefix} + } + return controllerutil.SetOwnerReference(fabric, intf, r.Scheme) + }) + if err != nil { + return fmt.Errorf("reconciling interface %s: %w", name, err) + } + if res == controllerutil.OperationResultCreated { + r.Recorder.Eventf(fabric, nil, "Normal", "InterfaceCreated", "Reconcile", "Created loopback interface %s", name) + } + return nil +} + +// devicesToFabrics is a [handler.MapFunc] that enqueues all Fabrics whose +// spec.deviceSelector matches the labels of the changed Device. +func (r *FabricReconciler) devicesToFabrics(ctx context.Context, obj client.Object) []ctrl.Request { + device, ok := obj.(*v1alpha1.Device) + if !ok { + panic(fmt.Sprintf("Expected a Device but got a %T", obj)) + } + + log := ctrl.LoggerFrom(ctx) + + fabricList := &evpnv1alpha1.FabricList{} + if err := r.List(ctx, fabricList, client.InNamespace(device.Namespace)); err != nil { + log.Error(err, "Failed to list Fabrics") + return nil + } + + var requests []ctrl.Request + for _, fabric := range fabricList.Items { + selector, err := metav1.LabelSelectorAsSelector(&fabric.Spec.DeviceSelector) + if err != nil { + log.Error(err, "Failed to parse deviceSelector", "fabric", fabric.Name) + continue + } + if selector.Matches(labels.Set(device.Labels)) { + log.V(2).Info("Enqueuing Fabric for reconciliation", "fabric", fabric.Name) + requests = append(requests, ctrl.Request{ + NamespacedName: client.ObjectKeyFromObject(&fabric), + }) + } + } + return requests +} diff --git a/internal/controller/evpn/fabric_controller_test.go b/internal/controller/evpn/fabric_controller_test.go index 7b06b12c7..2e232dcec 100644 --- a/internal/controller/evpn/fabric_controller_test.go +++ b/internal/controller/evpn/fabric_controller_test.go @@ -6,36 +6,308 @@ package evpn import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/client" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + corev1alpha1 "github.com/ironcore-dev/network-operator/api/core/v1alpha1" evpnv1alpha1 "github.com/ironcore-dev/network-operator/api/evpn/v1alpha1" + poolv1alpha1 "github.com/ironcore-dev/network-operator/api/pool/v1alpha1" ) +// Minimal 2-spine / 2-leaf EVPN/VXLAN topology used by the tests below. +// All four devices share the "topology.kubernetes.io/zone: test-zone" label; +// the "role" label drives per-role loopback allocation. +// +// (RR, RP) (RR, RP) +// spine-1 spine-2 +// | \_____________/ | +// | / \ | +// leaf-1 leaf-2 +// (VTEP) (VTEP) +// +// Loopback allocation: +// +// lo0 (Router-ID) — all 4 devices +// lo1 (primary VTEP) — leaf-1, leaf-2 +// lo2 (anycast VTEP) — leaf-1, leaf-2 +// lo100 (anycast RP) — 1 Claim shared across spine-1, spine-2 var _ = Describe("Fabric Controller", func() { - var fabric *evpnv1alpha1.Fabric - - BeforeEach(func() { - fabric = &evpnv1alpha1.Fabric{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "fabric-", - Namespace: metav1.NamespaceDefault, - }, - Spec: evpnv1alpha1.FabricSpec{}, - } - Expect(k8sClient.Create(ctx, fabric)).To(Succeed()) - }) + Context("When reconciling a resource", func() { + var ( + pool *poolv1alpha1.IPAddressPool + spine1 *corev1alpha1.Device + spine2 *corev1alpha1.Device + leaf1 *corev1alpha1.Device + leaf2 *corev1alpha1.Device + fabric *evpnv1alpha1.Fabric + ) - AfterEach(func() { - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, fabric))).To(Succeed()) - }) + BeforeEach(func() { + By("Creating an IPAddressPool for loopback allocation") + pool = &poolv1alpha1.IPAddressPool{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "loopback-pool-", + Namespace: metav1.NamespaceDefault, + }, + Spec: poolv1alpha1.IPAddressPoolSpec{ + Prefixes: []corev1alpha1.IPPrefix{corev1alpha1.MustParsePrefix("10.0.0.0/24")}, + }, + } + Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + + By("Creating spine-1 (route reflector, rendezvous point)") + spine1 = &corev1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "spine-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "test-zone", + "role": "spine", + }, + }, + Spec: corev1alpha1.DeviceSpec{ + Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.1:9339"}, + }, + } + Expect(k8sClient.Create(ctx, spine1)).To(Succeed()) + + By("Creating spine-2 (route reflector, rendezvous point)") + spine2 = &corev1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "spine-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "test-zone", + "role": "spine", + }, + }, + Spec: corev1alpha1.DeviceSpec{ + Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.2:9339"}, + }, + } + Expect(k8sClient.Create(ctx, spine2)).To(Succeed()) + + By("Creating leaf-1 (VTEP)") + leaf1 = &corev1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "leaf-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "test-zone", + "role": "leaf", + }, + }, + Spec: corev1alpha1.DeviceSpec{ + Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.1:9339"}, + }, + } + Expect(k8sClient.Create(ctx, leaf1)).To(Succeed()) + + By("Creating leaf-2 (VTEP)") + leaf2 = &corev1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "leaf-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "test-zone", + "role": "leaf", + }, + }, + Spec: corev1alpha1.DeviceSpec{ + Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.2:9339"}, + }, + } + Expect(k8sClient.Create(ctx, leaf2)).To(Succeed()) + + By("Creating the Fabric resource") + fabric = &evpnv1alpha1.Fabric{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "fabric-", + Namespace: metav1.NamespaceDefault, + }, + Spec: evpnv1alpha1.FabricSpec{ + DeviceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"topology.kubernetes.io/zone": "test-zone"}, + }, + Loopbacks: evpnv1alpha1.FabricLoopbacksSpec{ + IPAddressPoolRef: corev1alpha1.LocalObjectReference{Name: pool.Name}, + }, + Underlay: evpnv1alpha1.FabricUnderlaySpec{ + Protocol: evpnv1alpha1.UnderlayProtocolOSPF, + InterfaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"role": "fabric"}, + }, + Addressing: evpnv1alpha1.FabricUnderlayAddressingSpec{ + Unnumbered: true, + }, + }, + Overlay: evpnv1alpha1.FabricOverlaySpec{ + Protocol: evpnv1alpha1.OverlayProtocolIBGP, + IBGP: &evpnv1alpha1.FabricIBGPSpec{ + ASNumber: intstr.FromInt(65000), + RouteReflectors: []evpnv1alpha1.RouteReflectorGroup{ + { + Name: "spines", + DeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "spine"}}, + ClientDeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "leaf"}}, + }, + }, + }, + }, + BUM: evpnv1alpha1.FabricBUMSpec{ + Type: evpnv1alpha1.BUMTypeMulticast, + PIM: &evpnv1alpha1.FabricPIMSpec{ + AnycastRendezvousPoints: []evpnv1alpha1.AnycastRendezvousPoint{ + { + Name: "spine-rp", + MulticastGroups: []corev1alpha1.IPPrefix{corev1alpha1.MustParsePrefix("224.0.0.0/4")}, + DeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "spine"}}, + ClientDeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "leaf"}}, + }, + }, + }, + }, + VTEP: evpnv1alpha1.FabricVTEPSpec{ + DeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "leaf"}}, + }, + }, + } + Expect(k8sClient.Create(ctx, fabric)).To(Succeed()) + }) + + AfterEach(func() { + By("Deleting the Fabric resource") + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, fabric))).To(Succeed()) + + By("Deleting the Device resources") + for _, d := range []*corev1alpha1.Device{spine1, spine2, leaf1, leaf2} { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, d))).To(Succeed()) + } + + By("Deleting the IPAddressPool resource") + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, pool))).To(Succeed()) + + By("Deleting all Claims created for the Fabric") + Expect(k8sClient.DeleteAllOf(ctx, &poolv1alpha1.Claim{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) + + By("Verifying all Claims are deleted") + Eventually(func(g Gomega) { + list := &poolv1alpha1.ClaimList{} + g.Expect(k8sClient.List(ctx, list, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) + g.Expect(list.Items).To(BeEmpty()) + }).Should(Succeed()) + }) + + It("Should create lo0 Claims for all fabric devices, lo1/lo2 Claims for VTEP devices, and one lo100 Claim per RP group", func() { + By("Verifying the controller adds a finalizer") + Eventually(func(g Gomega) { + f := &evpnv1alpha1.Fabric{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name, Namespace: metav1.NamespaceDefault}, f)).To(Succeed()) + g.Expect(controllerutil.ContainsFinalizer(f, evpnv1alpha1.FinalizerName)).To(BeTrue()) + }).Should(Succeed()) + + By("Verifying lo0 Claims are created for all 4 fabric devices") + for _, d := range []*corev1alpha1.Device{spine1, spine2, leaf1, leaf2} { + Eventually(func(g Gomega) { + claim := &poolv1alpha1.Claim{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + d.Name + "-lo0", Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) + g.Expect(claim.Spec.PoolRef.Name).To(Equal(pool.Name)) + g.Expect(claim.Spec.PoolRef.Kind).To(Equal("IPAddressPool")) + }).Should(Succeed()) + } + + By("Verifying lo1 and lo2 Claims are created only for leaf (VTEP) devices") + for _, d := range []*corev1alpha1.Device{leaf1, leaf2} { + for _, id := range []string{"lo1", "lo2"} { + Eventually(func(g Gomega) { + claim := &poolv1alpha1.Claim{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + d.Name + "-" + id, Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) + }).Should(Succeed()) + } + } + + By("Verifying a single lo100 Claim is created for the spine-rp RP group") + Eventually(func(g Gomega) { + claim := &poolv1alpha1.Claim{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-spine-rp-lo100", Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) + }).Should(Succeed()) + + By("Verifying lo0 Interfaces are created for all 4 fabric devices once Claims are allocated") + for _, d := range []*corev1alpha1.Device{spine1, spine2, leaf1, leaf2} { + Eventually(func(g Gomega) { + intf := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + d.Name + "-lo0", Namespace: metav1.NamespaceDefault}, intf)).To(Succeed()) + g.Expect(intf.Spec.Type).To(Equal(corev1alpha1.InterfaceTypeLoopback)) + g.Expect(intf.Spec.DeviceRef.Name).To(Equal(d.Name)) + g.Expect(intf.Spec.Name).To(Equal("lo0")) + g.Expect(intf.Spec.AdminState).To(Equal(corev1alpha1.AdminStateUp)) + g.Expect(intf.Spec.Description).To(Equal("Router-ID, BGP Source")) + g.Expect(intf.Spec.IPv4).NotTo(BeNil()) + g.Expect(intf.Spec.IPv4.Addresses).To(HaveLen(1)) + }).Should(Succeed()) + } + + By("Verifying lo1 and lo2 Interfaces are created for leaf (VTEP) devices") + for _, d := range []*corev1alpha1.Device{leaf1, leaf2} { + for loIdx, id := range []string{"lo1", "lo2"} { + descriptions := []string{"Primary VTEP", "VTEP Anycast"} + Eventually(func(g Gomega) { + intf := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + d.Name + "-" + id, Namespace: metav1.NamespaceDefault}, intf)).To(Succeed()) + g.Expect(intf.Spec.Type).To(Equal(corev1alpha1.InterfaceTypeLoopback)) + g.Expect(intf.Spec.DeviceRef.Name).To(Equal(d.Name)) + g.Expect(intf.Spec.Name).To(Equal(id)) + g.Expect(intf.Spec.AdminState).To(Equal(corev1alpha1.AdminStateUp)) + g.Expect(intf.Spec.Description).To(Equal(descriptions[loIdx])) + g.Expect(intf.Spec.IPv4).NotTo(BeNil()) + g.Expect(intf.Spec.IPv4.Addresses).To(HaveLen(1)) + }).Should(Succeed()) + } + } + + By("Verifying the lo100 Interface is created on each spine (shared RP address)") + for _, d := range []*corev1alpha1.Device{spine1, spine2} { + Eventually(func(g Gomega) { + intf := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + d.Name + "-lo100", Namespace: metav1.NamespaceDefault}, intf)).To(Succeed()) + g.Expect(intf.Spec.Type).To(Equal(corev1alpha1.InterfaceTypeLoopback)) + g.Expect(intf.Spec.DeviceRef.Name).To(Equal(d.Name)) + g.Expect(intf.Spec.Name).To(Equal("lo100")) + g.Expect(intf.Spec.AdminState).To(Equal(corev1alpha1.AdminStateUp)) + g.Expect(intf.Spec.Description).To(Equal("Rendezvous Point")) + g.Expect(intf.Spec.IPv4).NotTo(BeNil()) + g.Expect(intf.Spec.IPv4.Addresses).To(HaveLen(1)) + }).Should(Succeed()) + } + }) + + It("Should set the Fabric as controller owner of each Claim", func() { + By("Verifying the lo0 Claim for leaf-1 has the Fabric as controller owner") + Eventually(func(g Gomega) { + claim := &poolv1alpha1.Claim{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + leaf1.Name + "-lo0", Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) + g.Expect(claim.OwnerReferences).To(ContainElement( + SatisfyAll( + HaveField("Kind", "Fabric"), + HaveField("Name", fabric.Name), + HaveField("Controller", HaveValue(BeTrue())), + ), + )) + }).Should(Succeed()) + }) - It("Should successfully reconcile a Fabric", func() { - By("Updating the status") - Eventually(func(g Gomega) { - current := &evpnv1alpha1.Fabric{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fabric), current)).To(Succeed()) - }).Should(Succeed()) + It("Should set the Fabric to Ready", func() { + By("Verifying the Fabric Ready condition is True once all phases are complete") + Eventually(func(g Gomega) { + f := &evpnv1alpha1.Fabric{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name, Namespace: metav1.NamespaceDefault}, f)).To(Succeed()) + g.Expect(f.Status.Conditions).NotTo(BeEmpty()) + g.Expect(f.Status.Conditions[0].Type).To(Equal(corev1alpha1.ReadyCondition)) + g.Expect(f.Status.Conditions[0].Status).To(Equal(metav1.ConditionTrue)) + }).Should(Succeed()) + }) }) }) diff --git a/internal/controller/evpn/suite_test.go b/internal/controller/evpn/suite_test.go index 35a43c3d7..4fdbfb5ff 100644 --- a/internal/controller/evpn/suite_test.go +++ b/internal/controller/evpn/suite_test.go @@ -5,6 +5,7 @@ package evpn import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -22,8 +23,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + corev1alpha1 "github.com/ironcore-dev/network-operator/api/core/v1alpha1" evpnv1alpha1 "github.com/ironcore-dev/network-operator/api/evpn/v1alpha1" + poolv1alpha1 "github.com/ironcore-dev/network-operator/api/pool/v1alpha1" + poolcontroller "github.com/ironcore-dev/network-operator/internal/controller/pool" + "github.com/ironcore-dev/network-operator/internal/deviceutil" + "github.com/ironcore-dev/network-operator/internal/provider" // +kubebuilder:scaffold:imports ) @@ -52,9 +59,9 @@ var _ = BeforeSuite(func() { ctx, cancel = context.WithCancel(ctrl.SetupSignalHandler()) - var err error - err = evpnv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) + Expect(evpnv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(corev1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(poolv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) // +kubebuilder:scaffold:scheme @@ -74,8 +81,10 @@ var _ = BeforeSuite(func() { Expect(cfg).NotTo(BeNil()) k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - Logger: GinkgoLogr, + Scheme: scheme.Scheme, + Logger: GinkgoLogr, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", }) Expect(err).NotTo(HaveOccurred()) @@ -95,9 +104,28 @@ var _ = BeforeSuite(func() { Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), Recorder: recorder, + Provider: NewProvider, }).SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) + err = (&poolcontroller.IPAddressPoolReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + + err = (&poolcontroller.IPAddressReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(ctx, k8sManager) + Expect(err).NotTo(HaveOccurred()) + + err = (&poolcontroller.ClaimReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(ctx, k8sManager) + Expect(err).NotTo(HaveOccurred()) + go func() { defer GinkgoRecover() err := k8sManager.Start(ctx) @@ -139,3 +167,29 @@ func getFirstFoundEnvTestBinaryDir() string { } return "" } + +var _ provider.InterfaceProvider = (*Provider)(nil) + +type Provider struct{} + +func NewProvider() provider.Provider { return &Provider{} } + +func (p *Provider) Connect(context.Context, *deviceutil.Connection) error { return nil } +func (p *Provider) Disconnect(context.Context, *deviceutil.Connection) error { return nil } +func (p *Provider) EnsureInterface(context.Context, *provider.EnsureInterfaceRequest) error { + return nil +} + +func (p *Provider) DeleteInterface(context.Context, *provider.InterfaceRequest) error { return nil } + +func (p *Provider) GetInterfaceStatus(context.Context, *provider.InterfaceRequest) (provider.InterfaceStatus, error) { + return provider.InterfaceStatus{}, nil +} + +func (p *Provider) InterfaceNameEqual(_ context.Context, a, b string) (bool, error) { + return a == b, nil +} + +func (p *Provider) LoopbackInterfaceName(id int) (string, error) { + return fmt.Sprintf("lo%d", id), nil +} diff --git a/internal/provider/cisco/iosxr/provider.go b/internal/provider/cisco/iosxr/provider.go index a9ca3d70c..e97174070 100644 --- a/internal/provider/cisco/iosxr/provider.go +++ b/internal/provider/cisco/iosxr/provider.go @@ -373,6 +373,10 @@ func (p *Provider) InterfaceNameEqual(_ context.Context, a, b string) (bool, err return a == b, nil } +func (p *Provider) LoopbackInterfaceName(id int) (string, error) { + return fmt.Sprintf("Loopback%d", id), nil +} + func init() { provider.Register("cisco-iosxr-gnmi", NewProvider) } diff --git a/internal/provider/cisco/nxos/provider.go b/internal/provider/cisco/nxos/provider.go index d270637cb..ee2278302 100644 --- a/internal/provider/cisco/nxos/provider.go +++ b/internal/provider/cisco/nxos/provider.go @@ -1431,6 +1431,10 @@ func (p *Provider) InterfaceNameEqual(_ context.Context, a, b string) (bool, err return shortA == shortB, nil } +func (p *Provider) LoopbackInterfaceName(id int) (string, error) { + return fmt.Sprintf("Loopback%d", id), nil +} + var ErrInterfaceNotFound = errors.New("one or more interfaces do not exist") func (p *Provider) EnsureInterfacesExist(ctx context.Context, interfaces []*v1alpha1.Interface) (names []string, err error) { diff --git a/internal/provider/openconfig/provider.go b/internal/provider/openconfig/provider.go index 424d2e690..04ab42fbd 100644 --- a/internal/provider/openconfig/provider.go +++ b/internal/provider/openconfig/provider.go @@ -205,6 +205,10 @@ func (p *Provider) InterfaceNameEqual(_ context.Context, a, b string) (bool, err return a == b, nil } +func (p *Provider) LoopbackInterfaceName(id int) (string, error) { + return fmt.Sprintf("lo%d", id), nil +} + func init() { provider.Register("openconfig", NewProvider) } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index f1845132e..2f253b47e 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -89,6 +89,8 @@ type InterfaceProvider interface { GetInterfaceStatus(context.Context, *InterfaceRequest) (InterfaceStatus, error) // InterfaceNameEqual reports whether two interface names refer to the same interface on the provider. InterfaceNameEqual(context.Context, string, string) (bool, error) + // LoopbackInterfaceName returns the vendor-specific interface name for a loopback with the given numeric ID. + LoopbackInterfaceName(id int) (string, error) } type EnsureInterfaceRequest struct {