controller.go

package nodedeployment

import (
	"context"
	"fmt"
	"time"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/record"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/builder"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/predicate"

	seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1"
	"github.com/sei-protocol/sei-k8s-controller/internal/controller/observability"
	"github.com/sei-protocol/sei-k8s-controller/internal/planner"
)

const (
	groupFinalizerName = "sei.io/seinodedeployment-finalizer"
	controllerName     = "seinodedeployment"
	statusPollInterval = 30 * time.Second
	fieldOwner         = client.FieldOwner("seinodedeployment-controller")
)

// SeiNodeDeploymentReconciler reconciles a SeiNodeDeployment object.
type SeiNodeDeploymentReconciler struct {
	client.Client
	Scheme   *runtime.Scheme
	Recorder record.EventRecorder

	// ControllerSA is the SPIFFE principal of the controller's ServiceAccount.
	// It is auto-injected into every AuthorizationPolicy to ensure the
	// controller can always reach the seictl sidecar.
	ControllerSA string

	// GatewayName, GatewayNamespace, and GatewayDomain identify the platform
	// Gateway for HTTPRoute parentRefs and hostname derivation.
	// Read from SEI_GATEWAY_NAME / SEI_GATEWAY_NAMESPACE / SEI_GATEWAY_DOMAIN.
	GatewayName         string
	GatewayNamespace    string
	GatewayDomain       string
	GatewayPublicDomain string

	// PlanExecutor drives group-level task plans (e.g. genesis assembly).
	PlanExecutor planner.PlanExecutor[*seiv1alpha1.SeiNodeDeployment]
}

// +kubebuilder:rbac:groups=sei.io,resources=seinodedeployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=sei.io,resources=seinodedeployments/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=sei.io,resources=seinodedeployments/finalizers,verbs=update
// +kubebuilder:rbac:groups=sei.io,resources=seinodes,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=sei.io,resources=seinodes/status,verbs=get
// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=security.istio.io,resources=authorizationpolicies,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete

func (r *SeiNodeDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	logger := log.FromContext(ctx)

	group := &seiv1alpha1.SeiNodeDeployment{}
	if err := r.Get(ctx, req.NamespacedName, group); err != nil {
		if apierrors.IsNotFound(err) {
			return ctrl.Result{}, nil
		}
		return ctrl.Result{}, err
	}

	if !group.DeletionTimestamp.IsZero() {
		return r.handleDeletion(ctx, group)
	}

	if err := r.ensureFinalizer(ctx, group); err != nil {
		return ctrl.Result{}, err
	}

	// Snapshot the status before any reconciliation mutates it in memory.
	// Conditions set during networking/monitoring reconciliation are captured
	// in the diff when updateStatus patches against this base.
	statusBase := client.MergeFromWithOptions(group.DeepCopy(), client.MergeFromWithOptimisticLock{})
	ns, name := group.Namespace, group.Name

	// Networking runs before node creation so that the external P2P
	// address (from the LoadBalancer ingress) is known at plan build time.
	if err := timeSubstep("reconcileNetworking", func() error {
		return r.reconcileNetworking(ctx, group)
	}); err != nil {
		logger.Error(err, "reconciling networking")
		observability.ReconcileErrorsTotal.WithLabelValues(controllerName, ns, name).Inc()
		return ctrl.Result{}, fmt.Errorf("reconciling networking: %w", err)
	}

	// Gate: wait for the LoadBalancer to provision an external address
	// before creating nodes so the P2P address is baked into the plan.
	if r.hasExternalService(group) && r.resolveExternalP2PAddress(ctx, group) == "" {
		logger.Info("waiting for LoadBalancer to provision external address")
		setCondition(group, seiv1alpha1.ConditionExternalServiceReady, metav1.ConditionFalse,
			"LoadBalancerPending", "Waiting for LoadBalancer to provision external P2P address")
		if err := r.updateStatus(ctx, group, statusBase); err != nil {
			return ctrl.Result{}, fmt.Errorf("updating status: %w", err)
		}
		return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
	}

	if err := timeSubstep("reconcileSeiNodes", func() error {
		return r.reconcileSeiNodes(ctx, group)
	}); err != nil {
		logger.Error(err, "reconciling SeiNodes")
		observability.ReconcileErrorsTotal.WithLabelValues(controllerName, ns, name).Inc()
		return ctrl.Result{}, fmt.Errorf("reconciling SeiNodes: %w", err)
	}

	planResult, planErr := r.reconcilePlan(ctx, group, statusBase)
	if planErr != nil {
		logger.Error(planErr, "reconciling plan")
		observability.ReconcileErrorsTotal.WithLabelValues(controllerName, ns, name).Inc()
		return ctrl.Result{}, fmt.Errorf("reconciling plan: %w", planErr)
	}
	if shouldRequeue(planResult) {
		if err := r.updateStatus(ctx, group, statusBase); err != nil {
			return ctrl.Result{}, fmt.Errorf("updating status: %w", err)
		}
		return planResult, nil
	}

	if err := timeSubstep("reconcileMonitoring", func() error {
		return r.reconcileMonitoring(ctx, group)
	}); err != nil {
		logger.Error(err, "reconciling monitoring")
		observability.ReconcileErrorsTotal.WithLabelValues(controllerName, ns, name).Inc()
		return ctrl.Result{}, fmt.Errorf("reconciling monitoring: %w", err)
	}

	if err := r.updateStatus(ctx, group, statusBase); err != nil {
		return ctrl.Result{}, fmt.Errorf("updating status: %w", err)
	}

	emitGroupPhase(ns, name, group.Status.Phase)
	emitGroupReplicas(ns, name, group.Spec.Replicas, group.Status.ReadyReplicas)
	emitGroupConditions(ns, name, group.Status.Conditions)

	return ctrl.Result{RequeueAfter: statusPollInterval}, nil
}

func (r *SeiNodeDeploymentReconciler) ensureFinalizer(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) error {
	if controllerutil.ContainsFinalizer(group, groupFinalizerName) {
		return nil
	}
	patch := client.MergeFrom(group.DeepCopy())
	controllerutil.AddFinalizer(group, groupFinalizerName)
	return r.Patch(ctx, group, patch)
}

func (r *SeiNodeDeploymentReconciler) handleDeletion(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) (ctrl.Result, error) {
	if !controllerutil.ContainsFinalizer(group, groupFinalizerName) {
		return ctrl.Result{}, nil
	}

	patch := client.MergeFrom(group.DeepCopy())
	group.Status.Phase = seiv1alpha1.GroupPhaseTerminating
	if err := r.Status().Patch(ctx, group, patch); err != nil {
		return ctrl.Result{}, fmt.Errorf("setting terminating status: %w", err)
	}

	policy := group.Spec.DeletionPolicy
	if policy == "" {
		policy = seiv1alpha1.DeletionPolicyDelete
	}

	if policy == seiv1alpha1.DeletionPolicyRetain {
		r.Recorder.Event(group, corev1.EventTypeNormal, "RetainResources", "Orphaning child SeiNodes and networking resources")
		if err := r.orphanChildSeiNodes(ctx, group); err != nil {
			return ctrl.Result{}, fmt.Errorf("orphaning child SeiNodes: %w", err)
		}
		if err := r.orphanNetworkingResources(ctx, group); err != nil {
			return ctrl.Result{}, fmt.Errorf("orphaning networking resources: %w", err)
		}
	} else {
		if err := r.deleteNetworkingResources(ctx, group); err != nil {
			r.Recorder.Eventf(group, corev1.EventTypeWarning, "DeleteFailed", "Failed to clean up networking resources: %v", err)
			return ctrl.Result{}, fmt.Errorf("cleaning up networking: %w", err)
		}
	}

	cleanupGroupMetrics(group.Namespace, group.Name)

	finalizerPatch := client.MergeFrom(group.DeepCopy())
	controllerutil.RemoveFinalizer(group, groupFinalizerName)
	return ctrl.Result{}, r.Patch(ctx, group, finalizerPatch)
}

func shouldRequeue(result ctrl.Result) bool {
	return result.RequeueAfter > 0
}

// SetupWithManager sets up the controller with the Manager.
func (r *SeiNodeDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&seiv1alpha1.SeiNodeDeployment{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
		Owns(&seiv1alpha1.SeiNode{}, builder.WithPredicates(childPhaseChangedPredicate())).
		Owns(&corev1.Service{}).
		Named(controllerName).
		Complete(r)
}