disable HAC + set noobaa-core term grace to 1

- HAC can be enabled with the env HAC_ENABLED=true
- Also changed the default grace period of the HAC when enabled from 0 to 1. This change avoids force deletion of the pods, to prevent multiple containers running concurrently.
- The grace period can be set to 0 with the env HAC_FORCE_DELETE=true
- Also setting the terminationGracePeriodSeconds of noobaa-core to 1. By default this is 30 seconds. Since we don't have any handler in noobaa-core for SIGETRM, there isn't a need to wait 30 seconds before killing the container.

Signed-off-by: Danny Zaken <dannyzaken@gmail.com>
(cherry picked from commit 38294af)
(cherry picked from commit b7652a4)
(cherry picked from commit 1e0e4ee)
(cherry picked from commit e8592ca)
(cherry picked from commit b3e5a89)
(cherry picked from commit 45f36a5)

Author: dannyzaken

.github/workflows/run_hac_test.yml

@@ -1,5 +1,5 @@
 name: HAC on KIND cluster Test
-on: [push, pull_request, workflow_dispatch]
+on: [workflow_dispatch]
 
 jobs:
   run-hac-test:

pkg/controller/add_hac.go

@@ -1,10 +1,15 @@
 package controller
 
 import (
+	"os"
+
 	hac "github.com/noobaa/noobaa-operator/v5/pkg/controller/ha"
 )
 
 func init() {
-	// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
-	AddToManagerFuncs = append(AddToManagerFuncs, hac.Add)
+	hacEnabled := os.Getenv("HAC_ENABLED")
+	if hacEnabled == "true" {
+		// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
+		AddToManagerFuncs = append(AddToManagerFuncs, hac.Add)
+	}
 }

pkg/controller/ha/ha_controller.go

@@ -71,6 +71,7 @@ func nodeNotReadyPredicate() predicate.Predicate {
 
 // Add creates a nodewatcher Controller and adds it to the Manager.
 func Add(mgr manager.Manager) error {
+	util.Logger().Info("Adding high availability controller (HAC)")
 
 	opts := controller.Options{
 		MaxConcurrentReconciles: 1,

pkg/hac/hac.go

@@ -5,6 +5,7 @@ package hac
 
 import (
 	"context"
+	"os"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -43,8 +44,12 @@ func (pd *PodDeleter) DeletePodsOnNode() error {
 		return errors.Errorf("failed to list noobaa pods on the node %v in namespace %v", pd.NodeName, options.Namespace)
 	}
 
-	// delete the found pods
-	var gracePeriod int64 = 0
+	// delete the found pods. by default using a 1 second grace period.
+	var gracePeriod int64 = 1
+	if os.Getenv("HAC_FORCE_DELETE") == "true" {
+		// if HAC_FORCE_DELETE is set to true, use a 0 second grace period.
+		gracePeriod = 0
+	}
 	deleteOpts := client.DeleteOptions{GracePeriodSeconds: &gracePeriod}
 	for _, pod := range podList.Items {
 		if err := pd.Client.Delete(context.Background(), &pod, &deleteOpts); err != nil {

pkg/system/phase2_creating.go

@@ -445,6 +445,10 @@ func (r *Reconciler) SetDesiredCoreApp() error {
 	r.CoreApp.Spec.ServiceName = r.ServiceMgmt.Name
 
 	podSpec := &r.CoreApp.Spec.Template.Spec
+	// set the termination grace period for noobaa-core pod.
+	// For now we set it to 1 second. A better approach should be to implement a graceful shutdown for the noobaa-core pod when SIGTERM is received.
+	terminationGracePeriodSeconds := int64(1)
+	podSpec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds
 	podSpec.ServiceAccountName = "noobaa-core"
 	coreImageChanged := false