Add EnsureNetworksAnnotation as an alternative for CreateNetworksAnnotation

For BGP setup there is the need to set the default gateway
to the additional interface defined via the multus annotations.
To allow this a user can configure `ipam.gateway` in the NAD.
CreateNetworksAnnotation() will override the pod network default
route by reading the NAD. If `ipam.gateway` is defined and not "",
it gets set on the networks annotation as the `default-route`.

Jira: https://issues.redhat.com/browse/OSPRH-8680

Signed-off-by: Martin Schuppert <mschuppert@redhat.com>

Author: stuggi

modules/common/networkattachment/networkattachment.go

@@ -17,14 +17,17 @@ limitations under the License.
 package networkattachment
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/jsonpath"
 )
 
 // GetNADWithName - Get network-attachment-definition with name in namespace
@@ -49,6 +52,7 @@ func GetNADWithName(
 
 // CreateNetworksAnnotation returns pod annotation for network-attachment-definition list
 // e.g. k8s.v1.cni.cncf.io/networks: '[{"name": "internalapi", "namespace": "openstack"},{"name": "storage", "namespace": "openstack"}]'
+// NOTE: Deprecated, use EnsureNetworksAnnotation
 func CreateNetworksAnnotation(namespace string, nads []string) (map[string]string, error) {
 
 	netAnnotations := []networkv1.NetworkSelectionElement{}
@@ -136,3 +140,67 @@ func VerifyNetworkStatusFromAnnotation(
 
 	return networkReady, networkAttachmentStatus, nil
 }
+
+// EnsureNetworksAnnotation returns pod annotation for network-attachment-definition list
+// e.g. k8s.v1.cni.cncf.io/networks: '[{"name": "internalapi", "namespace": "openstack"},{"name": "storage", "namespace": "openstack"}]'
+// If `ipam.gateway` is defined in the NAD, the annotation will contain the `default-route` for that network:
+// e.g. k8s.v1.cni.cncf.io/networks: '[{"name":"internalapi","namespace":"openstack","interface":"internalapi","default-route":["10.1.2.200"]}]'
+func EnsureNetworksAnnotation(
+	nadList []networkv1.NetworkAttachmentDefinition,
+) (map[string]string, error) {
+
+	annotationString := map[string]string{}
+	netAnnotations := []networkv1.NetworkSelectionElement{}
+	for _, nad := range nadList {
+		gateway := ""
+
+		var data interface{}
+		if err := json.Unmarshal([]byte(nad.Spec.Config), &data); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal JSON data: %w", err)
+		}
+
+		// use jsonpath to parse the cni config
+		jp := jsonpath.New(nad.Name)
+		jp.AllowMissingKeys(true) // Allow missing keys, for when no gateway configured
+
+		// Parse the JSONPath template, for now just `ipam.gateway`
+		err := jp.Parse(`{.ipam.gateway}`)
+		if err != nil {
+			return annotationString, fmt.Errorf("parse template error: %w", err)
+		}
+
+		buf := new(bytes.Buffer)
+		// get the gateway from the config
+		err = jp.Execute(buf, data)
+		if err != nil {
+			return annotationString, fmt.Errorf("parse execute template against nad %+v error: %w", nad.Spec.Config, err)
+		}
+
+		gateway = buf.String()
+
+		gatewayReq := []net.IP{}
+		if gateway != "" {
+			gatewayReq = append(gatewayReq, net.ParseIP(gateway))
+
+		}
+
+		netAnnotations = append(
+			netAnnotations,
+			networkv1.NetworkSelectionElement{
+				Name:             nad.Name,
+				Namespace:        nad.Namespace,
+				InterfaceRequest: GetNetworkIFName(nad.Name),
+				GatewayRequest:   gatewayReq,
+			},
+		)
+	}
+
+	networks, err := json.Marshal(netAnnotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode networks %v into json: %w", nadList, err)
+	}
+
+	annotationString[networkv1.NetworkAttachmentAnnot] = string(networks)
+
+	return annotationString, nil
+}

modules/common/networkattachment/networkattachment_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	. "github.com/onsi/gomega"
 )
@@ -169,3 +170,124 @@ func TestGetNetworkIFName(t *testing.T) {
 		})
 	}
 }
+
+func TestEnsureNetworksAnnotation(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		nadList []networkv1.NetworkAttachmentDefinition
+		want    map[string]string
+	}{
+		{
+			name:    "No network",
+			nadList: []networkv1.NetworkAttachmentDefinition{},
+			want:    map[string]string{networkv1.NetworkAttachmentAnnot: "[]"},
+		},
+		{
+			name: "Single network",
+			nadList: []networkv1.NetworkAttachmentDefinition{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "one", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "internalapi",
+  "type": "macvlan",
+  "master": "internalapi",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.17.0.0/24",
+    "range_start": "172.17.0.30",
+    "range_end": "172.17.0.70"
+  }
+}
+`,
+					},
+				},
+			},
+			want: map[string]string{networkv1.NetworkAttachmentAnnot: "[{\"name\":\"one\",\"namespace\":\"foo\",\"interface\":\"one\"}]"},
+		},
+		{
+			name: "Multiple networks",
+			nadList: []networkv1.NetworkAttachmentDefinition{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "one", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "internalapi",
+  "type": "macvlan",
+  "master": "internalapi",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.17.0.0/24",
+    "range_start": "172.17.0.30",
+    "range_end": "172.17.0.70"
+  }
+}
+`,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "two", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "tenant",
+  "type": "macvlan",
+  "master": "tenant",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.19.0.0/24",
+    "range_start": "172.19.0.30",
+    "range_end": "172.19.0.70"
+  }
+}
+`,
+					},
+				},
+			},
+			want: map[string]string{networkv1.NetworkAttachmentAnnot: "[{\"name\":\"one\",\"namespace\":\"foo\",\"interface\":\"one\"},{\"name\":\"two\",\"namespace\":\"foo\",\"interface\":\"two\"}]"},
+		},
+		{
+			name: "With gateway defined",
+			nadList: []networkv1.NetworkAttachmentDefinition{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "one", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "internalapi",
+  "type": "macvlan",
+  "master": "internalapi",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.17.0.0/24",
+    "range_start": "172.17.0.30",
+    "range_end": "172.17.0.70",
+    "gateway": "172.17.0.1"
+  }
+}
+`,
+					},
+				},
+			},
+			want: map[string]string{networkv1.NetworkAttachmentAnnot: "[{\"name\":\"one\",\"namespace\":\"foo\",\"interface\":\"one\",\"default-route\":[\"172.17.0.1\"]}]"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			networkAnnotation, err := EnsureNetworksAnnotation(tt.nadList)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(networkAnnotation).To(HaveLen(len(tt.want)))
+			g.Expect(networkAnnotation).To(BeEquivalentTo(tt.want))
+		})
+	}
+}