fix: Switched back to using ELB instead of NLB due to various issues. Also added the ability to modify proxy protocol and traffic policy settings and return the nginx request id to the front end. (#45)

Author: bmonkman

modules/kubernetes/ingress_nginx/README.md

@@ -25,9 +25,11 @@ No requirements.
 | chart\_version | The version of helm chart to use. | `string` | `"3.25.0"` | no |
 | connection\_idle\_timeout | The amount of time the load balancer will keep an idle connection open for. The value of nginx upstream-keepalive-timeout will also be set to this value + 5. If it were shorter than the LB timeout it could cause intermittent 502s. | `number` | `55` | no |
 | enable\_metrics | Enable prometheus metrics support, including adding a ServiceMonitor. | `bool` | n/a | yes |
+| external\_traffic\_policy | The external traffic policy to apply to the ingress service. Cluster will open a valid NodePort on all nodes even if they aren't running an ingress pod and kubernetes will handle sending the traffic to the correct pod. Local will only have valid NodePorts on the nodes running ingress pods. | `string` | `"Cluster"` | no |
 | namespace | Namespace to create the ingress in. | `string` | `"ingress-nginx"` | no |
 | replica\_count | Number of replicas of the ingress controller to create. Should be 2 or more in production. | `number` | `2` | no |
-| use\_network\_load\_balancer | Use an AWS NLB to load balance traffic to the cluster. Recommended. If false, will create a Classic Load Balancer. | `bool` | `true` | no |
+| use\_network\_load\_balancer | Use an AWS NLB to load balance traffic to the cluster. If false, will create a Classic Load Balancer. NLB is not recommended at this time due to some connection issues. | `bool` | `false` | no |
+| use\_proxy\_protocol | If true, will enable proxy protocol support between the Load Balancer and the nginx ingress controller. This allows nginx to know the IP of the client when using an ELB. | `bool` | `true` | no |
 
 ## Outputs
 

modules/kubernetes/ingress_nginx/main.tf

@@ -24,14 +24,14 @@ EOF
   configmap_defaults = {
     "proxy-real-ip-cidr"         = "0.0.0.0/0"
     "use-forwarded-headers"      = "true"
-    "use-proxy-protocol"         = "false"
+    "use-proxy-protocol"         = tostring(var.use_proxy_protocol)
     "log-format-escape-json"     = "true"
     "log-format-upstream"        = replace(local.log_format, "\n", "")
     "generate-request-id"        = "true"
     "upstream-keepalive-timeout" = var.connection_idle_timeout + 5
   }
 
-  # Anti-affinity rules to apply. Will instruct k8s to try to not schedule 2 pods on the same node if possible.
+  # Anti-affinity rules to apply. Will instruct k8s to try not to schedule 2 pods on the same node if possible.
   pod_anti_affinity = {
     podAntiAffinity : {
       preferredDuringSchedulingIgnoredDuringExecution : [
@@ -75,14 +75,17 @@ EOF
           namespace : "metrics"
         }
       }
+      addHeaders : { "X-Request-Id" : "$request_id" }
+
       service : {
-        externalTrafficPolicy : "Local"
+        externalTrafficPolicy : var.external_traffic_policy
 
         annotations : {
           "service.beta.kubernetes.io/aws-load-balancer-backend-protocol" : "tcp"
           "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" : var.connection_idle_timeout
           "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled" : "true"
           "service.beta.kubernetes.io/aws-load-balancer-type" : var.use_network_load_balancer ? "nlb" : "elb"
+          "service.beta.kubernetes.io/aws-load-balancer-proxy-protocol" : var.use_proxy_protocol ? "*" : "false" # "*" is the only value that enables proxy protocol on the LB
         }
       }
 

modules/kubernetes/ingress_nginx/variables.tf

@@ -28,9 +28,9 @@ variable "enable_metrics" {
 }
 
 variable "use_network_load_balancer" {
-  description = "Use an AWS NLB to load balance traffic to the cluster. Recommended. If false, will create a Classic Load Balancer."
+  description = "Use an AWS NLB to load balance traffic to the cluster. If false, will create a Classic Load Balancer. NLB is not recommended at this time due to some connection issues."
   type        = bool
-  default     = true
+  default     = false
 }
 variable "connection_idle_timeout" {
   description = "The amount of time the load balancer will keep an idle connection open for. The value of nginx upstream-keepalive-timeout will also be set to this value + 5. If it were shorter than the LB timeout it could cause intermittent 502s."
@@ -43,3 +43,20 @@ variable "apply_pod_anti_affinity" {
   type        = bool
   default     = true
 }
+
+variable "use_proxy_protocol" {
+  description = "If true, will enable proxy protocol support between the Load Balancer and the nginx ingress controller. This allows nginx to know the IP of the client when using an ELB."
+  type        = bool
+  default     = true
+}
+
+variable "external_traffic_policy" {
+  description = "The external traffic policy to apply to the ingress service. Cluster will open a valid NodePort on all nodes even if they aren't running an ingress pod and kubernetes will handle sending the traffic to the correct pod. Local will only have valid NodePorts on the nodes running ingress pods."
+  type        = string
+  default     = "Cluster"
+
+  validation {
+    condition     = (var.external_traffic_policy == "Local" || var.external_traffic_policy == "Cluster")
+    error_message = "Invalid value for external_traffic_policy. Valid values are Local or Cluster."
+  }
+}