fix: Remove log. prefix in kibana fluentd logs, add cert-manager servicemonitor (#192)

Author: bmonkman

templates/kubernetes/terraform/environments/prod/main.tf

@@ -76,7 +76,8 @@ module "kubernetes" {
     # ["Test User 2", "10.10.99.202/32", "h2jMuaXNIlx7Z0a3owWFjPsAA8B+ZpQH3FbZK393+08="],
   ]
 
-  domain_name                 = local.domain_name
+  domain_name = local.domain_name
+
   <% if eq (index .Params `userAuth`) "yes" %>user_auth = [
     {
       name                          = local.project

templates/kubernetes/terraform/environments/stage/main.tf

@@ -75,7 +75,8 @@ module "kubernetes" {
     # ["Test User 2", "10.10.199.202/32", "h2jMuaXNIlx7Z0a3owWFjPsAA8B+ZpQH3FbZK393+08="],
   ]
 
-  domain_name                 = local.domain_name
+  domain_name = local.domain_name
+
   <% if eq (index .Params `userAuth`) "yes" %>user_auth = [
     {
       name                          = local.project

templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf

@@ -97,6 +97,22 @@ resource "helm_release" "cert_manager" {
     name  = "installCRDs"
     value = true
   }
+  set {
+    name  = "prometheus.enabled"
+    value = var.metrics_type == "prometheus"
+  }
+  set {
+    name  = "prometheus.servicemonitor.enabled"
+    value = var.metrics_type == "prometheus"
+  }
+  set {
+    name  = "prometheus.servicemonitor.labels.app"
+    value = var.metrics_type == "prometheus" ? "kube-prometheus-stack-prometheus" : ""
+  }
+  set {
+    name  = "prometheus.servicemonitor.namespace"
+    value = "metrics"
+  }
 }
 
 # Manually kubectl apply the cert-manager issuers, as the kubernetes terraform provider

templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf

@@ -49,6 +49,7 @@ resource "kubernetes_config_map" "nginx_configuration" {
     use-proxy-protocol     = "false"
     log-format-escape-json = "true"
     log-format-upstream    = replace(local.log_format, "\n", "")
+    generate-request-id    = "true"
   }
 }
 

templates/kubernetes/terraform/modules/kubernetes/logging/kibana/files/fluentd-application.conf

@@ -8,35 +8,36 @@
         pattern ^nginx
         tag     logs.kubernetes.nginx
       </rule>
-      # Add a tag for your application
-      <rule>
-        key     $.kubernetes.container_name
-        pattern ^your-application
-        tag     logs.kubernetes.yourapplication
-      </rule>      # Catch all (have to actually change the tag (add "logs.") otherwise there is an infinite loop)
+      # # Add a tag for your application - enable this and the blocks below if you want fluentd to manually control the structure of your logs
+      # # Otherwise it will just parse out the structured json data and send it to elasticsearch
+      # <rule>
+      #   key     $.kubernetes.container_name
+      #   pattern ^your-application
+      #   tag     logs.kubernetes.yourapplication
+      # </rule>      # Catch all (have to actually change the tag (add "logs.") otherwise there is an infinite loop)
       <rule>
         key     log
         pattern ^(.*)$
         tag     logs.${tag}
       </rule>
     </match>
 
-    # This allows us to parse log lines that have json separately, but still keep the unstructured ones
-    <match {logs.kubernetes.yourapplication}>
-      @type rewrite_tag_filter
-      # Add a tag for messages that didn't have parseable json
-      <rule>
-        key     log
-        pattern /"message" *: *\{/
-        tag     ${tag}.hasJson
-      </rule>
-      # Catch all
-      <rule>
-        key     log
-        pattern ^(.*)$
-        tag     ${tag}.noJson
-      </rule>
-    </match>
+    # # This allows us to parse log lines that have json separately, but still keep the unstructured ones
+    # <match {logs.kubernetes.yourapplication}>
+    #   @type rewrite_tag_filter
+    #   # Add a tag for messages that didn't have parseable json
+    #   <rule>
+    #     key     log
+    #     pattern /"message" *: *\{/
+    #     tag     ${tag}.hasJson
+    #   </rule>
+    #   # Catch all
+    #   <rule>
+    #     key     log
+    #     pattern ^(.*)$
+    #     tag     ${tag}.noJson
+    #   </rule>
+    # </match>
 
 
     # Remove fields here we don't care to log
@@ -45,15 +46,15 @@
       remove_keys $.docker.container_id, $.kubernetes.master_url, $.kubernetes.pod_id, $.kubernetes.namespace_id
     </filter>
 
-    # Separate out messages with json into another key called data
-    # message":{
-    <filter logs.kubernetes.*.hasJson>
-      @type record_transformer
-      enable_ruby
-      <record>
-          log ${record["log"].gsub(/"message" *: *\{/, '"data": {')}
-      </record>
-    </filter>
+    # # Separate out messages with json into another key called data
+    # # message":{
+    # <filter logs.kubernetes.*.hasJson>
+    #   @type record_transformer
+    #   enable_ruby
+    #   <record>
+    #       log ${record["log"].gsub(/"message" *: *\{/, '"data": {')}
+    #   </record>
+    # </filter>
 
     # Parse nested fields as json
     <filter logs.kubernetes.**>
@@ -62,7 +63,7 @@
       key_name log
       reserve_data true
       remove_key_name_field true
-      inject_key_prefix log.
+      # inject_key_prefix log.
 
       <parse>
         @type multi_format
@@ -101,7 +102,7 @@
         http.request.bytes ${record["log.request_length"].to_i}
         http.response.bytes ${record["log.bytes_sent"].to_i}
         http.response.status_code ${record["log.status"].to_i}
-        trace.id ${record["log.http_x_correlation_id"]}
+        trace.id ${record["log.request_id"]}
         client.ip ${record["log.remote_addr"]}
         url.path ${record["log.path"]}
         url.query ${record["log.request_query"]}
@@ -110,7 +111,7 @@
       </record>
     </filter>
 
-    # Reformat your application logs into ECS format
+    # # Reformat your application logs into ECS format
     # <filter logs.kubernetes.*.hasJson>
     #   @id application-to-ecs-format
     #   @type record_transformer
@@ -122,7 +123,7 @@
     # </filter>
 
 
-    # Uncomment these to make debugging easier by printing to stdout in fluentd
+    # # Uncomment these to make debugging easier by printing to stdout in fluentd
     # <match logs.kubernetes.nginx>
     #   @type stdout
     # </match>