Merge pull request #22 from u-s-p/feature-modularize

Feature modularization, refactoring into separate config files

Author: bitsofinfo

0000_header.conf

@@ -0,0 +1,39 @@
+##########################################################
+##########################################################
+#
+# Example Modsecurity audit log ingestor
+# configuration for Logstash
+#
+# @author bitsofinfo.g[at]gmail.com
+# built/tested w logstash v1.3.x line
+#
+#
+# @see http://logstash.net/
+# @see https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats
+# @see http://bitsofinfo.wordpress.com/2013/09/19/logstash-for-modsecurity-audit-logs/
+#
+# @license http://www.apache.org/licenses/LICENSE-2.0
+#
+# @notes NOTE: this is not perfect and I am no Ruby expert
+#        however this worked when processing quite a bit of
+#        high volume mod-sec logs with lots of different
+#        variations in what A-K sections were and were not
+#        present. At a minimum its a good starting point
+#        to start tackling a complex log format.
+#
+# Be careful w/ the custom ruby filter blocks and be aware
+# of https://logstash.jira.com/browse/LOGSTASH-1375
+#
+# This config file for whatever reason will not run
+# if you try to add the "-- web" option onto the logstash
+# flat jar. This has been reported to the developers.
+# Recommend you run this without the "-- web" option and just
+# hook up Kibana separately.
+#
+# Enable the "-v" verbose option when starting logstash
+# to aid in debugging things. Disable the "-v" option
+# when running in real/non-debug environment
+#
+#
+##########################################################
+##########################################################

1000_input_stdin_example.conf

@@ -0,0 +1,13 @@
+input {
+
+  stdin {
+    type => "mod_security"
+
+    codec => multiline {
+      pattern => "^--[a-fA-F0-9]{8}-Z--$"
+      negate => true
+      what => previous
+    }
+  }
+
+}

1010_input_file_example.conf

@@ -0,0 +1,20 @@
+input {
+  file {
+    # IMPORTANT! set this correctly to the charset
+    # that your server writes these log files in
+    path => "/path/to/your/modsec/audit/logs/*.log"
+    type => "mod_security"
+
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # merge all modsec events for a given entity into the same event.
+    # so essentially the modsec -Z marker is used as the splitter
+    # which is the end of each modsec logical event in the logfile
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    codec => multiline {
+      charset => "US-ASCII"
+      pattern => "^--[a-fA-F0-9]{8}-Z--$"
+      negate => true
+      what => previous
+    }
+  }
+}

2000_filter_sections_split.conf

@@ -0,0 +1,45 @@
+filter {
+  if [type] == "mod_security" {
+
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Due to the complexity of the collapsed single string
+    # we get from multiline and the variance of exactly
+    # which modsec sections (A-K) may or may not be in each
+    # log entry, we run some custom ruby code that will
+    # split on each modsec "section" and store each found in
+    # new fields named "rawSection[A-K]" as appropriate, the value
+    # of each of these fields contains the raw un-parsed data
+    # from that modsec section. Sections that are non-existant
+    # will not have a key in "fields"
+    #
+    # A bit long and crazy yes, but after spending many hours
+    # just doing this w/ grok patterns, this ended up being the
+    # most reliable way to break up this in-consistent format into
+    # more usable blocks
+    #
+    # @see https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats
+    #
+    # READ the above to get a good understanding of the sections
+    # and which ones can actively contain data depending on your modsec
+    # version and environment!
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    ruby {
+      code => "
+          if !event['message'].nil?
+              modSecSectionData = event['message'].split(/(?:--[a-fA-F0-9]{8}-([A-Z])--)/)
+              modSecSectionData.shift
+              for i in 0..((modSecSectionData.length-1)/2)
+                  sectionName = 'rawSection'.concat(modSecSectionData.shift)
+                  sectionData = modSecSectionData.shift
+                  sectionName = sectionName.strip
+                  if !sectionData.nil?
+                      sectionData = sectionData.strip
+                  end
+                  event.to_hash.merge!(sectionName => sectionData)
+              end
+          end
+        "
+    }
+  }
+}

2010_filter_section_a_parse.conf

@@ -0,0 +1,14 @@
+filter {
+  if [type] == "mod_security" {
+
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Parse out fields from Section A (general event basics)
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    grok {
+      match => {
+        "rawSectionA" => "\[%{L1_TIMESTAMP:modsec_timestamp}\] %{DATA:uniqueId} %{IP:sourceIp} %{INT:sourcePort} %{IP:destIp} %{INT:destPort}"
+      }
+      patterns_dir => "./patterns/logstash_modsecurity_patterns"
+    }
+  }
+}

2020_filter_section_b_parse_request_line.conf

@@ -0,0 +1,19 @@
+filter {
+  if [type] == "mod_security" {
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Parse out fields from Section B (request related line 1)
+    # note line one could be garbage OR adhere to the
+    # httpMethod [space] uri [space] protocol pattern
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    # if a legit line... normal http request
+    if [rawSectionB] =~ /.+/ {
+      grok {
+        match => {
+          "rawSectionB" => [  "(?m)^%{DATA:httpMethod}\s(?<requestedUri>\S+)\s(?<incomingProtocol>[^\n]+)(?:\n(?<raw_requestHeaders>.+)?)?$",
+                              "(?<httpMethod>^(.*)$)" ]
+        }
+      }
+    }
+  }
+}

2021_filter_section_b_headers_key-value.conf

@@ -0,0 +1,30 @@
+filter {
+  if [type] == "mod_security" {
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Convert raw request headers into a key/value
+    # pair map
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    if [raw_requestHeaders] =~ /.+/ {
+      kv {
+        source => "raw_requestHeaders"
+        field_split => "\n"
+        value_split => ":"
+        target => "requestHeaders"
+      }
+
+
+      # trim leading/trailing hack  @see https://logstash.jira.com/browse/LOGSTASH-1369
+      ruby {
+        code => "
+            requestHeaders = event.to_hash['requestHeaders']
+            requestHeaders.each { |k, v|
+              if !v.nil? and v.is_a? String
+                requestHeaders[k] = v.strip
+              end
+            }
+          "
+      }
+    }
+  }
+}

2029_filter_section_b_example_header_Cookie.conf

@@ -0,0 +1,18 @@
+filter {
+  if [type] == "mod_security" {
+
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Example of looking for a specific Cookie and promoting
+    # it to a first class field
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    if [raw_requestHeaders] =~ /Cookie/ and [raw_requestHeaders] =~ /myCookie=.+\b/ {
+
+      grok {
+        match => {
+          "raw_requestHeaders" => "(?<myCookie>myCookie[^; \s]+)"
+        }
+      }
+    }
+  }
+}

2029_filter_section_b_example_header_X-Forwarded-For.conf

@@ -0,0 +1,22 @@
+filter {
+  if [type] == "mod_security" {
+
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Promote real request IP to a field if it exists
+    # in the request headers section
+    #
+    # NOTE this is an example of promoting a custom header to a first
+    # class field that might be set by a app firewall or other
+    # upstream proxy
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    if [raw_requestHeaders] =~ /X-Forwarded-For:/ {
+
+      grok {
+        match => {
+          "raw_requestHeaders" => "X-Forwarded-For: %{IPORHOST:XForwardedFor}"
+        }
+      }
+    }
+  }
+}

2029_filter_section_b_example_splitt_all_cockies.conf

@@ -0,0 +1,18 @@
+filter {
+  if [type] == "mod_security" {
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Example of splitting all Cookies from the requestHeader Cookie
+    # and promoting it to a first class field
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+      if [requestHeaders][Cookie] =~ /.+/ {
+          kv {
+              source => "[requestHeaders][Cookie]"
+              field_split => "; "
+              value_split => "="
+              target => "requestCookies"
+          }
+      }
+    }
+  }
+}