cloudflare-log-range

#!/bin/bash

# Script to iterate over a time range and download each hour of logs.

# Configure your email and API key below, or set them as environment variables

email="${CLOUDFLARE_EMAIL}"
apikey="${CLOUDFLARE_API_KEY}"

display_usage() {
  echo "Usage: $(basename ${0}) [-h] domain begin end"
  echo ""
  echo "-h      display this help text"
  echo "domain  the zone/domain to download logs for; e.g. example.com"
  echo "begin   the date from which downloaded logs should start"
  echo "end     the date up to which downloaded logs should go"
  echo ""
  echo "Both the begin and end dates should be in full ISO 8601 format,"
  echo "including seconds and the time zone. To see an example of this"
  echo "format, run: date --iso-8601=seconds"
  echo ""
  echo "Provide your API email and key as environment variables:"
  echo "  CLOUDFLARE_EMAIL"
  echo "  CLOUDFLARE_API_KEY"
  echo "Alternatively, you can enter them directly in the script."
  echo ""
  exit 1
}

if (( $# < 1 )); then
  display_usage
fi

if [ $1 = "-h" ]; then
  display_usage
  exit 0
fi

if (( $# < 3 )); then
  display_usage
fi

domain=$1
begin_date=$2
end_date=$3

# Configure requested fields; unmodified this will get all fields available.
# This is left as one huge line to make it easier to comment out.
fields="CacheCacheStatus,CacheResponseBytes,CacheResponseStatus,CacheTieredFill,ClientASN,ClientCountry,ClientDeviceType,ClientIP,ClientIPClass,ClientRequestBytes,ClientRequestHost,ClientRequestMethod,ClientRequestPath,ClientRequestProtocol,ClientRequestReferer,ClientRequestURI,ClientRequestUserAgent,ClientSSLCipher,ClientSSLProtocol,ClientSrcPort,EdgeColoID,EdgeEndTimestamp,EdgePathingOp,EdgePathingSrc,EdgePathingStatus,EdgeRateLimitAction,EdgeRateLimitID,EdgeRequestHost,EdgeResponseBytes,EdgeResponseCompressionRatio,EdgeResponseContentType,EdgeResponseStatus,EdgeServerIP,EdgeStartTimestamp,OriginIP,OriginResponseBytes,OriginResponseHTTPExpires,OriginResponseHTTPLastModified,OriginResponseStatus,OriginResponseTime,OriginSSLProtocol,ParentRayID,RayID,SecurityLevel,WAFAction,WAFFlags,WAFMatchedVar,WAFProfile,WAFRuleID,WAFRuleMessage,WorkerCPUTime,WorkerStatus,WorkerSubrequest,WorkerSubrequestCount,ZoneID"

# Verify the email and API key are configured
if [ -z ${email} ]; then
  echo "Your email is not configured! Provide your email as an envvar (or in the script)." 1>&2
  echo "" 1>&2
  display_usage
  exit 1
fi

if [ -z ${apikey} ]; then
  echo "Your API key is not configured! Provide your API key as an envvar (or in the script)." 1>&2
  echo "" 1>&2
  display_usage
  exit 1
fi

# Verify that the logshare-cli utility is available
if ! command -v logshare-cli > /dev/null; then
  echo "The logshare-cli utility is not available." 1>&2
  echo "Install the utility from here: https://github.com/cloudflare/logshare" 1>&2
fi

# Verify and display the begin date to be used
if date --iso-8601=seconds -d "${begin_date}" > /dev/null; then
  begin_date_unix=$(date -d "${begin_date}" +"%s")
  echo "Using beginning date: ${begin_date}" 1>&2
else
  echo "Begin date is not a valid date. Used: ${begin_date}" 1>&2
  echo "Valid example: $(date --iso-8601=seconds)" 1>&2
  exit 1
fi

# Verify and display the end date to be used
if date --iso-8601=seconds -d "${end_date}" > /dev/null; then
  end_date_unix=$(date -d "${end_date}" +"%s")
  echo "Using end date: ${end_date}" 1>&2
else
  echo "End date is not a valid date. Used: ${end_date}" 1>&2
  echo "Valid example: $(date --iso-8601=seconds)" 1>&2
  exit 1
fi

# Iterate over the hours in the time range and retrieve the logs for each hour
echo "Beginning download of logs, hour by hour..." 1>&2

# Iterate by hourly increments until we pass the end date
iter="${begin_date_unix}"
while (( iter < end_date_unix )); do
  # If the next hour would take us beyond the end date, use the end date instead
  if (( iter + 3600 > end_date_unix )); then
    iter_end="${end_date_unix}"
  else
    iter_end=$(( iter + 3600 ))
  fi
  logshare-cli --api-email "${email}" --api-key "${apikey}" --zone-name "${domain}" --fields "${fields}" --count -1 --start-time "${iter}" --end-time "${iter_end}"
  iter="${iter_end}"
done