ECS/JSON structured logging for Apache OFBiz.

.github/workflows/scorecard.yml

@@ -103,6 +103,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
         if: ${{ !env.ACT }}
-        uses: github/codeql-action/upload-sarif@e58424170fb0262c8d7ed60a2e84b9bffe205c67 # v2.16.2
+        uses: github/codeql-action/upload-sarif@29c712e0b7e56a70523cddd70c2cb49d3e3eb717 # v2.16.2
         with:
           sarif_file: results.sarif

build.gradle

@@ -116,6 +116,12 @@ ext.os = System.getProperty('os.name').toLowerCase()
 ext.gradlew = os.contains('windows') ? 'gradlew.bat' : './gradlew'
 ext.pluginsDir = "${rootDir}/plugins"
 
+// "./gradlew ofbiz" command reflects them directly, with no separate command-line flag to remember.
+def debugProperties = new Properties()
+file('framework/base/config/debug.properties').withInputStream { debugProperties.load(it) }
+ext.jsonLogsEnabled = debugProperties.getProperty('json.logs.enabled', 'false').toBoolean()
+ext.jsonLogsTemplate = debugProperties.getProperty('json.logs.template', 'classpath:templates/ecs-layout.json')
+
 application {
     mainClass = 'org.apache.ofbiz.base.start.Start'
     applicationDefaultJvmArgs = project.hasProperty('jvmArgs')
@@ -130,6 +136,8 @@ application {
             '--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED',
             // Allow libraries using the stable Foreign Function & Memory API (Apache SSHD, Tika)
             '--enable-native-access=ALL-UNNAMED',
+            "-Dofbiz.json.logs=${jsonLogsEnabled}",
+            "-Dofbiz.json.logs.template=${jsonLogsTemplate}",
             ]
 }
 

dependencies.gradle

@@ -51,6 +51,7 @@ dependencies {
     implementation 'org.apache.httpcomponents:httpclient-cache:4.5.14'
     implementation 'org.apache.logging.log4j:log4j-api:2.25.4' // the API of log4j 2
     implementation 'org.apache.logging.log4j:log4j-core:2.25.4' // Somehow needed by Buildbot to compile OFBizDynamicThresholdFilter.java
+    implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.25.4' // JSON/ECS structured logging profile (log4j2-json.xml)
     implementation 'org.apache.poi:poi:5.5.1'
     implementation 'org.apache.pdfbox:pdfbox:3.0.7'
     implementation 'org.apache.pdfbox:pdfbox-io:3.0.7'

framework/base/config/debug.properties

@@ -35,4 +35,60 @@ print.fatal=true
 ## File to display on webtools https://localhost:8443/webtools/control/LogView
 # log4j.appender.css.defaultFile=ofbiz.log
 ## RegExp use to filter file available on dropdown selection to diplay log on https://localhost:8443/webtools/control/FetchLog
-# log4j.appender.css.fileNameRegExp=[(ofbiz)|(error)].*
+# log4j.appender.css.fileNameRegExp=[(ofbiz)|(error)].*
+
+## Global switch for structured (ECS JSON) logging - default false, zero extra cost when disabled.
+## Set to true to have the default "./gradlew ofbiz" command write runtime/logs/ofbiz-json.log
+## alongside the normal text logs; no extra command-line flag needed.
+json.logs.enabled=false
+
+## Which JSON event template to use when structured logging is enabled above - a classpath
+## location, defaulting to the bundled ECS template. Point this at a template shipped in your own
+## custom component to switch formats without touching framework code or passing any JVM flag -
+## picked up automatically by the default "./gradlew ofbiz" command, same as json.logs.enabled
+## above.
+##
+## To define your own template in a component under plugins/, e.g. plugins/your-component/:
+##   1. plugins/your-component/ofbiz-component.xml needs <classpath type="dir" location="config"/>
+##   2. put the template at plugins/your-component/config/templates/your-template.json
+##   3. point this property at it: json.logs.template=classpath:templates/your-template.json
+## Note: classpath: resolution isn't scoped per-component - it finds the first matching resource
+## across the entire combined classpath, so a filename collision with ecs-layout.json or another
+## component's template is ambiguous/classpath-order-dependent. Use a distinctive name (e.g.
+## templates/mycompany-splunk-template.json, not templates/splunk-template.json) to avoid that.
+json.logs.template=classpath:templates/ecs-layout.json
+
+## How userLoginId is written into the log correlation context (see CorrelationValve in
+## framework/catalina). userLoginId is frequently PII (often an email address), and today it is
+## the only correlation field written into ofbiz-json.log's "labels" - the plain-text logs never
+## included it, so this setting only affects structured JSON logging.
+##   plain (default) - current behavior, raw userLoginId.
+##   hash  - deterministic PBKDF2 digest keyed with log.correlation.userLoginId.hash.pepper below;
+##           the same user always yields the same value, so correlation across requests still
+##           works, but the raw userLoginId isn't in the logs. Still deterministic/linkable, so
+##           this reduces exposure to a casual reader of the logs - it does not by itself make
+##           userLoginId stop being personal data under regulations like GDPR.
+##   off   - userLoginId is left out of the correlation context entirely. Use this, not hash, when
+##           the actual requirement is "no PII in logs" - correlate on requestId/visitId instead.
+## mode=hash REQUIRES the pepper property below to be set - OFBiz refuses to start with mode=hash
+## and no pepper configured, rather than silently hashing without one.
+log.correlation.userLoginId.mode=plain
+
+## Mandatory when log.correlation.userLoginId.mode=hash above, ignored otherwise. Deployment-
+## specific secret used as the PBKDF2 salt when hashing userLoginId. Treat this with the same
+## handling rigor as a database password: it's one shared secret for every user (a per-user random
+## salt would make the same user hash differently on every request, defeating the correlation this
+## field exists for), so if it leaks, every userLoginId ever written to the logs under this
+## deployment becomes crackable offline in one shot.
+log.correlation.userLoginId.hash.pepper=
+
+## Only used when log.correlation.userLoginId.mode=hash above, ignored otherwise. Length in bits of
+## the derived digest written into the logs - shorter values produce a shorter userLoginId field in
+## every log line. This does not weaken resistance against an attacker who holds the pepper (that's
+## governed by the iteration count and pepper secrecy alone); it only raises the odds of two
+## different users' digests colliding, which is negligible at the 128-bit default for any realistic
+## deployment. If you need to trace a specific user's activity, correlate via visitId against the
+## Visit entity instead of trying to reverse this digest.
+## Must be a positive number when mode=hash - OFBiz refuses to start with mode=hash and a zero,
+## negative, or otherwise invalid value here, rather than letting it fail on the first login attempt.
+log.correlation.userLoginId.hash.keyLengthBits=128

framework/base/config/log4j2.xml

@@ -29,6 +29,15 @@ under the License.
         <Property name="includeLocation_prod">false</Property>
         <Property name="logPattern">%date{DEFAULT} |%-20.20thread |%-30.30logger{1}${lineToken_${sys:ofbiz.env:-dev}}|%level{length=1}| %message%n</Property>
         <Property name="includeLocation">${includeLocation_${sys:ofbiz.env:-dev}}</Property>
+        <!--
+          Placeholder for the JSON event template used by the opt-in structured logging appender
+          below. Defaults to the bundled ECS template. To use a different schema (e.g. Splunk,
+          CloudWatch), ship a template JSON file in your own component's config/ directory and
+          point json.logs.template at it (classpath:your-template.json) in
+          framework/base/config/debug.properties - the default "./gradlew ofbiz" command picks
+          that up directly, no JVM flag or change to this file required.
+        -->
+        <Property name="jsonLogTemplateUri">${sys:ofbiz.json.logs.template:-classpath:templates/ecs-layout.json}</Property>
     </Properties>
     <OFBizDynamicThresholdFilter key="uri" onMatch="ACCEPT" onMismatch="DENY">
         <KeyValuePair key="/getJs" value="ERROR"/>
@@ -74,10 +83,40 @@ under the License.
             <DefaultRolloverStrategy fileIndex="min" max="10"/>
         </RollingFile>
 
+        <!--
+          Opt-in structured (ECS JSON) logging - disabled by default, so this contributes zero
+          extra disk/CPU cost unless explicitly enabled. Controlled by the json.logs.enabled
+          switch in framework/base/config/debug.properties (default false); the default
+          "./gradlew ofbiz" command already picks that up, no separate flag needed. When enabled,
+          writes runtime/logs/ofbiz-json.log alongside the plain-text files above, unchanged.
+          The Select/SystemPropertyArbiter below means this appender (and its AppenderRef further
+          down) simply doesn't exist in the resolved configuration when disabled.
+        -->
+        <Select>
+            <SystemPropertyArbiter propertyName="ofbiz.json.logs" propertyValue="true">
+                <RollingFile name="jsonFile" fileName="runtime/logs/ofbiz-json.log" filePattern="runtime/logs/ofbiz-json.log.%i">
+                    <JsonTemplateLayout eventTemplateUri="${jsonLogTemplateUri}">
+                        <EventTemplateAdditionalField key="service.name" value="ofbiz"/>
+                        <EventTemplateAdditionalField key="service.environment" value="${sys:ofbiz.env:-dev}"/>
+                    </JsonTemplateLayout>
+                    <Policies>
+                        <OnStartupTriggeringPolicy/>
+                        <SizeBasedTriggeringPolicy size="10 MB"/>
+                    </Policies>
+                    <DefaultRolloverStrategy fileIndex="min" max="30"/>
+                </RollingFile>
+            </SystemPropertyArbiter>
+        </Select>
+
         <Async name="async" includeLocation="${includeLocation}">
             <AppenderRef ref="ofbiz"/>
             <AppenderRef ref="stdout"/>
             <AppenderRef ref="error"/>
+            <Select>
+                <SystemPropertyArbiter propertyName="ofbiz.json.logs" propertyValue="true">
+                    <AppenderRef ref="jsonFile"/>
+                </SystemPropertyArbiter>
+            </Select>
         </Async>
     </Appenders>
 

framework/base/config/templates/ecs-layout.json

@@ -0,0 +1,49 @@
+{
+  "@timestamp": {
+    "$resolver": "timestamp",
+    "pattern": {
+      "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
+      "timeZone": "UTC"
+    }
+  },
+  "ecs.version": "1.2.0",
+  "log.level": {
+    "$resolver": "level",
+    "field": "name"
+  },
+  "message": {
+    "$resolver": "message",
+    "stringified": true
+  },
+  "process.thread.name": {
+    "$resolver": "thread",
+    "field": "name"
+  },
+  "log.logger": {
+    "$resolver": "logger",
+    "field": "name"
+  },
+  "labels": {
+    "$resolver": "mdc",
+    "flatten": false,
+    "stringified": true
+  },
+  "tags": {
+    "$resolver": "ndc"
+  },
+  "error.type": {
+    "$resolver": "exception",
+    "field": "className"
+  },
+  "error.message": {
+    "$resolver": "exception",
+    "field": "message"
+  },
+  "error.stack_trace": {
+    "$resolver": "exception",
+    "field": "stackTrace",
+    "stackTrace": {
+      "stringified": true
+    }
+  }
+}

framework/catalina/src/main/java/org/apache/ofbiz/catalina/container/CatalinaContainer.java

@@ -376,6 +376,9 @@ private static List<Valve> prepareTomcatEngineValves(Configuration.Property engi
             throws ContainerException {
         List<Valve> engineValves = new ArrayList<>();
 
+        // populates Log4j2 ThreadContext with requestId/visitId/userLoginId for log correlation
+        engineValves.add(new CorrelationValve());
+
         // configure the CrossSubdomainSessionValve
         if (ContainerConfig.getPropertyValue(engineConfig, "enable-cross-subdomain-sessions", false)) {
             engineValves.add(new CrossSubdomainSessionValve());

framework/catalina/src/main/java/org/apache/ofbiz/catalina/container/CorrelationFieldProvider.java

@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *******************************************************************************/
+package org.apache.ofbiz.catalina.container;
+
+import java.util.Map;
+
+import org.apache.catalina.connector.Request;
+import org.apache.ofbiz.entity.GenericValue;
+
+/**
+ * Extension point for {@link CorrelationValve}: lets a component contribute additional
+ * key/value pairs to the per-request logging correlation context (Log4j2 {@code ThreadContext}),
+ * beyond the core {@code requestId}/{@code visitId}/{@code userLoginId} fields the Valve always
+ * sets.
+ *
+ * <p>Implementations are discovered via {@link java.util.ServiceLoader} - a component adds one by
+ * providing an implementation class plus a
+ * {@code META-INF/services/org.apache.ofbiz.catalina.container.CorrelationFieldProvider} file
+ * naming it, with no framework code changes required. All discovered providers are called for
+ * every request and their fields merged together.
+ */
+public interface CorrelationFieldProvider {
+
+    /**
+     * Returns additional correlation fields for the current request, or {@code null}/empty if
+     * this provider has nothing to add (e.g. an anonymous request for a provider that only
+     * contributes fields for logged-in users).
+     * @param request the current request
+     * @param userLogin the logged-in user, or {@code null} if the request is anonymous
+     */
+    Map<String, String> getFields(Request request, GenericValue userLogin);
+}