From 1c858a2fa0d9889141e4b6edc053b8eeb7f9c7ef Mon Sep 17 00:00:00 2001
From: Gabor Roczei <1918366+roczei@users.noreply.github.com>
Date: Thu, 16 Jun 2022 14:01:35 +0200
Subject: [PATCH] [LIVY-1040] Migrate logging framework from Reload4j to Log4j2
Replaced the reload4j dependency with the Log4j2 library across all
modules.
Co-authored by Peter Toth and Gyorgy Gal:
- Added log4j-api, log4j-core, log4j-slf4j-impl and log4j-1.2-api
(backward-compat bridge) to dependencies
- Removed ch.qos.reload4j:reload4j from all module poms
- Migrated all log4j.properties test config files to log4j2.properties
using the new Log4j2 property-file syntax (rootLogger, appender.* keys)
Co-authored-by: Peter Toth <7253827+peter-toth@users.noreply.github.com>
Co-authored-by: Gyorgy Gal <27883675+gyogal@users.noreply.github.com>
---
api/src/test/resources/log4j2.properties | 43 +++++++
client-common/pom.xml | 2 +-
.../src/test/resources/log4j2.properties | 41 +++++++
...es.template => log4j2.properties.template} | 16 ++-
core/pom.xml | 18 ++-
.../conf/livy/log4j.properties | 33 ------
.../conf/livy/log4j2.properties | 43 +++++++
.../conf/master/log4j.properties | 35 ------
.../conf/master/log4j2.properties | 42 +++++++
.../conf/worker/log4j.properties | 35 ------
.../conf/worker/log4j2.properties | 43 +++++++
integration-test/pom.xml | 8 ++
.../{log4j.properties => log4j2.properties} | 45 +++++---
pom.xml | 106 +++++++++++++++++-
repl/pom.xml | 21 +++-
.../src/test/resources/log4j2.properties | 28 ++---
.../apache/livy/repl/ReplDriverSuite.scala | 2 +-
rsc/pom.xml | 21 +++-
rsc/src/test/resources/log4j.properties | 26 -----
.../src/test/resources/log4j2.properties | 28 ++---
scala-api/pom.xml | 2 +-
.../{log4j.properties => log4j2.properties} | 31 ++---
.../livy/scalaapi/ScalaClientTest.scala | 2 +-
server/pom.xml | 10 +-
.../{log4j.properties => log4j2.properties} | 35 +++---
thriftserver/client/pom.xml | 12 ++
thriftserver/server/pom.xml | 6 +-
.../{log4j.properties => log4j2.properties} | 36 +++---
thriftserver/session/pom.xml | 2 +-
29 files changed, 529 insertions(+), 243 deletions(-)
create mode 100644 api/src/test/resources/log4j2.properties
create mode 100644 client-http/src/test/resources/log4j2.properties
rename conf/{log4j.properties.template => log4j2.properties.template} (71%)
delete mode 100644 dev/docker/livy-dev-cluster/conf/livy/log4j.properties
create mode 100644 dev/docker/livy-dev-cluster/conf/livy/log4j2.properties
delete mode 100644 dev/docker/livy-dev-cluster/conf/master/log4j.properties
create mode 100644 dev/docker/livy-dev-cluster/conf/master/log4j2.properties
delete mode 100644 dev/docker/livy-dev-cluster/conf/worker/log4j.properties
create mode 100644 dev/docker/livy-dev-cluster/conf/worker/log4j2.properties
rename integration-test/src/test/resources/{log4j.properties => log4j2.properties} (51%)
rename client-http/src/test/resources/log4j.properties => repl/src/test/resources/log4j2.properties (63%)
delete mode 100644 rsc/src/test/resources/log4j.properties
rename repl/src/test/resources/log4j.properties => rsc/src/test/resources/log4j2.properties (63%)
rename scala-api/src/test/resources/{log4j.properties => log4j2.properties} (63%)
rename server/src/test/resources/{log4j.properties => log4j2.properties} (60%)
rename thriftserver/server/src/test/resources/{log4j.properties => log4j2.properties} (56%)
diff --git a/api/src/test/resources/log4j2.properties b/api/src/test/resources/log4j2.properties
new file mode 100644
index 000000000..7585fd387
--- /dev/null
+++ b/api/src/test/resources/log4j2.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
+
+# Tests that launch java subprocesses can set the "test.appender" system property to
+# "console" to avoid having the child process's logs overwrite the unit test's
+# log file.
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
+
+# Ignore messages below warning level from Jetty, because it's a bit verbose
+logger.livy_shaded.name = org.apache.livy.shaded
+logger.livy_shaded.level = info
+logger.spark_project_jetty.name = org.sparkproject.jetty
+logger.spark_project_jetty.level = warn
+logger.eclipse_jetty.name = org.eclipse.jetty
+logger.eclipse_jetty.level = warn
diff --git a/client-common/pom.xml b/client-common/pom.xml
index fd49c2ec0..6a3cd07f8 100644
--- a/client-common/pom.xml
+++ b/client-common/pom.xml
@@ -46,7 +46,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
provided
diff --git a/client-http/src/test/resources/log4j2.properties b/client-http/src/test/resources/log4j2.properties
new file mode 100644
index 000000000..b1efa04c2
--- /dev/null
+++ b/client-http/src/test/resources/log4j2.properties
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
+
+# Tests that launch java subprocesses can set the "test.appender" system property to
+# "console" to avoid having the child process's logs overwrite the unit test's
+# log file.
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
+
+# Ignore messages below warning level from Jetty, because it's a bit verbose
+logger.sparkproject.name = org.apache.livy.shaded
+logger.sparkproject.level = info
+logger.jetty.name = org.eclipse.jetty
+logger.jetty.level = warn
diff --git a/conf/log4j.properties.template b/conf/log4j2.properties.template
similarity index 71%
rename from conf/log4j.properties.template
rename to conf/log4j2.properties.template
index 70b67a6d5..3f9276cd8 100644
--- a/conf/log4j.properties.template
+++ b/conf/log4j2.properties.template
@@ -15,10 +15,14 @@
# limitations under the License.
#
# The default Livy logging configuration.
-log4j.rootCategory=INFO, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
-log4j.logger.org.eclipse.jetty=WARN
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = console
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
+
+logger.jetty.name = org.eclipse.jetty
+logger.jetty.level = warn
diff --git a/core/pom.xml b/core/pom.xml
index fca0600b7..8983f0d0a 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -38,8 +38,20 @@
- ch.qos.reload4j
- reload4j
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+ org.apache.logging.log4j
+ log4j-api
+
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+ org.apache.logging.log4j
+ log4j-1.2-api
@@ -59,7 +71,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
diff --git a/dev/docker/livy-dev-cluster/conf/livy/log4j.properties b/dev/docker/livy-dev-cluster/conf/livy/log4j.properties
deleted file mode 100644
index c3a2d09b0..000000000
--- a/dev/docker/livy-dev-cluster/conf/livy/log4j.properties
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-log4j.rootCategory=INFO, console, DRFA
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=/logs/livy-server.log
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n
-
-log4j.logger.org.eclipse.jetty=WARN
diff --git a/dev/docker/livy-dev-cluster/conf/livy/log4j2.properties b/dev/docker/livy-dev-cluster/conf/livy/log4j2.properties
new file mode 100644
index 000000000..5025fab20
--- /dev/null
+++ b/dev/docker/livy-dev-cluster/conf/livy/log4j2.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+property.livy.log.dir = /logs
+property.livy.log.file = livy-server.log
+
+rootLogger.level = info
+rootLogger.appenderRefs = stdout, drfa
+rootLogger.appenderRef.stdout.ref = stdout
+rootLogger.appenderRef.drfa.ref = drfa
+
+appender.stdout.type = Console
+appender.stdout.name = stdout
+appender.stdout.layout.type = PatternLayout
+appender.stdout.layout.pattern = %d %p %c{1} [%t]: %m%n
+
+appender.drfa.type = RollingFile
+appender.drfa.name = drfa
+appender.drfa.fileName = ${livy.log.dir}/${livy.log.file}
+appender.drfa.filePattern = ${livy.log.dir}/${livy.log.file}.%d{yyyy-MM-dd}
+appender.drfa.layout.type = PatternLayout
+appender.drfa.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+appender.drfa.policies.type = Policies
+appender.drfa.policies.time.type = TimeBasedTriggeringPolicy
+appender.drfa.policies.time.interval = 1
+
+logger.jetty.name = org.eclipse.jetty
+logger.jetty.level = warn
+
diff --git a/dev/docker/livy-dev-cluster/conf/master/log4j.properties b/dev/docker/livy-dev-cluster/conf/master/log4j.properties
deleted file mode 100644
index 644032d2d..000000000
--- a/dev/docker/livy-dev-cluster/conf/master/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-log4j.rootCategory=INFO, console, DRFA
-log4jspark.log.dir=/logs
-log4jspark.log.file=spark-master.log
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${log4jspark.log.dir}/${log4jspark.log.file}
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n
-
-log4j.logger.org.eclipse.jetty=WARN
diff --git a/dev/docker/livy-dev-cluster/conf/master/log4j2.properties b/dev/docker/livy-dev-cluster/conf/master/log4j2.properties
new file mode 100644
index 000000000..70db369b9
--- /dev/null
+++ b/dev/docker/livy-dev-cluster/conf/master/log4j2.properties
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+property.spark.log.dir = /logs
+property.spark.log.file = spark-master.log
+
+rootLogger.level = info
+rootLogger.appenderRefs = stdout, drfa
+rootLogger.appenderRef.stdout.ref = stdout
+rootLogger.appenderRef.drfa.ref = drfa
+
+appender.stdout.type = Console
+appender.stdout.name = stdout
+appender.stdout.layout.type = PatternLayout
+appender.stdout.layout.pattern = %d %p %c{1} [%t]: %m%n
+
+appender.drfa.type = RollingFile
+appender.drfa.name = drfa
+appender.drfa.fileName = ${spark.log.dir}/${spark.log.file}
+appender.drfa.filePattern = ${spark.log.dir}/${spark.log.file}.%d{yyyy-MM-dd}
+appender.drfa.layout.type = PatternLayout
+appender.drfa.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+appender.drfa.policies.type = Policies
+appender.drfa.policies.time.type = TimeBasedTriggeringPolicy
+appender.drfa.policies.time.interval = 1
+
+logger.jetty.name = org.eclipse.jetty
+logger.jetty.level = warn
diff --git a/dev/docker/livy-dev-cluster/conf/worker/log4j.properties b/dev/docker/livy-dev-cluster/conf/worker/log4j.properties
deleted file mode 100644
index 23c5ee5d2..000000000
--- a/dev/docker/livy-dev-cluster/conf/worker/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-log4j.rootCategory=INFO, console, DRFA
-log4jspark.log.dir=/logs
-log4jspark.log.file=spark-worker.log
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${log4jspark.log.dir}/${log4jspark.log.file}
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n
-
-log4j.logger.org.eclipse.jetty=WARN
diff --git a/dev/docker/livy-dev-cluster/conf/worker/log4j2.properties b/dev/docker/livy-dev-cluster/conf/worker/log4j2.properties
new file mode 100644
index 000000000..8003e403c
--- /dev/null
+++ b/dev/docker/livy-dev-cluster/conf/worker/log4j2.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+property.spark.log.dir = /logs
+property.spark.log.file = spark-worker.log
+
+rootLogger.level = info
+rootLogger.appenderRefs = stdout, drfa
+rootLogger.appenderRef.stdout.ref = stdout
+rootLogger.appenderRef.drfa.ref = drfa
+
+appender.stdout.type = Console
+appender.stdout.name = stdout
+appender.stdout.layout.type = PatternLayout
+appender.stdout.layout.pattern = %d %p %c{1} [%t]: %m%n
+
+appender.drfa.type = RollingFile
+appender.drfa.name = drfa
+appender.drfa.fileName = ${spark.log.dir}/${spark.log.file}
+appender.drfa.filePattern = ${spark.log.dir}/${spark.log.file}.%d{yyyy-MM-dd}
+appender.drfa.layout.type = PatternLayout
+appender.drfa.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+appender.drfa.policies.type = Policies
+appender.drfa.policies.time.type = TimeBasedTriggeringPolicy
+appender.drfa.policies.time.interval = 1
+
+logger.jetty.name = org.eclipse.jetty
+logger.jetty.level = warn
+
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index a803b3573..a38d5cd6a 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -215,6 +215,10 @@
log4j
log4j
+
+ org.slf4j
+ slf4j-api
+
org.slf4j
slf4j-log4j12
@@ -337,6 +341,10 @@
org.apache.logging.log4j
log4j-slf4j-impl
+
+ log4j
+ log4j
+
org.slf4j
slf4j-log4j12
diff --git a/integration-test/src/test/resources/log4j.properties b/integration-test/src/test/resources/log4j2.properties
similarity index 51%
rename from integration-test/src/test/resources/log4j.properties
rename to integration-test/src/test/resources/log4j2.properties
index 295df53b6..92d744179 100644
--- a/integration-test/src/test/resources/log4j.properties
+++ b/integration-test/src/test/resources/log4j2.properties
@@ -16,27 +16,36 @@
#
# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=DEBUG, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+rootLogger.level = debug
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
# Tests that launch java subprocesses can set the "test.appender" system property to
# "console" to avoid having the child process's logs overwrite the unit test's
# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t: %m%n
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
# Silence 3rd party libraries.
-log4j.logger.org.apache.livy.rsc=INFO
-log4j.logger.org.apache.livy.shaded=INFO
-log4j.logger.com.decodified=WARN
-log4j.logger.com.ning.http=WARN
-log4j.logger.org.apache.hadoop=WARN
-log4j.logger.org.eclipse.jetty=WARN
-log4j.logger.net.schmizz=WARN
+logger.livy_rsc.name = org.apache.livy.rsc
+logger.livy_rsc.level = info
+logger.livy_shaded.name = org.apache.livy.shaded
+logger.livy_shaded.level = info
+logger.decodified.name = com.decodified
+logger.decodified.level = warn
+logger.ning_http.name = com.ning.http
+logger.ning_http.level = warn
+logger.hadoop.name = org.apache.hadoop
+logger.hadoop.level = warn
+logger.schmizz.name = net.schmizz
+logger.schmizz.level = warn
+logger.jetty.name = org.eclipse.jetty
+logger.jetty.level = warn
diff --git a/pom.xml b/pom.xml
index fba6efcc8..16d26d6b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,7 +81,6 @@
2.10.1
compile
1.7.36
- 1.2.26
3.3.4
${spark.scala-2.12.version}
5.6.0
@@ -97,6 +96,7 @@
9.4.56.v20240826
4.13.1
0.9.3
+ 2.26.0
4.0.2
3.1.0
1.10.19
@@ -282,9 +282,25 @@
- ch.qos.reload4j
- reload4j
- ${reload4j.version}
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ ${log4j.version}
+
+
+ org.apache.logging.log4j
+ log4j-api
+ ${log4j.version}
+
+
+ org.apache.logging.log4j
+ log4j-core
+ ${log4j.version}
+
+
+
+ org.apache.logging.log4j
+ log4j-1.2-api
+ ${log4j.version}
@@ -378,6 +394,14 @@
org.slf4j
slf4j-log4j12
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -414,6 +438,14 @@
com.sun.jersey
jersey-server
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -443,6 +475,14 @@
com.sun.jersey
jersey-server
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -483,6 +523,14 @@
org.codehaus.jackson
jackson-xc
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -495,6 +543,18 @@
log4j
log4j
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -512,6 +572,18 @@
com.sun.jersey
jersey-core
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -566,6 +638,18 @@
org.codehaus.jackson
jackson-jaxrs
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -593,6 +677,18 @@
com.sun.jersey
jersey-client
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ ch.qos.reload4j
+ reload4j
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -857,7 +953,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
${slf4j.version}
diff --git a/repl/pom.xml b/repl/pom.xml
index b3e96324f..7103b1b18 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -84,8 +84,23 @@
- ch.qos.reload4j
- reload4j
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-api
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-core
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-1.2-api
provided
@@ -158,7 +173,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
provided
diff --git a/client-http/src/test/resources/log4j.properties b/repl/src/test/resources/log4j2.properties
similarity index 63%
rename from client-http/src/test/resources/log4j.properties
rename to repl/src/test/resources/log4j2.properties
index 9a1c19f26..ab02104c6 100644
--- a/client-http/src/test/resources/log4j.properties
+++ b/repl/src/test/resources/log4j2.properties
@@ -16,22 +16,24 @@
#
# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=INFO, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
# Tests that launch java subprocesses can set the "test.appender" system property to
# "console" to avoid having the child process's logs overwrite the unit test's
# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%t: %m%n
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
# Ignore messages below warning level from Jetty, because it's a bit verbose
-log4j.logger.org.apache.livy.shaded=INFO
-log4j.logger.org.eclipse.jetty=WARN
+logger.jetty.name = org.sparkproject.jetty
+logger.jetty.level = warn
diff --git a/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala b/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala
index da64577e4..e2a753af9 100644
--- a/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala
+++ b/repl/src/test/scala/org/apache/livy/repl/ReplDriverSuite.scala
@@ -39,7 +39,7 @@ class ReplDriverSuite extends FunSuite with LivyBaseUnitTestSuite {
test("start a repl session using the rsc") {
val client = new LivyClientBuilder()
- .setConf(SparkLauncher.DRIVER_MEMORY, "512m")
+ .setConf(SparkLauncher.DRIVER_MEMORY, "1024m")
.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, sys.props("java.class.path"))
.setConf(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, sys.props("java.class.path"))
.setConf(RSCConf.Entry.LIVY_JARS.key(), "")
diff --git a/rsc/pom.xml b/rsc/pom.xml
index c45d37619..c5f9659af 100644
--- a/rsc/pom.xml
+++ b/rsc/pom.xml
@@ -87,8 +87,23 @@
provided
- ch.qos.reload4j
- reload4j
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-api
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-core
+ provided
+
+
+ org.apache.logging.log4j
+ log4j-1.2-api
provided
@@ -118,7 +133,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
provided
diff --git a/rsc/src/test/resources/log4j.properties b/rsc/src/test/resources/log4j.properties
deleted file mode 100644
index 6143ee577..000000000
--- a/rsc/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-log4j.rootCategory=WARN, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
-
-# Silence some verbose logs.
-log4j.logger.org.spark-project.jetty=WARN
diff --git a/repl/src/test/resources/log4j.properties b/rsc/src/test/resources/log4j2.properties
similarity index 63%
rename from repl/src/test/resources/log4j.properties
rename to rsc/src/test/resources/log4j2.properties
index a54d27de9..ab02104c6 100644
--- a/repl/src/test/resources/log4j.properties
+++ b/rsc/src/test/resources/log4j2.properties
@@ -16,22 +16,24 @@
#
# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=INFO, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
# Tests that launch java subprocesses can set the "test.appender" system property to
# "console" to avoid having the child process's logs overwrite the unit test's
# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%t: %m%n
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
# Ignore messages below warning level from Jetty, because it's a bit verbose
-log4j.logger.org.spark-project.jetty=WARN
-org.spark-project.jetty.LEVEL=WARN
+logger.jetty.name = org.sparkproject.jetty
+logger.jetty.level = warn
diff --git a/scala-api/pom.xml b/scala-api/pom.xml
index 991fb4412..9c004d0c7 100644
--- a/scala-api/pom.xml
+++ b/scala-api/pom.xml
@@ -90,7 +90,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
provided
diff --git a/scala-api/src/test/resources/log4j.properties b/scala-api/src/test/resources/log4j2.properties
similarity index 63%
rename from scala-api/src/test/resources/log4j.properties
rename to scala-api/src/test/resources/log4j2.properties
index 7f24092ff..ab02104c6 100644
--- a/scala-api/src/test/resources/log4j.properties
+++ b/scala-api/src/test/resources/log4j2.properties
@@ -14,21 +14,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=INFO, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
+
# Tests that launch java subprocesses can set the "test.appender" system property to
# "console" to avoid having the child process's logs overwrite the unit test's
# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%t: %m%n
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
+
# Ignore messages below warning level from Jetty, because it's a bit verbose
-log4j.logger.org.spark-project.jetty=WARN
-org.spark-project.jetty.LEVEL=WARN
+logger.jetty.name = org.sparkproject.jetty
+logger.jetty.level = warn
diff --git a/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala
index a716f5856..48c80d07e 100644
--- a/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala
+++ b/scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTest.scala
@@ -182,7 +182,7 @@ object ScalaClientTest {
} else {
val classpath: String = System.getProperty("java.class.path")
conf.put("spark.app.name", "SparkClientSuite Remote App")
- conf.put(SparkLauncher.DRIVER_MEMORY, "512m")
+ conf.put(SparkLauncher.DRIVER_MEMORY, "1024m")
conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, classpath)
conf.put(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, classpath)
}
diff --git a/server/pom.xml b/server/pom.xml
index 29fe3555c..8b3ebeb97 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -164,10 +164,18 @@
log4j
log4j
+
+ org.slf4j
+ slf4j-api
+
org.slf4j
slf4j-log4j12
+
+ log4j
+ log4j
+
@@ -231,7 +239,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
diff --git a/server/src/test/resources/log4j.properties b/server/src/test/resources/log4j2.properties
similarity index 60%
rename from server/src/test/resources/log4j.properties
rename to server/src/test/resources/log4j2.properties
index 9195bd934..a5fe75df3 100644
--- a/server/src/test/resources/log4j.properties
+++ b/server/src/test/resources/log4j2.properties
@@ -16,24 +16,29 @@
#
# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=INFO, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
# Tests that launch java subprocesses can set the "test.appender" system property to
# "console" to avoid having the child process's logs overwrite the unit test's
# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%t: %m%n
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
# Silence some noisy libraries.
-log4j.logger.org.apache.http=WARN
-log4j.logger.org.apache.spark=INFO
-log4j.logger.org.eclipse.jetty=WARN
-log4j.logger.org.spark-project.jetty=WARN
+logger.http.name = org.apache.http
+logger.http.level = warn
+logger.spark.name = org.apache.spark
+logger.spark.level = info
+logger.jetty.name = org.sparkproject.jetty
+logger.jetty.level = warn
+
diff --git a/thriftserver/client/pom.xml b/thriftserver/client/pom.xml
index 2744d343d..7c6a5200c 100644
--- a/thriftserver/client/pom.xml
+++ b/thriftserver/client/pom.xml
@@ -39,6 +39,18 @@
org.apache.hbase
*
+
+ org.eclipse.jetty
+ apache-jsp
+
+
+ log4j
+ log4j
+
+
+ org.slf4j
+ slf4j-log4j12
+
diff --git a/thriftserver/server/pom.xml b/thriftserver/server/pom.xml
index 759a75174..17a9f70eb 100644
--- a/thriftserver/server/pom.xml
+++ b/thriftserver/server/pom.xml
@@ -103,7 +103,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
org.apache.thrift
@@ -141,6 +141,10 @@
org.apache.logging.log4j
log4j-slf4j-impl
+
+ log4j
+ log4j
+
org.slf4j
slf4j-log4j12
diff --git a/thriftserver/server/src/test/resources/log4j.properties b/thriftserver/server/src/test/resources/log4j2.properties
similarity index 56%
rename from thriftserver/server/src/test/resources/log4j.properties
rename to thriftserver/server/src/test/resources/log4j2.properties
index 72bab7053..23bf98e76 100644
--- a/thriftserver/server/src/test/resources/log4j.properties
+++ b/thriftserver/server/src/test/resources/log4j2.properties
@@ -16,24 +16,30 @@
#
# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=WARN, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+rootLogger.level = warn
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
# Tests that launch java subprocesses can set the "test.appender" system property to
# "console" to avoid having the child process's logs overwrite the unit test's
# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t: %m%n
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
# Enable DEBUG logs for Livy classes (with some exceptions).
-log4j.logger.org.apache.livy=DEBUG
-log4j.logger.org.apache.livy.rsc=INFO
-log4j.logger.org.apache.livy.shaded=INFO
-log4j.logger.org.apache.livy.thriftserver.LivyCLIService=INFO
+logger.livy.name = org.apache.livy
+logger.livy.level = debug
+logger.livy_rsc.name = org.apache.livy.rsc
+logger.livy_rsc.level = info
+logger.livy_shaded.name = org.apache.livy.shaded
+logger.livy_shaded.level = info
+logger.livy_cli_service.name = org.apache.livy.thriftserver.LivyCLIService
+logger.livy_cli_service.level = info
diff --git a/thriftserver/session/pom.xml b/thriftserver/session/pom.xml
index e7be3fe92..84ad78226 100644
--- a/thriftserver/session/pom.xml
+++ b/thriftserver/session/pom.xml
@@ -38,7 +38,7 @@
org.slf4j
- slf4j-reload4j
+ slf4j-api
provided