fix: null-check AppenderDynamicMBean setLayout instantiation

log4j-1.2-api/src/main/java/org/apache/log4j/jmx/AppenderDynamicMBean.java

@@ -187,6 +187,11 @@ public Object invoke(final String operationName, final Object params[], final St
         } else if (operationName.equals("setLayout")) {
             final Layout layout =
                     (Layout) OptionConverter.instantiateByClassName((String) params[0], Layout.class, null);
+            if (layout == null) {
+                cat.error("Could not instantiate layout class [" + params[0] + "] for appender ["
+                        + getAppenderName(appender) + "].");
+                return "Could not instantiate layout class.";
+            }
             appender.setLayout(layout);
             registerLayoutMBean(layout);
         }

log4j-1.2-api/src/test/java/org/apache/log4j/jmx/AppenderDynamicMBeanTest.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.log4j.jmx;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Regression for JMX {@code setLayout}: instantiateByClassName may return null.
+ */
+class AppenderDynamicMBeanTest {
+
+    @Test
+    void setLayoutDoesNotNpeWhenClassCannotBeInstantiated() throws Exception {
+        final ConsoleAppender appender = new ConsoleAppender();
+        appender.setName("jmx-layout-test");
+        final AppenderDynamicMBean mbean = new AppenderDynamicMBean(appender);
+
+        final Object result = assertDoesNotThrow(
+                () -> mbean.invoke("setLayout", new Object[] {"this.class.does.not.exist.MissingLayout"}, new String[] {
+                    String.class.getName()
+                }));
+        assertTrue(result == null || result.toString().contains("Could not instantiate"));
+        assertNull(appender.getLayout());
+    }
+
+    @Test
+    void setLayoutStillAttachesValidLayout() throws Exception {
+        final ConsoleAppender appender = new ConsoleAppender();
+        appender.setName("jmx-layout-valid");
+        final AppenderDynamicMBean mbean = new AppenderDynamicMBean(appender);
+
+        mbean.invoke("setLayout", new Object[] {PatternLayout.class.getName()}, new String[] {String.class.getName()});
+        assertNotNull(appender.getLayout());
+        assertTrue(appender.getLayout() instanceof PatternLayout);
+    }
+}

src/changelog/.2.x.x/fix_appender_dynamic_mbean_null_layout.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<entry xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns="https://logging.apache.org/xml/ns"
+       xsi:schemaLocation="https://logging.apache.org/xml/ns https://logging.apache.org/xml/ns/log4j-changelog-0.xsd"
+       type="fixed">
+  <issue id="4219" link="https://github.com/apache/logging-log4j2/pull/4219"/>
+  <description format="asciidoc">Fix NPE in log4j-1.2-api `AppenderDynamicMBean.setLayout` when the layout class cannot be instantiated</description>
+</entry>