[KYUUBI #2134] Respect Spark bundled log4j in extension modules

### _Why are the changes needed?_

As Spark 3.1/3.2 still use log4j 1.2, we should respect Spark bundled log4j in extension modules.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #2134 from pan3793/log.

Closes #2134

7429503 [Cheng Pan] Revert "reset mvn log level to info in CI"
694a159 [Cheng Pan] Revert "mvn --batch-mode"
ba72d84 [Cheng Pan] log4j.properties
39f12c9 [Cheng Pan] mvn --batch-mode
4792226 [Cheng Pan] reset mvn log level to info in CI
b337f71 [Cheng Pan] Respect Spark bundled log4j in extension modules

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Kent Yao <yao@apache.org>

dev/kyuubi-extension-spark-3-1/src/test/resources/log4j.properties

@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+log4j.rootLogger=INFO, CA, FA
+
+# Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold = FATAL
+
+# File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/unit-tests.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
+
+# Set the logger level of File Appender to WARN
+log4j.appender.FA.Threshold = DEBUG
+
+# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805
+log4j.appender.CA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.CA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.CA.filter.1.AcceptOnMatch=false
+
+log4j.appender.FA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.FA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.FA.filter.1.AcceptOnMatch=false

dev/kyuubi-extension-spark-3-2/src/test/resources/log4j.properties

@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+log4j.rootLogger=INFO, CA, FA
+
+# Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold = FATAL
+
+# File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/unit-tests.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
+
+# Set the logger level of File Appender to WARN
+log4j.appender.FA.Threshold = DEBUG
+
+# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805
+log4j.appender.CA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.CA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.CA.filter.1.AcceptOnMatch=false
+
+log4j.appender.FA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.FA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.FA.filter.1.AcceptOnMatch=false

dev/kyuubi-extension-spark-common/src/test/resources/log4j.properties

@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+log4j.rootLogger=INFO, CA, FA
+
+# Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold = FATAL
+
+# File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/unit-tests.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
+
+# Set the logger level of File Appender to WARN
+log4j.appender.FA.Threshold = DEBUG
+
+# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805
+log4j.appender.CA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.CA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.CA.filter.1.AcceptOnMatch=false
+
+log4j.appender.FA.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.FA.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.FA.filter.1.AcceptOnMatch=false

pom.xml

@@ -455,8 +455,8 @@
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
                     <!--
-                      The module is only used in Kyuubi Spark Extensions, so we don't care about which
-                      version of Log4j it depends on.
+                      The module is only used in Kyuubi Spark Extensions, we should respect
+                      the Spark bundled log4j.
                      -->
                 </exclusions>
             </dependency>
@@ -1503,6 +1503,7 @@
                         </environmentVariables>
                         <systemProperties>
                             <log4j.ignoreTCL>true</log4j.ignoreTCL>
+                            <log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
                             <log4j2.configurationFile>file:src/test/resources/log4j2-test.properties</log4j2.configurationFile>
                             <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
                             <spark.driver.memory>1g</spark.driver.memory>