appcon compatibility (#4420)

casionone · web-flow · commit 8d2179123065 · 2023-03-28T21:35:24.000+08:00
* engine bml resource compatible prefix v format
diff --git a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md
@@ -3,7 +3,7 @@
 | module name(模块名) | error code(错误码)  | describe(描述) |enumeration name(枚举)| Exception Class(类名)|
 | -------- | -------- | ----- |-----|-----|
 |linkis-engineconn-plugin-core |10001|Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)|FAILED_CREATE_ELR|EngineconnCoreErrorCodeSummary|
-|linkis-engineconn-plugin-core |10001|The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary|
+|linkis-engineconn-plugin-core |10001|The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary|
 |linkis-engineconn-plugin-core |10001|EngineTypeLabel are requested(需要参数 EngineTypeLabel)|ETL_REQUESTED|EngineconnCoreErrorCodeSummary|
 |linkis-engineconn-plugin-core |20000|Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)|CANNOT_INSTANCE_ECE|EngineconnCoreErrorCodeSummary|
 |linkis-engineconn-plugin-core |20000|Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)|CANNOT_DEFAULT_EF|EngineconnCoreErrorCodeSummary|
diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
@@ -199,7 +199,11 @@ object SecurityFilter {
 
   def getLoginUsername(req: HttpServletRequest): String = {
     if (Configuration.IS_TEST_MODE.getValue) {
-      ServerConfiguration.BDP_TEST_USER.getValue;
+      val testUser = ServerConfiguration.BDP_TEST_USER.getValue
+      if (StringUtils.isBlank(testUser)) {
+        throw new IllegalUserTicketException("Need to set test user when enable test module")
+      }
+      testUser
     } else {
       getLoginUser(req).getOrElse(
         throw new IllegalUserTicketException(ILLEGAL_USER_TOKEN.getErrorDesc)
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java
@@ -23,7 +23,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode {
   FAILED_CREATE_ELR(10001, "Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)"),
   EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION(
       10001,
-      "The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"),
+      "The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"),
   ETL_REQUESTED(10001, "EngineTypeLabel are requested(需要参数 EngineTypeLabel)"),
   CANNOT_INSTANCE_ECE(20000, "Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)"),
 
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala
@@ -35,4 +35,8 @@ object EngineConnPluginConfiguration {
   val ENABLED_BML_UPLOAD_FAILED_EXIT: CommonVars[Boolean] =
     CommonVars("wds.linkis.engineconn.bml.upload.failed.enable", true)
 
+  // for third party eg appconn/datax, if all update, can set to false then to remove
+  val EC_BML_VERSION_MAY_WITH_PREFIX_V: CommonVars[Boolean] =
+    CommonVars("linkis.engineconn.bml.version.may.with.prefix", true)
+
 }
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala
@@ -17,6 +17,7 @@
 
 package org.apache.linkis.engineplugin.server.localize
 
+import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration
 import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration.ENGINE_CONN_HOME
 import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator.NO_VERSION_MARK
 import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException
@@ -53,13 +54,37 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou
     val engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile.getPath
     logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome)
     val engineConnPackageHomeFile = new File(engineConnPackageHome)
+
     if (!engineConnPackageHomeFile.exists()) {
-      throw new EngineConnPluginErrorException(
-        ENGINE_VERSION_NOT_FOUND.getErrorCode,
-        MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType)
-      )
+      if (
+          !version.startsWith(
+            "v"
+          ) && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue
+      ) {
+        val versionOld = "v" + version
+        val engineConnPackageHomeOld = Paths.get(engineConnDistHome, versionOld).toFile.getPath
+        logger.info(
+          "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" + engineConnPackageHomeOld
+        )
+        val engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld)
+        if (!engineConnPackageHomeFileOld.exists()) {
+          throw new EngineConnPluginErrorException(
+            ENGINE_VERSION_NOT_FOUND.getErrorCode,
+            MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType)
+          )
+        } else {
+          engineConnPackageHomeOld
+        }
+      } else {
+        throw new EngineConnPluginErrorException(
+          ENGINE_VERSION_NOT_FOUND.getErrorCode,
+          MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType)
+        )
+      }
+    } else {
+      engineConnPackageHome
     }
-    engineConnPackageHome
+
   }
 
   private def checkEngineConnDistHome(engineConnPackageHomePath: String): Unit = {
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala
@@ -201,8 +201,6 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo
         engineConnBmlResource.setFileName(localizeResource.fileName)
         engineConnBmlResource.setFileSize(localizeResource.fileSize)
         engineConnBmlResource.setLastModified(localizeResource.lastModified)
-        if (version.startsWith("v")) engineConnBmlResource.setVersion(version.substring(1))
-        else engineConnBmlResource.setVersion(version)
         engineConnBmlResource.setVersion(version)
         engineConnBmlResourceDao.save(engineConnBmlResource)
       } else {
@@ -241,9 +239,18 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo
   ): EngineConnResource = {
     val engineConnType = engineConnBMLResourceRequest.getEngineConnType
     val version = engineConnBMLResourceRequest.getVersion
-    val engineConnBmlResources = asScalaBufferConverter(
+    var engineConnBmlResources = asScalaBufferConverter(
       engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version)
     )
+    if (
+        engineConnBmlResources.asScala.size == 0 && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue
+    ) {
+      logger.info("Try to get engine conn bml resource with prefex v")
+      engineConnBmlResources = asScalaBufferConverter(
+        engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version)
+      )
+    }
+
     val confBmlResourceMap = engineConnBmlResources.asScala
       .find(_.getFileName == LaunchConstants.ENGINE_CONN_CONF_DIR_NAME + ".zip")
       .map(parseToBmlResource)
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
@@ -710,7 +710,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ
             var heartbeatMsgMetrics = ""
             Utils.tryAndWarn {
               val oldMetrics = nodeMetricManagerPersistence.getNodeMetrics(ecNode)
-              if (StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) {
+              if (oldMetrics != null && StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) {
                 heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg
               }
             }
diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json
@@ -629,7 +629,7 @@
         "versionList": "Version List",
         "rollback": "Rollback",
         "checkEngineConnTypeAndVersion": "Please select the engine type and version",
-        "upload": "Please click the button to upload the engine plug-in"
+        "upload": "Please click the button to upload the engine plugin"
       }
     }
   }

Original file line number	Diff line number	Diff line change
`@@ -35,4 +35,8 @@ object EngineConnPluginConfiguration {`
`35`	`35`	`val ENABLED_BML_UPLOAD_FAILED_EXIT: CommonVars[Boolean] =`
`36`	`36`	`CommonVars("wds.linkis.engineconn.bml.upload.failed.enable", true)`
`37`	`37`
	`38`	`+ // for third party eg appconn/datax, if all update, can set to false then to remove`
	`39`	`+ val EC_BML_VERSION_MAY_WITH_PREFIX_V: CommonVars[Boolean] =`
	`40`	`+ CommonVars("linkis.engineconn.bml.version.may.with.prefix", true)`
	`41`	`+`
`38`	`42`	`}`
Original file line number	Diff line number	Diff line change
`@@ -710,7 +710,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ`
`710`	`710`	`var heartbeatMsgMetrics = ""`
`711`	`711`	`Utils.tryAndWarn {`
`712`	`712`	`val oldMetrics = nodeMetricManagerPersistence.getNodeMetrics(ecNode)`
`713`		`- if (StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) {`
	`713`	`+ if (oldMetrics != null && StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) {`
`714`	`714`	`heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg`
`715`	`715`	`}`
`716`	`716`	`}`
Original file line number	Diff line number	Diff line change
`@@ -629,7 +629,7 @@`
`629`	`629`	`"versionList": "Version List",`
`630`	`630`	`"rollback": "Rollback",`
`631`	`631`	`"checkEngineConnTypeAndVersion": "Please select the engine type and version",`
`632`		`- "upload": "Please click the button to upload the engine plug-in"`
	`632`	`+ "upload": "Please click the button to upload the engine plugin"`
`633`	`633`	`}`
`634`	`634`	`}`
`635`	`635`	`}`