Ignore SSM work dir and trigger fetch operation when a file is moved to covered dir (#2067)

Author: rui-mo

conf/smart-default.xml

@@ -49,6 +49,16 @@
     </description>
   </property>
 
+  <property>
+    <name>smart.work.dir</name>
+    <value>/system/ssm</value>
+    <description>
+      This directory is used as a work directory for SSM to store tmp files and data.
+      The default path is "/system/ssm", and SSM will ignore files under the work directory.
+      No more than one directory can be set as the work directory.
+    </description>
+  </property>
+
   <property>
     <name>smart.server.rpc.address</name>
     <value>0.0.0.0:7042</value>

smart-client/src/main/java/org/smartdata/client/SmartClient.java

@@ -81,7 +81,6 @@ private void initialize(InetSocketAddress address) throws IOException {
     singleIgnoreList = new ConcurrentHashMap<>(200);
     ignoreAccessEventDirs = ((SmartConf) conf).getIgnoreDir();
     fetchAccessEventDirs = ((SmartConf) conf).getCoverDir();
-
   }
 
   private void checkOpen() throws IOException {

smart-common/src/main/java/org/smartdata/conf/SmartConf.java

@@ -34,7 +34,6 @@
 import java.util.Scanner;
 import java.util.Set;
 
-
 /**
  * SSM related configurations as well as HDFS configurations.
  */
@@ -53,8 +52,13 @@ public SmartConf() {
         SmartConfKeys.SMART_IGNORE_DIRS_KEY);
     Collection<String> fetchDirs = this.getTrimmedStringCollection(
         SmartConfKeys.SMART_COVER_DIRS_KEY);
+
     ignoreList = new ArrayList<>();
     coverList = new ArrayList<>();
+    String tmpDir = this.get(
+        SmartConfKeys.SMART_WORK_DIR_KEY, SmartConfKeys.SMART_WORK_DIR_DEFAULT);
+    tmpDir = tmpDir + (tmpDir.endsWith("/") ? "" : "/");
+    ignoreList.add(tmpDir);
     for (String s : ignoreDirs) {
       ignoreList.add(s + (s.endsWith("/") ? "" : "/"));
     }
@@ -86,7 +90,7 @@ public void setIgnoreDir(ArrayList<String> ignoreDirs) {
     for (String s : ignoreDirs) {
       ignoreList.add(s + (s.endsWith("/") ? "" : "/"));
     }
-}
+  }
 
   public Set<String> init(String fileName, SmartConf conf) {
     String hostName = "";

smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java

@@ -181,6 +181,8 @@ public class SmartConfKeys {
    * For more than one directories, they should be separated by ",".
    */
   public static final String SMART_COVER_DIRS_KEY = "smart.cover.dirs";
+  public static final String SMART_WORK_DIR_KEY = "smart.work.dir";
+  public static final String SMART_WORK_DIR_DEFAULT = "/system/ssm/";
 
   // Target cluster
   public static final String SMART_STORAGE_INFO_UPDATE_INTERVAL_KEY =

smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java

@@ -295,6 +295,11 @@ public void run() {
           throw new IOException();
         }
 
+        if(!status.isDir()) {
+          this.addFileStatus(convertToFileInfo(status, parent));
+          numFilesFetched.incrementAndGet();
+        }
+
         if (status.isDir()) {
           if (startAfter == null) {
             FileInfo internal = convertToFileInfo(status, "");

smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java

@@ -48,7 +48,8 @@ public class ErasureCodingScheduler extends ActionSchedulerService {
   public static final String listecActionID = "listec";
   public static final List<String> actions =
       Arrays.asList(ecActionID, unecActionID, checkecActionID, listecActionID);
-  public static final String EC_DIR = "/system/ssm/ec_tmp/";
+  public static String EC_DIR;
+  public static final String EC_TMP_DIR = "ec_tmp/";
   public static final String EC_TMP = "-ecTmp";
   public static final String EC_POLICY = "-policy";
   private Set<String> fileLock;
@@ -66,6 +67,10 @@ public ErasureCodingScheduler(SmartContext context, MetaStore metaStore) {
     if (this.throttleInMb > 0) {
       this.rateLimiter = RateLimiter.create(throttleInMb);
     }
+    String ssmTmpDir = conf.get(
+        SmartConfKeys.SMART_WORK_DIR_KEY, SmartConfKeys.SMART_WORK_DIR_DEFAULT);
+    ssmTmpDir = ssmTmpDir + (ssmTmpDir.endsWith("/") ? "" : "/");
+    ErasureCodingScheduler.EC_DIR = ssmTmpDir + EC_TMP_DIR;
   }
 
   public List<String> getSupportedActions() {

smart-server/src/test/java/org/smartdata/server/engine/cmdlet/TestCompressionReadWrite.java

@@ -124,7 +124,7 @@ public void testCompressEmptyFile() throws Exception {
         + " -bufSize " + bufSize + " -compressImpl " + compressionImpl);
 
     waitTillActionDone(cmdId);
-    Thread.sleep(600);
+    Thread.sleep(5000);
     // metastore  test
     FileState fileState = metaStore.getFileState(fileName);
     Assert.assertEquals(FileState.FileType.COMPRESSION, fileState.getFileType());