[Bug fix] The partition table is cleaned incorrectly when set only device TTL for tree mode (#17123)

Author: CRZbulabula

integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java

@@ -50,7 +50,7 @@ public class IoTDBPartitionTableAutoCleanIT {
 
   private static final int TEST_REPLICATION_FACTOR = 1;
   private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
-  private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
+  private static final long TEST_TTL_CHECK_INTERVAL_IN_MS = 5_00;
 
   private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
       new TTimePartitionSlot()
@@ -68,7 +68,7 @@ public void setUp() throws Exception {
         .setSchemaReplicationFactor(TEST_REPLICATION_FACTOR)
         .setDataReplicationFactor(TEST_REPLICATION_FACTOR)
         .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL)
-        .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL);
+        .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL_IN_MS);
 
     // Init 1C1D environment
     EnvFactory.getEnv().initClusterEnvironment(1, 1);
@@ -135,6 +135,59 @@ public void testAutoCleanPartitionTableForTreeModel() throws Exception {
     Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!");
   }
 
+  @Test
+  public void testAutoCleanTakesNoEffectsForTreeDeviceTTL() throws Exception {
+    try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      // Create databases and insert test data
+      for (int i = 0; i < 3; i++) {
+        String databaseName = String.format("%s%d", TREE_DATABASE_PREFIX, i);
+        statement.execute(String.format("CREATE DATABASE %s", databaseName));
+        statement.execute(
+            String.format(
+                "CREATE TIMESERIES %s.s WITH DATATYPE=INT64,ENCODING=PLAIN", databaseName));
+        // Insert expired data
+        statement.execute(
+            String.format(
+                "INSERT INTO %s(timestamp, s) VALUES (%d, %d)",
+                databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1));
+        // Insert existed data
+        statement.execute(
+            String.format(
+                "INSERT INTO %s(timestamp, s) VALUES (%d, %d)",
+                databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
+        // Create an empty device and set a TTL.
+        // This TTL should not trigger the auto cleaner,
+        // since the database does not have a TTL.
+        statement.execute(
+            String.format(
+                "CREATE TIMESERIES %s.m.empty WITH DATATYPE=INT64,ENCODING=PLAIN", databaseName));
+        statement.execute(String.format("SET TTL TO %s.m.** %d", databaseName, TEST_TTL));
+      }
+    }
+
+    TDataPartitionReq req = new TDataPartitionReq();
+    for (int i = 0; i < 3; i++) {
+      req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>());
+    }
+    try (SyncConfigNodeIServiceClient client =
+        (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+      for (int retry = 0; retry < 10; retry++) {
+        // Ensure the partitions are not cleaned
+        boolean partitionTableAutoCleaned = false;
+        TDataPartitionTableResp resp = client.getDataPartitionTable(req);
+        if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) {
+          partitionTableAutoCleaned =
+              resp.getDataPartitionTable().entrySet().stream()
+                  .flatMap(e1 -> e1.getValue().entrySet().stream())
+                  .allMatch(e2 -> e2.getValue().size() == 1);
+        }
+        Assert.assertFalse(partitionTableAutoCleaned);
+        TimeUnit.SECONDS.sleep(1);
+      }
+    }
+  }
+
   @Test
   public void testAutoCleanPartitionTableForTableModel() throws Exception {
     try (final Connection connection =

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java

@@ -130,6 +130,20 @@ public int getTTLCount() {
     return ttlInfo.getTTLCount();
   }
 
+  /**
+   * Get the maximum ttl of the corresponding database level.
+   *
+   * @param database the path of the database.
+   * @return the maximum ttl of the corresponding database level.
+   */
+  public long getDatabaseLevelTTL(final String database) {
+    final long ttl = ttlInfo.getDatabaseLevelTTL(database);
+    return ttl == Long.MAX_VALUE || ttl < 0
+        ? ttl
+        : CommonDateTimeUtils.convertMilliTimeWithPrecision(
+            ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
+  }
+
   /**
    * Get the maximum ttl of the subtree of the corresponding database.
    *

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java

@@ -158,6 +158,23 @@ public int getTTLCount() {
     }
   }
 
+  /**
+   * Get the maximum ttl of the corresponding database level.
+   *
+   * @param database the path of the database.
+   * @return the maximum ttl of the corresponding database level.
+   */
+  public long getDatabaseLevelTTL(final String database) {
+    lock.readLock().lock();
+    try {
+      return ttlCache.getDatabaseLevelTTL(database);
+    } catch (IllegalPathException e) {
+      return TTLCache.NULL_TTL;
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
   /**
    * Get the maximum ttl of the subtree of the corresponding database.
    *

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java

@@ -61,10 +61,20 @@ protected void periodicExecute(Env env) {
     List<String> databases = configManager.getClusterSchemaManager().getDatabaseNames(null);
     Map<String, Long> databaseTTLMap = new TreeMap<>();
     for (String database : databases) {
-      long databaseTTL =
-          PathUtils.isTableModelDatabase(database)
-              ? configManager.getClusterSchemaManager().getDatabaseMaxTTL(database)
-              : configManager.getTTLManager().getDatabaseMaxTTL(database);
+      long databaseTTL;
+      if (PathUtils.isTableModelDatabase(database)) {
+        // For table mode, the auto cleaner takes effect
+        // when the maximum TTL among tables is less than Long.MAX_VALUE.
+        // Because the database-level TTL do not affect data in table mode.
+        databaseTTL = configManager.getClusterSchemaManager().getDatabaseMaxTTL(database);
+      } else {
+        databaseTTL = configManager.getTTLManager().getDatabaseLevelTTL(database);
+        if (0 < databaseTTL && databaseTTL < Long.MAX_VALUE) {
+          // For tree mode, the auto cleaner takes effect only when the database-level TTL is set.
+          // Subsequently, we employ the maximum TTL among all time series in this database.
+          databaseTTL = configManager.getTTLManager().getDatabaseMaxTTL(database);
+        }
+      }
       databaseTTLMap.put(database, databaseTTL);
     }
     LOGGER.info(

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java

@@ -175,6 +175,35 @@ public long getLastNodeTTL(String[] nodes) {
     return node.ttl;
   }
 
+  /**
+   * Get the maximum ttl of the corresponding database level.
+   *
+   * @param database the path of the database.
+   * @return the maximum ttl of the corresponding database level.
+   * @throws IllegalPathException if the database path is illegal.
+   */
+  public long getDatabaseLevelTTL(String database) throws IllegalPathException {
+    long curTTL = NULL_TTL;
+    // Get global TTL root.** if exists
+    CacheNode curNode = ttlCacheTree.searchChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
+    if (curNode != null && curNode.ttl < Long.MAX_VALUE) {
+      curTTL = curNode.ttl;
+    }
+    // Compare database TTL if exists
+    curNode = ttlCacheTree.searchChild(database);
+    if (curNode != null && curNode.ttl < Long.MAX_VALUE) {
+      curTTL = Math.max(curTTL, curNode.ttl);
+    }
+    // Compare database.** TTL if exists
+    curNode =
+        ttlCacheTree.searchChild(
+            database + IoTDBConstant.PATH_SEPARATOR + IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
+    if (curNode != null && curNode.ttl < Long.MAX_VALUE) {
+      curTTL = Math.max(curTTL, curNode.ttl);
+    }
+    return curTTL;
+  }
+
   /**
    * Get the maximum ttl of the subtree of the corresponding database.
    *