Merged PR 1751463: [TTL] use index hints to force ordered index scan on composite TTL indexes

Jayanta Mondal · shuaitian-git · commit 6b95258b2473 · 2025-08-13T15:53:16.000+08:00
### Does this PR have any customer impact?

### Type (Feature, Refactoring, Bugfix, DevOps, Testing, Perf, etc)

### Does it involve schema level changes? (Table, Column, Index, UDF, etc level changes)

### Are you introducing any new config? If yes, do you have tests with and without them being set?

### ChangeLog (Refer [Template](../oss/CHANGELOG.md))

### Description

In this PR, for TTL indexes, created as a composite index, we force an ordered scan while querying for eligible rows using index hints.

Composite indexes support ordered scan on the index allowing the scan to be really efficient on the index. Without composite index, the planner uses a bitmap index scan which requires fetching all index pages and creating a bitmap. If there are a lot of documents to deleted, this can put a lot of pressure on the disk IOPS due to repeated fetching of large amount of index pages.

When TTL index is created as a composite index, we effectively bypass the planner via the use of index hints and forces postgres to use the composite index. When index hint is provided the planner pins an ordered scan on the index provided as a hint.

Below is an example of how an additional condition is added, as an index hint, to the query for fetching purge eligible rows:

```sql
SELECT ctid FROM mongo_data.documents_1963502_19635007
                WHERE bson_dollar_lt(document, '{ "ttl" : { "$date" : { "$numberLong" : "1657900030775" } } }'::bson)
                AND helio_api_internal.bson_dollar_index_hint(document, 'ttl_index'::text, '{"key": {"ttl": 1}}'::bson, isSparse)
```
diff --git a/internal/pg_documentdb_distributed/src/test/regress/bin/expect_normalize.sed b/internal/pg_documentdb_distributed/src/test/regress/bin/expect_normalize.sed
@@ -11,4 +11,6 @@
 s/^-[+-]{2,}$/---------------------------------------------------------------------/g
 # Replace the values of the time system variables ($$NOW) with a constant
 s/\"sn\" : \{ \"\$date\" : \{ \"\$numberLong\" : \"[0-9]*\" \} \}/\"sn\" : NOW_SYS_VARIABLE/g
-s/\"now\" : \{ \"\$date\" : \{ \"\$numberLong\" : \"[0-9]*\" \} \}/\"now\" : NOW_SYS_VARIABLE/g
+s/\"now\" : \{ \"\$date\" : \{ \"\$numberLong\" : \"[0-9]*\" \} \}/\"now\" : NOW_SYS_VARIABLE/g
+s/TTL job elapsed time: [+-]?[0-9]*\.?[0-9]+([eE][+-]?[0-9]+)?ms,/TTL job elapsed time:<redacted>/g
+s/expiry_cutoff=[0-9]*,/expiry_cutoff=<redacted>/g
diff --git a/internal/pg_documentdb_distributed/src/test/regress/bin/normalize.sed b/internal/pg_documentdb_distributed/src/test/regress/bin/normalize.sed
@@ -25,3 +25,5 @@ s/\"sn\" : \{ \"\$date\" : \{ \"\$numberLong\" : \"[0-9]*\" \} \}/\"sn\" : NOW_S
 s/documentdb_api_catalog.shard_key_and_document/shard_key_and_document/g
 s/documentdb_api_internal.generate_unique_shard_document/generate_unique_shard_document/g
 s/documentdb_core.bson/bson/g
+s/TTL job elapsed time: [+-]?[0-9]*\.?[0-9]+([eE][+-]?[0-9]+)?ms,/TTL job elapsed time:<redacted>/g
+s/expiry_cutoff=[0-9]*,/expiry_cutoff=<redacted>/g
diff --git a/internal/pg_documentdb_distributed/src/test/regress/expected/commands_create_ttl_indexes.out b/internal/pg_documentdb_distributed/src/test/regress/expected/commands_create_ttl_indexes.out
diff --git a/internal/pg_documentdb_distributed/src/test/regress/sql/commands_create_ttl_indexes.sql b/internal/pg_documentdb_distributed/src/test/regress/sql/commands_create_ttl_indexes.sql
@@ -115,6 +115,21 @@ AND (dist.shardid = get_shard_id_for_distribution_column(logicalrelid, coll.coll
 AND coll.collection_id >= 20000 AND coll.collection_id < 21000 -- added to reduce test flakiness
 ORDER BY shardid ASC; -- added to reduce test flakiness
 
+-- Delete all other indexes from previous tests to reduce flakiness
+WITH deleted AS (
+  DELETE FROM documentdb_api_catalog.collection_indexes
+  WHERE collection_id != 20000
+  RETURNING 1
+) SELECT true FROM deleted UNION ALL SELECT true LIMIT 1;
+
+SELECT
+    collection_id,
+    (index_spec).index_key, (index_spec).index_name,
+    (index_spec).index_expire_after_seconds as ttl_expiry,
+    (index_spec).index_is_sparse as is_sparse,
+    (index_spec).index_name as index_name
+FROM documentdb_api_catalog.collection_indexes WHERE (index_spec).index_expire_after_seconds > 0;
+
 -- 10.b. Call ttl task procedure with a batch size of 0 --
 BEGIN;
 Set citus.log_remote_commands to on; -- Will print Citus rewrites of the queries
@@ -311,4 +326,131 @@ CALL documentdb_api_internal.delete_expired_rows(10);
   -- 3000 ms does 70 iterations locally. So document count should be well below 9900.
 SELECT count(*) < 9900 from documentdb_api.collection('db', 'ttlRepeatedDeletes');
 SELECT count(*) < 9900 from documentdb_api.collection('db', 'ttlRepeatedDeletes2');
+END;
+
+
+-- 21. TTL index with forced ordered scan via index hints
+
+set documentdb.enableExtendedExplainPlans to on;
+SET documentdb.enableNewCompositeIndexOpClass to on;
+set documentdb_rum.preferOrderedIndexScan to on;
+
+-- if documentdb_extended_rum exists, set alternate index handler
+SELECT pg_catalog.set_config('documentdb.alternate_index_handler_name', 'extended_rum', false), extname FROM pg_extension WHERE extname = 'documentdb_extended_rum';
+
+-- Delete all other indexes from previous tests to reduce flakiness
+SELECT documentdb_api.drop_collection('db', 'ttlcoll'), documentdb_api.drop_collection('db', 'ttlcoll1'), documentdb_api.drop_collection('db', 'ttlcoll2'),
+documentdb_api.drop_collection('db', 'ttlcoll3'),documentdb_api.drop_collection('db', 'ttlRepeatedDeletes'),documentdb_api.drop_collection('db', 'ttlRepeatedDeletes2');
+
+-- make sure jobs are scheduled and disable it to avoid flakiness on the test as it could run on its schedule and delete documents before we run our commands in the test
+select cron.unschedule(jobid) from cron.job where jobname like '%ttl_task%';
+
+-- 1. Populate collection with a set of documents with different combination of $date fields --
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 0, "ttl" : { "$date": { "$numberLong": "-1000" } } }', NULL);
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 1, "ttl" : { "$date": { "$numberLong": "0" } } }', NULL);
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 2, "ttl" : { "$date": { "$numberLong": "100" } } }', NULL);
+    -- Documents with date older than when the test was written
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 3, "ttl" : { "$date": { "$numberLong": "1657900030774" } } }', NULL);
+    -- Documents with date way in future
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 4, "ttl" : { "$date": { "$numberLong": "2657899731608" } } }', NULL);
+    -- Documents with date array
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 5, "ttl" : [{ "$date": { "$numberLong": "100" }}] }', NULL);
+    -- Documents with date array, should be deleted based on min timestamp
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 6, "ttl" : [{ "$date": { "$numberLong": "100" }}, { "$date": { "$numberLong": "2657899731608" }}] }', NULL);
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 7, "ttl" : [true, { "$date": { "$numberLong": "100" }}, { "$date": { "$numberLong": "2657899731608" }}] }', NULL);
+    -- Documents with non-date ttl field
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 8, "ttl" : true }', NULL);
+    -- Documents with non-date ttl field
+SELECT documentdb_api.insert_one('db','ttlCompositeOrderedScan', '{ "_id" : 9, "ttl" : "would not expire" }', NULL);
+
+SELECT COUNT(documentdb_api.insert_one('db', 'ttlCompositeOrderedScan', FORMAT('{ "_id": %s, "ttl": { "$date": { "$numberLong": "1657900030774" } } }', i, i)::documentdb_core.bson)) FROM generate_series(10, 10000) AS i;
+
+--  Create TTL Index --
+SET documentdb.enableExtendedExplainPlans to on;
+SET documentdb.enableNewCompositeIndexOpClass to on;
+SET documentdb.enableIndexOrderbyPushdown to on;
+SELECT documentdb_api_internal.create_indexes_non_concurrently('db', '{"createIndexes": "ttlCompositeOrderedScan", "indexes": [{"key": {"ttl": 1}, "enableCompositeTerm": true, "name": "ttl_index", "v" : 1, "expireAfterSeconds": 5, "sparse": true}]}', true);
+
+select
+    collection_id,
+    (index_spec).index_key, (index_spec).index_name,
+    (index_spec).index_expire_after_seconds as ttl_expiry,
+    (index_spec).index_is_sparse as is_sparse,
+    (index_spec).index_name as index_name
+from documentdb_api_catalog.collection_indexes where (index_spec).index_expire_after_seconds > 0;
+
+\d  documentdb_data.documents_20006
+
+--  List All indexes --
+SELECT bson_dollar_unwind(cursorpage, '$cursor.firstBatch') FROM documentdb_api.list_indexes_cursor_first_page('db','{ "listIndexes": "ttlCompositeOrderedScan" }') ORDER BY 1;
+SELECT count(*) from ( SELECT shard_key_value, object_id, document  from documentdb_api.collection('db', 'ttlCompositeOrderedScan') order by object_id) as a;
+
+--  Call ttl purge procedure with a batch size of 100
+BEGIN;
+SET client_min_messages TO LOG;
+SET LOCAL documentdb.logTTLProgressActivity to on;
+CALL documentdb_api_internal.delete_expired_rows(100);
+RESET client_min_messages;
+END;
+
+BEGIN;
+SET client_min_messages TO LOG;
+SET LOCAL documentdb.useIndexHintsForTTLTask to off;
+SET LOCAL documentdb.logTTLProgressActivity to on;
+CALL documentdb_api_internal.delete_expired_rows(100);
+RESET client_min_messages;
+END;
+
+--  Check what documents are left after purging
+SELECT count(*) from ( SELECT shard_key_value, object_id, document  from documentdb_api.collection('db', 'ttlCompositeOrderedScan') order by object_id) as a;
+
+
+--  TTL indexes behaves like normal indexes that are used in queries (cx can provide .hint() to force)
+BEGIN;
+SET LOCAL documentdb.enableIndexOrderbyPushdown to on;
+SET LOCAL documentdb.enableNewCompositeIndexOpClass to on;
+EXPLAIN(costs off) SELECT object_id FROM documentdb_data.documents_20006
+		WHERE bson_dollar_eq(document, '{ "ttl" : { "$date" : { "$numberLong" : "100" } } }'::documentdb_core.bson)
+        LIMIT 100;
+END;
+
+--  Check the query to fetch the eligible TTL indexes uses IndexScan.
+
+BEGIN;
+SET LOCAL documentdb.enableIndexOrderbyPushdown to on;
+SET LOCAL documentdb.enableNewCompositeIndexOpClass to on;
+EXPLAIN(analyze on, verbose on, costs off, timing off, summary off) SELECT ctid FROM documentdb_data.documents_20006_2000105
+                WHERE bson_dollar_lt(document, '{ "ttl" : { "$date" : { "$numberLong" : "1754515365000" } } }'::documentdb_core.bson)
+                AND documentdb_api_internal.bson_dollar_index_hint(document, 'ttl_index'::text, '{"key": {"ttl": 1}}'::documentdb_core.bson, true)
+        LIMIT 100;
+END;
+
+--  Shard collection
+SELECT documentdb_api.shard_collection('db', 'ttlCompositeOrderedScan', '{ "_id": "hashed" }', false);
+
+--  Check TTL deletes work on sharded (should delete 800 docs, 100 for each shard)
+SELECT count(*) from ( SELECT shard_key_value, object_id, document  from documentdb_api.collection('db', 'ttlCompositeOrderedScan') order by object_id) as a;
+CALL documentdb_api_internal.delete_expired_rows(100);
+SELECT count(*) from ( SELECT shard_key_value, object_id, document  from documentdb_api.collection('db', 'ttlCompositeOrderedScan') order by object_id) as a;
+
+
+--  Check for Ordered Indes Scan on the ttl index 
+
+BEGIN;
+SET LOCAL documentdb.enableIndexOrderbyPushdown to on;
+SET LOCAL documentdb.enableNewCompositeIndexOpClass to on;
+EXPLAIN(analyze on, verbose on, costs off, timing off, summary off) SELECT ctid FROM documentdb_data.documents_20006_2000124
+                WHERE bson_dollar_lt(document, '{ "ttl" : { "$date" : { "$numberLong" : "1657900030775" } } }'::documentdb_core.bson)
+                AND documentdb_api_internal.bson_dollar_index_hint(document, 'ttl_index'::text, '{"key": {"ttl": 1}}'::documentdb_core.bson, true)
+        LIMIT 100;
+END;
+
+BEGIN;
+SET LOCAL documentdb.enableIndexOrderbyPushdown to on;
+SET LOCAL documentdb.enableNewCompositeIndexOpClass to on;
+SET client_min_messages TO INFO;
+EXPLAIN(COSTS OFF, ANALYZE ON, SUMMARY OFF, TIMING OFF) SELECT ctid FROM documentdb_data.documents_20006_2000122
+                WHERE bson_dollar_lt(document, '{ "ttl" : { "$date" : { "$numberLong" : "1657900030775" } } }'::documentdb_core.bson)
+                AND documentdb_api_internal.bson_dollar_index_hint(document, 'ttl_index'::text, '{"key": {"ttl": 1}}'::documentdb_core.bson, true)
+        LIMIT 100;
 END;
diff --git a/pg_documentdb/include/commands/create_indexes.h b/pg_documentdb/include/commands/create_indexes.h
@@ -19,6 +19,8 @@
 #include "operators/bson_expression.h"
 #include "vector/vector_spec.h"
 
+#define MAX_INDEX_OPTIONS_LENGTH 1500
+
 /*
  * Used with the ERRCODE_DOCUMENTDB_INDEXBUILDABORTED error code.
  */
diff --git a/pg_documentdb/src/commands/create_indexes.c b/pg_documentdb/src/commands/create_indexes.c
@@ -65,10 +65,6 @@
 #include "vector/vector_utilities.h"
 #include "index_am/index_am_utils.h"
 
-
-#define MAX_INDEX_OPTIONS_LENGTH 1500
-
-
 /* Return value of TryCreateCollectionIndexes */
 typedef struct
 {
diff --git a/pg_documentdb/src/configs/background_job_configs.c b/pg_documentdb/src/configs/background_job_configs.c
@@ -68,6 +68,9 @@ bool LogTTLProgressActivity = DEFAULT_LOG_TTL_PROGRESS_ACTIVITY;
 #define DEFAULT_FORCE_INDEX_SCAN_TTL_TASK true
 bool ForceIndexScanForTTLTask = DEFAULT_FORCE_INDEX_SCAN_TTL_TASK;
 
+#define DEFAULT_USE_INDEX_HINTS_TTL_TASK true
+bool UseIndexHintsForTTLTask = DEFAULT_USE_INDEX_HINTS_TTL_TASK;
+
 void
 InitializeBackgroundJobConfigurations(const char *prefix, const char *newGucPrefix)
 {
@@ -96,6 +99,13 @@ InitializeBackgroundJobConfigurations(const char *prefix, const char *newGucPref
 		NULL, &ForceIndexScanForTTLTask, DEFAULT_FORCE_INDEX_SCAN_TTL_TASK,
 		PGC_USERSET, 0, NULL, NULL, NULL);
 
+	DefineCustomBoolVariable(
+		psprintf("%s.useIndexHintsForTTLTask", prefix),
+		gettext_noop(
+			"Whether to force ordered Index Scan via Index Hints for TTL task"),
+		NULL, &UseIndexHintsForTTLTask, DEFAULT_USE_INDEX_HINTS_TTL_TASK,
+		PGC_USERSET, 0, NULL, NULL, NULL);
+
 	DefineCustomIntVariable(
 		psprintf("%s.TTLPurgerStatementTimeout", prefix),
 		gettext_noop(
@@ -122,7 +132,7 @@ InitializeBackgroundJobConfigurations(const char *prefix, const char *newGucPref
 	DefineCustomBoolVariable(
 		psprintf("%s.repeatPurgeIndexesForTTLTask", newGucPrefix),
 		gettext_noop(
-			"Wether to keep deleting documents in batches until `TTLTaskMaxRunTimeInMS` is reach per TTL task invocation."),
+			"Whether to keep deleting documents in batches until `TTLTaskMaxRunTimeInMS` is reach per TTL task invocation."),
 		NULL,
 		&RepeatPurgeIndexesForTTLTask,
 		DEFAULT_REPEAT_PURGE_INDEXES_FOR_TTL_TASK,
diff --git a/pg_documentdb/src/ttl/ttl_index.c b/pg_documentdb/src/ttl/ttl_index.c
