chg: [v6.7] update delete old indexes + doc + add reindexing tool

Author: Terrtia

HOWTO.md

@@ -98,3 +98,24 @@ meilisearch = True
 meilisearch_url = http://<meilisearch_url>        # URL where Meilisearch is running (e.g., `http://127.0.0.1:7700`)  
 meilisearch_key = <meilisearch_db_password>       # Meilisearch master key or API key
 ```
+
+### Reindex after upgrading to v6.7
+
+> ⚠️ Meilisearch indexes created with AIL versions **before v6.7** are **automatically removed during the update**.  
+> Search indexing was refactored, so the indexes must be rebuilt using the new schema.
+
+Use `tools/reindex_meilisearch.py` to rebuild the indexes:
+
+```bash
+# Reindex everything
+python3 tools/reindex_meilisearch.py --type all
+
+# Reindex only one dataset
+python3 tools/reindex_meilisearch.py --type messages
+```
+
+If you want to **reset and recreate the indexes** before reindexing:
+
+```bash
+python3 tools/reindex_meilisearch.py --reset --type all
+```

bin/lib/search_engine.py

@@ -62,8 +62,6 @@ def load_indexes_names():
 
 
 INDEX_NAMES = load_indexes_names()
-
-
 def get_indexes_names():
     return INDEX_NAMES
 
@@ -399,6 +397,20 @@ def index_file_names():
         if document:
             Engine.update(index, document)
 
+
+INDEXING_FUNCTIONS = {
+    'all': index_all,
+    'crawled': index_crawled,
+    'chats': index_chats,
+    'messages': index_messages,
+    'user_accounts': index_user_accounts,
+    'images_descriptions': index_images_descriptions,
+    'screenshots_descriptions': index_screenshots_descriptions,
+    'domains_descriptions': index_domains_descriptions,
+    'titles': index_titles,
+    'file_names': index_file_names,
+}
+
 ## --INDEXER-- ##
 
 

tools/reindex_meilisearch.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Reindex Meilisearch data from AIL.
+
+By default, this script indexes one or more datasets without deleting existing
+indexes. If `--reset` is provided, it first deletes and recreates all indexes.
+In both cases, index creation/configuration is ensured before indexing starts.
+"""
+
+import argparse
+import os
+import sys
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import search_engine
+
+def ensure_indexes_ready(reset=False):
+    """Ensure indexes exist and are configured before indexing."""
+    engine = search_engine.Engine
+
+    if reset:
+        engine._delete_all()
+        engine.create_indexes()
+    else:
+        existing = set(engine.get_indexes())
+        required = set(search_engine.get_indexes_names())
+        for index_name in sorted(required - existing):
+            engine._create_index(index_name)
+            engine.setup_indexes_searchable_filterable_sortable()
+
+
+def run_indexing(index_type):
+    indexing_fct = search_engine.INDEXING_FUNCTIONS[index_type]
+    indexing_fct()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Optionally reset Meilisearch indexes and index one dataset or everything.'
+    )
+    parser.add_argument(
+        '-t',
+        '--type',
+        dest='index_type',
+        default='all',
+        choices=sorted(search_engine.INDEXING_FUNCTIONS.keys()),
+        help='Dataset type to index (default: all)',
+    )
+    parser.add_argument(
+        '--reset',
+        action='store_true',
+        help='Delete and recreate all indexes before indexing',
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    if args.reset:
+        print('Resetting indexes and preparing configuration...')
+    else:
+        print('Preparing indexes and configuration (no reset)...')
+
+    if not search_engine.is_meilisearch_enabled():
+        raise RuntimeError('Meilisearch is disabled in configuration')
+
+    if not search_engine.Engine.is_up():
+        raise RuntimeError('Meilisearch is not reachable')
+
+    ensure_indexes_ready(reset=args.reset)
+    print('Indexes are ready.')
+
+    print(f'Start indexing: {args.index_type}')
+    run_indexing(args.index_type)
+    print('Indexing finished.')
+
+
+if __name__ == '__main__':
+    main()