From 09d49b77b9033676070c482a4c69881bec13ef29 Mon Sep 17 00:00:00 2001 From: Ahmath-Gadji Date: Thu, 9 Jul 2026 09:12:29 +0000 Subject: [PATCH 1/2] feat(indexation): disable topic tagging by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Topic tagging ran on every indexed document, spending one LLM call per file to produce tags that nothing consumes yet — they are not exposed in the API and not used in retrieval. The admin UI already ships the toggle as disabled with a "Coming soon" hint. Make it opt-in instead: - IndexationPipelineConfig.enable_topic_tagging defaults to False - the seeded `default` indexation preset sets it False explicitly - _required_llm_names no longer resolves an LLM endpoint for an absent key - _replace_topic_tags_if_needed treats an absent key as disabled, so a re-index under a preset that never enabled tagging clears tags left behind by an earlier run - the admin UI toggle falls back to off, matching the backend Existing deployments are unaffected: seed_defaults only inserts when a preset type has no rows, so a stored preset with tagging on keeps it on. --- openrag/core/config/indexation_pipeline.py | 6 ++++-- openrag/services/orchestrators/preset_service.py | 2 +- openrag/services/workers/indexer_actor.py | 5 ++++- openrag/services/workers/indexer_pool.py | 2 +- tests/unit/core/config/test_pipeline_configs.py | 5 +++++ .../services/orchestrators/test_preset_service.py | 11 +++++++++++ tests/unit/services/workers/test_indexer_pool.py | 5 +++-- ui/src/pages/admin/presets.tsx | 2 +- 8 files changed, 30 insertions(+), 8 deletions(-) diff --git a/openrag/core/config/indexation_pipeline.py b/openrag/core/config/indexation_pipeline.py index e0ecb0f6b..5b53c3908 100644 --- a/openrag/core/config/indexation_pipeline.py +++ b/openrag/core/config/indexation_pipeline.py @@ -52,8 +52,10 @@ class IndexationPipelineConfig(BaseModel): default_factory=lambda: ["person", "organization", "location", "event"], ) - # Topic tagging - enable_topic_tagging: bool = True + # Topic tagging. Off by default: the generated tags are not yet surfaced in + # the API or used for retrieval, so enabling it only buys an extra LLM call + # per document. Partitions that want the tags opt in on their preset. + enable_topic_tagging: bool = False max_topic_tags: int = Field(default=7, ge=1, le=50) topic_tagging_llm: str | None = None diff --git a/openrag/services/orchestrators/preset_service.py b/openrag/services/orchestrators/preset_service.py index 8eb07a661..3e835df02 100644 --- a/openrag/services/orchestrators/preset_service.py +++ b/openrag/services/orchestrators/preset_service.py @@ -52,7 +52,7 @@ # spin up that backend's Ray pool — e.g. forcing marker on a # pymupdf-configured deployment. Named presets below opt in explicitly. "enable_entity_extraction": True, - "enable_topic_tagging": True, + "enable_topic_tagging": False, }, "legal": { "chunking": {"name": "recursive_splitter", "chunk_size": 1024, "chunk_overlap_rate": 0.25}, diff --git a/openrag/services/workers/indexer_actor.py b/openrag/services/workers/indexer_actor.py index 40fa9f2cb..3af5ffb1e 100644 --- a/openrag/services/workers/indexer_actor.py +++ b/openrag/services/workers/indexer_actor.py @@ -159,7 +159,10 @@ async def _replace_topic_tags_if_needed( return has_topic_tags = "topic_tags" in row - topic_tagging_disabled = indexation_config is not None and indexation_config.get("enable_topic_tagging") is False + # Mirrors IndexationPipelineConfig.enable_topic_tagging: an absent key means + # disabled, so a re-index under a preset that never enabled tagging still + # clears tags left behind by an earlier run. + topic_tagging_disabled = indexation_config is not None and not indexation_config.get("enable_topic_tagging", False) if not has_topic_tags and not topic_tagging_disabled: return diff --git a/openrag/services/workers/indexer_pool.py b/openrag/services/workers/indexer_pool.py index 28cd794f3..d41cd21cf 100644 --- a/openrag/services/workers/indexer_pool.py +++ b/openrag/services/workers/indexer_pool.py @@ -358,7 +358,7 @@ def _required_llm_names(indexation_config: dict[str, Any] | None) -> list[str]: names: list[str] = [] if indexation_config.get("enable_contextualization"): names.append(indexation_config.get("contextualization_llm") or "default") - if indexation_config.get("enable_topic_tagging", True): + if indexation_config.get("enable_topic_tagging", False): names.append(indexation_config.get("topic_tagging_llm") or "default") return names diff --git a/tests/unit/core/config/test_pipeline_configs.py b/tests/unit/core/config/test_pipeline_configs.py index 3461274cc..206b7c7ba 100644 --- a/tests/unit/core/config/test_pipeline_configs.py +++ b/tests/unit/core/config/test_pipeline_configs.py @@ -41,6 +41,11 @@ def test_indexation_pipeline_rejects_unknown_contextualization_mode(): IndexationPipelineConfig(contextualization_mode="verbose") +def test_indexation_pipeline_topic_tagging_defaults_off(): + """Topic tagging is opt-in: tags are not yet surfaced or used in retrieval.""" + assert IndexationPipelineConfig().enable_topic_tagging is False + + def test_retrieval_pipeline_rejects_unknown_type(): """Retrieval presets reject unsupported retrieval modes.""" with pytest.raises(ValidationError): diff --git a/tests/unit/services/orchestrators/test_preset_service.py b/tests/unit/services/orchestrators/test_preset_service.py index a4d601a16..5e81e4fe6 100644 --- a/tests/unit/services/orchestrators/test_preset_service.py +++ b/tests/unit/services/orchestrators/test_preset_service.py @@ -155,6 +155,17 @@ async def test_seed_defaults_inserts_six_presets_when_empty(): assert len(upserts) == 6 # 3 indexation + 3 retrieval +@pytest.mark.asyncio +async def test_seed_defaults_indexation_leaves_topic_tagging_off(): + repo = _FakePresetRepo() + svc = _make_service(repo) + await svc.seed_defaults() + + idx = [r for r in repo._store.values() if r["preset_type"] == "indexation"] + assert idx # sanity: indexation presets were seeded + assert all(r["config"].get("enable_topic_tagging", False) is False for r in idx) + + @pytest.mark.asyncio async def test_seed_defaults_retrieval_inherits_reranker_enabled(): from core.config.root import Settings diff --git a/tests/unit/services/workers/test_indexer_pool.py b/tests/unit/services/workers/test_indexer_pool.py index 9faf8fad9..80a816dbd 100644 --- a/tests/unit/services/workers/test_indexer_pool.py +++ b/tests/unit/services/workers/test_indexer_pool.py @@ -480,9 +480,10 @@ def test_required_llm_names_mirrors_pipeline_selection() -> None: from services.workers.indexer_pool import _required_llm_names assert _required_llm_names(None) == [] - # Topic tagging defaults on, contextualization defaults off. - assert _required_llm_names({}) == ["default"] + # Both topic tagging and contextualization default off. + assert _required_llm_names({}) == [] assert _required_llm_names({"enable_topic_tagging": False}) == [] + assert _required_llm_names({"enable_topic_tagging": True}) == ["default"] assert _required_llm_names( { "enable_contextualization": True, diff --git a/ui/src/pages/admin/presets.tsx b/ui/src/pages/admin/presets.tsx index a8fd28b7d..f6c5de3cd 100644 --- a/ui/src/pages/admin/presets.tsx +++ b/ui/src/pages/admin/presets.tsx @@ -406,7 +406,7 @@ function IndexationPresetForm({ /> toggleFeature("enable_topic_tagging", "topic_tagging_llm", on)} // Postponed: tags are generated but not yet surfaced or used in // retrieval, so the control is disabled until the feature ships. From 9d00e76c2548a28eece55db4297e7a178b1fdee6 Mon Sep 17 00:00:00 2001 From: Ahmath-Gadji Date: Wed, 15 Jul 2026 07:49:23 +0000 Subject: [PATCH 2/2] test(indexation): cover absent-key topic-tag purge path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _replace_topic_tags_if_needed now treats an absent enable_topic_tagging key as disabled, so a re-index under a preset that never enabled tagging (legal/finance) purges tags left by an earlier run. The existing disabled test passed explicit enable_topic_tagging=False, which also purged under the old 'is False' semantics, leaving the new absent-key path untested — reverting it to 'is False' kept the suite green. Pin the absent-key case. --- .../services/workers/test_indexer_worker.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/unit/services/workers/test_indexer_worker.py b/tests/unit/services/workers/test_indexer_worker.py index 29b48590d..3e56f731a 100644 --- a/tests/unit/services/workers/test_indexer_worker.py +++ b/tests/unit/services/workers/test_indexer_worker.py @@ -557,6 +557,40 @@ async def run(self, row: dict[str, Any]) -> dict[str, Any]: assert repo.inserted == [] +@pytest.mark.asyncio +async def test_process_file_deletes_topic_tags_when_tagging_key_absent(tmp_path: Path) -> None: + """An absent ``enable_topic_tagging`` key means disabled (mirrors the config + default), so a re-index under a preset that never enabled tagging — e.g. + ``legal``/``finance`` — still purges tags left behind by an earlier run. + Guards against reverting the absent-key semantics to explicit ``is False``.""" + path = tmp_path / "doc.txt" + path.write_bytes(b"content") + + class UntaggedPipeline: + async def run(self, row: dict[str, Any]) -> dict[str, Any]: + row["stored_count"] = 1 + row["stage"] = "stored" + return row + + repo = FakeTopicTagRepo() + worker = IndexerWorker( + pipeline=UntaggedPipeline(), + task_state_manager=_fake_tsm(), + topic_tag_repo=repo, + ) + + await worker.process_file( + task_id="t-absent-tags", + path=str(path), + metadata={"file_id": "f1"}, + partition="tenant-a", + indexation_config={"enable_image_captioning": True}, + ) + + assert repo.deleted == [("f1", "tenant-a")] + assert repo.inserted == [] + + @pytest.mark.asyncio async def test_process_file_rejects_malformed_topic_tags_before_delete(tmp_path: Path) -> None: path = tmp_path / "doc.txt"