fix: keep arXiv preprints apart from unlinked published records

citeforge/identity.py

@@ -191,7 +191,12 @@ def _enrichment_identity(left: _RecordView, right: _RecordView) -> IdentityEvide
         and len(txt.parse_authors_any(left.authors)) >= 2
         and len(txt.parse_authors_any(right.authors)) >= 2
     )
-    if not preprint_pair and not external_ids_match(left.fields, right.fields) and not strong_authors:
+    # A one-sided arXiv id is unknown identity, as in _disk_identity, so it never
+    # opens the composite gate for a candidate from another work.
+    unlinked_arxiv = left.arxiv_id != right.arxiv_id
+    if unlinked_arxiv or (
+        not preprint_pair and not external_ids_match(left.fields, right.fields) and not strong_authors
+    ):
         return _evidence(left, right, False, IdentityReason.GATE_CLOSED, title_score=score)
     composite = txt.compute_dedup_score(left.fields, right.fields, count_preprint_xor=not preprint_pair)
     return _evidence(
@@ -227,6 +232,10 @@ def _candidate_identity(left: _RecordView, right: _RecordView, candidate_doi: st
 
 def _disk_identity(left: _RecordView, right: _RecordView) -> IdentityEvidence:
     score = _title_score(left, right)
+    # The merge strips eprint from every published record, so an arXiv id on one
+    # side only is unknown identity, not agreement. Without equal ids, a relaxed
+    # preprint title score joined VISLA (2404.16365) to SugarCrepe++ (2406.11171).
+    arxiv_compatible = left.arxiv_id == right.arxiv_id or score >= cfg.SIM_FILE_DUPLICATE_THRESHOLD
     if left.doi and right.doi:
         if left.doi == right.doi or doi_bases_match(left.doi, right.doi):
             reason = IdentityReason.DOI_EXACT if left.doi == right.doi else IdentityReason.DOI_VERSION
@@ -240,7 +249,11 @@ def _disk_identity(left: _RecordView, right: _RecordView) -> IdentityEvidence:
             if left.arxiv_id and right.arxiv_id and left.arxiv_id != right.arxiv_id:
                 return _evidence(left, right, False, IdentityReason.ARXIV_CONFLICT, title_score=score)
             composite = txt.compute_dedup_score(left.fields, right.fields, count_preprint_xor=False)
-            verdict = score >= cfg.SIM_PREPRINT_TITLE_THRESHOLD and composite >= cfg.SIM_DEDUP_COMPOSITE_THRESHOLD
+            verdict = (
+                arxiv_compatible
+                and score >= cfg.SIM_PREPRINT_TITLE_THRESHOLD
+                and composite >= cfg.SIM_DEDUP_COMPOSITE_THRESHOLD
+            )
             return _evidence(left, right, verdict, IdentityReason.PREPRINT_PAIR, title_score=score, composite=composite)
         return _evidence(left, right, False, IdentityReason.DOI_CONFLICT, title_score=score)
     if external_ids_match(left.fields, right.fields) and score >= cfg.SIM_PREPRINT_TITLE_THRESHOLD:
@@ -268,15 +281,20 @@ def _disk_identity(left: _RecordView, right: _RecordView) -> IdentityEvidence:
         if txt.authors_overlap(left.authors, right.authors):
             return _evidence(left, right, True, IdentityReason.TRUNCATED_TITLE, title_score=score)
     if (
-        score >= 0.6
+        arxiv_compatible
+        and score >= 0.6
         and len(txt.parse_authors_any(left.authors)) >= 2
         and len(txt.parse_authors_any(right.authors)) >= 2
     ):
         overlap = txt.author_overlap_ratio(left.authors, right.authors)
         composite = txt.compute_dedup_score(left.fields, right.fields)
         if overlap >= 0.9 and composite >= cfg.SIM_DEDUP_COMPOSITE_THRESHOLD:
             return _evidence(left, right, True, IdentityReason.COMPOSITE, title_score=score, composite=composite)
-    if score >= cfg.SIM_PREPRINT_TITLE_THRESHOLD and txt.authors_overlap(left.authors, right.authors):
+    if (
+        arxiv_compatible
+        and score >= cfg.SIM_PREPRINT_TITLE_THRESHOLD
+        and txt.authors_overlap(left.authors, right.authors)
+    ):
         left_journal = str(left.fields.get("journal") or "").lower()
         right_journal = str(right.fields.get("journal") or "").lower()
         left_preprint = bool(left.doi and is_secondary_doi(left.doi)) or any(

tests/test_save_entry.py

@@ -374,6 +374,7 @@ def test_composite_boundary_flips_verdict(tmp_path: Path, authors: tuple[str, st
 
     assert len(_bib_files(author_dir)) == expected_files
 
+
 def test_year_change_renames_the_file_to_match_the_new_year(tmp_path: Path) -> None:
     """A record whose year changed must not keep the old year in its filename.
 
@@ -453,3 +454,62 @@ def test_validated_same_doi_metadata_replaces_equal_field_baseline(tmp_path: Pat
     saved = _read(author_dir, Path(path).name)
     assert "Khaleda Papry" in saved
     assert "year = {2025}" in saved
+
+
+# The committed Milios records. VISLA is arXiv 2404.16365 and SugarCrepe++ is
+# arXiv 2406.11171, two works; the published record carries no arXiv id because
+# the merge strips eprint fields once a published DOI is known.
+_VISLA = factories.entry(
+    type="misc",
+    key="Dumpala2024:VislaBenchmark",
+    title="VISLA Benchmark: Evaluating Embedding Sensitivity to Semantic and Lexical Alterations",
+    author="Sri Harsha Dumpala and Aman Jaiswal and Chandramouli Shama Sastry and Evangelos Milios "
+    "and Sageev Oore and Hassan Sajjad",
+    year="2024",
+    howpublished="arXiv:2404.16365",
+    doi="10.48550/arxiv.2404.16365",
+    url="https://arxiv.org/abs/2404.16365",
+    eprint="2404.16365",
+    archiveprefix="arXiv",
+)
+_SUGARCREPE = factories.inproceedings(
+    key="Dumpala2024:SugarcrepeDataset",
+    title="SUGARCREPE++ Dataset: Vision-Language Model Sensitivity to Semantic and Lexical Alterations",
+    author="Sri Dumpala and Aman Jaiswal and Chandramouli Sastry and Evangelos Milios and Sageev Oore "
+    "and Hassan Sajjad",
+    year="2024",
+    booktitle="Advances in Neural Information Processing Systems 37",
+    publisher="Neural Information Processing Systems Foundation, Inc. (NeurIPS)",
+    pages="17972-18018",
+    doi="10.52202/079017-0571",
+    url="https://doi.org/10.52202/079017-0571",
+)
+
+
+@pytest.mark.parametrize(
+    ("on_disk", "incoming"), [(_VISLA, _SUGARCREPE), (_SUGARCREPE, _VISLA)], ids=["published_in", "preprint_in"]
+)
+def test_published_record_never_absorbs_a_distinct_arxiv_preprint(
+    tmp_path: Path, on_disk: dict[str, object], incoming: dict[str, object]
+) -> None:
+    """A 0.71 title score with no linking identifier is two works, so both files stay."""
+    author_dir = _author_dir(tmp_path)
+    factories.write_bib(author_dir, on_disk, "on_disk.bib")
+
+    _, was_written = save_entry_to_file(str(tmp_path), AUTHOR_ID, incoming)
+
+    assert was_written is True
+    texts = [_read(author_dir, name).lower() for name in _bib_files(author_dir)]
+    assert len(texts) == 2, texts
+    assert any("10.48550/arxiv.2404.16365" in text for text in texts)
+    assert any("10.52202/079017-0571" in text for text in texts)
+
+
+def test_enrichment_never_adopts_a_distinct_work_as_the_published_version() -> None:
+    """A Crossref candidate carries no arXiv id either, so enrichment must not merge it into VISLA."""
+    from citeforge.identity import IdentityContext, IdentityReason, evaluate_identity
+
+    evidence = evaluate_identity(_VISLA, _SUGARCREPE, context=IdentityContext.ENRICHMENT)
+
+    assert evidence.verdict is False
+    assert evidence.reason is IdentityReason.GATE_CLOSED