Merge pull request #434 from TranslatorSRI/fix-synonyms-compression

Actually running the synonym compression code in Babel v1.10.0 (#424) revealed that it wasn't working properly. This PR should fix that.

Author: gaurav

Snakefile

@@ -74,3 +74,12 @@ rule clean_downloads:
     shell:
         "rm -rf {params.dir}/*"
 
+# Sometimes a synonyms file is available as a .gz file, but not as the .txt file itself.
+# This rule is here so that Snakemake knows how to uncompress it if needed.
+rule uncompress_synonym_file:
+    input:
+        config['output_directory'] + '/synonyms/{synonym_file}.txt.gz'
+    output:
+        config['output_directory'] + '/synonyms/{synonym_file}.txt'
+    shell:
+        'gunzip {input} -c > {output}'

src/createcompendia/leftover_umls.py

@@ -12,7 +12,7 @@
 from src.categories import ACTIVITY, AGENT, DEVICE, DRUG, FOOD, SMALL_MOLECULE, PHYSICAL_ENTITY, PUBLICATION, PROCEDURE
 
 
-def write_leftover_umls(compendia, mrconso, mrsty, synonyms, umls_compendium, umls_synonyms, report, done, biolink_version):
+def write_leftover_umls(compendia, mrconso, mrsty, synonyms, umls_compendium, umls_synonyms, report, biolink_version):
     """
     Search for "leftover" UMLS concepts, i.e. those that are defined and valid in MRCONSO but are not
     mapped to a concept in Babel.
@@ -26,12 +26,11 @@ def write_leftover_umls(compendia, mrconso, mrsty, synonyms, umls_compendium, um
     :param umls_compendium: The UMLS compendium file to write out.
     :param umls_synonyms: The synonyms file to generate for this compendium.
     :param report: The report file to write out.
-    :param done: The done file to write out.
     :return: Nothing.
     """
 
     logging = Logger()
-    logging.info(f"write_leftover_umls({compendia}, {mrconso}, {mrsty}, {synonyms}, {umls_compendium}, {umls_synonyms}, {report}, {done})")
+    logging.info(f"write_leftover_umls({compendia}, {mrconso}, {mrsty}, {synonyms}, {umls_compendium}, {umls_synonyms}, {report}, {biolink_version})")
 
     # For now, we have many more UMLS entities in MRCONSO than in the compendia, so
     # we'll make an in-memory list of those first. Once that flips, this should be
@@ -244,9 +243,4 @@ def umls_type_to_biolink_type(umls_tui):
         logging.info(f"Wrote out {count_synonym_objs} synonym objects into the leftover UMLS synonyms file.")
         reportf.write(f"Wrote out {count_synonym_objs} synonym objects into the leftover UMLS synonyms file.\n")
 
-
-    # Write out `done` file.
-    with open(done, 'w') as outf:
-        outf.write(f"done\n{datetime.now()}\n")
-
-    logging.info("Complete")
+    logging.info("Complete")

src/exporters/duckdb_exporters.py

@@ -97,7 +97,7 @@ def export_compendia_to_parquet(compendium_filename, clique_parquet_filename, du
         )
 
 
-def export_synonyms_to_parquet(synonyms_filename, duckdb_filename, synonyms_parquet_filename):
+def export_synonyms_to_parquet(synonyms_filename_gz, duckdb_filename, synonyms_parquet_filename):
     """
     Export a synonyms file to a DuckDB directory.
 
@@ -115,7 +115,7 @@ def export_synonyms_to_parquet(synonyms_filename, duckdb_filename, synonyms_parq
 
     with setup_duckdb(duckdb_filename) as db:
         # Step 1. Load the entire synonyms file.
-        synonyms_jsonl = db.read_json(synonyms_filename, format='newline_delimited')
+        synonyms_jsonl = db.read_json(synonyms_filename_gz, format='newline_delimited')
 
         # Step 2. Create a Cliques table with all the cliques from this file.
         #db.sql("CREATE TABLE Cliques (curie TEXT PRIMARY KEY, label TEXT, clique_identifier_count INT, biolink_type TEXT)")

src/exporters/sapbert.py

@@ -35,24 +35,24 @@
 LOWERCASE_ALL_NAMES = True
 
 
-def convert_synonyms_to_sapbert(synonym_filename, sapbert_filename_gzipped):
+def convert_synonyms_to_sapbert(synonym_filename_gz, sapbert_filename_gzipped):
     """
     Convert a synonyms file to the training format for SAPBERT (https://github.com/RENCI-NER/sapbert).
 
     Based on the code in https://github.com/TranslatorSRI/babel-validation/blob/f21b1b308e54ec0af616f2c24f7e2738ac4c261c/src/main/scala/org/renci/babel/utils/converter/Converter.scala#L107-L207
 
-    :param synonym_filename: The compendium file to convert.
+    :param synonym_filename_gz: The compendium file to convert.
     :param sapbert_filename_gzipped: The SAPBERT training file to generate.
     """
 
-    logger.info(f"convert_synonyms_to_sapbert({synonym_filename}, {sapbert_filename_gzipped})")
+    logger.info(f"convert_synonyms_to_sapbert({synonym_filename_gz}, {sapbert_filename_gzipped})")
 
     # For now, the simplest way to identify the DrugChemicalConflated file is by name.
     # In this case we still generate DrugChemicalConflated.txt, but we also generate
     # DrugChemicalConflatedSmaller.txt, which ignores cliques whose preferred label is
     # longer than config['demote_labels_longer_than'].
     generate_smaller_filename = None
-    if GENERATE_DRUG_CHEMICAL_SMALLER_FILE and synonym_filename.endswith('/DrugChemicalConflated.txt'):
+    if GENERATE_DRUG_CHEMICAL_SMALLER_FILE and synonym_filename_gz.endswith('/DrugChemicalConflated.txt.gz'):
         generate_smaller_filename = sapbert_filename_gzipped.replace('.txt.gz', 'Smaller.txt.gz')
 
     # Make the output directories if they don't exist.
@@ -67,7 +67,7 @@ def convert_synonyms_to_sapbert(synonym_filename, sapbert_filename_gzipped):
     count_entry = 0
     count_training_rows = 0
     count_smaller_rows = 0
-    with open(synonym_filename, "r", encoding="utf-8") as synonymf, gzip.open(sapbert_filename_gzipped, "wt", encoding="utf-8") as sapbertf:
+    with gzip.open(synonym_filename_gz, "rt", encoding="utf-8") as synonymf, gzip.open(sapbert_filename_gzipped, "wt", encoding="utf-8") as sapbertf:
         for input_line in synonymf:
             count_entry += 1
             entry = json.loads(input_line)
@@ -136,14 +136,14 @@ def convert_synonyms_to_sapbert(synonym_filename, sapbert_filename_gzipped):
                         generate_smaller_file.write(line)
                         count_smaller_rows += 1
 
-    logger.info(f"Converted {synonym_filename} to SAPBERT training file {synonym_filename}: " +
+    logger.info(f"Converted {synonym_filename_gz} to SAPBERT training file {synonym_filename_gz}: " +
                 f"read {count_entry} entries and wrote out {count_training_rows} training rows.")
 
     # Close SmallerFile if needed.
     if generate_smaller_file:
         generate_smaller_file.close()
         percentage = count_smaller_rows / float(count_training_rows) * 100
-        logger.info(f"Converted {synonym_filename} to smaller SAPBERT training file {generate_smaller_filename}: " +
+        logger.info(f"Converted {synonym_filename_gz} to smaller SAPBERT training file {generate_smaller_filename}: " +
                     f"read {count_entry} entries and wrote out {count_smaller_rows} training rows ({percentage:.2f}%).")
 
 

src/snakefiles/cell_line.snakefile

@@ -1,6 +1,7 @@
 import src.datahandlers.clo as clo
 import src.createcompendia.cell_line as cell_line
 import src.assess_compendia as assessments
+import src.snakefiles.util as util
 
 ### Cell Line
 
@@ -26,7 +27,7 @@ rule cell_line_compendia:
         icrdf_filename=config['download_directory']+'/icRDF.tsv',
     output:
         config['output_directory']+"/compendia/CellLine.txt",
-        config['output_directory']+"/synonyms/CellLine.txt"
+        temp(config['output_directory']+"/synonyms/CellLine.txt")
     run:
         cell_line.build_compendia(input.ids,input.icrdf_filename)
 
@@ -49,9 +50,11 @@ rule check_cell_line:
 rule cell_line:
     input:
         config['output_directory']+'/reports/cell_line_completeness.txt',
-        config['output_directory'] + "/synonyms/CellLine.txt",
-        config['output_directory'] + "/reports/CellLine.txt"
+        config['output_directory'] + "/reports/CellLine.txt",
+        cell_line_synonyms=config['output_directory'] + "/synonyms/CellLine.txt",
     output:
+        config['output_directory'] + "/synonyms/CellLine.txt.gz",
         x=config['output_directory']+'/reports/cell_line_done'
-    shell:
-        "echo 'done' >> {output.x}"
+    run:
+        util.gzip_files([input.cell_line_synonyms])
+        util.write_done(output.x)

src/snakefiles/drugchemical.snakefile

@@ -56,20 +56,17 @@ rule drugchemical_conflated_synonyms:
     input:
         drugchemical_conflation=[config['output_directory']+'/conflation/DrugChemical.txt'],
         chemical_compendia=expand("{do}/compendia/{co}", do=config['output_directory'], co=config['chemical_outputs']),
-        chemical_synonyms=expand("{do}/synonyms/{co}", do=config['output_directory'], co=config['chemical_outputs']),
+        chemical_synonyms_gz=expand("{do}/synonyms/{co}.gz", do=config['output_directory'], co=config['chemical_outputs']),
     output:
-        drugchemical_conflated=config['output_directory']+'/synonyms/DrugChemicalConflated.txt',
+        drugchemical_conflated_gz=config['output_directory']+'/synonyms/DrugChemicalConflated.txt.gz',
     run:
-        synonymconflation.conflate_synonyms(input.chemical_synonyms, input.chemical_compendia, input.drugchemical_conflation, output.drugchemical_conflated)
+        synonymconflation.conflate_synonyms(input.chemical_synonyms_gz, input.chemical_compendia, input.drugchemical_conflation, output.drugchemical_conflated_gz)
 
 rule drugchemical:
     input:
         config['output_directory']+'/conflation/DrugChemical.txt',
-        config['output_directory']+'/synonyms/DrugChemicalConflated.txt',
-        chemical_synonyms=expand("{do}/synonyms/{co}", do=config['output_directory'], co=config['chemical_outputs']),
+        config['output_directory']+'/synonyms/DrugChemicalConflated.txt.gz',
     output:
-        chemical_synonyms_gzipped=expand("{do}/synonyms/{co}.gz", do=config['output_directory'], co=config['chemical_outputs']),
-        x=config['output_directory']+'/reports/drugchemical_done'
+        done=config['output_directory']+'/reports/drugchemical_done'
     run:
-        util.gzip_files(input.chemical_synonyms)
-        util.write_done(output.x)
+        util.write_done(output.done)

src/snakefiles/duckdb.snakefile

@@ -44,7 +44,7 @@ rule export_all_synonyms_to_duckdb:
 # Generic rule for generating the Parquet files for a particular compendia file.
 rule export_synonyms_to_duckdb:
     input:
-        synonyms_file=config['output_directory'] + "/synonyms/{filename}.txt",
+        synonyms_file=config['output_directory'] + "/synonyms/{filename}.txt.gz",
     output:
         duckdb_filename=temp(config['output_directory'] + "/duckdb/duckdbs/filename={filename}/synonyms.duckdb"),
         synonyms_parquet_filename=config['output_directory'] + "/duckdb/parquet/filename={filename}/Synonyms.parquet",

src/snakefiles/exports.snakefile

@@ -49,8 +49,8 @@ rule export_all_to_sapbert_training:
 # Generic rule for generating the KGX files for a particular compendia file.
 rule generate_sapbert_training_data:
     input:
-        synonym_file=config['output_directory'] + "/synonyms/{filename}",
+        synonym_file_gz=config['output_directory'] + "/synonyms/{filename}.gz",
     output:
         sapbert_training_data_file=config['output_directory'] + "/sapbert-training-data/{filename}.gz",
     run:
-        sapbert.convert_synonyms_to_sapbert(input.synonym_file, output.sapbert_training_data_file)
+        sapbert.convert_synonyms_to_sapbert(input.synonym_file_gz, output.sapbert_training_data_file)

src/snakefiles/leftover_umls.snakefile

@@ -31,14 +31,15 @@ rule leftover_umls:
         umls_compendium = config['output_directory'] + "/compendia/umls.txt",
         umls_synonyms = temp(config['output_directory'] + "/synonyms/umls.txt"),
         report = config['output_directory'] + "/reports/umls.txt",
-        done = config['output_directory'] + "/reports/umls_done"
     run:
-        write_leftover_umls(input.input_compendia, input.mrconso, input.mrsty, input.synonyms, output.umls_compendium, output.umls_synonyms, output.report, output.done, config['biolink_version'])
+        write_leftover_umls(input.input_compendia, input.mrconso, input.mrsty, input.synonyms, output.umls_compendium, output.umls_synonyms, output.report, config['biolink_version'])
 
 rule compress_umls:
     input:
         umls_synonyms = config['output_directory'] + "/synonyms/umls.txt",
     output:
         umls_synonyms_gzipped = config['output_directory'] + "/synonyms/umls.txt.gz",
+        done = config['output_directory'] + "/reports/umls_done",
     run:
-        util.gzip_files([input.umls_synonyms])
+        util.gzip_files([input.umls_synonyms])
+        util.write_done(output.done)

src/snakefiles/publications.snakefile

@@ -1,5 +1,6 @@
 import src.createcompendia.publications as publications
 import src.assess_compendia as assessments
+from src.snakefiles import util
 
 ### PubMed
 
@@ -54,6 +55,8 @@ rule generate_pubmed_compendia:
         icrdf_filename=config['download_directory'] + '/icRDF.tsv',
     output:
         publication_compendium = config['output_directory'] + '/compendia/Publication.txt',
+        # We generate an empty Publication Synonyms files, but we still need to generate one.
+        publication_synonyms_gz = config['output_directory'] + '/synonyms/Publication.txt.gz',
     run:
         publications.generate_compendium(
             [input.pmid_doi_concord_file],
@@ -62,6 +65,10 @@ rule generate_pubmed_compendia:
             output.publication_compendium,
             input.icrdf_filename
         )
+        # generate_compendium() will generate an (empty) Publication.txt.gz file, but we need
+        # to compress it.
+        publication_synonyms = os.path.splitext(output.publication_synonyms_gz)[0]
+        util.gzip_files([publication_synonyms])
 
 rule check_publications_completeness:
     input:
@@ -82,8 +89,7 @@ rule check_publications:
 rule publications:
     input:
         config['output_directory']+'/reports/publication_completeness.txt',
-        # No synonyms for Publication.txt yet.
-        # synonyms=expand("{od}/synonyms/{ap}", od = config['output_directory'], ap = config['publication_outputs']),
+        synonyms = expand("{od}/synonyms/{ap}.gz", od = config['output_directory'], ap = config['publication_outputs']),
         reports = expand("{od}/reports/{ap}",od=config['output_directory'], ap = config['publication_outputs'])
     output:
         x=config['output_directory']+'/reports/publications_done'