Escat tier

config/schema.yaml

@@ -384,6 +384,41 @@ variant biomarker for treatment:
         data_source: str
         Tier: str
 
+biomarker_for_treatment_level_IA:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_IA
+
+biomarker_for_treatment_level_IC:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_IC
+
+biomarker_for_treatment_level_II:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_II
+
+biomarker_for_treatment_level_IIIA:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_IIIA
+
+biomarker_for_treatment_level_IIIB:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_IIIB
+
+biomarker_for_treatment_level_IVA:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_IVA
+
+biomarker_for_treatment_level_IVB:
+    is_a: variant biomarker for treatment
+    represented_as: edge
+    input_label: biomarker_for_treatment_level_IVB
+
 ## METADATA
 
 patient has chronic illness:

make.sh

@@ -4,30 +4,30 @@ work_dir=$(pwd)
 
 CONFIG="config/neo4j.yaml"
 if [[ -z "$1" ]] ; then
-    echo "ERROR, usage: $0 <DECIDER_snapshot_dir> [config] [debug|sub-sample_percentage]" >&2
+    echo "ERROR, usage: $0 <DECIDER_snapshot_dir> [config] [debug|xdebug|sub-sample_percentage]" >&2
     echo "    config defaults to: $CONFIG" >&2
     exit 2
 fi
 
 if [[ -z "$2" ]] ; then
     echo "Selecting Neo4j output configuration." >&2
-elif [[ "$2" == "debug" ]] ; then
-    echo "ERROR, if DEBUG MODE, usage: $0 <DECIDER_data_dir> <config> debug" >&2
+elif [[ "$2" == "debug" || "$2" == "xdebug" ]] ; then
+    echo "ERROR, if DEBUG MODE, usage: $0 <DECIDER_data_dir> <config> <debug|xdebug>" >&2
     exit 2
 else
     echo "Selecting output configuration: $2" >&2
     CONFIG="$2"
 fi
 
-set -e
-set -o pipefail
+# set -e
+# set -o pipefail
 
 decider_dir="$(realpath $1)"
 data_dir="$(realpath $1/..)"
 
 sub_sample=""
 if [[ -n "$3" ]] ; then
-    if [[ "$3" == "debug" ]] ; then
+    if [[ "$3" == "debug" || "$3" == "xdebug" ]] ; then
         echo "DEBUG MODE" >&2
         data_dir="$(realpath $decider_dir/../../data_debug)"
         decider_dir="$(realpath $decider_dir/../../data_debug/DECIDER_debug)"
@@ -62,13 +62,14 @@ else
     NEO_USER=""
 fi
 
-# py_args="-O" # Optimize = remove asserts and optimize bytecode.
-py_args="" # Optimize = remove asserts and optimize bytecode. # HERE
+py_args="-O" # Optimize = remove asserts and optimize bytecode.
 weave_args="-v INFO" # Default, for having clean progress bars.
 if [[ "$3" == "debug" ]] ; then
+    py_args=""
+    weave_args="--debug -v INFO"
+elif [[ "$3" == "xdebug"  ]] ; then
     py_args=""
     weave_args="--debug -v DEBUG"
-    # weave_args="--debug -v INFO"
 fi
 
 
@@ -84,9 +85,9 @@ echo "Weave data..." >&2
 
 cmd="uv run python3 ${py_args} $script_dir/weave.py
     --config $CONFIG
-    --structural-variants-2                 $decider_dir/structural_variants_2.csv
     --clinical                              $decider_dir/clinical_export.xlsx
     --short-mutations-local                 $decider_dir/short_mutations_local.csv 
+    --structural-variants-2                 $decider_dir/structural_variants_2.csv
     --short-mutations-external              $decider_dir/short_mutations_external.csv 
     --copy-number-amplifications-local      $decider_dir/cnas_local.csv
     --copy-number-amplifications-external   $decider_dir/cnas_external.csv 
@@ -99,14 +100,17 @@ cmd="uv run python3 ${py_args} $script_dir/weave.py
     --oncokb-gene-status                    $decider_dir/oncokb_gene_status_info.csv
     ${sub_sample}
     ${weave_args}"
-
 echo "Weaving command:" >&2
 echo "$cmd" >&2
 
 $cmd > last_biocypher_import.sh
 
+if [[ -z "$(cat last_biocypher_import.sh)" ]] ; then
+    echo "ERROR: The weave command returned nothing." >&2
+    exit 1
+fi
 
-if [[ "$CONFIG" == "config/neo4j.yaml" ]] ; then
+if [[ "$CONFIG" == *"neo4j"* ]] ; then
     echo "Stop Neo4j server..." >&2
     neo_version=$(neo4j-admin --version | cut -d. -f 1)
     if [[ "$neo_version" -eq 4 ]]; then
@@ -120,6 +124,12 @@ if [[ "$CONFIG" == "config/neo4j.yaml" ]] ; then
     chmod a+x  $(cat last_biocypher_import.sh)
     ${NEO_USER} $SHELL $(cat last_biocypher_import.sh)
 
+    err=$?
+    if [[ $err -gt 0 ]] ; then
+        echo "Import in Neo4j failed on error: $err"
+        exit $err
+    fi
+
     echo "Restart Neo4j..." >&2
     $server start
     sleep 5

oncodashkb/adapters/clinical.yaml

@@ -320,3 +320,5 @@ transformers:
     #     to_object: disease
     #     via_relation: patient_has_chronic_illness
 
+metadata:
+    - data_source: DECIDER_clinical

oncodashkb/adapters/oncokb.yaml

@@ -9,20 +9,23 @@ row:
             - CNA:
                 to_subject: copy_number_amplification
 transformers:
-    - replace:
-        column: treatment
-        to_object: treatment
-        via_relation: variant_biomarker_for_treatment
-        forbidden: ';'
-        substitute: ' + '
-    - split_translate:
-        column: treatment
-        to_object: drug
-        via_relation: variant_biomarker_for_treatment
-        separator: ";"
-        translations_file: ./data/OT/drug_molecule/part-00000-871f412e-aec4-4d33-a50d-feee532ddcd2-c000.snappy.parquet
-        translate_from: name
-        translate_to: id
+    # - replace:
+    # # replaced by user-defined transformer with escat level included in the name of the relation
+    #     column: treatment
+    #     to_object: treatment
+    #     via_relation: variant_biomarker_for_treatment
+    #     forbidden: ';'
+    #     substitute: ' + '
+    # 
+    # - split_translate:
+    # # Not used anymore
+    #     column: treatment
+    #     to_object: drug
+    #     via_relation: variant_biomarker_for_treatment
+    #     separator: ";"
+    #     translations_file: ./data/OT/drug_molecule/part-00000-871f412e-aec4-4d33-a50d-feee532ddcd2-c000.snappy.parquet
+    #     translate_from: name
+    #     translate_to: id
     - split_translate:
         column: treatment
         from_subject: treatment
@@ -44,20 +47,36 @@ transformers:
     #     translate_from: name
     #     translate_to: id
     ## Variant biomarker for treatment
-    - map:
-        column: level_of_evidence
-        to_property: level_of_evidence
-        for_object: variant_biomarker_for_treatment
+    # - map:
+    #  # Replaced by escat_tier_transformer
+    #     column: level_of_evidence
+    #     to_property: level_of_evidence
+    #     for_object: variant_biomarker_for_treatment
+    - escat_tier_transformer
     - replace:
         column: citations
         to_property: citations
-        for_object: variant_biomarker_for_treatment
+        for_object:
+          - variant_biomarker_for_treatment_level_IA
+          - variant_biomarker_for_treatment_level_IC
+          - variant_biomarker_for_treatment_level_II
+          - variant_biomarker_for_treatment_level_IIIA
+          - variant_biomarker_for_treatment_level_IIIB
+          - variant_biomarker_for_treatment_level_IVA
+          - variant_biomarker_for_treatment_level_IVB
         forbidden: ';'
         substitute: ','
     - map:
         column: tumorType
         to_property: tumorType
-        for_object: variant_biomarker_for_treatment
+        for_object:
+          - variant_biomarker_for_treatment_level_IA
+          - variant_biomarker_for_treatment_level_IC
+          - variant_biomarker_for_treatment_level_II
+          - variant_biomarker_for_treatment_level_IIIA
+          - variant_biomarker_for_treatment_level_IIIB
+          - variant_biomarker_for_treatment_level_IVA
+          - variant_biomarker_for_treatment_level_IVB
 
 metadata:
         - data_source: oncokb_annotation
@@ -105,4 +124,4 @@ validate:
       dtype: str
       nullable: true
       checks:
-        str_matches: ^[ABCD]\([^\d]+\)$
+        str_matches: ^[ABCD]\([^\d]+\)$

oncodashkb/adapters/template__copy_number_amplifications_external.yaml

@@ -6,6 +6,7 @@ row:
         translate_from: Patient card::Patient cohort code_Patient Card
         translate_to: Patient card::Publication code
         index_col: 0
+        on_unknown_value: skip
         usecols: [0,1,2]
 transformers:
     # Nodes
@@ -45,7 +46,7 @@ transformers:
             Loss-of-function: LoF
             Likely Loss-of-function: LoF
             Gain-of-function: GoF
-            Likely Gain-of-function: LoF
+            Likely Gain-of-function: GoF
             Act: GoF
             Inconclusive: Inconclusive
             ambiguous: ambiguous
@@ -82,7 +83,7 @@ transformers:
             Loss-of-function: LoF
             Likely Loss-of-function: LoF
             Gain-of-function: GoF
-            Likely Gain-of-function: LoF
+            Likely Gain-of-function: GoF
             Act: GoF
             Inconclusive: Inconclusive
             ambiguous: ambiguous
@@ -91,11 +92,12 @@ transformers:
         columns: gene_role
         to_property: gene_role
         for_object: variant_causes_gene_status
+        on_unknown_value: skip
         translations:
             Loss-of-function: LoF
             Likely Loss-of-function: LoF
             Gain-of-function: GoF
-            Likely Gain-of-function: LoF
+            Likely Gain-of-function: GoF
             Act: GoF
             Inconclusive: Inconclusive
             ambiguous: ambiguous
@@ -170,4 +172,4 @@ transformers:
             - gene_status_affects_gene
 
 metadata:
-        - data_source: copy_number_amplifications_external
+        - data_source: DECIDER_copy_number_amplifications_external

oncodashkb/adapters/template__copy_number_amplifications_local.yaml

@@ -6,6 +6,7 @@ row:
         translate_from: Patient card::Patient cohort code_Patient Card
         translate_to: Patient card::Publication code
         index_col: 0
+        on_unknown_value: skip
         usecols: [0,1,2]
 transformers:
     - translate_sample_ids :
@@ -51,4 +52,4 @@ transformers:
         for_object: sample_carries_variant
 
 metadata:
-        - data_source: copy_number_amplifications_local
+        - data_source: DECIDER_copy_number_amplifications_local

oncodashkb/adapters/template__short_mutations_external.yaml

@@ -6,6 +6,7 @@ row:
         translate_from: Patient card::Patient cohort code_Patient Card
         translate_to: Patient card::Publication code
         index_col: 0
+        on_unknown_value: skip
         usecols: [0,1,2]
 transformers:
     # Nodes
@@ -42,7 +43,7 @@ transformers:
             Loss-of-function: LoF
             Likely Loss-of-function: LoF
             Gain-of-function: GoF
-            Likely Gain-of-function: LoF
+            Likely Gain-of-function: GoF
             Inconclusive: Inconclusive
             Unknown: Unknown
             Likely Neutral: Likely Neutral
@@ -64,25 +65,26 @@ transformers:
         for_object: gene_status
         to_property: ensembl_id_gene_status
         format_string: "{ensembl_id}:{gene_role}"
-        column_to_translate: 
+        column_to_translate:
             - gene_role
         translations:
             Loss-of-function: LoF
             Likely Loss-of-function: LoF
             Gain-of-function: GoF
-            Likely Gain-of-function: LoF
+            Likely Gain-of-function: GoF
             Inconclusive: Inconclusive
             Unknown: Unknown
             Likely Neutral: Likely Neutral
     - translate:
         columns: gene_role
         to_property: gene_role
         for_object: gene_status
+        on_unknown_value: skip
         translations:
             Loss-of-function: LoF
             Likely Loss-of-function: LoF
             Gain-of-function: GoF
-            Likely Gain-of-function: LoF
+            Likely Gain-of-function: GoF
             Inconclusive: Inconclusive
             Unknown: Unknown
             Likely Neutral: Likely Neutral
@@ -158,4 +160,4 @@ transformers:
             - variant_causes_gene_status
             - gene_status_affects_gene
 metadata:
-        - data_source: short_mutations_external
+        - data_source: DECIDER_short_mutations_external

oncodashkb/adapters/template__short_mutations_local.yaml

@@ -6,6 +6,7 @@ row:
         translate_from: Patient card::Patient cohort code_Patient Card
         translate_to: Patient card::Publication code
         index_col: 0
+        on_unknown_value: skip
         usecols: [0,1,2]
 transformers:
     - translate_sample_ids :
@@ -97,4 +98,4 @@ transformers:
             - patient_carries_sample
             - sample_carries_variant
 metadata:
-        - data_source: short_mutations_local
+        - data_source: DECIDER_short_mutations_local

oncodashkb/adapters/template__structural_variants.yaml

@@ -6,6 +6,7 @@ row:
         translate_from: Patient card::Patient cohort code_Patient Card
         translate_to: Patient card::Publication code
         index_col: 0
+        on_unknown_value: skip
         usecols: [0,1,2]
 transformers:
     # Nodes
@@ -67,6 +68,7 @@ transformers:
         translations_file: data/HGNC/hgnc_complete_set.txt
         translate_from: symbol
         translate_to: ensembl_gene_id
+        on_unknown_value: skip
         sep: "\t"
     # Properties
     ## Sample
@@ -112,6 +114,7 @@ transformers:
         columns: Gene_type
         to_property: gene_role
         for_object: gene_status
+        on_unknown_value: skip
         translations:
             gain: GoF
             loss: LoF
@@ -163,4 +166,4 @@ transformers:
             - gene_status_affects_gene
 
 metadata:
-        - data_source: structural_variants_placeholder
+        - data_source: DECIDER_structural_variants_placeholder

oncodashkb/adapters/template__structural_variants_2.yaml

@@ -117,5 +117,5 @@ transformers:
             - gene_status_affects_gene
 
 metadata:
-        - data_source: structural_variants
+        - data_source: DECIDER_structural_variants