Merge pull request #85 from openproblems-bio/jalil

multiple new metrics added

Author: janursa

scripts/sync_resources.sh

@@ -25,4 +25,5 @@ set -e
 # aws s3 sync  resources/grn_benchmark/prior s3://openproblems-data/resources/grn/grn_benchmark/prior --delete
 # aws s3 sync  resources/extended_data/ s3://openproblems-data/resources/grn/extended_data --delete
 # aws s3 sync resources/results/experiment  s3://openproblems-data/resources/grn/results/experiment --delete 
-aws s3 sync resources_test  s3://openproblems-data/resources_test/grn/ --delete 
+# aws s3 sync resources_test  s3://openproblems-data/resources_test/grn/ --delete 
+aws s3 sync  resources/grn_benchmark/ground_truth s3://openproblems-data/resources/grn/grn_benchmark/ground_truth 

src/methods/geneformer/config.vsh.yaml

@@ -53,6 +53,7 @@ resources:
     path: script.py
   - path: /src/utils/util.py
     dest: util.py
+  - path: helper.py
 
 engines:
   - type: docker

src/methods/geneformer/helper.py.py src/methods/geneformer/helper.pysrc/methods/geneformer/helper.py.py renamed to src/methods/geneformer/helper.py

@@ -54,4 +54,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [twodaytime, veryhighmem, highcpu]
+      label: [twodaytime, veryveryhighmem, highcpu]

src/methods/scenicplus/config.vsh.yaml

@@ -11,9 +11,11 @@
 
 from regression_2.helper import main as main_reg2
 from ws_distance.helper import main as main_ws_distance
-from sem.helper import main as main_sem
-from vc_v2.helper import main as main_vc_v2
-from tf_recovery import main as main_tf_rec
+from experimental.sem.helper import main as main_sem
+# from vc_v2.helper import main as main_vc_v2
+from tf_recovery.helper import main as main_tf_rec
+from tf_binding.helper import main as main_tf_binding
+from replica_consistency.helper import main as main_replica_consistency
 
 
 def main(par):
@@ -27,6 +29,22 @@ def main(par):
             tf_rec = pd.DataFrame()
         print("tf_rec done: ", tf_rec)
         rr_store.append(tf_rec)
+    if True:
+        try:
+            tf_binding = main_tf_binding(par)
+        except Exception as e:
+            print(f"Error in main_tf_binding metrics: {e}")
+            tf_binding = pd.DataFrame()
+        print("tf_binding done: ", tf_binding)
+        rr_store.append(tf_binding)
+    if True:
+        try:
+            rr_replica = main_replica_consistency(par)
+        except Exception as e:
+            print(f"Error in replica consistency metrics: {e}")
+            rr_replica = pd.DataFrame()
+        print("replica consistency done: ", rr_replica)
+        rr_store.append(rr_replica)
 
 
     # if True:
@@ -47,14 +65,14 @@ def main(par):
     # rr_store.append(rr_reg2)
     # print("reg2 done: ", rr_reg2)
 
-    # if False:
-    #     try:
-    #         rr_sem = main_sem(par)
-    #     except Exception as e:
-    #         print(f"Error in sem metrics: {e}")
-    #         rr_sem = pd.DataFrame()
-    #     print("sem done: ", rr_sem)
-    #     rr_store.append(rr_sem)
+    if True:
+        try:
+            rr_sem = main_sem(par)
+        except Exception as e:
+            print(f"Error in sem metrics: {e}")
+            rr_sem = pd.DataFrame()
+        print("sem done: ", rr_sem)
+        rr_store.append(rr_sem)
 
 
     # try:

src/metrics/all_metrics/helper.py

@@ -73,7 +73,7 @@ def main(par):
     if dataset_id not in DATASET_GROUPS:
         raise ValueError(f"Dataset {dataset_id} not found in DATASET_GROUPS")
 
-    anchor_cols = DATASET_GROUPS[dataset_id].get('anchors', ['donor_id', 'plate_name'])
+    anchor_cols = DATASET_GROUPS[dataset_id]['anchors']
     print(f"Using anchor variables: {anchor_cols}")
 
     # Manage layer
@@ -115,7 +115,7 @@ def main(par):
     gene_mask = np.logical_or(np.any(A, axis=1), np.any(A, axis=0))
     in_degrees = np.sum(A != 0, axis=0)
     out_degrees = np.sum(A != 0, axis=1)
-    idx = np.argsort(np.maximum(out_degrees, in_degrees))[:-1000]
+    idx = np.argsort(np.maximum(out_degrees, in_degrees))[:-2000]
     gene_mask[idx] = False
     X = X[:, gene_mask]
     X = X.toarray() if isinstance(X, csr_matrix) else X
@@ -142,9 +142,9 @@ def main(par):
         X_test = X[~mask, :]
 
         # Standardize features
-        #scaler = StandardScaler()
-        #X_train = scaler.fit_transform(X_train)
-        #X_test = scaler.transform(X_test)
+        scaler = StandardScaler()
+        X_train = scaler.fit_transform(X_train)
+        X_test = scaler.transform(X_test)
 
         for j in tqdm.tqdm(range(X_train.shape[1])):
 
@@ -178,7 +178,7 @@ def main(par):
             baseline_scores.append(np.mean(coefs))
     scores = np.array(scores)
     baseline_scores = np.array(baseline_scores)
-
+    reg3_lift = np.mean(scores) / (np.mean(baseline_scores) + 1e-6)
     p_value = wilcoxon(baseline_scores, scores, alternative="greater").pvalue
     p_value = max(p_value, 1e-300)
 
@@ -189,7 +189,8 @@ def main(par):
 
     # Return results as DataFrame
     results = {
-        'regression_3': [final_score]
+        'reg3_precision': [reg3_lift],
+        'reg3_balanced': [final_score]
     }
 
     df_results = pd.DataFrame(results)

src/metrics/experimental/recovery_2/helper.py

@@ -20,8 +20,8 @@ datasets=('op' 'parsebioscience' "300BCG"   "adamson"  "replogle" "xaira_HEK293T
 datasets=('op'  ) #"300BCG" "ibd" 'parsebioscience', 'xaira_HEK293T'
 
 # methods to process
-methods=( "pearson_corr" "positive_control" "negative_control" "ppcor" "portia" "scenic" "grnboost" "scprint" "scenicplus" "celloracle" "scglue" "figr" "granie")
-methods=( "pearson_corr" "negative_control" "positive_control" )
+methods=( "scprint"  "pearson_corr" "positive_control" "negative_control" "ppcor" "portia" "scenic" "grnboost" "scenicplus" "celloracle" "scglue" "figr" "granie")
+# methods=( "pearson_corr" "negative_control" "positive_control" )
 
 # temporary file to collect CSV rows
 combined_csv="${save_dir}/reg2_scores.csv"
@@ -47,7 +47,6 @@ for dataset in "${datasets[@]}"; do
             --prediction "$prediction" \
             --evaluation_data "$evaluation_data" \
             --regulators_consensus "resources/grn_benchmark/prior/regulators_consensus_${dataset}.json" \
-            --group_specific cell_type \
             --score "$score"
 
         # Extract metrics from the .h5ad and append to CSV