Merge pull request #108 from RBL-NCI/activeDev

Active dev

Author: slsevilla

config/cluster_config.yaml

@@ -2,15 +2,15 @@
 __default__:
     gres: lscratch:96
     mem: 40g
-    partition: norm
-    time: 00-02:00:00
+    partition: ccr,norm
+    time: 00-08:00:00
     threads: 32
     output: .%j.{wildcards}.out
     error: .%j.{wildcards}.err
 
 qc_barcode:
-    threads: 3
-    mem: 3g
+    threads: 4
+    mem: 100g
     time: 00-04:00:00
 
 demultiplex:
@@ -20,16 +20,16 @@ demultiplex:
 
 remove_adaptors:
     threads: 16
-    time: 1-00:00:00
+    time: 01-00:00:00
     mem: 32g
 
 qc_fastq_pre:
-    threads: 3
+    threads: 4
     mem: 3g
     time: 00-03:00:00
 
 qc_fastq_post:
-    threads: 3
+    threads: 4
     mem: 3g
     time: 00-03:00:00
 
@@ -38,90 +38,147 @@ qc_screen_validator:
     time: 00-03:00:00
 
 split_files:
-    threads: 3
+    threads: 4
     mem: 3g
     time: 00-03:00:00
 
 novoalign:
     mem: 50g
+    threads: 32
     time: 10-00:00:00
 
 cleanup_conversion:
-    threads: 5
+    threads: 8
     gres: lscratch:256
-    mem: 30g
-    time: 00-3:00:00
+    mem: 200g
+    time: 01-00:00:00
 
 merge_unmapped_splits:
+    threads: 8
     time: 01-00:00:00
-    mem: 75g
+    mem: 200g
 
 create_bam_mm_unique:
-    threads: 6
+    threads: 8
     gres: lscratch:256
-    mem: 30g
-    time: 09-00:00:00
+    mem: 200g
+    time: 01-00:00:00
 
 merge_splits_unique_mm:
-    mem: 512g
-    time: 03-00:00:00
-    partition: largemem
+    threads: 32
+    mem: 200g
+    time: 01-00:00:00
 
 merge_mm_and_unique:
-    threads: 2
+    threads: 8
     gres: lscratch:256
-    mem: 5g
-    time: 02-00:00:00
+    mem: 200g
+    time: 01-00:00:00
 
 qc_alignment:
-    mem: 10g
+    threads: 4
+    mem: 200g
 
 qc_troubleshoot:
-    threads: 3
+    threads: 4
     mem: 3g
 
 dedup:
-    threads: 2
-    mem: 64g
+    threads: 8
+    mem: 200g
     gres: lscratch:256
-    time: 01-00:00:00
+    time: 02-00:00:00
 
 create_beds_safs:
-    mem: 350g
-    gres: lscratch:256
+    mem: 200g
+    gres: lscratch:512
+    threads: 8
+
+bgzip_beds:
+    mem: 100g
     threads: 4
-    partition: largemem
 
 project_annotations:
     threads: 2
     mem: 10g
     time: 00-01:00:00
 
-peak_annotations:
-    threads: 3
+peak_junctions:
+    threads: 4
     gres: lscratch:128
     mem: 30g
-    time: 00-12:00:00
+    time: 04-00:00:00
+    
+peak_Transcripts:
+    threads: 4
+    gres: lscratch:128
+    mem: 30g
+    time: 04-00:00:00    
+
+peak_ExonIntron:
+    threads: 4
+    gres: lscratch:128
+    mem: 30g
+    time: 04-00:00:00 
 
+peak_RMSK:
+    threads: 4
+    gres: lscratch:128
+    mem: 30g
+    time: 04-00:00:00 
+    
 annotation_report:
-    mem: 10g
+    threads: 4
+    gres: lscratch:128
+    mem: 30g
+    time: 00-12:00:00
 
+MANORM_beds:
+    threads: 4
+    mem: 30g
+
+DIFFBIND_beds:
+    threads: 4
+    mem: 30g
+    
 MANORM_analysis:
     threads: 4
     mem: 30g
+    time: 04-00:00:00
+
+DIFFBIND_preprocess:
+    threads: 4
+    mem: 30g
+
+DIFFBIND_analysis:
+    threads: 4
+    mem: 30g
 
 MANORM_post_processing:
     threads: 2
-    mem: 2g
-    time: 00-01:00:00
+    mem: 30g
+    time: 00-12:00:00
+
+DIFFBIND_report:
+    threads: 4
+    mem: 30g
 
 MANORM_RMD:
     threads: 2
-    mem: 3g
-    time: 00-01:00:00
+    mem: 30g
+    time: 00-02:00:00
 
 mapq_recalc:
-    mem: 1TB
-    gres: lscratch:256
-    partition: largemem
-    time: 00-06:00:00
+    threads: 8
+    mem: 200g
+    gres: lscratch:512
+    time: 00-12:00:00
+
+mapq_stats:
+    threads: 8
+    mem: 200g
+
+feature_counts:
+    threads: 8
+    mem: 200g
+    

config/snakemake_config.yaml

@@ -1,28 +1,33 @@
+#########################################################################################
 # Global configuration file for the pipeline
+#########################################################################################
+
+#########################################################################################
+#Folders and Paths
+#########################################################################################
 #path to iCLIP directory
 sourceDir: "PIPELINE_HOME"
-
 #path to output directory
 outputDir: "OUTPUT_DIR"
-
 #path to manifest files
-sampleManifest: "OUTPUT_DIR/manifest/samples.tsv"
-multiplexManifest: "OUTPUT_DIR/manifest/multiplex.tsv"
-contrastManifest: "OUTPUT_DIR/manifest/contrasts.tsv"
-
+sampleManifest: "OUTPUT_DIR/manifests/samples.tsv"
+multiplexManifest: "OUTPUT_DIR/manifests/multiplex.tsv"
+contrastManifest: "OUTPUT_DIR/manifests/contrasts.tsv"
 #path to fastq files
 fastqDir: "/path/to/fastq/files/"
 
+########################################################################################
 #user parameters
-filterlength: 20 #minimum read length to include in analysis [any int >20]
-multiplexflag: "Y" #whether samples are multiplexed ["Y","N"]
+#########################################################################################
+filterlength: 20 #minimum read length to include in analysis [any int >20] this is also the novoalign l parameter
+multiplexflag: "N" #whether samples are multiplexed ["Y","N"]
 mismatch: 1 #if multiplexed, number of bp mismatches allowed in demultiplexing [1,2,3]
-reference: "hg38" #reference organism ["hg38","mm10"]
+reference: "mm10" #reference organism ["hg38","mm10"]
 spliceaware: "Y" #whether to run include splice_aware feature for alignment ["Y","N"]
 includerRNA: "Y" #if spliceaware, include refseq rRNA's in annotations ["Y", "N"]
 spliceBPlength: 75 #if spliceaware Y, length of splice index to use [50, 75, 150]
 splicejunction: "Y" #if spliceaware Y, include splice junctions in peak calls for DE_METHOD MANORM or DIFFBIND ["Y", "N"]
-condenseexon: "Y" #if spliceaware Y, if there are multiple peaks in the same transcript, whether to combine into one feature ["Y", "N"]
+AnnoAnchor: "max_total" #Annotations for spliced peaks will be based on either 5' most region or region with max reads ["max","5prime"]
 mincount: 3 #minimum number of reads to count as a peak [1,2,3]
 ntmerge: 10 #minimum distance of nucleotides to merge peaks [any integer >=1, default 10]
 peakid: "ALL" #report peaks for unique peaks only or unique and fractional mm ["unique","all"]
@@ -32,7 +37,37 @@ pval: 0.005 #if DEmethod, pval cutoff for significance
 fc: 1 #if DEmethod, fold change cut off for significance
 splitSampleNChunks: 96 #split samples into N chunks to speed up compute heavy tasks like alignment [1 - 99]
 
+#########################################################################################
+# novoalign parameters "test5" --> new iCLIP pipeline defaults
+#########################################################################################
+novoalign_x: 6
+novoalign_g: 40
+novoalign_s: 2
+novoalign_t: "20,3"
+novoalign_R: 5
+novoalign_r_mode: "EXHAUSTIVE"
+novoalign_r_int: 999 # integer represents number of alignments to be reported out
+
+#########################################################################################
+# novoalign parameters old iCLIP pipeline defaults
+#########################################################################################
+#novoalign_x: 4
+#novoalign_g: 20
+#novoalign_s: 1
+#novoalign_t: "15,3"
+#novoalign_R: 0
+#novoalign_r_mode: "EXHAUSTIVE"
+#novoalign_r_int: 999 # integer represents number of alignments to be reported out
+
+#########################################################################################
+# Useq parameters
+#########################################################################################
+useq_a: 50000
+useq_n: 999 # trying to match this to novoalign_r_exhaustive_int .... was set to 25 earlier
+
+#########################################################################################
 #modules, container parameters
+#########################################################################################
 containerDir: "/data/CCBR_Pipeliner/iCLIP/container"
 fastq_val: "/data/CCBR_Pipeliner/iCLIP/bin/fastQValidator"
 bedtools: "bedtools/2.29.2"
@@ -44,7 +79,7 @@ manorm: "manorm/1.1.4"
 multiqc: "multiqc/1.9"
 novocraft: "novocraft/4.03.01"
 perl: "perl/5.24.3"
-python: "python/3.7"
+python: "python/3.8"
 Qt: "Qt/5.13.2"
 singularity: "singularity"
 samtools: "samtools/1.11"

manifests/contrasts_example_diffbind.tsv

@@ -0,0 +1,2 @@
+group,background
+Ro_Clip,Control_Clip

manifests/contrasts_example.tsv manifests/contrasts_example_manorm.tsvmanifests/contrasts_example.tsv renamed to manifests/contrasts_example_manorm.tsv

@@ -1,2 +1,2 @@
-file_name	multiplex
-test_1.fastq.gz	test_1
+file_name,multiplex
+test_1.fastq.gz,test_1

manifests/multiplex_example.tsv

@@ -1,3 +1,3 @@
-multiplex	sample	group	barcode	adaptor
-test_1	Ro_Clip	CLIP	NNNTGGCNN	AGATCGGAAGAGCGGTTCAG
-test_1	Control_Clip	CNTRL	NNNCGGANN	AGATCGGAAGAGCGGTTCAG
+multiplex,sample,group,barcode,adaptor
+test_1,Ro_Clip,CLIP,NNNTGGCNN,AGATCGGAAGAGCGGTTCAG
+test_1,Control_Clip,CNTRL,NNNCGGANN,AGATCGGAAGAGCGGTTCAG