🚧 Kids First: Chromosome 18 - Proof of Concept

dbt_project/dbt_project.yml

@@ -37,6 +37,11 @@ models:
           +schema: inc_brainpower_src
         int:
           +schema: inc_brainpower_int
+      chr_18:
+        src:
+          +schema: inc_chr_18_src
+        int:
+          +schema: inc_chr_18_int
     kids_first:
       +tags:
       - study_stage
@@ -49,6 +54,9 @@ models:
           +schema: kf_sd_1nns3k8v_src
         int:
           +schema: kf_sd_1nns3k8v_int
+      chr_18:
+        +tags:
+        - chr_18
     access:
       +tags:
       - access_stage

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_clinical.sql

@@ -0,0 +1,118 @@
+{{ config(materialized='table') }}
+
+with base_data as (
+    select * from {{ ref('kf_chr_18_src_chr_18_phenotype_data') }}
+),
+phenotype_list as (
+    select phenotype from (
+        values 
+            ('chiari_2'), ('pituitary_gland_abn'), ('cleft_palate'), ('hearing_loss'), 
+            ('tetralogy_of_fallot'), ('conductive_hearing_loss'), ('holoprosencephaly'), ('choanal_atresia'),
+            ('aural_atresia'), ('midface_hypoplasia'), ('dysmyelination'), ('polymicrogyria'),
+            ('cryptorchidism'), ('sensorineural_hearing_loss'), ('cleft_lip'), ('imperforate_anus'),
+            ('microcephaly'), ('vertical_talus'), ('hypospadias'), ('rocker_bottom_foot'),
+            ('chiari_1'), ('ptosis'), ('strabismus'), ('bilateral_coxa_valga'),
+            ('cardiac_nos'), ('corpus_callosum_abn'), ('pulmonary_valve_stenosis'), ('metatarsus_varus'),
+            ('decreased_white_matter_volume'), ('macrocephaly'), ('suprasellar_cyst'), ('hydrocephalus'),
+            ('micropenis'), ('anal_stenosis'), ('congenital_heart_malformation'), ('bilateral_choanal_atresia'),
+            ('dandy_walker_malformation'), ('dandy_walker'), ('congenital_heart_defect'), ('aural_stenosis'),
+            ('kidney_malform'), ('pituitary_fossa_enlargement'), ('submucous_cleft'), ('virchow_robins_spaces'),
+            ('retinal_coloboma'), ('corneal_staphyloma'), ('club_foot'), ('macroglossia'),
+            ('dandy_walker_variant'), ('coanal_atresia'), ('occipatal_lobe_abn'), ('asd'),
+            ('vsd'), ('pectus_excavatum'), ('peripherial_pulmonary_stenosis'), ('sagittal_stenosis'),
+            ('hydronephrosis'), ('patent_ductus_arteriosus'), ('duanes_syndrome'), ('laryngomalacia'),
+            ('metatarsus_adductus'), ('ptosisbilateral'), ('microphallus'), ('sacral_agenesis'),
+            ('situs_inversus')
+    ) as t(phenotype)
+),
+unpivoted_phenotypes as (
+    select
+        bd.family_id,
+        bd.participant_id,
+        bd.family_relationship_target_participant_id,
+        bd.aliquot_id,
+        bd.consent_group,
+        bd.age_at_sample,
+        bd.sample_composition,
+        bd.race,
+        bd.sex,
+        bd.ethnicity,
+        bd.analyte_type,
+        bd.age_at_phenotype,
+        bd.affected_status,
+        bd.vital_status,
+        bd.sample_tissue_type,
+        pl.phenotype
+    from base_data bd
+    cross join phenotype_list pl
+    where case pl.phenotype
+        when 'chiari_2' then bd.chiari_2
+        when 'pituitary_gland_abn' then bd.pituitary_gland_abn
+        when 'cleft_palate' then bd.cleft_palate
+        when 'hearing_loss' then bd.hearing_loss
+        when 'tetralogy_of_fallot' then bd.tetralogy_of_fallot
+        when 'conductive_hearing_loss' then bd.conductive_hearing_loss
+        when 'holoprosencephaly' then bd.holoprosencephaly
+        when 'choanal_atresia' then bd.choanal_atresia
+        when 'aural_atresia' then bd.aural_atresia
+        when 'midface_hypoplasia' then bd.midface_hypoplasia
+        when 'dysmyelination' then bd.dysmyelination
+        when 'polymicrogyria' then bd.polymicrogyria
+        when 'cryptorchidism' then bd.cryptorchidism
+        when 'sensorineural_hearing_loss' then bd.sensorineural_hearing_loss
+        when 'cleft_lip' then bd.cleft_lip
+        when 'imperforate_anus' then bd.imperforate_anus
+        when 'microcephaly' then bd.microcephaly
+        when 'vertical_talus' then bd.vertical_talus
+        when 'hypospadias' then bd.hypospadias
+        when 'rocker_bottom_foot' then bd.rocker_bottom_foot
+        when 'chiari_1' then bd.chiari_1
+        when 'ptosis' then bd.ptosis
+        when 'strabismus' then bd.strabismus
+        when 'bilateral_coxa_valga' then bd.bilateral_coxa_valga
+        when 'cardiac_nos' then bd.cardiac_nos
+        when 'corpus_callosum_abn' then bd.corpus_callosum_abn
+        when 'pulmonary_valve_stenosis' then bd.pulmonary_valve_stenosis
+        when 'metatarsus_varus' then bd.metatarsus_varus
+        when 'decreased_white_matter_volume' then bd.decreased_white_matter_volume
+        when 'macrocephaly' then bd.macrocephaly
+        when 'suprasellar_cyst' then bd.suprasellar_cyst
+        when 'hydrocephalus' then bd.hydrocephalus
+        when 'micropenis' then bd.micropenis
+        when 'anal_stenosis' then bd.anal_stenosis
+        when 'congenital_heart_malformation' then bd.congenital_heart_malformation
+        when 'bilateral_choanal_atresia' then bd.bilateral_choanal_atresia
+        when 'dandy_walker_malformation' then bd.dandy_walker_malformation
+        when 'dandy_walker' then bd.dandy_walker
+        when 'congenital_heart_defect' then bd.congenital_heart_defect
+        when 'aural_stenosis' then bd.aural_stenosis
+        when 'kidney_malform' then bd.kidney_malform
+        when 'pituitary_fossa_enlargement' then bd.pituitary_fossa_enlargement
+        when 'submucous_cleft' then bd.submucous_cleft
+        when 'virchow_robins_spaces' then bd.virchow_robins_spaces
+        when 'retinal_coloboma' then bd.retinal_coloboma
+        when 'corneal_staphyloma' then bd.corneal_staphyloma
+        when 'club_foot' then bd.club_foot
+        when 'macroglossia' then bd.macroglossia
+        when 'dandy_walker_variant' then bd.dandy_walker_variant
+        when 'coanal_atresia' then bd.coanal_atresia
+        when 'occipatal_lobe_abn' then bd.occipatal_lobe_abn
+        when 'asd' then bd.asd
+        when 'vsd' then bd.vsd
+        when 'pectus_excavatum' then bd.pectus_excavatum
+        when 'peripherial_pulmonary_stenosis' then bd.peripherial_pulmonary_stenosis
+        when 'sagittal_stenosis' then bd.sagittal_stenosis
+        when 'hydronephrosis' then bd.hydronephrosis
+        when 'patent_ductus_arteriosus' then bd.patent_ductus_arteriosus
+        when 'duanes_syndrome' then bd.duanes_syndrome
+        when 'laryngomalacia' then bd.laryngomalacia
+        when 'metatarsus_adductus' then bd.metatarsus_adductus
+        when 'ptosisbilateral' then bd.ptosisbilateral
+        when 'microphallus' then bd.microphallus
+        when 'sacral_agenesis' then bd.sacral_agenesis
+        when 'situs_inversus' then bd.situs_inversus
+    end = 'present'
+)
+
+select * from unpivoted_phenotypes
+

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_s3.sql

@@ -0,0 +1,12 @@
+{{ config(materialized='table') }}
+
+    select
+    kf_index,
+    size,
+    key,
+    etag,
+    bucket,
+    lastmodified,
+    storageclass
+    from {{ ref('kf_chr_18_src_s3_scrape_cody') }}
+

dbt_project/models/kids_first/chr_18/int/kf_chr_18_int_sequencing.sql

@@ -0,0 +1,19 @@
+{{ config(materialized='table') }}
+
+    select
+    kf_index,
+    type,
+    sample,
+    file_name,
+    file_type,
+    file_description,
+    file_size,
+    md5sum,
+    library_id,
+    library_strategy,
+    library_layout,
+    library_source,
+    platform,
+    instrument_model
+    from {{ source('chr_18', 'ha_delivery_manifest_6735') }}
+

dbt_project/models/kids_first/chr_18/src/Clinical_Data/kf_chr_18_src_chr_18_phenotype_data.sql

@@ -0,0 +1,86 @@
+{{ config(materialized='table') }}
+
+    select
+    ROW_NUMBER() OVER () AS kf_index,
+    "Family ID"::text as family_id,
+    "Participant ID"::text as participant_id,
+    "Family Relationship Target Participant ID"::text as family_relationship_target_participant_id,
+    "alliquot_id"::text as aliquot_id,
+    "Consent Group"::text as consent_group,
+    "Age at Sample"::text as age_at_sample,
+    "Sample Composition"::text as sample_composition,
+    "Race"::text as race,
+    "Sex"::text as sex,
+    "Ethnicity"::text as ethnicity,
+    "Analyte Type"::text as analyte_type,
+    "Age at Phenotype"::text as age_at_phenotype,
+    "Affected Status"::text as affected_status,
+    "Vital Status"::text as vital_status,
+    "chiari 2"::text as chiari_2,
+    "pituitary gland abn"::text as pituitary_gland_abn,
+    "cleft palate"::text as cleft_palate,
+    "hearing loss"::text as hearing_loss,
+    "tetralogy of fallot"::text as tetralogy_of_fallot,
+    "conductive hearing loss"::text as conductive_hearing_loss,
+    "holoprosencephaly"::text as holoprosencephaly,
+    "choanal atresia"::text as choanal_atresia,
+    "aural atresia"::text as aural_atresia,
+    "midface hypoplasia"::text as midface_hypoplasia,
+    "dysmyelination"::text as dysmyelination,
+    "polymicrogyria"::text as polymicrogyria,
+    "cryptorchidism"::text as cryptorchidism,
+    "sensorineural hearing loss"::text as sensorineural_hearing_loss,
+    "cleft lip"::text as cleft_lip,
+    "imperforate anus"::text as imperforate_anus,
+    "microcephaly"::text as microcephaly,
+    "vertical talus"::text as vertical_talus,
+    "hypospadias"::text as hypospadias,
+    "rocker bottom foot"::text as rocker_bottom_foot,
+    "chiari 1"::text as chiari_1,
+    "ptosis"::text as ptosis,
+    "strabismus"::text as strabismus,
+    "bilateral coxa valga"::text as bilateral_coxa_valga,
+    "cardiac_nos"::text as cardiac_nos,
+    "corpus callosum abn"::text as corpus_callosum_abn,
+    "pulmonary valve stenosis"::text as pulmonary_valve_stenosis,
+    "metatarsus varus"::text as metatarsus_varus,
+    "decreased white matter volume"::text as decreased_white_matter_volume,
+    "macrocephaly"::text as macrocephaly,
+    "suprasellar cyst"::text as suprasellar_cyst,
+    "hydrocephalus"::text as hydrocephalus,
+    "micropenis"::text as micropenis,
+    "anal stenosis"::text as anal_stenosis,
+    "congenital heart malformation"::text as congenital_heart_malformation,
+    "bilateral choanal atresia"::text as bilateral_choanal_atresia,
+    "dandy walker malformation"::text as dandy_walker_malformation,
+    "dandy walker"::text as dandy_walker,
+    "congenital heart defect"::text as congenital_heart_defect,
+    "aural stenosis"::text as aural_stenosis,
+    "kidney malform"::text as kidney_malform,
+    "pituitary fossa enlargement"::text as pituitary_fossa_enlargement,
+    "submucous cleft"::text as submucous_cleft,
+    "virchow robins spaces"::text as virchow_robins_spaces,
+    "retinal coloboma"::text as retinal_coloboma,
+    "corneal staphyloma"::text as corneal_staphyloma,
+    "club foot"::text as club_foot,
+    "macroglossia"::text as macroglossia,
+    "dandy walker variant"::text as dandy_walker_variant,
+    "coanal atresia"::text as coanal_atresia,
+    "occipatal lobe abn"::text as occipatal_lobe_abn,
+    "asd"::text as asd,
+    "vsd"::text as vsd,
+    "pectus excavatum"::text as pectus_excavatum,
+    "peripherial pulmonary stenosis"::text as peripherial_pulmonary_stenosis,
+    "sagittal stenosis"::text as sagittal_stenosis,
+    "hydronephrosis"::text as hydronephrosis,
+    "patent ductus arteriosus"::text as patent_ductus_arteriosus,
+    "duanes syndrome"::text as duanes_syndrome,
+    "laryngomalacia"::text as laryngomalacia,
+    "metatarsus adductus"::text as metatarsus_adductus,
+    "ptosisbilateral"::text as ptosisbilateral,
+    "microphallus"::text as microphallus,
+    "sacral agenesis"::text as sacral_agenesis,
+    "situs inversus"::text as situs_inversus,
+    "Sample Tissue Type"::text as sample_tissue_type
+    from {{ source('chr_18', 'chr_18_phenotype_data') }}
+

dbt_project/models/kids_first/chr_18/src/S3_Bucket_Scrape/kf_chr_18_src_s3_scrape_cody.sql

@@ -0,0 +1,12 @@
+{{ config(materialized='table') }}
+
+    select
+    ROW_NUMBER() OVER () AS kf_index,
+    "Size"::integer as size,
+    "Key"::text as key,
+    "ETag"::text as etag,
+    "Bucket"::text as bucket,
+    "LastModified"::text as lastmodified,
+    "StorageClass"::text as storageclass
+    from {{ source('chr_18', 's3_scrape_cody') }}
+

dbt_project/models/kids_first/chr_18/src/Sequencing_Data/kf_chr_18_src_ha_delivery_manifest_6735.sql

@@ -0,0 +1,19 @@
+{{ config(materialized='table') }}
+
+    select
+    ROW_NUMBER() OVER () AS kf_index,
+    "type"::text as type,
+    "sample.sample_id"::text as sample,
+    "file_name"::text as file_name,
+    "file_type"::text as file_type,
+    "file_description"::text as file_description,
+    "file_size"::integer as file_size,
+    "md5sum"::text as md5sum,
+    "library_id"::text as library_id,
+    "library_strategy"::text as library_strategy,
+    "library_layout"::text as library_layout,
+    "library_source"::text as library_source,
+    "platform"::text as platform,
+    "instrument_model"::text as instrument_model
+    from {{ source('chr_18', 'ha_delivery_manifest_6735') }}
+