feature/model_creation_stu_cte_prog_assoc (#140)

* pulled in student cte program associations base and stage models

* made requested updates from pull request

* added base and stage models to yaml file

* updated base/stage yaml files with additional information

* updated with remaining columns from v5.2 handbook

* created model that flattens program services

* removed var from new models

* updated all cte models

* updated yaml files to enable variables

* updated change log with creation of base, stage and secondary stage model for StudentCTEProgramAssociation

Author: mberrien-fitzsimons

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Unreleased
+
 ## New features
+- Add base/stage model for `StudentCTEProgramAssociation`
+- Add stage model `stg_ef3__stu_cte__program_services` to flatten program services list for `StudentCTEProgramAssociation`
+
 ## Under the hood
 ## Fixes
 - Add handling of multiple academic subjects per course for DS 5.0 compatibility; new `stg_ef3__courses_academic_subjects` model and new `v_academic_subjects` columns in `base_ef3__courses` and `stg_ef3__courses` models.

models/staging/edfi_3/base/_edfi_3__base.yml

@@ -123,6 +123,10 @@ models:
   - name: base_ef3__student_contact_associations
     config:
       tags: ['core']
+  - name: base_ef3__student_cte_program_associations
+    config:
+      tags: ['cte']
+      enabled: "{{ var('src:program:cte:enabled', True) }}"
   - name: base_ef3__student_discipline_incident_associations
     config:
       tags: ['discipline']

models/staging/edfi_3/base/base_ef3__student_cte_program_associations.sql

@@ -0,0 +1,53 @@
+with source_stu_programs as (
+    {{ source_edfi3('student_cte_program_associations') }}
+),
+
+renamed as (
+    select 
+        -- generic columns
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                                                    as record_guid, 
+        ods_version, 
+        data_model_version,
+        v:studentReference:studentUniqueId::int                                         as student_unique_id,
+        v:educationOrganizationReference:educationOrganizationId::int                   as ed_org_id,
+        v:educationOrganizationReference:link:rel::string                               as ed_org_type,
+        v:programReference:educationOrganizationId::int                                 as program_ed_org_id,
+        v:beginDate::date                                                               as program_enroll_begin_date, 
+        v:endDate::date                                                                 as program_enroll_end_date, 
+        v:programReference:programName::string                                          as program_name,
+
+        v:nonTraditionalGenderStatus::boolean                                           as non_traditional_gender_status,
+        v:privateCTEProgram::boolean                                                    as private_cte_program,
+        v:ServedOutsideOfRegularSession::boolean                                        as served_outside_of_regular_session,
+
+        -- descriptors
+        {{ extract_descriptor('v:technicalSkillsAssessmentDescriptor') }}               as technical_skills_assessment,
+        {{ extract_descriptor('v:programReference:programTypeDescriptor') }}            as program_type,
+        {{ extract_descriptor('v:ReasonExitedDescriptor:ReasonExitedDescriptorId') }}   as reason_exited,
+
+        -- references
+        v:educationOrganizationReference                                                as education_organization_reference,
+        v:programReference                                                              as program_reference, 
+        v:studentReference                                                              as student_reference,
+
+        -- lists
+        v:cteProgramServices                                                            as v_cte_program_services, 
+        v:ctePrograms                                                                   as v_cte_programs, 
+        v:programParticipationStatuses                                                  as v_program_participation_statuses, 
+        v:services                                                                      as v_services,
+
+        -- edfi extensions
+        v:_ext as v_ext
+        
+    from source_stu_programs
+)
+
+select * from renamed

models/staging/edfi_3/stage/_edfi_3__stage.yml

@@ -536,6 +536,11 @@ models:
   - name: stg_ef3__staffs__identification_codes
     config:
       tags: ['core']
+  
+  - name: stg_ef3__stu_cte__program_services
+    config:
+      tags: ['cte']
+      enabled: "{{ var('src:program:cte:enabled', True) }}"
 
   - name: stg_ef3__stu_ed_org__addresses
     config:
@@ -664,6 +669,11 @@ models:
       - name: k_contact
         tests:
           *ref_k_contact
+  
+  - name: stg_ef3__student_cte_program_associations
+    config:
+      tags: ['cte']
+      enabled: "{{ var('src:program:cte:enabled', True) }}"
 
   - name: stg_ef3__student_discipline_incident_behavior_associations
     config:

models/staging/edfi_3/stage/stg_ef3__stu_cte__program_services.sql

@@ -0,0 +1,30 @@
+with stage_stu_programs as (
+    select * from {{ ref('stg_ef3__student_cte_program_associations') }}
+),
+
+flattened as (
+    select 
+        tenant_code,
+        api_year,
+        k_student,
+        k_student_xyear,
+        k_program,
+        k_lea,
+        k_school,
+
+        program_enroll_begin_date,
+        program_enroll_end_date,
+        {{ extract_descriptor('value:cteProgramServiceDescriptor::string') }} as program_service,
+        value:primaryIndicator::boolean         as primary_indicator, 
+        value:serviceBeginDate::date            as service_begin_date,
+        value:serviceEndDate::date              as service_end_date,
+        value:cipCode::string                   as cip_code,
+
+        -- edfi extensions
+        value:_ext as v_ext
+
+    from stage_stu_programs,
+        lateral flatten(input => parse_json(v_cte_program_services))
+)
+
+select * from flattened

models/staging/edfi_3/stage/stg_ef3__student_cte_program_associations.sql

@@ -0,0 +1,27 @@
+with base_stu_programs as (
+    select * from {{ ref('base_ef3__student_cte_program_associations') }}
+),
+
+keyed as (
+    select 
+        {{ gen_skey('k_student') }},
+        {{ gen_skey('k_student_xyear') }},
+        {{ gen_skey('k_program') }},
+        {{ edorg_ref(annualize=False) }},
+        api_year as school_year,
+        base_stu_programs.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+
+    from base_stu_programs
+),
+
+deduped as (
+    {{ dbt_utils.deduplicate(
+        relation='keyed',
+        partition_by='k_student, k_program, program_enroll_begin_date, school_year',
+        order_by='last_modified_timestamp desc, pull_timestamp desc'
+    ) }}
+)
+
+select * from deduped
+where not is_deleted