Create Base Models for Food Service and Migrant Education Programs (#144)

* added 2 program base models

* added additional columns from handbook

* updated begin and end date column name

* created staging models

* updated models

* updated yaml files with model names

* puled in CTE models and updated yml file to account for them.

* removed column from base model

* removed cte files before PR

* updated changelog with new base/stage models

* updated YAML files with enable/disable ability

* fixed issues w. models

* updated stage.yml file

* added cte to stage models

* brought in cte stage files for testing

* removed duplicate cte models from stage yml file

* added program associations enable/disable to changelog.

* updated alphabetical order of stage file

* fixed compilation issues

* updated _stage.yml file

* shorten changelog

---------

Co-authored-by: rlittle08 <rlittle@edanalytics.org>

Author: mberrien-fitzsimons

CHANGELOG.md

@@ -1,7 +1,10 @@
 # Unreleased
 
 ## New features
+- Add base/stage models `base_ef3__student_school_food_service_program_association`, `stg_ef3__student_school_food_service_program_association`, `stg_ef3__stu_school_food_service__program_services`
+- Add base/stage models `base_ef3__student_migrant_education_program_associations`, `stg_ef3__student_migrant_education_program_associations`, `stg_ef3__stu_migrant_edu__program_services`
 - Add `stg_ef3__students__other_names`
+
 ## Under the hood
 ## Fixes
 

models/staging/edfi_3/base/_edfi_3__base.yml

@@ -150,6 +150,10 @@ models:
     config:
       tags: ['language_instruction']
       enabled: "{{ var('src:program:language_instruction:enabled', True) }}"
+  - name: base_ef3__student_migrant_education_program_associations
+    config:
+      tags: ['migrant_education']
+      enabled: "{{ var('src:program:migrant_education:enabled', True) }}"
   - name: base_ef3__student_parent_associations
     config:
       tags: ['core']
@@ -162,6 +166,10 @@ models:
   - name: base_ef3__student_school_attendance_events
     config:
       tags: ['core']
+  - name: base_ef3__student_school_food_service_program_association
+    config:
+      tags: ['food_service']
+      enabled: "{{ var('src:program:food_service:enabled', True) }}"
   - name: base_ef3__student_section_associations
     config:
       tags: ['core']

models/staging/edfi_3/base/base_ef3__student_migrant_education_program_associations.sql

@@ -0,0 +1,54 @@
+with source_stu_programs as (
+    {{ source_edfi3('student_migrant_education_program_associations') }}
+),
+
+renamed as (
+    select
+        -- generic columns
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                                                                as record_guid, 
+        ods_version, 
+        data_model_version, 
+        v:studentReference:studentUniqueId::string                                                  as student_unique_id, 
+        v:educationOrganizationReference:educationOrganizationId::int                               as ed_org_id,
+        v:educationOrganizationReference:link:rel::string                                           as ed_org_type,
+        v:beginDate::date                                                                           as program_enroll_begin_date,
+        v:endDate::date                                                                             as program_enroll_end_date, 
+        v:programReference:programName::string                                                      as program_name, 
+
+        v:priorityForServices::boolean                                                              as priority_for_service, 
+
+        v:lastQualifyingMove::date                                                                  as last_qualifying_move, 
+        v:usInitialEntry::date                                                                      as us_initial_entry,
+        v:usMostRecentEntry::date                                                                   as us_most_recent_entry,
+        v:USInitialSchoolEntry::date                                                                as us_initial_school_entry,
+        v:qualifyingArrivalDate::date                                                               as qualifying_arrival_date,
+        v:stateResidencyDate::date                                                                  as state_residency_date,
+        v:eligibilityExpirationDate::date                                                           as eligibility_expiration_date,
+        -- descriptors
+        {{ extract_descriptor('v:programReference:programTypeDescriptor') }}                        as program_type,
+        {{ extract_descriptor('v:continuationOfServicesReasonDescriptor') }}                        as continuation_of_services_reason,
+
+        -- references
+        v:educationOrganizationReference                                                            as education_organization_reference,
+        v:programReference                                                                          as program_reference, 
+        v:studentReference                                                                          as student_reference,
+
+        -- lists
+        v:migrantEducationProgramServices                                                           as v_migrant_education_program_services,
+        v:programParticipationStatuses                                                              as v_program_participation_statuses,
+
+        -- edfi extensions
+        v:_ext                                                                                      as v_ext
+    from source_stu_programs
+)
+
+select * from renamed
+

models/staging/edfi_3/base/base_ef3__student_school_food_service_program_association.sql

@@ -0,0 +1,49 @@
+with source_stu_program as (
+    {{ source_edfi3('student_school_food_service_program_associations') }}
+), 
+
+renamed as (
+    select 
+        -- generic columns
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                                                as record_guid, 
+        ods_version::string,
+        data_model_version,
+        v:studentReference:studentUniqueId::string                                  as student_unique_id,
+        v:educationOrganizationReference:educationOrganizationId::int               as ed_org_id,
+        v:educationOrganizationReference:link:rel::string                           as ed_org_type,
+        v:programReference:educationOrganizationId::int                             as program_ed_org_id, 
+        v:beginDate::date                                                           as program_enroll_begin_date,
+        v:endDate::date                                                             as program_enroll_end_date,
+        v:programReference:programName::string                                      as program_name,
+
+        v:directCertification::boolean                                              as direct_certification,
+        v:servedOutsideOfRegularSession::boolean                                    as served_outside_of_regular_session,
+
+        -- descriptors
+        {{ extract_descriptor('v:programReference:programTypeDescriptor') }}        as program_type,
+        {{ extract_descriptor('v:reasonExitedDescriptor') }}                        as reason_exited,
+
+        -- references
+        v:educationOrganizationReference                                            as education_organization_reference,
+        v:programReference                                                          as program_reference,
+        v:studentReference                                                          as student_reference,
+
+        -- lists
+        v:programParticipationStatuses                                              as v_program_participation_statuses,
+        v:schoolFoodServiceProgramServices                                          as v_school_food_service_program_services,
+
+        -- edfi extensions
+        v:_ext                                                                      as v_ext
+
+    from source_stu_program
+)
+
+select * from renamed

models/staging/edfi_3/stage/_edfi_3__stage.yml

@@ -536,7 +536,7 @@ models:
   - name: stg_ef3__staffs__identification_codes
     config:
       tags: ['core']
-  
+
   - name: stg_ef3__stu_cte__program_services
     config:
       tags: ['cte']
@@ -593,6 +593,16 @@ models:
       tags: ['language_instruction']
       enabled: "{{ var('src:program:language_instruction:enabled', True) }}"
 
+  - name: stg_ef3__stu_migrant_edu__program_services
+    config: 
+      tags: ['migrant_education']
+      enabled: "{{ var('src:program:migrant_education:enabled', True) }}"
+
+  - name: stg_ef3__stu_school_food_service__program_services
+    config: 
+      tags: ['food_service']
+      enabled: "{{ var('src:program:food_service:enabled', True) }}"
+
   - name: stg_ef3__stu_spec_ed__disabilities
     config:
       tags: ['special_ed']
@@ -669,7 +679,7 @@ models:
       - name: k_contact
         tests:
           *ref_k_contact
-  
+
   - name: stg_ef3__student_cte_program_associations
     config:
       tags: ['cte']
@@ -741,6 +751,11 @@ models:
         tests:
           *ref_k_program
 
+  - name: stg_ef3__student_migrant_education_program_associations
+    config: 
+      tags: ['migrant_education']
+      enabled: "{{ var('src:program:migrant_education:enabled', True) }}"
+
   - name: stg_ef3__student_objective_assessments
     config:
       tags: ['assessment']
@@ -807,6 +822,11 @@ models:
         tests:
           *ref_k_session
 
+  - name: stg_ef3__student_school_food_service_program_association
+    config: 
+      tags: ['food_service']
+      enabled: "{{ var('src:program:food_service:enabled', True) }}"
+
   - name: stg_ef3__student_section_associations
     config:
       tags: ['core']

models/staging/edfi_3/stage/stg_ef3__stu_migrant_edu__program_services.sql

@@ -0,0 +1,29 @@
+with stage_stu_programs as (
+    select * from {{ ref('stg_ef3__student_migrant_education_program_associations') }}
+),
+
+flattened as (
+    select
+        tenant_code,
+        api_year,
+        k_student,
+        k_student_xyear,
+        k_program,
+        k_lea,
+        k_school,
+
+        program_enroll_begin_date,
+        program_enroll_end_date,
+        {{ extract_descriptor('value:migrantEducationProgramServiceDescriptor::string') }} as program_service,
+        value:primaryIndicator::boolean as primary_indicator,
+        value:serviceBeginDate::date    as service_begin_date,
+        value:serviceEndDate::date      as service_end_date,
+
+        -- edfi extensions
+        value:_ext as v_ext
+
+    from stage_stu_programs
+        {{ json_flatten('v_migrant_education_program_services') }}
+)
+
+select * from flattened

models/staging/edfi_3/stage/stg_ef3__stu_school_food_service__program_services.sql

@@ -0,0 +1,29 @@
+with stage_stu_programs as (
+    select * from {{ ref('stg_ef3__student_school_food_service_program_association') }}
+),
+
+flattened as (
+    select
+        tenant_code,
+        api_year,
+        k_student,
+        k_student_xyear,
+        k_program,
+        k_lea,
+        k_school,
+
+        program_enroll_begin_date,
+        program_enroll_end_date,
+        {{ extract_descriptor('value:schoolFoodServiceProgramServiceDescriptor::string') }} as program_service,
+        value:primaryIndicator::boolean as primary_indicator,
+        value:serviceBeginDate::date    as service_begin_date,
+        value:serviceEndDate::date      as service_end_date,
+
+        -- edfi extensions
+        value:_ext as v_ext
+
+    from stage_stu_programs
+        {{ json_flatten('v_school_food_service_program_services') }}
+)
+
+select * from flattened

models/staging/edfi_3/stage/stg_ef3__student_migrant_education_program_associations.sql

@@ -0,0 +1,27 @@
+with base_stu_programs as (
+    select * from {{ ref('base_ef3__student_migrant_education_program_associations') }}
+),
+
+keyed as (
+    select 
+        {{ gen_skey('k_student') }},
+        {{ gen_skey('k_student_xyear') }},
+        {{ gen_skey('k_program') }},
+        {{ edorg_ref(annualize=False) }},
+        api_year as school_year,
+        base_stu_programs.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+
+    from base_stu_programs
+),
+
+deduped as (
+    {{ dbt_utils.deduplicate(
+        relation='keyed',
+        partition_by='k_student, k_program, program_enroll_begin_date, school_year',
+        order_by='last_modified_timestamp desc, pull_timestamp desc'
+    ) }}
+)
+
+select * from deduped
+where not is_deleted

models/staging/edfi_3/stage/stg_ef3__student_school_food_service_program_association.sql

@@ -0,0 +1,27 @@
+with base_stu_programs as (
+    select * from {{ ref('base_ef3__student_school_food_service_program_association') }}
+),
+
+keyed as (
+    select 
+        {{ gen_skey('k_student') }},
+        {{ gen_skey('k_student_xyear') }},
+        {{ gen_skey('k_program') }},
+        {{ edorg_ref(annualize=False) }},
+        api_year as school_year,
+        base_stu_programs.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+
+    from base_stu_programs
+),
+
+deduped as (
+    {{ dbt_utils.deduplicate(
+        relation='keyed',
+        partition_by='k_student, k_program, program_enroll_begin_date, school_year',
+        order_by='last_modified_timestamp desc, pull_timestamp desc'
+    ) }}
+)
+
+select * from deduped
+where not is_deleted