Draft updated extract_extension macro and test tpdm_extension model a… (#149)

* Draft updated extract_extension macro and test tpdm_extension model and column

* Combine lines

* Add support for list of predefined extensions

* Add tpdm/epdm predefined credentials exts

* Fix if defined check

* Change flag name, condense if statement

* Add epdm:credentials predefined extension config block

* Remove enabled extensions list

* Explicitly disable epdm predefined extension

* Change variable name

* Complete epdm predefned extension config

* rename to tpdm for consistency

* Update changelog

---------

Co-authored-by: rlittle08 <rlittle@edanalytics.org>

Author: ryanaguilar

CHANGELOG.md

@@ -2,6 +2,8 @@
 - Updated `gen_skey.sql` macro. Changed the order of `k_survey_section` column list. Changed survey_section_title from snake case to camel case. 
 - Corrected `stg_ef3__survey_section_responses.sql` model `k_survey_section_response` variable. previously missing `survey_response_id` variable. 
 ## New features
+- Updated `extract_extension.sql` macro to allow for extensions to be defined in the edu_edfi_source `dbt_project.yml` and enabled per project.
+- Added the tpdm/epdm predefined extension to `dbt_project.yml'
 ## Under the hood
 ## Fixes
 
@@ -265,4 +267,4 @@
 ## Fixes
 
 # edu_edfi_source v0.1.0
-Initial release
+Initial release

dbt_project.yml

@@ -43,6 +43,45 @@ models:
 
 vars:
   extensions: null
+  predefined_extensions:
+    tpdm:
+      stg_ef3__credentials:
+        tpdm_certification_route_descriptor:
+          name: 'tpdm:certificationRouteDescriptor'
+          dtype: 'varchar'
+        tpdm_credential_status_descriptor:
+          name: 'tpdm:credentialStatusDescriptor'
+          dtype: 'varchar'
+        tpdm_educator_role_descriptor:
+          name: 'tpdm:educatorRoleDescriptor'
+          dtype: 'varchar'
+        tpdm_board_certification_indicator:
+          name: 'tpdm:boardCertificationIndicator'
+          dtype: 'boolean'
+        tpdm_credential_certification_title:
+          name: 'tpdm:certificationTitle'
+          dtype: 'varchar'
+        tpdm_credential_status_date:
+          name: 'tpdm:credentialStatusDate'
+          dtype: 'timestamp'
+        tpdm_person_reference_person_id:
+          name: 'tpdm:personReference:PersonId'
+          dtype: 'varchar'
+        tpdm_v_student_academic_records:
+          name: 'tpdm:studentAcademicRecords'
+          dtype: 'variant'
+      stg_ef3__schools:
+        tpdm_post_secondary_institution_reference_post_secondary_institution_id:
+          name: 'tpdm:postSecondaryInstitutionReference:postSecondaryInstitutionId'
+          dtype: 'int'
+      stg_ef3__survey_responses:
+        tpdm_person_reference_person_id:
+          name: 'tpdm:personReference:PersonId'
+          dtype: 'varchar'
+
 
   # student discipline participation codes that map to non-offenders
   'edu:discipline:non_offender_codes': ['Victim', 'Witness', 'Reporter']
+
+  # epdm extension default disabled
+  'src:predefined_extensions:tpdm:enabled': False

macros/extract_extension.sql

@@ -12,19 +12,39 @@
     {{ edu_edfi_source.extract_extension_list(model_name) }}
 {% else %}
   {# if `model_name` IS a singleton, use var of its single model name #}
-  {%- set extensions =  var('extensions')[model_name]  -%}
 
-  {%- if extensions is defined and extensions|length > 0 -%},{% endif -%}
+  {%- set extensions = var('extensions', {}).get(model_name, {}) -%}
+  {%- set predefined_extensions = {} -%}
+  {%- for k,v in var('predefined_extensions', {}).items() -%}
+    {# If predefined extension flag is enabled, add it to extensions list. #}
+    {%- if var('src:predefined_extensions:' ~ k ~ ':enabled') -%}
+      {%- set _ = predefined_extensions.update(v.get(model_name, {})) -%}
+    {%- endif -%}
+  {%- endfor -%}
+
+  {# merge the two extension lists #}
+  {% set all_extensions = {} %}
+  {% for k, v in extensions.items() %}
+    {% set _ = all_extensions.update({k: v}) %}
+  {% endfor %}
+  {%- if predefined_extensions is defined and predefined_extensions|length > 0 -%}
+    {% for k, v in predefined_extensions.items() %}
+      {% set _ = all_extensions.update({k: v}) %}
+    {% endfor %}
+  {%- endif -%}
+  
+
+  {%- if all_extensions is defined and all_extensions|length > 0 -%},{% endif -%}
 
-  {%- for ext in extensions %}
+  {%- for ext in all_extensions %}
 
     {# If flatten (as is done in stg models), pull out metadata from dbt_project var and use to flatten json into columns #}
     {%- if flatten %}
 
-      {%- set ext_native_name =  extensions[ext].name  -%}
+      {%- set ext_native_name =  all_extensions[ext].name  -%}
       {%- set full_ext_native_name =  'v_ext:' + ext_native_name  -%}
-      {%- set ext_dtype =  extensions[ext].dtype  -%}
-      {%- set ext_extract_descriptor =  extensions[ext].extract_descriptor  -%}
+      {%- set ext_dtype =  all_extensions[ext].dtype  -%}
+      {%- set ext_extract_descriptor =  all_extensions[ext].extract_descriptor  -%}
 
       {%- if ext_extract_descriptor %}
         {{extract_descriptor(full_ext_native_name + '::string')}}::{{ext_dtype}} as {{ext}}