Skip to content

Commit d5bfb36

Browse files
authored
Dcv 3288 dbt coves setup improvements (#7)
* DCV-3288 dbt-coves setup improvements * Remove miscommitted workflows * Changes after review with Noel * Add setup repo * Attempt Fix CI * Add services to ci * New data file + ci changes * New data file + ci changes * Remove answers, move ci_provider to data file * DCV-3288 complete data.yml file * New data entry * New data entry * Add trust flag * DCV-3288 finish setup changes * New data entries
1 parent 93808df commit d5bfb36

34 files changed

Lines changed: 143 additions & 53 deletions

File tree

.github/workflows/main.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ jobs:
4040
run: pip install -r ci_requirements.txt
4141

4242
- name: Run Copier
43-
run: |
44-
copier --defaults -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True -d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True -d airflow_profile_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions" -d dags_path="orchestrate/dags" -d setup_precommit=True -d use_sqlfluff=True -d use_yamllint=True -d use_dbt_checkpoint=True -a ../copier-answers.yml copy ./ generated/
43+
run: copier copy --data-file data.yml ./ generated/ --trust
4544

4645
# run dbt debug and dbt compile
4746
- name: Run dbt commands

ci_requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
dbt-coves==1.8.15
1+
git+https://github.com/datacoves/dbt-coves.git@support-pydantic-2.0
22
dbt-core==1.8.9
33
dbt-snowflake==1.8.4
44
pre-commit==3.7.1

copier-answers.yml

Lines changed: 0 additions & 9 deletions
This file was deleted.

copier.yml

Lines changed: 105 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
1+
_tasks:
2+
- echo 'Hello template'
3+
# - command: ["git init"]
4+
# only run on project setup
5+
# when: "{{ _copier_operation == 'copy' }}"
6+
7+
_message_before_copy: |
8+
Thanks for generating a project using our template.
9+
10+
_message_after_copy: |
11+
Your project "{{ dbt_project_name }}" has been created successfully!
12+
13+
_message_after_update: |
14+
Your project "{{ dbt_project_name }}" has been updated successfully!
15+
In case there
16+
117
_exclude:
2-
- copier-answers.yml
318
- ci_requirements.txt
419
- test_load_dagbag.py
520
- copier.yaml
@@ -9,64 +24,119 @@ _exclude:
924
- "*.py[co]"
1025
- __pycache__
1126
- .DS_Store
27+
- data.yml
28+
- answers.yml
1229

13-
ci_provider:
30+
services:
1431
type: str
32+
multiselect: true
33+
help: What services do you want to setup?
1534
choices:
16-
- github
17-
- gitlab
18-
when: "{{setup_ci_cd}}"
19-
default: github
20-
help: CI/CD provider
35+
Base dbt project: setup_dbt_project
36+
dbt profile for automated runs: setup_dbt_profile
37+
setup_ci_cd: setup_ci_cd
38+
Linting with SQLFluff, dbt-checkpoint and-or YMLLint: setup_precommit
39+
Sample Airflow DAGs: setup_airflow_dags
40+
dbt-coves config and-or templates: setup_dbtcoves
41+
42+
# dbt project related vars
43+
dbt_project_dir:
44+
type: str
45+
when: "{{'setup_dbt_project' in services}}"
46+
default: "{{ datacoves_env.get('DATACOVES__DBT_HOME', '.') }}"
47+
help: dbt project directory
48+
49+
dbt_project_name:
50+
type: str
51+
when: "{{'setup_dbt_project' in services}}"
52+
help: What is the name of the dbt project?
2153

54+
# Airflow related vars
2255
dags_path:
2356
type: str
24-
when: "{{airflow_dags_confirm_path}}"
25-
default: "{{tentative_dags_path}}"
57+
when: "{{'setup_airflow_dags' in services or 'setup_ci_cd' in services}}"
58+
default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DAGS_PATH', 'orchestrate/dags') }}"
2659
help: Airflow DAGs path
2760

61+
airflow_artifacts_path:
62+
type: str
63+
when: "{{'setup_ci_cd' in services}}"
64+
default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DAGS_PATH', 'orchestrate/dags').split('/')[0] }}"
65+
help: Airflow Artifacts path for unit tests
66+
2867
yml_dags_path:
2968
type: str
30-
when: "{{yml_dags_confirm_path}}"
31-
default: "{{tentative_yml_dags_path}}"
69+
when: "{{'setup_airflow_dags' in services and yml_dags_confirm_path}}"
70+
default: "{{datacoves_env.get('DATACOVES__AIRFLOW_DAGS_YML_PATH', 'orchestrate/dags_yml_definitions')}}"
3271
help: Airflow YML Dags path
3372

73+
# dbt-coves related vars
74+
add_dbtcoves_config_and_templates:
75+
type: bool
76+
default: false
77+
when: "{{not 'setup_dbtcoves' in services and 'setup_dbt_project' in services}}"
78+
help: Add dbt-coves config and templates?
79+
80+
setup_dbtcoves_config:
81+
type: bool
82+
default: true
83+
when: "{{'setup_dbtcoves' in services}}"
84+
help: Setup dbt-coves config?
85+
86+
setup_dbtcoves_templates:
87+
type: bool
88+
default: true
89+
when: "{{'setup_dbtcoves' in services}}"
90+
help: Setup dbt-coves templates?
91+
92+
# CI / CD Related vars
93+
94+
ci_provider:
95+
type: str
96+
choices:
97+
- github
98+
- gitlab
99+
when: "{{'setup_ci_cd' in services}}"
100+
default: github
101+
help: CI/CD provider
102+
34103
use_dbt_checkpoint:
35104
type: bool
36105
default: false
37-
when: "{{setup_precommit}}"
106+
when: "{{'setup_precommit' in services}}"
38107
help: use dbt-checkpoint?
39108

40109
use_sqlfluff:
41110
type: bool
42111
default: false
43-
when: "{{setup_precommit}}"
112+
when: "{{'setup_precommit' in services}}"
44113
help: use Sqlfluff?
45114

46115
use_yamllint:
47116
type: bool
48117
default: false
49-
when: "{{setup_precommit}}"
118+
when: "{{'setup_precommit' in services}}"
50119
help: use Yamllint?
51120

52-
add_dbtcoves_config_and_templates:
53-
type: bool
54-
default: false
55-
when: "{{not setup_dbt_coves and setup_dbt_project}}"
56-
help: Add dbt-coves config and templates?
121+
dbt_checkpoint_version:
122+
type: str
123+
help: What dbt-checkpoint version are you using?
124+
default: "{{ datacoves_env.get('DATACOVES__DBT_CHECKPOINT_VERSION', 'v2.0.6') }}"
125+
when: "{{use_dbt_checkpoint}}"
57126

58-
setup_dbtcoves_config:
59-
type: bool
60-
default: true
61-
when: "{{setup_dbt_coves}}"
62-
help: Setup dbt-coves config?
127+
yamllint_version:
128+
type: str
129+
help: What yamllint version are you using?
130+
default: "{{ datacoves_env.get('DATACOVES__YAMLLINT_VERSION', 'v1.36.2') }}"
131+
when: "{{use_yamllint}}"
63132

64-
setup_dbtcoves_templates:
65-
type: bool
66-
default: true
67-
when: "{{setup_dbt_coves}}"
68-
help: Setup dbt-coves templates?
133+
sqlfluff_version:
134+
type: str
135+
help: What sqlfluff version are you using?
136+
default: "{{ datacoves_env.get('DATACOVES__SQLFLUFF_VERSION', '3.1.1') }}"
137+
when: "{{use_sqlfluff}}"
69138

139+
# dbt related vars
70140
dbt_adapter:
71141
type: str
72142
choices:
@@ -75,15 +145,11 @@ dbt_adapter:
75145
- redshift
76146
- databricks
77147
help: What dbt adapter are you using?
78-
when: "{{not datacoves_dbt_adapter and (setup_dbt_profile or setup_ci_cd or setup_sqlfluff)}}"
79-
default: snowflake
80-
81-
dbt_checkpoint_version:
82-
type: str
83-
help: What dbt-checkpoint version are you using?
84-
when: "{{use_dbt_checkpoint and ask_dbt_checkpoint_version}}"
148+
when: "{{not datacoves_dbt_adapter and ('setup_dbt_profile' in services or setup_ci_cd or setup_sqlfluff)}}"
149+
default: "{{ datacoves_env.get('DATACOVES__DBT_ADAPTER', 'snowflake') }}"
85150

86-
yamllint_version:
151+
dbt_profiles_path:
87152
type: str
88-
help: What yamllint version are you using?
89-
when: "{{use_yamllint and ask_yamllint_version}}"
153+
when: "{{ 'setup_dbt_profile' in services or 'setup_ci_cd' in services }}"
154+
default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DBT_PROFILE_PATH', 'automate/dbt') }}"
155+
help: dbt Profile Path for automated runs

data.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
services:
2+
- setup_dbt_project
3+
- setup_dbt_profile
4+
- setup_ci_cd
5+
- setup_precommit
6+
- setup_airflow_dags
7+
- setup_dbtcoves
8+
dbt_project_dir: "."
9+
dbt_project_name: "mock_project"
10+
is_new_project: true
11+
dbt_profiles_path: "automate/dbt"
12+
yml_dags_path: "orchestrate/dag_yml_definitions"
13+
dags_path: "orchestrate/dags"
14+
use_sqlfluff: true
15+
use_yamllint: true
16+
use_dbt_checkpoint: true
17+
ci_provider: "github"
18+
datacoves_env:
19+
DATACOVES__AIRFLOW_DBT_PROFILE_PATH: "automate/dbt"
20+
DATACOVES__DBT_HOME: "."
21+
DATACOVES__AIRFLOW_DAGS_PATH: "orchestrate/dags"
22+
DATACOVES__AIRFLOW_DAGS_YML_PATH: "orchestrate/dags_yml_definitions"
23+
airflow_artifacts_path: "orchestrate"
24+
setup_dbtcoves_config: true
25+
setup_dbtcoves_templates: true
26+
dbt_checkpoint_version: v2.0.6
27+
yamllint_version: "v1.36.2"
28+
sqlfluff_version: "3.1.1"
29+
dbt_adapter: "snowflake"
30+
add_dbtcoves_config_and_templates: true
31+
dbt_core_version: "1.8.9"
32+
dbt_adapter_version: "1.8.4"

{% if setup_airflow_dag%}{{ dags_path }}{% endif %}/sample_dag.py renamed to {% if 'setup_airflow_dags' in services %}{{ dags_path }}{% endif %}/sample_dag.py

File renamed without changes.

{% if setup_airflow_dag %}{{ yml_dags_path }}{% endif %}/sample_dag.yml renamed to {% if 'setup_airflow_dags' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml

File renamed without changes.

{% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} renamed to {% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %}

File renamed without changes.

{% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja renamed to {% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja

File renamed without changes.

{% if setup_dbt_profile %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja renamed to {% if 'setup_dbt_profile' in services %}{{ dbt_profiles_path }}{%endif%}/profiles.yml.jinja

File renamed without changes.

0 commit comments

Comments
 (0)