Skip to content

Commit 95497f5

Browse files
committed
add custom layer to view transform artifacts
1 parent ed088f0 commit 95497f5

2 files changed

Lines changed: 9 additions & 7 deletions

File tree

src/pasteur/kedro/pipelines/main.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,10 +125,11 @@ def generate_pipelines(
125125

126126
# Create view transform pipeline that can run as part of ingest
127127
if view.fit_global:
128-
pipe_fit = create_fit_pipeline(view, all_types, modules, 'view')
129-
+ create_transform_pipeline(
128+
pipe_fit = create_fit_pipeline(
129+
view, all_types, modules, "view"
130+
) + create_transform_pipeline(
130131
view,
131-
'view',
132+
"view",
132133
all_types,
133134
)
134135
else:

src/pasteur/kedro/pipelines/transform.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,31 +175,32 @@ def create_transform_pipeline(
175175
),
176176
]
177177

178+
layer = "view_transformed" if split == "view" else "split_transformed"
178179
outputs.append(
179180
D(
180-
"split_transformed",
181+
layer,
181182
f"{view}.{split}.ctx_{t}",
182183
["view", view, split, "ctx", t],
183184
type="multi",
184185
)
185186
)
186187
outputs.append(
187188
D(
188-
"split_transformed",
189+
layer,
189190
f"{view}.{split}.bst_{t}",
190191
["view", view, split, "bst", t],
191192
)
192193
)
193194
outputs.append(
194195
D(
195-
"split_transformed",
196+
layer,
196197
f"{view}.{split}.ids_{t}",
197198
["view", view, split, "ids", t],
198199
)
199200
)
200201

201202
for enc in types:
202-
if enc in ("bst", "raw"):
203+
if enc in ("bst", "raw") or split == "view":
203204
continue
204205

205206
table_nodes += [

0 commit comments

Comments
 (0)