Merge branch 'main' of https://github.com/Jakee4488/MLOps_Databricks_Project

Author: Jakee4488

notebooks/marvel_data_preprocessing.py notebooks/1.marvel_data_preprocessing.pynotebooks/marvel_data_preprocessing.py renamed to notebooks/1.marvel_data_preprocessing.py

@@ -1,14 +1,15 @@
 # Databricks notebook source
-
 # MAGIC %pip install -e ..
 # MAGIC %restart_python
 
 # COMMAND ----------
-# from pathlib import Path
-# import sys
-# sys.path.append(str(Path.cwd().parent / 'src'))
+
+from pathlib import Path
+import sys
+sys.path.append(str(Path.cwd().parent / 'src'))
 
 # COMMAND ----------
+
 import pandas as pd
 import yaml
 from loguru import logger
@@ -38,6 +39,7 @@
 logger.info(df[config.target].value_counts())
 
 # COMMAND ----------
+
 # Load the Marvel characters dataset
 
 data_processor = DataProcessor(df, config, spark)
@@ -55,11 +57,11 @@
 logger.info("Test set shape: %s", X_test.shape)
 
 # COMMAND ----------
+
 # Save to catalog
 logger.info("Saving data to catalog")
 data_processor.save_to_catalog(X_train, X_test)
 
 # Enable change data feed (only once!)
 logger.info("Enable change data feed")
 data_processor.enable_change_data_feed()
-# COMMAND ----------