Data ingestion component

Author: austinLorenzMccoy

logs/01_15_2025_13_06_05.log

@@ -0,0 +1,17 @@
+from networksecurity.components.data_ingestion import DataIngestion
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+from networksecurity.entity.config_entity import DataIngestionConfig
+from networksecurity.entity.config_entity import Training_pipeline_config
+import sys
+
+if __name__ == "__main__":
+    try:
+       training_pipeline_config=Training_pipeline_config()
+       dataingestionconfig=DataIngestionConfig(training_pipeline_config)
+       data_ingestion=DataIngestion(data_ingestion_config=dataingestionconfig)
+       logging.info("starting data ingestion")
+       dataingestionartifact=data_ingestion.initiate_data_ingestion()
+       print(dataingestionartifact)
+    except Exception as e:
+        raise NetworkSecurityException(e, sys) 

logs/01_15_2025_13_08_16.log

@@ -0,0 +1,84 @@
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+from networksecurity.entity.config_entity import DataIngestionConfig
+from networksecurity.entity.artifact_entity import DataIngestionArtifact
+import os
+import sys
+import numpy as np
+import pandas as pd
+import pymongo
+from typing import List
+from sklearn.model_selection import train_test_split
+from dotenv import load_dotenv
+
+load_dotenv()
+uri = os.getenv('MONGODB_URI')
+
+class DataIngestion:
+    def __init__(self, data_ingestion_config: DataIngestionConfig):
+        try:
+            self.data_ingestion_config = data_ingestion_config
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def export_collection_as_dataframe(self):
+        try:
+            database_name = self.data_ingestion_config.database_name
+            collection_name = self.data_ingestion_config.collection_name
+            self.mongo_client = pymongo.MongoClient(uri)
+            collection = self.mongo_client[database_name][collection_name]
+
+            df = pd.DataFrame(list(collection.find()))
+            if "_id" in df.columns.to_list():
+                df = df.drop(columns=["_id"], axis=1)
+
+            df.replace("", np.nan, inplace=True)
+            return df
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def export_data_into_feature_store(self, df: pd.DataFrame, feature_store_file_path: str):
+        try:
+            dir_path = os.path.dirname(feature_store_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+            df.to_csv(feature_store_file_path, index=False, header=True)
+            return df  # Ensure df is returned
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def split_data_as_train_test(self, df: pd.DataFrame):
+        try:
+            train_set, test_set = train_test_split(
+                df, test_size=self.data_ingestion_config.train_test_split_ratio
+            )
+            logging.info("Performed train test split on the dataframe")
+
+            dir_path = os.path.dirname(self.data_ingestion_config.training_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+
+            train_set.to_csv(
+                self.data_ingestion_config.training_file_path, index=False, header=True
+            )
+            test_set.to_csv(
+                self.data_ingestion_config.test_file_path, index=False, header=True  # Changed from testing_file_path to test_file_path
+            )
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def initiate_data_ingestion(self):
+        logging.info("Entered the data ingestion method or component")
+        try:
+            dataframe = self.export_collection_as_dataframe()
+            dataframe = self.export_data_into_feature_store(
+                dataframe, self.data_ingestion_config.feature_store_file_path
+            )
+            self.split_data_as_train_test(dataframe)
+            data_ingestion_artifact = DataIngestionArtifact(
+                feature_store_file_path=self.data_ingestion_config.feature_store_file_path,
+                train_file_path=self.data_ingestion_config.training_file_path,
+                test_file_path=self.data_ingestion_config.test_file_path,  # Changed from testing_file_path to test_file_path
+            )
+            return data_ingestion_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)

logs/01_15_2025_13_09_45.log

@@ -0,0 +1,25 @@
+import os
+import sys
+import numpy as np
+import pandas as pd
+
+"""
+defining common constant variable for training pipeline
+"""
+TARGET_COLUMN = "text"
+PIPELINE_NAME: str = "networksecurity"
+ARTIFACT_DIR: str = os.path.join(os.getcwd(), "artifact")
+FILE_NAME: str = "cyber_threat_intelligence_train.csv"
+
+TRAIN_FILE_NAME: str = "train.csv"
+TEST_FILE_NAME: str = "test.csv"
+
+"""
+Data Ingestion related# Data Ingestion related constant start with DATA_INGESTION VAR NAME
+"""
+DATA_INGESTION_COLLECTION_NAME: str = "network_data"
+DATA_INGESTION_DATABASE_NAME: str = "AUSTINAI"
+DATA_INGESTION_DIR_NAME: str = "data_ingestion"
+DATA_INGESTION_FEATURE_STORE_DIR: str = "feature_store"
+DATA_INGESTION_INGESTED_DIR: str = "ingested"
+DATA_INGESTION_TRAIN_TEST_SPLIT_RATIO: float = 0.2