Add complete ML pipeline with model training, testing, and API components

Author: austinLorenzMccoy

Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first to leverage Docker cache
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application
+COPY . .
+
+# Install the package in development mode
+RUN pip install -e .
+
+# Set environment variables
+ENV PYTHONPATH=/app
+ENV PYTHONUNBUFFERED=1
+
+# Expose ports for FastAPI and MLflow UI
+EXPOSE 8000
+EXPOSE 5000
+
+# Command to run when container starts
+# Use uvicorn to serve the FastAPI application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

data_schema/schema.yaml

@@ -1,65 +1,67 @@
+#data_schema/schema.yaml
+
 columns:
-  -having_IP_Address: int64
-  -URL_Length: int64
-  -Shortining_Service: int64
-  -having_At_Symbol: int64
-  -double_slash_redirecting: int64
-  -Prefix_Suffix: int64
-  -having_Sub_Domain: int64
-  -SSLfinal_State: int64
-  -Domain_registeration_length: int64
-  -Favicon: int64
-  -port: int64
-  -HTTPS_token: int64
-  -Request_URL: int64
-  -URL_of_Anchor: int64
-  -Links_in_tags: int64
-  -SFH: int64
-  -Submitting_to_email: int64
-  -Abnormal_URL: int64
-  -Redirect: int64
-  -on_mouseover: int64
-  -RightClick: int64
-  -popUpWidnow: int64
-  -Iframe: int64
-  -age_of_domain: int64
-  -DNSRecord: int64
-  -web_traffic: int64
-  -Page_Rank: int64
-  -Google_Index: int64
-  -Links_pointing_to_page: int64
-  -Statistical_report: int64
-  -Result: int64
+  having_IP_Address: int64
+  URL_Length: int64
+  Shortining_Service: int64
+  having_At_Symbol: int64
+  double_slash_redirecting: int64
+  Prefix_Suffix: int64
+  having_Sub_Domain: int64
+  SSLfinal_State: int64
+  Domain_registeration_length: int64
+  Favicon: int64
+  port: int64
+  HTTPS_token: int64
+  Request_URL: int64
+  URL_of_Anchor: int64
+  Links_in_tags: int64
+  SFH: int64
+  Submitting_to_email: int64
+  Abnormal_URL: int64
+  Redirect: int64
+  on_mouseover: int64
+  RightClick: int64
+  popUpWidnow: int64
+  Iframe: int64
+  age_of_domain: int64
+  DNSRecord: int64
+  web_traffic: int64
+  Page_Rank: int64
+  Google_Index: int64
+  Links_pointing_to_page: int64
+  Statistical_report: int64
+  Result: int64
 
 numerical_columns:
-  -having_IP_Address
-  -URL_Length
-  -Shortining_Service
-  -having_At_Symbol
-  -double_slash_redirecting
-  -Prefix_Suffix
-  -having_Sub_Domain
-  -SSLfinal_State
-  -Domain_registeration_length
-  -Favicon
-  -port
-  -HTTPS_token
-  -Request_URL
-  -URL_of_Anchor
-  -Links_in_tags
-  -SFH
-  -Submitting_to_email
-  -Abnormal_URL
-  -Redirect
-  -on_mouseover
-  -RightClick
-  -popUpWidnow
-  -Iframe
-  -age_of_domain
-  -DNSRecord
-  -web_traffic
-  -Page_Rank
-  -Google_Index
-  -Links_pointing_to_page
-  -Statistical_report
-  -Result
+  - having_IP_Address
+  - URL_Length
+  - Shortining_Service
+  - having_At_Symbol
+  - double_slash_redirecting
+  - Prefix_Suffix
+  - having_Sub_Domain
+  - SSLfinal_State
+  - Domain_registeration_length
+  - Favicon
+  - port
+  - HTTPS_token
+  - Request_URL
+  - URL_of_Anchor
+  - Links_in_tags
+  - SFH
+  - Submitting_to_email
+  - Abnormal_URL
+  - Redirect
+  - on_mouseover
+  - RightClick
+  - popUpWidnow
+  - Iframe
+  - age_of_domain
+  - DNSRecord
+  - web_traffic
+  - Page_Rank
+  - Google_Index
+  - Links_pointing_to_page
+  - Statistical_report
+  - Result

main.py

@@ -1,34 +1,33 @@
 from networksecurity.components.data_ingestion import DataIngestion
 from networksecurity.components.data_validation import DataValidation
 from networksecurity.components.data_transformation import DataTransformation
+from networksecurity.components.model_trainer import ModelTrainer
+from networksecurity.entity.config_entity import DataIngestionConfig, DataValidationConfig, DataTransformationConfig, ModelTrainerConfig, TrainingPipelineConfig
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
-from networksecurity.entity.config_entity import DataIngestionConfig, DataValidationConfig,DataTransformationConfig
-from networksecurity.entity.config_entity import Training_pipeline_config
 import sys
 
 if __name__ == "__main__":
     try:
-       training_pipeline_config=Training_pipeline_config()
-       dataingestionconfig=DataIngestionConfig(training_pipeline_config)
-       data_ingestion=DataIngestion(data_ingestion_config=dataingestionconfig)
-       logging.info("starting data ingestion")
-       dataingestionartifact=data_ingestion.initiate_data_ingestion()
-       logging.info("data ingestion completed")
-       print(dataingestionartifact)
-       data_validation_config=DataValidationConfig(training_pipeline_config)
-       data_validation=DataValidation(dataingestionartifact,data_validation_config)
-       logging.info("starting data validation")
-       data_validation_artifact=data_validation.initiate_data_validation()
-       logging.info("data validation completed")
-       print(data_validation_artifact)
-       datatransformationconfig=DataTransformationConfig(training_pipeline_config)
-       data_transformation=DataTransformation(data_validation_artifact, datatransformationconfig)
-       logging.info("starting data transformation")
-       data_transformation_artifact=data_transformation.initiate_data_transformation()
-       logging.info("data transformation completed")
-       print(data_transformation_artifact)
+        # Initialize configurations
+        training_pipeline_config = TrainingPipelineConfig()
+        data_ingestion_config = DataIngestionConfig(training_pipeline_config)
+        data_validation_config = DataValidationConfig(training_pipeline_config)
+        data_transformation_config = DataTransformationConfig(training_pipeline_config)
+        model_trainer_config = ModelTrainerConfig(training_pipeline_config)
 
+        # Run pipeline
+        data_ingestion = DataIngestion(data_ingestion_config)
+        data_ingestion_artifact = data_ingestion.initiate_data_ingestion()
+
+        data_validation = DataValidation(data_ingestion_artifact, data_validation_config)
+        data_validation_artifact = data_validation.initiate_data_validation()
+
+        data_transformation = DataTransformation(data_validation_artifact, data_transformation_config)
+        data_transformation_artifact = data_transformation.initiate_data_transformation()
+
+        model_trainer = ModelTrainer(model_trainer_config, data_transformation_artifact)
+        model_trainer_artifact = model_trainer.initiate_model_trainer()
 
     except Exception as e:
-        raise NetworkSecurityException(e, sys) 
+        raise NetworkSecurityException(e, sys)

networksecurity/components/data_ingestion.py

@@ -1,84 +1,76 @@
-from networksecurity.exception.exception import NetworkSecurityException
-from networksecurity.logging.logger import logging
-from networksecurity.entity.config_entity import DataIngestionConfig
-from networksecurity.entity.artifact_entity import DataIngestionArtifact
 import os
 import sys
 import numpy as np
 import pandas as pd
 import pymongo
-from typing import List
 from sklearn.model_selection import train_test_split
 from dotenv import load_dotenv
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+from networksecurity.entity.config_entity import DataIngestionConfig
+from networksecurity.entity.artifact_entity import DataIngestionArtifact
 
 load_dotenv()
 uri = os.getenv('MONGODB_URI')
 
 class DataIngestion:
     def __init__(self, data_ingestion_config: DataIngestionConfig):
-        try:
-            self.data_ingestion_config = data_ingestion_config
-        except Exception as e:
-            raise NetworkSecurityException(e, sys)
+        self.data_ingestion_config = data_ingestion_config
 
-    def export_collection_as_dataframe(self):
+    def export_collection_as_dataframe(self) -> pd.DataFrame:
+        """
+        Export MongoDB collection to a DataFrame.
+        """
         try:
             database_name = self.data_ingestion_config.database_name
             collection_name = self.data_ingestion_config.collection_name
-            self.mongo_client = pymongo.MongoClient(uri)
-            collection = self.mongo_client[database_name][collection_name]
-
-            df = pd.DataFrame(list(collection.find()))
-            if "_id" in df.columns.to_list():
-                df = df.drop(columns=["_id"], axis=1)
-
+            logging.info(f"Connecting to MongoDB: {database_name}.{collection_name}")
+            
+            client = pymongo.MongoClient(uri)
+            collection = client[database_name][collection_name]
+            cursor = collection.find({})
+            
+            df = pd.DataFrame(list(cursor))
+            if "_id" in df.columns:
+                df.drop(columns=["_id"], axis=1, inplace=True)
+            
             df.replace("", np.nan, inplace=True)
+            logging.info(f"DataFrame shape: {df.shape}")
             return df
-
-        except Exception as e:
-            raise NetworkSecurityException(e, sys)
-
-    def export_data_into_feature_store(self, df: pd.DataFrame, feature_store_file_path: str):
-        try:
-            dir_path = os.path.dirname(feature_store_file_path)
-            os.makedirs(dir_path, exist_ok=True)
-            df.to_csv(feature_store_file_path, index=False, header=True)
-            return df  # Ensure df is returned
         except Exception as e:
             raise NetworkSecurityException(e, sys)
 
-    def split_data_as_train_test(self, df: pd.DataFrame):
+    def split_data_as_train_test(self, df: pd.DataFrame) -> None:
+        """
+        Split data into train and test sets.
+        """
         try:
             train_set, test_set = train_test_split(
-                df, test_size=self.data_ingestion_config.train_test_split_ratio
-            )
-            logging.info("Performed train test split on the dataframe")
-
-            dir_path = os.path.dirname(self.data_ingestion_config.training_file_path)
-            os.makedirs(dir_path, exist_ok=True)
-
-            train_set.to_csv(
-                self.data_ingestion_config.training_file_path, index=False, header=True
-            )
-            test_set.to_csv(
-                self.data_ingestion_config.test_file_path, index=False, header=True  # Changed from testing_file_path to test_file_path
+                df, test_size=self.data_ingestion_config.train_test_split_ratio, random_state=42
             )
+            logging.info(f"Train set shape: {train_set.shape}, Test set shape: {test_set.shape}")
+            
+            os.makedirs(os.path.dirname(self.data_ingestion_config.training_file_path), exist_ok=True)
+            train_set.to_csv(self.data_ingestion_config.training_file_path, index=False, header=True)
+            test_set.to_csv(self.data_ingestion_config.test_file_path, index=False, header=True)
         except Exception as e:
             raise NetworkSecurityException(e, sys)
 
-    def initiate_data_ingestion(self):
-        logging.info("Entered the data ingestion method or component")
+    def initiate_data_ingestion(self) -> DataIngestionArtifact:
+        """
+        Main method to initiate data ingestion.
+        """
         try:
-            dataframe = self.export_collection_as_dataframe()
-            dataframe = self.export_data_into_feature_store(
-                dataframe, self.data_ingestion_config.feature_store_file_path
-            )
-            self.split_data_as_train_test(dataframe)
-            data_ingestion_artifact = DataIngestionArtifact(
+            logging.info("Starting data ingestion")
+            df = self.export_collection_as_dataframe()
+            self.split_data_as_train_test(df)
+            
+            artifact = DataIngestionArtifact(
                 feature_store_file_path=self.data_ingestion_config.feature_store_file_path,
                 train_file_path=self.data_ingestion_config.training_file_path,
-                test_file_path=self.data_ingestion_config.test_file_path,  # Changed from testing_file_path to test_file_path
+                test_file_path=self.data_ingestion_config.test_file_path
             )
-            return data_ingestion_artifact
+            logging.info("Data ingestion completed successfully")
+            return artifact
         except Exception as e:
-            raise NetworkSecurityException(e, sys)
+            raise NetworkSecurityException(e, sys)