data validation completed

Author: austinLorenzMccoy

=4.0

@@ -0,0 +1,6 @@
+Collecting pymongo
+  Using cached pymongo-4.10.1-cp310-cp310-macosx_10_9_x86_64.whl.metadata (22 kB)
+Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in ./netsec_env/lib/python3.10/site-packages (from pymongo) (1.16.0)
+Using cached pymongo-4.10.1-cp310-cp310-macosx_10_9_x86_64.whl (835 kB)
+Installing collected packages: pymongo
+Successfully installed pymongo-4.10.1

data_schema/schema.yaml

@@ -0,0 +1,65 @@
+columns:
+  -having_IP_Address: int64
+  -URL_Length: int64
+  -Shortining_Service: int64
+  -having_At_Symbol: int64
+  -double_slash_redirecting: int64
+  -Prefix_Suffix: int64
+  -having_Sub_Domain: int64
+  -SSLfinal_State: int64
+  -Domain_registeration_length: int64
+  -Favicon: int64
+  -port: int64
+  -HTTPS_token: int64
+  -Request_URL: int64
+  -URL_of_Anchor: int64
+  -Links_in_tags: int64
+  -SFH: int64
+  -Submitting_to_email: int64
+  -Abnormal_URL: int64
+  -Redirect: int64
+  -on_mouseover: int64
+  -RightClick: int64
+  -popUpWidnow: int64
+  -Iframe: int64
+  -age_of_domain: int64
+  -DNSRecord: int64
+  -web_traffic: int64
+  -Page_Rank: int64
+  -Google_Index: int64
+  -Links_pointing_to_page: int64
+  -Statistical_report: int64
+  -Result: int64
+
+numerical_columns:
+  -having_IP_Address
+  -URL_Length
+  -Shortining_Service
+  -having_At_Symbol
+  -double_slash_redirecting
+  -Prefix_Suffix
+  -having_Sub_Domain
+  -SSLfinal_State
+  -Domain_registeration_length
+  -Favicon
+  -port
+  -HTTPS_token
+  -Request_URL
+  -URL_of_Anchor
+  -Links_in_tags
+  -SFH
+  -Submitting_to_email
+  -Abnormal_URL
+  -Redirect
+  -on_mouseover
+  -RightClick
+  -popUpWidnow
+  -Iframe
+  -age_of_domain
+  -DNSRecord
+  -web_traffic
+  -Page_Rank
+  -Google_Index
+  -Links_pointing_to_page
+  -Statistical_report
+  -Result

main.py

@@ -1,7 +1,8 @@
 from networksecurity.components.data_ingestion import DataIngestion
+from networksecurity.components.data_validation import DataValidation
 from networksecurity.exception.exception import NetworkSecurityException
 from networksecurity.logging.logger import logging
-from networksecurity.entity.config_entity import DataIngestionConfig
+from networksecurity.entity.config_entity import DataIngestionConfig, DataValidationConfig
 from networksecurity.entity.config_entity import Training_pipeline_config
 import sys
 
@@ -12,6 +13,15 @@
        data_ingestion=DataIngestion(data_ingestion_config=dataingestionconfig)
        logging.info("starting data ingestion")
        dataingestionartifact=data_ingestion.initiate_data_ingestion()
+       logging.info("data ingestion completed")
        print(dataingestionartifact)
+       data_validation_config=DataValidationConfig(training_pipeline_config)
+       data_validation=DataValidation(dataingestionartifact,data_validation_config)
+       logging.info("starting data validation")
+       data_validation_artifact=data_validation.initiate_data_validation()
+       logging.info("data validation completed")
+       print(data_validation_artifact)
+
+
     except Exception as e:
         raise NetworkSecurityException(e, sys) 

networksecurity/components/__pycache__/data_validation.cpython-310.pyc

@@ -0,0 +1,113 @@
+from networksecurity.entity.artifact_entity import DataIngestionArtifact, DataValidationArtifact
+from networksecurity.entity.config_entity import DataValidationConfig
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+from networksecurity.constants import SCHEMA_FILE_PATH
+from scipy.stats import ks_2samp
+import os
+import pandas as pd
+import sys
+from networksecurity.utils.main_utils import read_yaml_file, write_yaml_file
+
+class DataValidation:
+    def __init__(self, data_ingestion_artifact: DataIngestionArtifact,
+                 data_validation_config: DataValidationConfig):
+        try:
+            self.data_ingestion_artifact = data_ingestion_artifact
+            self.data_validation_config = data_validation_config
+            self.schema_config = read_yaml_file(file_path=SCHEMA_FILE_PATH)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    @staticmethod
+    def read_data(file_path) -> pd.DataFrame:
+        try:
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def validate_number_of_columns(self, dataframe: pd.DataFrame) -> bool:
+        try:
+            number_of_columns = len(self.schema_config["columns"])
+            logging.info(f"Number of columns in dataframe: {len(dataframe.columns)}")
+            logging.info(f"Required number of columns as per schema: {number_of_columns}")
+            if len(dataframe.columns) == number_of_columns:
+                return True
+            return False
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def detect_drift_in_data(self, base_df, current_df, threshold=0.5) -> bool:
+        try:
+            status = True
+            report = {}
+            for column in base_df.columns:
+                d1 = base_df[column]
+                d2 = current_df[column]
+                is_same_dist = ks_2samp(d1, d2)
+                if threshold <= is_same_dist.pvalue:
+                    is_found = False
+                else:
+                    is_found = True
+                    status = False
+                report.update({column: {
+                    "p_value": float(is_same_dist.pvalue),
+                    "drift_status": is_found
+                }})
+            
+            drift_report_file_path = os.path.join(
+                self.data_validation_config.drift_report_file_path,
+                "drift_report.json"
+            )
+
+            # Create directory
+            dir_path = os.path.dirname(drift_report_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+            write_yaml_file(file_path=drift_report_file_path, content=report, replace=True)
+            return status
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def initiate_data_validation(self) -> DataValidationArtifact:
+        try:
+            error_message = ""
+            train_file_path = self.data_ingestion_artifact.train_file_path
+            test_file_path = self.data_ingestion_artifact.test_file_path
+
+            # Read data from train and test file location
+            train_df = DataValidation.read_data(train_file_path)
+            test_df = DataValidation.read_data(test_file_path)
+
+            # Validate number of columns
+            status = self.validate_number_of_columns(dataframe=train_df)
+            if not status:
+                error_message = f"{error_message}Train dataframe does not contain all columns."
+                logging.info(f"{error_message}Train dataframe does not contain all columns.")
+            status = self.validate_number_of_columns(dataframe=test_df)
+            if not status:
+                error_message = f"{error_message}Test dataframe does not contain all columns."
+                logging.info(f"{error_message}Test dataframe does not contain all columns.")
+
+            # Check data drift
+            status = self.detect_drift_in_data(base_df=train_df, current_df=test_df)
+            
+            # Create directory for valid data
+            dir_path = os.path.dirname(self.data_validation_config.valid_train_file_path)
+            os.makedirs(dir_path, exist_ok=True)
+
+            train_df.to_csv(self.data_validation_config.valid_train_file_path, index=False, header=True)
+            test_df.to_csv(self.data_validation_config.valid_test_file_path, index=False, header=True)
+
+            data_validation_artifact = DataValidationArtifact(
+                validation_status=status,
+                valid_train_file_path=self.data_validation_config.valid_train_file_path,
+                valid_test_file_path=self.data_validation_config.valid_test_file_path,
+                invalid_train_file_path=self.data_validation_config.invalid_train_file_path,
+                invalid_test_file_path=self.data_validation_config.invalid_test_file_path,
+                drift_report_file_path=self.data_validation_config.drift_report_file_path
+            )
+            
+            return data_validation_artifact
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)

networksecurity/components/data_validation.py

@@ -1,3 +1,5 @@
+# networksecurity/constants/Training_pipeline/__init__.py
+
 import os
 import sys
 import numpy as np
@@ -14,12 +16,31 @@
 TRAIN_FILE_NAME: str = "train.csv"
 TEST_FILE_NAME: str = "test.csv"
 
+SCHEMA_FILE_PATH = os.path.join("data_schema", "schema.yaml")
+
 """
-Data Ingestion related# Data Ingestion related constant start with DATA_INGESTION VAR NAME
+Data Ingestion related constant start with DATA_INGESTION VAR NAME
 """
 DATA_INGESTION_COLLECTION_NAME: str = "network_data"
 DATA_INGESTION_DATABASE_NAME: str = "AUSTINAI"
 DATA_INGESTION_DIR_NAME: str = "data_ingestion"
 DATA_INGESTION_FEATURE_STORE_DIR: str = "feature_store"
 DATA_INGESTION_INGESTED_DIR: str = "ingested"
-DATA_INGESTION_TRAIN_TEST_SPLIT_RATIO: float = 0.2
+DATA_INGESTION_TRAIN_TEST_SPLIT_RATIO: float = 0.2
+
+"""
+Data Validation related constant start with DATA_VALIDATION VAR NAME
+"""
+DATA_VALIDATION_DIR_NAME: str = "data_validation"
+DATA_VALIDATION_VALID_DIR: str = "validated"
+DATA_VALIDATION_INVALID_DIR: str = "invalid"
+DATA_VALIDATION_DRIFT_REPORT_DIR: str = "drift_report"
+DATA_VALIDATION_DRIFT_REPORT_FILE_NAME: str = "report.yaml"
+
+# Adding the missing constants
+DATA_VALIDATION_TRAIN_FILE_NAME: str = "train.csv"
+DATA_VALIDATION_TEST_FILE_NAME: str = "test.csv"
+
+"""
+Data Transformation related constant start with DATA_TRANSFORMATION VAR NAME
+"""

networksecurity/constants/Training_pipeline/__init__.py

@@ -0,0 +1,49 @@
+# networksecurity/constants/__init__.py
+
+from networksecurity.constants.Training_pipeline import (
+    TARGET_COLUMN,
+    PIPELINE_NAME,
+    ARTIFACT_DIR,
+    FILE_NAME,
+    TRAIN_FILE_NAME,
+    TEST_FILE_NAME,
+    SCHEMA_FILE_PATH,
+    # Data Ingestion Constants
+    DATA_INGESTION_COLLECTION_NAME,
+    DATA_INGESTION_DATABASE_NAME,
+    DATA_INGESTION_DIR_NAME,
+    DATA_INGESTION_FEATURE_STORE_DIR,
+    DATA_INGESTION_INGESTED_DIR,
+    DATA_INGESTION_TRAIN_TEST_SPLIT_RATIO,
+    # Data Validation Constants
+    DATA_VALIDATION_DIR_NAME,
+    DATA_VALIDATION_VALID_DIR,
+    DATA_VALIDATION_INVALID_DIR,
+    DATA_VALIDATION_DRIFT_REPORT_DIR,
+    DATA_VALIDATION_DRIFT_REPORT_FILE_NAME,
+    DATA_VALIDATION_TRAIN_FILE_NAME,
+    DATA_VALIDATION_TEST_FILE_NAME,
+)
+
+__all__ = [
+    "TARGET_COLUMN",
+    "PIPELINE_NAME",
+    "ARTIFACT_DIR",
+    "FILE_NAME",
+    "TRAIN_FILE_NAME",
+    "TEST_FILE_NAME",
+    "SCHEMA_FILE_PATH",
+    "DATA_INGESTION_COLLECTION_NAME",
+    "DATA_INGESTION_DATABASE_NAME",
+    "DATA_INGESTION_DIR_NAME",
+    "DATA_INGESTION_FEATURE_STORE_DIR",
+    "DATA_INGESTION_INGESTED_DIR",
+    "DATA_INGESTION_TRAIN_TEST_SPLIT_RATIO",
+    "DATA_VALIDATION_DIR_NAME",
+    "DATA_VALIDATION_VALID_DIR",
+    "DATA_VALIDATION_INVALID_DIR",
+    "DATA_VALIDATION_DRIFT_REPORT_DIR",
+    "DATA_VALIDATION_DRIFT_REPORT_FILE_NAME",
+    "DATA_VALIDATION_TRAIN_FILE_NAME",
+    "DATA_VALIDATION_TEST_FILE_NAME",
+]