ETLpipeline on MongoDB completed

Author: austinLorenzMccoy

.DS_Store

@@ -0,0 +1 @@
+[2025-01-15 13:06:05,458] 29 root - INFO - MongoDB connection established successfully

Network_Data/cyber_threat_intelligence_train.csv

@@ -0,0 +1 @@
+[2025-01-15 13:08:16,939] 29 root - INFO - MongoDB connection established successfully

logs/01_15_2025_13_06_05.log

@@ -0,0 +1 @@
+[2025-01-15 13:09:45,851] 29 root - INFO - MongoDB connection established successfully

logs/01_15_2025_13_08_16.log

@@ -0,0 +1,70 @@
+import os
+import sys
+import json
+
+from dotenv import load_dotenv
+# Load environment variables
+load_dotenv()
+
+# Get MongoDB URI from environment variable
+uri = os.getenv('MONGODB_URI')
+
+print("MongoDB URI:", uri)
+
+import certifi #root certificate for http connection and store in ca(certificate authority)
+ca=certifi.where()
+
+import pandas as pd
+import numpy as np
+import pymongo
+from networksecurity.exception.exception import NetworkSecurityException
+from networksecurity.logging.logger import logging
+
+class NetworkDataExtract():
+    def __init__(self):
+        try:
+            self.client = pymongo.MongoClient(uri, tlsCAFile=ca)
+            self.db = self.client['network_security']
+            self.collection = self.db['network_data']
+            logging.info("MongoDB connection established successfully")
+        except Exception as e:
+            logging.error("Error while connecting to MongoDB")
+            raise NetworkSecurityException(e, sys) from e
+        
+    def csv_to_json_converter(self, file_path):
+        try:
+            df = pd.read_csv(file_path)
+            if df.empty:
+                raise ValueError("CSV file is empty.")
+            df.reset_index(drop=True, inplace=True)
+            records = list(json.loads(df.T.to_json()).values())
+            return records
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+        
+    def insert_data_mongodb(self, records, database, collection):
+        try:
+            # Initialize MongoDB client and access specified database and collection
+            mongo_client = pymongo.MongoClient(uri, tlsCAFile=ca)
+            db = mongo_client[database]
+            collection = db[collection]
+            
+            # Insert records into the collection
+            collection.insert_many(records)
+            return len(records)
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+        
+if __name__ == "__main__":
+    file_path = "Network_Data/cyber_threat_intelligence_train.csv"
+    database = "AUSTINAI"
+    collection = "network_data"
+
+    network_data_extract = NetworkDataExtract()
+    records = network_data_extract.csv_to_json_converter(file_path)
+    num_records = network_data_extract.insert_data_mongodb(records, database, collection)
+    print(f"Inserted {num_records} records into MongoDB")
+
+        

logs/01_15_2025_13_09_45.log

@@ -3,5 +3,8 @@ pandas
 numpy
 pymongo
 certifi
+pymongo[srv]==3.6
 
--e .    
+
+
+## -e .    

networksecurity/exception/__pycache__/__init__.cpython-310.pyc

@@ -13,8 +13,10 @@ def get_requirements() -> List[str]:
     requirements: List[str] = []
     try:
         with open('requirements.txt', 'r') as file:
+            # process each line
             for line in file:
                 requirement = line.strip()
+                #ignore empty lines and -e.
                 if requirement and requirement != '-e .':
                     requirements.append(requirement)
     except FileNotFoundError: