Merge pull request #17 from SiriwatHuntra/main

Fix, last version (ver.2)

Author: Kariusdi

Regularization/scratching/Ridge_ass2.py

@@ -1,35 +1,54 @@
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
-from sklearn.model_selection import train_test_split
 from sklearn.preprocessing import PolynomialFeatures
 from sklearn.linear_model import Ridge
 from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import KFold
 
 def polyRidge(degree):
     # Create polynomial features
     poly = PolynomialFeatures(degree)  # Adjust degree as needed
     X_poly = poly.fit_transform(X)
-    # Split data into training and testing sets
-    X_train, X_test, y_train, y_test = train_test_split(X_poly, y, test_size=0.2, random_state=42)
-    # Create and train the Ridge regression model
-    ridge = Ridge(alpha=100)  # Adjust alpha for regularization strength
-    ridge.fit(X_train, y_train)
-
-    # Make predictions on the test and training sets
-    y_pred_train = ridge.predict(X_train)
-    y_pred_test = ridge.predict(X_test)
-
-    # Evaluate the model
-    # Calculate E-train and E-test (RMSE)
-    mse_train = mean_squared_error(y_train, y_pred_train)
-    mse_test = mean_squared_error(y_test, y_pred_test)
-    rmse_train = np.sqrt(mse_train)
-    rmse_test = np.sqrt(mse_test)
+
+    # Split data for KFold cross-validation
+    kf = KFold(n_splits=10, shuffle=True, random_state=42)  # Adjust n_splits
+
+    # Initialize variables
+    mse_train_list = []
+    mse_test_list = []
+
+    # Perform KFold cross-validation
+    for train_index, test_index in kf.split(X_poly):
+        X_train, X_test = X_poly[train_index], X_poly[test_index]
+        y_train, y_test = y[train_index], y[test_index]
+
+        # Create and train the Ridge regression model
+        ridge = Ridge(alpha=100000)  # Adjust alpha for regularization strength
+        ridge.fit(X_train, y_train)
+
+        # Make predictions on the test and training sets
+        y_pred_train = ridge.predict(X_train)
+        y_pred_test = ridge.predict(X_test)
+
+        # Evaluate the model
+        mse_train = mean_squared_error(y_train, y_pred_train)
+        mse_test = mean_squared_error(y_test, y_pred_test)
+
+        mse_train_list.append(mse_train)
+        mse_test_list.append(mse_test)
+
+    # Estimate bias (average training error)
+    E_train = np.sqrt(np.mean(mse_train_list))
+
+    # Estimate variance (average difference between training and test error)
+    E_var = np.sqrt(np.mean(np.square(np.array(mse_test_list) - np.mean(mse_train_list))))
+
+    # Estimated E_out (sum of bias and variance)
+    E_out = E_train + E_var
 
     # Print results and return errors
-    print(f"Degree={degree}: E_train: {rmse_train:.4f}, E_test: {rmse_test:.4f}")
-    return rmse_train, rmse_test
+    return E_train, E_out
 
 def generate_sin():
     np.random.seed(42)
@@ -49,19 +68,19 @@ def import_csv(path):
 degrees = np.arange(1, 11)  # Adjust the range as needed
 
 # Initialize lists to store errors
-E_train_list = []
-E_test_list = []
+E_out_list = []
+E_in_List = []
 
 # Call the function for each degree and store errors
 for deg in degrees:
-  rmse_train, rmse_test = polyRidge(deg)
-  E_train_list.append(rmse_train)
-  E_test_list.append(rmse_test)
+  E_out, E_in = polyRidge(deg)
+  E_out_list.append(E_out)
+  E_in_List.append(E_in)
 
 # Plot the results
 plt.figure(figsize=(8, 6))
-plt.plot(degrees, E_train_list, label="E_train", marker='o', linestyle='-')
-plt.plot(degrees, E_test_list, label="E_test", marker='s', linestyle='-')
+plt.plot(degrees, E_out_list, label="E_Out", marker='o', linestyle='-')
+plt.plot(degrees, E_in_List, label="E_In", marker='s', linestyle='-')
 plt.xlabel("Model Complexity (Degree)")
 plt.ylabel("RMSE")
 plt.title("Model Complexity vs. E_train and E_test")

Regularization/scratching/lamda.py

@@ -0,0 +1,59 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import Ridge
+from sklearn.metrics import mean_squared_error
+
+def generate_sin():
+    np.random.seed(42)
+    X = np.random.rand(100, 1) * 10
+    Y = np.sin(X) + np.random.randn(100) / 10
+    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=0)
+    return X_train, Y_train, X_test, Y_test
+
+def import_csv(path):
+    df = pd.read_csv(path)
+    X = df['Height'].values.reshape(-1, 1)  # Reshape for sklearn
+    Y = df['Weight'].values
+    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=0)
+    return X_train, Y_train, X_test, Y_test
+
+def ridge_regression(X_train, Y_train, X_test, Y_test, alpha):
+    model = Ridge(alpha=alpha)
+    model.fit(X_train, Y_train)
+
+    train_rmse = mean_squared_error(Y_train, model.predict(X_train))
+    test_rmse = mean_squared_error( Y_test, model.predict(X_test))
+
+    return train_rmse, test_rmse
+
+def plot_rmse_vs_alpha(alphas, train_rmse, test_rmse):
+
+    plt.figure(figsize=(8, 6))
+    plt.plot(alphas, train_rmse, label="Train", marker='o', linestyle='-')
+    plt.plot(alphas, test_rmse, label="Test", marker='s', linestyle='-')
+    plt.xlabel("Model Complexity (Log scale)")
+    plt.xscale('log')
+    plt.ylabel("RMSE")
+    plt.title("Model Complexity vs. E_train and E_test")
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+    plt.show()
+
+X_train, Y_train, X_test, Y_test = import_csv("Regularization/dataset/HeightWeight.csv")
+#X_train, Y_train, X_test, Y_test = generate_sin()
+
+alphas = np.arange(1, 100000, 100)
+E_train = []
+E_test = []
+
+for alpha_ in alphas:
+    train_rmse, test_rmse = ridge_regression(X_train, Y_train, X_test, Y_test, alpha_)
+    E_train.append(np.sqrt(train_rmse))
+    E_test.append(np.sqrt(test_rmse))
+
+plot_rmse_vs_alpha(alphas, E_train, E_test)
+
+