new hyperparameter outcomes

Author: robin-janssen

datasets/primordial/surrogates_config.py

@@ -5,76 +5,148 @@
 
 @dataclass
 class MultiONetConfig:
-    """Model config for MultiONet for the simple_ode dataset"""
+    """Model config for MultiONet for the primordial dataset"""
 
-    # Naive configuration
-    loss_function: nn.Module = nn.MSELoss()
-    optimizer: str = "adamw"
+    # primordial_final, trial 196
     scheduler: str = "schedulefree"
-    branch_hidden_layers: int = 5
-    hidden_size: int = 256
-    output_factor: int = 200
-    trunk_hidden_layers: int = 5
-    learning_rate: float = 1e-04
-    regularization_factor: float = 1e-04
-    activation: nn.Module = nn.ReLU()
+    optimizer: str = "AdamW"
+    loss_function: nn.Module = nn.SmoothL1Loss()
+    beta: float = 0.845
+    branch_hidden_layers: int = 3
+    hidden_size: int = 160
+    output_factor: int = 130
+    trunk_hidden_layers: int = 2
+    learning_rate: float = 9.6e-04
+    regularization_factor: float = 0.283
+    activation: nn.Module = nn.LeakyReLU()
 
 
 @dataclass
 class LatentNeuralODEConfig:
-    """Model config for LatentNeuralODE for the simple_ode dataset"""
+    """Model config for LatentNeuralODE for the primordial dataset"""
 
-    # Naive configuration
-    loss_function: nn.Module = nn.MSELoss()
-    optimizer: str = "adamw"
+    # primordial_final, trial 243
     scheduler: str = "schedulefree"
-    ode_tanh_reg: bool = False
-    latent_features: int = 10
-    coder_layers: int = 3
-    coder_width: int = 256
-    ode_layers: int = 3
-    ode_width: int = 256
-    learning_rate: float = 1e-04
-    regularization_factor: float = 1e-04
-    activation: nn.Module = nn.ReLU()
+    optimizer: str = "SGD"
+    loss_function: nn.Module = nn.MSELoss()
+    momentum: float = 0.085
+    latent_features: int = 9
+    coder_layers: int = 1
+    coder_width: int = 580
+    ode_tanh_reg: bool = True
+    ode_layers: int = 6
+    ode_width: int = 480
+    learning_rate: float = 9.68e-5
+    regularization_factor: float = 0.0544
+    activation: nn.Module = nn.SiLU()
 
 
 @dataclass
 class FullyConnectedConfig:
-    """Model config for FullyConnected for the simple_ode dataset"""
+    """Model config for FullyConnected for the primordial dataset"""
 
-    # Naive configuration
+    # primordial_final, trial 174
+    scheduler: str = "poly"
+    optimizer: str = "adam"
     loss_function: nn.Module = nn.MSELoss()
-    optimizer: str = "adamw"
-    scheduler: str = "schedulefree"
-    # other params from primordial_tuning, trial 63
-    hidden_size: int = 256
-    num_hidden_layers: int = 5
-    learning_rate: float = 1e-04
-    regularization_factor: float = 1e-04
+    momentum: float = 0.0132
+    degree: int = 4
+    latent_features: int = 8
+    coder_layers: int = 2
+    coder_width: int = 470
+    learning_rate: float = 1.77e-04
+    regularization_factor: float = 9.20e-03
     activation: nn.Module = nn.ReLU()
 
 
 @dataclass
 class LatentPolyConfig:
-    """Model config for LatentPoly for the simple_ode dataset"""
+    """Model config for LatentPoly for the primordial dataset"""
 
-    # Naive configuration
-    loss_function: nn.Module = nn.MSELoss()
-    optimizer: str = "adamw"
+    # primordial_final, trial 31
     scheduler: str = "schedulefree"
-    degree: int = 3
-    latent_features: int = 10
-    coder_layers: int = 3
-    coder_width: int = 256
-    learning_rate: float = 1e-04
-    regularization_factor: float = 1e-04
-    activation: nn.Module = nn.ReLU()
+    optimizer: str = "SGD"
+    loss_function: nn.Module = nn.SmoothL1Loss()
+    beta: float = 3.73
+    hidden_size: int = 470
+    num_hidden_layers: int = 5
+    learning_rate: float = 1.27e-03
+    regularization_factor: float = 3.23e-05
+    activation: nn.Module = nn.ELU()
+
+
+# @dataclass
+# class MultiONetConfig:
+#     """Model config for MultiONet for the primordial dataset"""
+
+#     # Naive configuration
+#     loss_function: nn.Module = nn.MSELoss()
+#     optimizer: str = "adamw"
+#     scheduler: str = "schedulefree"
+#     branch_hidden_layers: int = 5
+#     hidden_size: int = 256
+#     output_factor: int = 200
+#     trunk_hidden_layers: int = 5
+#     learning_rate: float = 1e-04
+#     regularization_factor: float = 1e-04
+#     activation: nn.Module = nn.ReLU()
+
+
+# @dataclass
+# class LatentNeuralODEConfig:
+#     """Model config for LatentNeuralODE for the primordial dataset"""
+
+#     # Naive configuration
+#     loss_function: nn.Module = nn.MSELoss()
+#     optimizer: str = "adamw"
+#     scheduler: str = "schedulefree"
+#     ode_tanh_reg: bool = False
+#     latent_features: int = 10
+#     coder_layers: int = 3
+#     coder_width: int = 256
+#     ode_layers: int = 3
+#     ode_width: int = 256
+#     learning_rate: float = 1e-04
+#     regularization_factor: float = 1e-04
+#     activation: nn.Module = nn.ReLU()
+
+
+# @dataclass
+# class FullyConnectedConfig:
+#     """Model config for FullyConnected for the primordial dataset"""
+
+#     # Naive configuration
+#     loss_function: nn.Module = nn.MSELoss()
+#     optimizer: str = "adamw"
+#     scheduler: str = "schedulefree"
+#     # other params from primordial_tuning, trial 63
+#     hidden_size: int = 256
+#     num_hidden_layers: int = 5
+#     learning_rate: float = 1e-04
+#     regularization_factor: float = 1e-04
+#     activation: nn.Module = nn.ReLU()
+
+
+# @dataclass
+# class LatentPolyConfig:
+#     """Model config for LatentPoly for the primordial dataset"""
+
+#     # Naive configuration
+#     loss_function: nn.Module = nn.MSELoss()
+#     optimizer: str = "adamw"
+#     scheduler: str = "schedulefree"
+#     degree: int = 3
+#     latent_features: int = 10
+#     coder_layers: int = 3
+#     coder_width: int = 256
+#     learning_rate: float = 1e-04
+#     regularization_factor: float = 1e-04
+#     activation: nn.Module = nn.ReLU()
 
 
 # @dataclass
 # class MultiONetConfig:
-#     """Model config for MultiONet for the simple_ode dataset"""
+#     """Model config for MultiONet for the primordial dataset"""
 
 #     # primordial_tuning, trial 18
 #     scheduler: str = "poly"
@@ -93,7 +165,7 @@ class LatentPolyConfig:
 
 # @dataclass
 # class LatentNeuralODEConfig:
-#     """Model config for LatentNeuralODE for the simple_ode dataset"""
+#     """Model config for LatentNeuralODE for the primordial dataset"""
 
 #     # primordial_tuning, trial 186
 #     scheduler: str = "poly"
@@ -114,7 +186,7 @@ class LatentPolyConfig:
 
 # @dataclass
 # class FullyConnectedConfig:
-#     """Model config for FullyConnected for the simple_ode dataset"""
+#     """Model config for FullyConnected for the primordial dataset"""
 
 #     # primordial_tuning, trial 63
 #     scheduler: str = "poly"
@@ -131,7 +203,7 @@ class LatentPolyConfig:
 
 # @dataclass
 # class LatentPolyConfig:
-#     """Model config for LatentPoly for the simple_ode dataset"""
+#     """Model config for LatentPoly for the primordial dataset"""
 
 #     # primordial_tuning, trial 176
 #     scheduler: str = "poly"

datasets/primordial_parametric/surrogates_config.py

@@ -5,59 +5,136 @@
 
 @dataclass
 class MultiONetConfig:
-    """Model config for MultiONet for the simple_ode dataset"""
-
-    # primordialparams, trial 56
-
-    branch_hidden_layers: int = 2
-    trunk_hidden_layers: int = 7
-    hidden_size: int = 164
-    output_factor: int = 63
-    learning_rate: float = 4e-5
-    activation: nn.Module = nn.GELU()
+    """Model config for MultiONet for the primordial_parametric dataset"""
+
+    # primordial_parametric_final_multionet, trial 18
+    scheduler: str = "poly"
+    optimizer: str = "AdamW"
+    loss_function: nn.Module = nn.MSELoss()
+    activation: nn.Module = nn.PReLU()
+    branch_hidden_layers: int = 5
+    hidden_size: int = 50
+    learning_rate: float = 0.0018
+    output_factor: int = 74
     params_branch: bool = True
+    poly_power: float = 0.64
+    regularization_factor: float = 0.00114
+    trunk_hidden_layers: bool = True
 
 
 @dataclass
 class LatentNeuralODEConfig:
-    """Model config for LatentNeuralODE for the simple_ode dataset"""
-
-    # primordialparams, trial 75
-
-    latent_features: int = 5
-    coder_layers: int = 4
-    coder_width: int = 268
-    learning_rate: float = 4e-5
-    ode_layers: int = 9
-    ode_width: int = 125
-    ode_tanh_reg: bool = False
-    activation: nn.Module = nn.Tanh()
-    model_version: str = "v2"
+    """Model config for LatentNeuralODE for the primordial_parametric dataset"""
+
+    # primordial_parametric_final_latentneuralode, trial 234
+    scheduler: str = "cosine"
+    optimizer: str = "SGD"
+    loss_function: nn.Module = nn.MSELoss()
+    activation: nn.Module = nn.SiLU()
+    coder_layers: bool = True
+    coder_width: int = 360
     encode_params: bool = False
+    eta_min: float = 0.00191
+    latent_features: int = 10
+    learning_rate: float = 2.04e-05
+    momentum: float = 0.823
+    ode_layers: int = 8
+    ode_tanh_reg: bool = True
+    ode_width: int = 220
+    regularization_factor: float = 1.87e-05
 
 
 @dataclass
 class FullyConnectedConfig:
-    """Model config for FullyConnected for the simple_ode dataset"""
+    """Model config for FullyConnected for the primordial_parametric dataset"""
 
-    # primordialparams, trial 40
-
-    hidden_size: int = 629
-    num_hidden_layers: int = 1
-    learning_rate: float = 6e-5
-    activation: nn.Module = nn.LeakyReLU()
+    # primordial_parametric_final_fullyconnected, trial 1
+    scheduler: str = "poly"
+    optimizer: str = "AdamW"
+    loss_function: nn.Module = nn.SmoothL1Loss()
+    activation: nn.Module = nn.Mish()
+    beta: float = 6.69
+    hidden_size: int = 470
+    learning_rate: float = 0.00129
+    num_hidden_layers: int = 3
+    poly_power: float = 0.899
+    regularization_factor: float = 0.0144
 
 
 @dataclass
 class LatentPolyConfig:
-    """Model config for LatentPoly for the simple_ode dataset"""
+    """Model config for LatentPoly for the primordial_parametric dataset"""
+
+    # primordial_parametric_final_latentpoly, trial 16
+    scheduler: str = "poly"
+    optimizer: str = "AdamW"
+    loss_function: nn.Module = nn.MSELoss()
+    activation: nn.Module = nn.ELU()
+    coder_layers: bool = True
+    coder_width: int = 700
+    coeff_network: bool = False
+    degree: int = 3
+    latent_features: int = 7
+    learning_rate: float = 1.33e-05
+    poly_power: float = 0.804
+    regularization_factor: float = 0.0174
 
-    # primordialparams, trial 32
 
-    latent_features: int = 10
-    degree: int = 7
-    learning_rate: float = 5e-4
-    coder_layers: int = 2
-    coder_width: int = 267
-    activation: nn.Module = nn.LeakyReLU()
-    coeff_network: bool = False
+# @dataclass
+# class MultiONetConfig:
+#     """Model config for MultiONet for the simple_ode dataset"""
+
+#     # primordialparams, trial 56
+
+#     branch_hidden_layers: int = 2
+#     trunk_hidden_layers: int = 7
+#     hidden_size: int = 164
+#     output_factor: int = 63
+#     learning_rate: float = 4e-5
+#     activation: nn.Module = nn.GELU()
+#     params_branch: bool = True
+
+
+# @dataclass
+# class LatentNeuralODEConfig:
+#     """Model config for LatentNeuralODE for the simple_ode dataset"""
+
+#     # primordialparams, trial 75
+
+#     latent_features: int = 5
+#     coder_layers: int = 4
+#     coder_width: int = 268
+#     learning_rate: float = 4e-5
+#     ode_layers: int = 9
+#     ode_width: int = 125
+#     ode_tanh_reg: bool = False
+#     activation: nn.Module = nn.Tanh()
+#     model_version: str = "v2"
+#     encode_params: bool = False
+
+
+# @dataclass
+# class FullyConnectedConfig:
+#     """Model config for FullyConnected for the simple_ode dataset"""
+
+#     # primordialparams, trial 40
+
+#     hidden_size: int = 629
+#     num_hidden_layers: int = 1
+#     learning_rate: float = 6e-5
+#     activation: nn.Module = nn.LeakyReLU()
+
+
+# @dataclass
+# class LatentPolyConfig:
+#     """Model config for LatentPoly for the simple_ode dataset"""
+
+#     # primordialparams, trial 32
+
+#     latent_features: int = 10
+#     degree: int = 7
+#     learning_rate: float = 5e-4
+#     coder_layers: int = 2
+#     coder_width: int = 267
+#     activation: nn.Module = nn.LeakyReLU()
+#     coeff_network: bool = False