Merge pull request mala-project#659 from nerkulec/rename_validation_settings

Rename validation settings (ref mala-project#636)

Author: RandomDefaultUser

docs/source/advanced_usage/hyperparameters.rst

@@ -114,7 +114,7 @@ a physical validation metric such as
 
       .. code-block:: python
 
-            parameters.running.after_training_metric = "band_energy"
+            parameters.running.final_validation_metric = "band_energy"
 
 Advanced optimization algorithms
 ********************************

docs/source/advanced_usage/trainingmodel.rst

@@ -71,13 +71,13 @@ is directly outputted by MALA. By default, this validation loss gives the
 mean squared error between LDOS prediction and actual value. From a purely
 ML point of view, this is fine; however, the correctness of the LDOS itself
 does not hold much physical virtue. Thus, MALA implements physical validation
-metrics to be accessed before and after the training routine.
+metrics which can be evaluated for example after the training.
 
 Specifically, when setting
 
       .. code-block:: python
 
-            parameters.running.after_training_metric = "band_energy"
+            parameters.running.final_validation_metric = "band_energy"
 
 the error in the band energy between actual and predicted LDOS will be
 calculated and printed before and after network training (in meV/atom).
@@ -212,7 +212,7 @@ in the file ``advanced/ex03_tensor_board``. Simply select a logger prior to trai
       .. code-block:: python
 
             parameters.running.logger = "tensorboard"
-            parameters.running.logging_dir = "mala_vis"
+            parameters.running.logging_dir = "mala_logs"
 
 or
 
@@ -224,14 +224,14 @@ or
                   entity="your_wandb_entity"
             )
             parameters.running.logger = "wandb"
-            parameters.running.logging_dir = "mala_vis"
+            parameters.running.logging_dir = "mala_logs"
 
 where ``logging_dir`` specifies some directory in which to save the
 MALA logging data. You can also select which metrics to record via
 
       .. code-block:: python
 
-            parameters.validation_metrics = ["ldos", "dos", "density", "total_energy"]
+            parameters.logging_metrics = ["ldos", "dos", "density", "total_energy"]
 
 Full list of available metrics:
       - "ldos": MSE of the LDOS.
@@ -249,14 +249,14 @@ To save time and resources you can specify the logging interval via
 
       .. code-block:: python
 
-            parameters.running.validate_every_n_epochs = 10
+            parameters.running.logging_metrics_interval = 10
 
 If you want to monitor the degree to which the model overfits to the training data,
 you can use the option
 
       .. code-block:: python
             
-            parameters.running.validate_on_training_data = True
+            parameters.running.log_metrics_on_train_set = True
 
 MALA will evaluate the validation metrics on the training set as well as the validation set.
 

examples/advanced/ex03_tensor_board.py

@@ -27,8 +27,8 @@
 # files into.
 parameters.running.logger = "tensorboard"
 parameters.running.logging_dir = "mala_vis"
-parameters.running.validation_metrics = ["ldos", "band_energy"]
-parameters.running.validate_every_n_epochs = 5
+parameters.running.logging_metrics = ["ldos", "band_energy"]
+parameters.running.logging_metrics_interval = 5
 
 data_handler = mala.DataHandler(parameters)
 data_handler.add_snapshot(

examples/advanced/ex06_distributed_hyperparameter_optimization.py

@@ -42,14 +42,14 @@
 parameters.hyperparameters.number_training_per_trial = 3
 
 # Hyperparameter optimization can be further refined by using ensemble training
-# at each step and by using a different metric then the validation loss
+# at each step and by using a different metric then the test metric
 # (e.g. the band energy). It is recommended not to use the ensemble training
 # method in Single-GPU use, as it naturally drastically decreases performance.
-# For this small example setting, using the band energy as the after training
+# For this small example setting, using the band energy as the test
 # metric is not recommended, since the small data size makes
 # an accurate hyperparameter search difficult. For larger systems, enabling
 # this option is recommended.
-# parameters.running.after_training_metric = "band_energy"
+# parameters.running.final_validation_metric = "band_energy"
 
 data_handler = mala.DataHandler(parameters)
 

mala/common/parameters.py

@@ -1038,8 +1038,8 @@ class ParametersRunning(ParametersBase):
             - "tensorboard": Tensorboard logger.
             - "wandb": Weights and Biases logger.
 
-    validation_metrics : list
-        List of metrics to be used for validation. Default is ["ldos"].
+    logging_metrics : list
+        List of metrics to be used for logging. Default is ["ldos"].
         Possible options are:
 
             - "ldos": MSE of the LDOS.
@@ -1049,15 +1049,21 @@ class ParametersRunning(ParametersBase):
             - "total_energy_actual_fe": Total energy computed with ground truth Fermi energy.
             - "fermi_energy": Fermi energy.
             - "density": Electron density.
-            - "density_relative": Rlectron density (MAPE).
+            - "density_relative": Electron density (MAPE).
             - "dos": Density of states.
             - "dos_relative": Density of states (MAPE).
+            
+        The units for energy metrics are meV/atom.
+        Selected metrics are evalauted every `logging_metrics_interval` (see below) epochs.
+        To use the energy metrics the validation snapshots need not be shuffled.
+        Note that evaluating the energy metrics takes considerably longer than just LDOS
+        and therefore it is discouraged.
 
-    validate_on_training_data : bool
-        Whether to validate on the training data as well. Default is False.
+    log_metrics_on_train_set : bool
+        Whether to also log metrics evaluated on the training set. Default is False.
 
-    validate_every_n_epochs : int
-        Determines how often validation is performed. Default is 1.
+    logging_metrics_interval : int
+        Determines how often (in the unit of epochs) metrics are logged. Default is 1.
 
     training_log_interval : int
         Determines how often detailed performance info is printed during
@@ -1103,8 +1109,8 @@ def __init__(self):
         self.learning_rate_scheduler = None
         self.learning_rate_decay = 0.1
         self.learning_rate_patience = 0
-        self._during_training_metric = "ldos"
-        self._after_training_metric = "ldos"
+        self._validation_metric = "ldos"
+        self._final_validation_metric = "ldos"
         # self.use_compression = False
         self.num_workers = 0
         self.use_shuffling_for_samplers = True
@@ -1116,9 +1122,9 @@ def __init__(self):
         self.logging_dir = "./mala_logging"
         self.logging_dir_append_date = True
         self.logger = None
-        self.validation_metrics = ["ldos"]
-        self.validate_on_training_data = False
-        self.validate_every_n_epochs = 1
+        self.logging_metrics = ["ldos"]
+        self.log_metrics_on_train_set = False
+        self.logging_metrics_interval = 1
         self.inference_data_grid = [0, 0, 0]
         self.use_mixed_precision = False
         self.use_graphs = False
@@ -1137,60 +1143,75 @@ def _update_ddp(self, new_ddp):
             New DDP setting.
         """
         super(ParametersRunning, self)._update_ddp(new_ddp)
-        self.during_training_metric = self.during_training_metric
-        self.after_training_metric = self.after_training_metric
+        self.validation_metric = self.validation_metric
+        self.final_validation_metric = self.final_validation_metric
 
     @property
-    def during_training_metric(self):
+    def validation_metric(self):
         """
-        Control the metric used during training.
+        Control the metric used for validation.
 
-        Metric for evaluated on the validation set during training.
+        Metric to be evaluated on the validation set during training.
         Default is "ldos", meaning that the regular loss on the LDOS will be
-        used as a metric. Possible options are "band_energy" and
-        "total_energy". For these, the band resp. total energy of the
-        validation snapshots will be calculated and compared to the provided
-        DFT results. Of these, the mean average error in eV/atom will be
-        calculated.
-        """
-        return self._during_training_metric
+        used as a metric.
+        
+        Possible options are:
 
-    @during_training_metric.setter
-    def during_training_metric(self, value):
+            - "ldos": MSE of the LDOS.
+            - "band_energy": Band energy.
+            - "band_energy_actual_fe": Band energy computed with ground truth Fermi energy.
+            - "total_energy": Total energy.
+            - "total_energy_actual_fe": Total energy computed with ground truth Fermi energy.
+            - "fermi_energy": Fermi energy.
+            - "density": Electron density.
+            - "density_relative": Electron density (MAPE).
+            - "dos": Density of states.
+            - "dos_relative": Density of states (MAPE).
+        
+        The units for energy metrics are meV/atom.
+        Selected metric is evalauted after every epoch on the validation set.
+        The validation metric is used as a criterion for early stopping and also
+        for checkpointing the best model.
+        Note that evaluating the energy metrics takes considerably longer than LDOS
+        and therefore it is discouraged.
+        """
+        return self._validation_metric
+
+    @validation_metric.setter
+    def validation_metric(self, value):
         if value != "ldos":
             if self._configuration["ddp"]:
                 raise Exception(
                     "Currently, MALA can only operate with the "
                     '"ldos" metric for ddp runs.'
                 )
-            if value not in self.validation_metrics:
-                self.validation_metrics.append(value)
-        self._during_training_metric = value
+            if value not in self.logging_metrics:
+                self.logging_metrics.append(value)
+        self._validation_metric = value
 
     @property
-    def after_training_metric(self):
+    def final_validation_metric(self):
         """
-        Get the metric used during training.
+        Metric for final model evaluation.
 
-        Metric for evaluated on the validation and test set before and after
-        training. Default is "LDOS", meaning that the regular loss on the LDOS
-        will be used as a metric. Possible options are "band_energy" and
-        "total_energy". For these, the band resp. total energy of the
-        validation snapshots will be calculated and compared to the provided
-        DFT results. Of these, the mean average error in eV/atom will be
-        calculated.
+        This metric is evaluated on the validation set after training.
+        Available options are the same as for `validation_metric`.
+        Default is "LDOS", meaning that MSE of the LDOS
+        will be used as a metric.
+        The final validation metric is used as a target
+        for hyperparameter optimization.
         """
-        return self._after_training_metric
+        return self._final_validation_metric
 
-    @after_training_metric.setter
-    def after_training_metric(self, value):
+    @final_validation_metric.setter
+    def final_validation_metric(self, value):
         if value != "ldos":
             if self._configuration["ddp"]:
                 raise Exception(
                     "Currently, MALA can only operate with the "
                     '"ldos" metric for ddp runs.'
                 )
-        self._after_training_metric = value
+        self._final_validation_metric = value
 
     @property
     def use_graphs(self):

mala/network/trainer.py

@@ -460,24 +460,24 @@ def train_network(self):
                         total_batch_id += 1
 
             dataset_fractions = ["validation"]
-            if self.parameters.validate_on_training_data:
+            if self.parameters.log_metrics_on_train_set:
                 dataset_fractions.append("train")
-            validation_metrics = ["ldos"]
+            logging_metrics = ["ldos"]
             if (
                 epoch != 0
-                and (epoch - 1) % self.parameters.validate_every_n_epochs == 0
+                and (epoch - 1) % self.parameters.logging_metrics_interval == 0
             ):
-                validation_metrics = self.parameters.validation_metrics
-            errors = self._validate_network(
-                dataset_fractions, validation_metrics
+                logging_metrics = self.parameters.logging_metrics
+            errors = self._evaluate_metrics(
+                dataset_fractions, logging_metrics
             )
             for dataset_fraction in dataset_fractions:
                 for metric in errors[dataset_fraction]:
                     errors[dataset_fraction][metric] = np.mean(
                         np.abs(errors[dataset_fraction][metric])
                     )
             vloss = errors["validation"][
-                self.parameters.during_training_metric
+                self.parameters.validation_metric
             ]
             if self.parameters_full.use_ddp:
                 vloss = self.__average_validation(
@@ -589,17 +589,17 @@ def train_network(self):
         ############################
         # CALCULATE FINAL METRICS
         ############################
-        if self.parameters.after_training_metric in errors["validation"]:
+        if self.parameters.final_validation_metric in errors["validation"]:
             self.final_validation_loss = errors["validation"][
-                self.parameters.after_training_metric
+                self.parameters.final_validation_metric
             ]
         else:
-            final_errors = self._validate_network(
-                ["validation"], [self.parameters.after_training_metric]
+            final_errors = self._evaluate_metrics(
+                ["validation"], [self.parameters.final_validation_metric]
             )
             vloss = np.mean(
                 final_errors["validation"][
-                    self.parameters.after_training_metric
+                    self.parameters.final_validation_metric
                 ]
             )
 
@@ -619,7 +619,7 @@ def train_network(self):
             self._training_data_loaders.cleanup()
             self._validation_data_loaders.cleanup()
 
-    def _validate_network(self, data_set_fractions, metrics):
+    def _evaluate_metrics(self, data_set_fractions, metrics):
         """
         Validate a network, using train or validation data.
 
@@ -770,11 +770,11 @@ def __calculate_validation_error_ldos_only(self, data_loaders):
         """
         Calculate the validation error for the LDOS only.
 
-        This is a specialization of _validate_network that ONLY computes
+        This is a specialization of _evaluate_metrics that ONLY computes
         one error, namely the LDOS error. It is more efficient, especially
         in the distributed case, than the implementation called from
-        _validate_network. As such it is mostly "legacy" code for now, until
-        we adapt _validate_network.
+        _evaluate_metrics. As such it is mostly "legacy" code for now, until
+        we adapt _evaluate_metrics.
 
         Parameters
         ----------