Rename validation and logging related parameters (refs mala-project#636)

Author: nerkulec

docs/source/advanced_usage/hyperparameters.rst

@@ -114,7 +114,7 @@ a physical validation metric such as
 
       .. code-block:: python
 
-            parameters.running.after_training_metric = "band_energy"
+            parameters.running.final_validation_metric = "band_energy"
 
 Advanced optimization algorithms
 ********************************

docs/source/advanced_usage/trainingmodel.rst

@@ -71,13 +71,13 @@ is directly outputted by MALA. By default, this validation loss gives the
 mean squared error between LDOS prediction and actual value. From a purely
 ML point of view, this is fine; however, the correctness of the LDOS itself
 does not hold much physical virtue. Thus, MALA implements physical validation
-metrics to be accessed before and after the training routine.
+metrics which can be evaluated for example after the training.
 
 Specifically, when setting
 
       .. code-block:: python
 
-            parameters.running.after_training_metric = "band_energy"
+            parameters.running.final_validation_metric = "band_energy"
 
 the error in the band energy between actual and predicted LDOS will be
 calculated and printed before and after network training (in meV/atom).
@@ -231,7 +231,7 @@ MALA logging data. You can also select which metrics to record via
 
       .. code-block:: python
 
-            parameters.validation_metrics = ["ldos", "dos", "density", "total_energy"]
+            parameters.logging_metrics = ["ldos", "dos", "density", "total_energy"]
 
 Full list of available metrics:
       - "ldos": MSE of the LDOS.
@@ -249,14 +249,14 @@ To save time and resources you can specify the logging interval via
 
       .. code-block:: python
 
-            parameters.running.validate_every_n_epochs = 10
+            parameters.running.logging_metrics_freq = 10
 
 If you want to monitor the degree to which the model overfits to the training data,
 you can use the option
 
       .. code-block:: python
             
-            parameters.running.validate_on_training_data = True
+            parameters.running.log_metrics_on_train_set = True
 
 MALA will evaluate the validation metrics on the training set as well as the validation set.
 

examples/advanced/ex03_tensor_board.py

@@ -27,8 +27,8 @@
 # files into.
 parameters.running.logger = "tensorboard"
 parameters.running.logging_dir = "mala_vis"
-parameters.running.validation_metrics = ["ldos", "band_energy"]
-parameters.running.validate_every_n_epochs = 5
+parameters.running.logging_metrics = ["ldos", "band_energy"]
+parameters.running.logging_metrics_freq = 5
 
 data_handler = mala.DataHandler(parameters)
 data_handler.add_snapshot(

examples/advanced/ex06_distributed_hyperparameter_optimization.py

@@ -42,14 +42,14 @@
 parameters.hyperparameters.number_training_per_trial = 3
 
 # Hyperparameter optimization can be further refined by using ensemble training
-# at each step and by using a different metric then the validation loss
+# at each step and by using a different metric then the test metric
 # (e.g. the band energy). It is recommended not to use the ensemble training
 # method in Single-GPU use, as it naturally drastically decreases performance.
-# For this small example setting, using the band energy as the after training
+# For this small example setting, using the band energy as the test
 # metric is not recommended, since the small data size makes
 # an accurate hyperparameter search difficult. For larger systems, enabling
 # this option is recommended.
-# parameters.running.after_training_metric = "band_energy"
+# parameters.running.final_validation_metric = "band_energy"
 
 data_handler = mala.DataHandler(parameters)
 

mala/common/parameters.py

@@ -1038,7 +1038,7 @@ class ParametersRunning(ParametersBase):
             - "tensorboard": Tensorboard logger.
             - "wandb": Weights and Biases logger.
 
-    validation_metrics : list
+    logging_metrics : list
         List of metrics to be used for validation. Default is ["ldos"].
         Possible options are:
 
@@ -1053,10 +1053,10 @@ class ParametersRunning(ParametersBase):
             - "dos": Density of states.
             - "dos_relative": Density of states (MAPE).
 
-    validate_on_training_data : bool
+    log_metrics_on_train_set : bool
         Whether to validate on the training data as well. Default is False.
 
-    validate_every_n_epochs : int
+    logging_metrics_freq : int
         Determines how often validation is performed. Default is 1.
 
     training_log_interval : int
@@ -1103,8 +1103,8 @@ def __init__(self):
         self.learning_rate_scheduler = None
         self.learning_rate_decay = 0.1
         self.learning_rate_patience = 0
-        self._during_training_metric = "ldos"
-        self._after_training_metric = "ldos"
+        self._validation_metric = "ldos"
+        self._final_validation_metric = "ldos"
         # self.use_compression = False
         self.num_workers = 0
         self.use_shuffling_for_samplers = True
@@ -1116,9 +1116,9 @@ def __init__(self):
         self.logging_dir = "./mala_logging"
         self.logging_dir_append_date = True
         self.logger = None
-        self.validation_metrics = ["ldos"]
-        self.validate_on_training_data = False
-        self.validate_every_n_epochs = 1
+        self.logging_metrics = ["ldos"]
+        self.log_metrics_on_train_set = False
+        self.logging_metrics_freq = 1
         self.inference_data_grid = [0, 0, 0]
         self.use_mixed_precision = False
         self.use_graphs = False
@@ -1137,11 +1137,11 @@ def _update_ddp(self, new_ddp):
             New DDP setting.
         """
         super(ParametersRunning, self)._update_ddp(new_ddp)
-        self.during_training_metric = self.during_training_metric
-        self.after_training_metric = self.after_training_metric
+        self.validation_metric = self.validation_metric
+        self.final_validation_metric = self.final_validation_metric
 
     @property
-    def during_training_metric(self):
+    def validation_metric(self):
         """
         Control the metric used during training.
 
@@ -1153,22 +1153,22 @@ def during_training_metric(self):
         DFT results. Of these, the mean average error in eV/atom will be
         calculated.
         """
-        return self._during_training_metric
+        return self._validation_metric
 
-    @during_training_metric.setter
-    def during_training_metric(self, value):
+    @validation_metric.setter
+    def validation_metric(self, value):
         if value != "ldos":
             if self._configuration["ddp"]:
                 raise Exception(
                     "Currently, MALA can only operate with the "
                     '"ldos" metric for ddp runs.'
                 )
-            if value not in self.validation_metrics:
-                self.validation_metrics.append(value)
-        self._during_training_metric = value
+            if value not in self.logging_metrics:
+                self.logging_metrics.append(value)
+        self._validation_metric = value
 
     @property
-    def after_training_metric(self):
+    def final_validation_metric(self):
         """
         Get the metric used during training.
 
@@ -1180,17 +1180,17 @@ def after_training_metric(self):
         DFT results. Of these, the mean average error in eV/atom will be
         calculated.
         """
-        return self._after_training_metric
+        return self._final_validation_metric
 
-    @after_training_metric.setter
-    def after_training_metric(self, value):
+    @final_validation_metric.setter
+    def final_validation_metric(self, value):
         if value != "ldos":
             if self._configuration["ddp"]:
                 raise Exception(
                     "Currently, MALA can only operate with the "
                     '"ldos" metric for ddp runs.'
                 )
-        self._after_training_metric = value
+        self._final_validation_metric = value
 
     @property
     def use_graphs(self):

mala/network/trainer.py

@@ -460,24 +460,24 @@ def train_network(self):
                         total_batch_id += 1
 
             dataset_fractions = ["validation"]
-            if self.parameters.validate_on_training_data:
+            if self.parameters.log_metrics_on_train_set:
                 dataset_fractions.append("train")
-            validation_metrics = ["ldos"]
+            logging_metrics = ["ldos"]
             if (
                 epoch != 0
-                and (epoch - 1) % self.parameters.validate_every_n_epochs == 0
+                and (epoch - 1) % self.parameters.logging_metrics_freq == 0
             ):
-                validation_metrics = self.parameters.validation_metrics
-            errors = self._validate_network(
-                dataset_fractions, validation_metrics
+                logging_metrics = self.parameters.logging_metrics
+            errors = self._evaluate_metrics(
+                dataset_fractions, logging_metrics
             )
             for dataset_fraction in dataset_fractions:
                 for metric in errors[dataset_fraction]:
                     errors[dataset_fraction][metric] = np.mean(
                         np.abs(errors[dataset_fraction][metric])
                     )
             vloss = errors["validation"][
-                self.parameters.during_training_metric
+                self.parameters.validation_metric
             ]
             if self.parameters_full.use_ddp:
                 vloss = self.__average_validation(
@@ -589,17 +589,17 @@ def train_network(self):
         ############################
         # CALCULATE FINAL METRICS
         ############################
-        if self.parameters.after_training_metric in errors["validation"]:
+        if self.parameters.final_validation_metric in errors["validation"]:
             self.final_validation_loss = errors["validation"][
-                self.parameters.after_training_metric
+                self.parameters.final_validation_metric
             ]
         else:
-            final_errors = self._validate_network(
-                ["validation"], [self.parameters.after_training_metric]
+            final_errors = self._evaluate_metrics(
+                ["validation"], [self.parameters.final_validation_metric]
             )
             vloss = np.mean(
                 final_errors["validation"][
-                    self.parameters.after_training_metric
+                    self.parameters.final_validation_metric
                 ]
             )
 
@@ -619,7 +619,7 @@ def train_network(self):
             self._training_data_loaders.cleanup()
             self._validation_data_loaders.cleanup()
 
-    def _validate_network(self, data_set_fractions, metrics):
+    def _evaluate_metrics(self, data_set_fractions, metrics):
         """
         Validate a network, using train or validation data.
 
@@ -770,11 +770,11 @@ def __calculate_validation_error_ldos_only(self, data_loaders):
         """
         Calculate the validation error for the LDOS only.
 
-        This is a specialization of _validate_network that ONLY computes
+        This is a specialization of _evaluate_metrics that ONLY computes
         one error, namely the LDOS error. It is more efficient, especially
         in the distributed case, than the implementation called from
-        _validate_network. As such it is mostly "legacy" code for now, until
-        we adapt _validate_network.
+        _evaluate_metrics. As such it is mostly "legacy" code for now, until
+        we adapt _evaluate_metrics.
 
         Parameters
         ----------