Fixed gray-box result output bugs

Author: DimitriWeiss

.gitignore

@@ -7,16 +7,19 @@ __pycache__/
 *.so
 
 # Folders
-rtac/logs/Cadical_ReACTR
-rtac/logs/TSP_Q_ReACTR
-rtac/logs/TSP_RT_ReACTR
 rtac/logs/Cadical_CPPL
+rtac/logs/Cadicalpp_CPPL
 rtac/logs/Cadicalpp_CPPL_gb
+rtac/logs/Cadicalpp_ReACTR
+rtac/logs/Cadicalpp_ReACTRpp
+rtac/logs/Cadicalpp_ReACTR_gb
+rtac/logs/Cadicalpp_ReACTRpp_gb
+rtac/logs/Cadical_ReACTR
 rtac/logs/TSP_Qpp_ReACTRpp
+rtac/logs/TSP_Q_ReACTR
 rtac/logs/TSP_RT_CPPL
 rtac/logs/TSP_RTpp_ReACTRpp
-rtac/logs/Cadicalpp_ReACTR_gb
-rtac/logs/Cadicalpp_ReACTRpp_gb
+rtac/logs/TSP_RT_ReACTR
 rtac/data/instances/power_law_easy
 rtac/data/solvers/cadical
 

.idea/.name

@@ -466,8 +466,9 @@ def insert_in_pool(self, configs):
 
             for c, d in zip(configs.values(), self.discard):
                 disc_id = list(self.pool.keys())[d]
-                print('Replaced cotender', disc_id,
-                      'by contender generated via', c.gen, '\n')
+                if self.scenario.verbosity == 2:
+                    print('Replaced contender', disc_id,
+                          'by contender generated via', c.gen, '\n')
 
     def skill_and_confidence(self):
         with threadpool_limits(limits=1):

rtac/ac_functionalities/ranking/cppl.py

@@ -254,15 +254,16 @@ def term_list(self, pred, cores, verbosity):
         worse_than_avg = \
             [core for core, rank in ranks.items() if rank >= avg_rank]
 
-        if verbosity == 2:
-            print("Better counts:", dict(better_counts))
-            print("Ranks:", ranks)
-            print("Average rank:", avg_rank)
-            print("Worse than average:", worse_than_avg)
-
-        if avg_rank != 1.0:
+        if len(set(better_counts.values())) == 1:
             return []
         else:
+            if verbosity == 2:
+                print('\n\n')
+                print('GRAY BOX PAIRWSE COMPARISON STATS:')
+                print("Better counts:", dict(better_counts))
+                print("Ranks:", ranks)
+                print("Average rank:", avg_rank)
+                print("Worse than average:", worse_than_avg)
             return worse_than_avg
 
 

rtac/ac_functionalities/ranking/gray_box.py

@@ -106,7 +106,8 @@ def manage_pool(self) -> None:
     def select_contenders(self) -> None:
         """Select contenders for next tournament/problem instance."""
 
-    def process_tourn(self, rtac_data: RTACData, instance: str = None, tourn_nr: int = None) -> str:
+    def process_tourn(self, rtac_data: RTACData, instance: str = None,
+                      tourn_nr: int = None) -> str:
         """Manage result processing.
 
         :param rtac_data: Object containing data and objects necessary
@@ -146,11 +147,12 @@ def get_contender_dict(self) -> dict[str: Configuration]:
     def result_summary_terminal(self, results, tourn_nr=None):
         if tourn_nr:
             self.tourn_nr = tourn_nr
-        if self.scenario.verbosity == 2:
+        if self.scenario.verbosity == 2 and self.scenario.experimental:
             if not self.scenario.objective_min:
                 unit = 'seconds'
             else:
                 unit = 'objective value'
+
             self.time_sum += round(min(results), 3)
             len_str = len('Instance nr. ' + str(self.tourn_nr) + ' : ' + str(
                 round(self.time_sum, 3)
@@ -233,7 +235,8 @@ def get_winner(self, times: list[float], res: list[float]) \
 
         return winner, ranks
 
-    def process_results(self, rtac_data: RTACData, instance: str = None, tourn_nr: int = None) -> None:
+    def process_results(self, rtac_data: RTACData, instance: str = None,
+                        tourn_nr: int = None) -> None:
         """Perform tournament result processing necessary to replace contenders
         in pool and select contenders for next tournament/problem instance
         according to ReACTR implementation.
@@ -248,10 +251,15 @@ def process_results(self, rtac_data: RTACData, instance: str = None, tourn_nr: i
             if self.rtac_data.ta_res[core] == self.huge_float:
                 self.rtac_data.ta_res_time[core] = self.scenario.timeout
 
+        if not self.scenario.objective_min:
+            results = list(self.rtac_data.ta_res_time[:])
+        else:
+            results = list(self.rtac_data.ta_res[:])
+
+        self.result_summary_terminal(results)
+
         times = list(self.rtac_data.ta_res_time[:])
 
-        self.result_summary_terminal(times)
-        
         self.rtac_data.newtime = min(times)
         res = list(self.rtac_data.ta_res[:])
         self.rtac_data.best_res = min(res)
@@ -457,57 +465,66 @@ def get_winner(self, times: list[float], res: list[float]) \
         else:
             winner = res.index(min(res))
 
-        interim_sorted = [[self.rtac_data.interim[j][i]
-                          for j in range(self.scenario.number_cores)]
-                          for i, _ in enumerate(self.rtac_data.interim[0])]
-
-        interim_sorted = np.array(interim_sorted)
-        interim_sorted = interim_sorted.astype(float)
-
-        for i, isort in enumerate(interim_sorted):
-            if self.rtac_data.interim_meaning[i] is \
-                    InterimMeaning.decrease:
-                interim_sorted[i] = rankdata(isort,
-                                             method='dense',
-                                             nan_policy="propagate")
-            elif self.rtac_data.interim_meaning[i] is \
-                    InterimMeaning.increase:
-                interim_sorted[i] = rankdata([-1 * i if i is not None
-                                              else None for i in isort],
-                                             method='dense',
-                                             nan_policy="propagate")
-
-        ranks = [0 for core in range(self.scenario.number_cores)]
-
-        for _, isort in enumerate(interim_sorted):
-            for r, _ in enumerate(ranks):
-                ranks[r] += isort[r]
-
-        if self.scenario.objective_min:
-            res_ranks = rankdata(res, method='dense', nan_policy="propagate")
-
-            duplicates = []
-
-            for rank in sorted(set(res_ranks)):
-                duplicates.append(self.duplicates(res_ranks, rank))
-            
-            for duplicate in duplicates:
-                if len(duplicate) > 1:
-                    interim_ranks = [ranks[dup] for dup in duplicate]
-                    tie_winner = \
-                        duplicate[interim_ranks.index(min(interim_ranks))]
-                    interim_ranks = rankdata(interim_ranks, method='dense')
-                    interim_ranks -= min(interim_ranks)
-                    for d, ir in zip(duplicate, interim_ranks):
-                        for r, _ in enumerate(res_ranks):
-                            if d == r and r != winner and r != tie_winner:
-                                res_ranks[r] += interim_ranks[ir]
-                            else:
-                                res_ranks[r] += max(interim_ranks)
-
-            ranks = res_ranks
-
-        ranks = rankdata(ranks, method='dense')
+        if not any(elem is None for sublist in self.rtac_data.interim
+                   for elem in sublist):
+
+            ranks = [0 for core in range(self.scenario.number_cores)]
+
+            interim_sorted = [[self.rtac_data.interim[j][i]
+                              for j in range(self.scenario.number_cores)]
+                              for i, _ in enumerate(self.rtac_data.interim[0])]
+
+            interim_sorted = np.array(interim_sorted)
+            interim_sorted = interim_sorted.astype(float)
+
+            for i, isort in enumerate(interim_sorted):
+                if self.rtac_data.interim_meaning[i] is \
+                        InterimMeaning.decrease:
+                    interim_sorted[i] = rankdata(isort,
+                                                 method='dense',
+                                                 nan_policy="propagate")
+                elif self.rtac_data.interim_meaning[i] is \
+                        InterimMeaning.increase:
+                    interim_sorted[i] = rankdata([-1 * i if i is not None
+                                                  else None for i in isort],
+                                                 method='dense',
+                                                 nan_policy="propagate")
+
+            for _, isort in enumerate(interim_sorted):
+                for r, _ in enumerate(ranks):
+                    ranks[r] += isort[r]
+
+            if self.scenario.objective_min:
+                res_ranks = rankdata(res, method='dense', nan_policy="propagate")
+
+                duplicates = []
+
+                for rank in sorted(set(res_ranks)):
+                    duplicates.append(self.duplicates(res_ranks, rank))
+                
+                for duplicate in duplicates:
+                    if len(duplicate) > 1:
+                        interim_ranks = [ranks[dup] for dup in duplicate]
+                        if not all(np.isnan(ir) for ir in interim_ranks):
+                            tie_winner = \
+                                duplicate[interim_ranks.index(min(interim_ranks))]
+                            interim_ranks = rankdata(interim_ranks, method='dense')
+                            interim_ranks -= min(interim_ranks)
+                            for d, ir in zip(duplicate, interim_ranks):
+                                for r, _ in enumerate(res_ranks):
+                                    if d == r and r != winner and r != tie_winner:
+                                        res_ranks[r] += interim_ranks[ir]
+                                    elif np.isnan(max(interim_ranks)):
+                                        res_ranks[r] += self.scenario.number_cores
+                                    else:
+                                        res_ranks[r] += max(interim_ranks)
+
+                ranks = res_ranks
+
+            ranks = rankdata(ranks, method='dense')
+
+        else:
+            ranks = [1 for core in range(self.scenario.number_cores)]
 
         ranks[winner] = 0
 
@@ -560,7 +577,8 @@ def init_cppl(self, scenario, pool, random_config_gen, contender_dict,
 
         queue.put((bandit, bandit_models))
 
-    def process_tourn(self, rtac_data: RTACData, instance: str = None, tourn_nr: int = None) -> str:
+    def process_tourn(self, rtac_data: RTACData, instance: str = None,
+                      tourn_nr: int = None) -> str:
         """Manage result processing.
 
         :param rtac_data: Object containing data and objects necessary
@@ -615,7 +633,8 @@ def cppl_process_results(self, contender_dict, scenario, rtac_data,
 
         queue.put((bandit, bandit_models, results, times))
 
-    def process_results(self, rtac_data: RTACData, instance: str = None, tourn_nr: int = None) -> None:
+    def process_results(self, rtac_data: RTACData, instance: str = None,
+                        tourn_nr: int = None) -> None:
         """Perform tournament result processing necessary to replace contenders
         in pool and select contenders for next tournament/problem instance.
 

rtac/ac_functionalities/result_processing.py

@@ -176,7 +176,7 @@ def __init__(self, scenario: argparse.Namespace, **kwargs) -> None:
         self.tournID = 0
         self.cores_start = Manager().list(
             [core for core in range(scenario.number_cores)])
-        self.early_start_tournament = False
+        self.early_start_tournament = Value('b', False)  # False
         self.event = Value('i', 0)
         self.newtime = Value('d', float(scenario.timeout))
         self.best_res = Value('d', huge_res)
@@ -242,12 +242,57 @@ def __init__(self, scenario: argparse.Namespace, rtacdata_init, **kwargs) -> Non
         :param scenario: Namespace containing all settings for the RTAC.
         :type scenario: argparse.Namespace
         """
+        self.scenario = scenario
         self.rec_data = {core: Manager().dict()
                          for core in range(scenario.number_cores)}
 
         self.RuntimeFeatures = Manager().list(
             [[] for core in range(scenario.number_cores)])
 
+    def early_rtac_copy(self):
+        early_rtac_data = self.__class__.__new__(self.__class__)
+
+        overrides = {
+            'ev': Event(),
+            'cores_start': self.cores_start,
+            'early_start_tournament': Value('b', False),
+            'event': Value('i', 0),
+            'newtime': Value('d', float(self.scenario.timeout)),
+            'best_res': Value('d', sys.float_info.max * 1e-100),
+            'winner': Manager().Value('c', 0),
+            'status': Array('i',
+                            [0 for core in range(self.scenario.number_cores)]),
+            'pids': Array('i',
+                          [0 for core in range(self.scenario.number_cores)]),
+            'substart':
+            Array('d', [0.0 for core in range(self.scenario.number_cores)]),
+            'substart_wall':
+            Array('d', [0.0 for core in range(self.scenario.number_cores)]),
+            'ta_res':
+            Array('d', [sys.float_info.max * 1e-100
+                        for core in range(self.scenario.number_cores)]),
+            'ta_res_time':
+            Array('d', [self.scenario.timeout * self.scenario.runtimePAR
+                        for core in range(self.scenario.number_cores)]),
+            'ta_rtac_time':
+            Array('d', [self.scenario.timeout * self.scenario.runtimePAR
+                        for core in range(self.scenario.number_cores)]),
+            'process':
+            ['process_{0}'.format(s)
+             for s in range(self.scenario.number_cores)],
+            'start': time.time()
+        }
+
+        # Selectively deepcopy or shallow copy attributes
+        for key, value in self.__dict__.items():
+            if key in overrides:
+                setattr(early_rtac_data, key, overrides[key])
+            else:
+                # Safe to deepcopy
+                setattr(early_rtac_data, key, copy.deepcopy(value))
+
+        return early_rtac_data
+
 
 def rtacdata_factory(scenario: argparse.Namespace, **kwargs) \
         -> AbstractRTACData:
@@ -272,6 +317,7 @@ class rtacdata_copy(rtacdata):
 
         rtacdata_init = rtacdata.__init__
         rtacdata_copy.__init__ = GBData.__init__
+        rtacdata_copy.early_rtac_copy = GBData.early_rtac_copy
 
         return rtacdata_copy(scenario, rtacdata_init, **kwargs)
 

rtac/ac_functionalities/rtac_data.py

@@ -276,7 +276,7 @@ def run(self, instance: str, config: Configuration,
         while self.running:
             # Avoid checking output excessively often (causes too much
             # overhead)
-            time.sleep(5e-6)
+            time.sleep(0.005)  # time.sleep(5e-6)
 
             ta_output = non_block_read(self.proc.stdout, self.logs)