feat: add a view script

Author: xionghuichen

RLA/easy_log/log_tools.py

@@ -28,39 +28,13 @@ def __init__(self, optional_log_type=None):
         if optional_log_type is not None:
             self.log_types.extend(optional_log_type)
 
-class DownloadLogTool(BasicLogTool):
-    def __init__(self, rla_config_path, proj_root, task, regex, *args, **kwargs):
-        fs = open(rla_config_path, encoding="UTF-8")
-        self.private_config = yaml.load(fs)
-        self.proj_root = proj_root
-        self.task_table_name = task
-        self.regex = regex
-
-    def _download_log(self, show=False):
-        for log_type in self.log_types:
-            root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex)
-            empty = True
-            for root_dir in glob.glob(root_dir_regex):
-
-                pass
-
-class DeleteLogTool(BasicLogTool):
-    def __init__(self, proj_root, task_table_name, regex, filter, *args, **kwargs):
-        self.proj_root = proj_root
-        self.task_table_name = task_table_name
-        self.regex = regex
-        assert isinstance(filter, Filter)
-        self.filter = filter
-        self.small_timestep_regs = []
-        super(DeleteLogTool, self).__init__(*args, **kwargs)
-
-    def _find_small_timestep_log(self):
-        root_dir_regex = osp.join(self.proj_root, LOG, self.task_table_name, self.regex)
+    def _find_small_timestep_log(self, proj_root, task_table_name, regex, timstep_upper_bound=np.inf, timestep_lower_bound=0):
+        small_timestep_regs = []
+        root_dir_regex = osp.join(proj_root, LOG, task_table_name, regex)
         for root_dir in glob.glob(root_dir_regex):
             print("searching dirs", root_dir)
             if os.path.exists(root_dir):
                 for file_list in os.walk(root_dir):
-
                     if re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]):
                         target_reg = re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]).group(0)
                     else:
@@ -77,7 +51,8 @@ def _find_small_timestep_log(self):
                             if file_list[1] == ['tb'] or os.path.exists(progress_csv_file): # in root of logdir
                                 if not os.path.exists(progress_csv_file) or os.path.getsize(progress_csv_file) == 0:
                                     print("[delete] find an experiment without progress.csv.", file_list[0])
-                                    self.small_timestep_regs.append(target_reg)
+                                    if timestep_lower_bound <= 0:
+                                        small_timestep_regs.append([target_reg, file_list[0]])
                                 else:
                                     try:
                                         reader = pd.read_csv(progress_csv_file, chunksize=100000, quoting=csv.QUOTE_NONE,
@@ -88,9 +63,9 @@ def _find_small_timestep_log(self):
                                             raw_df = pd.concat([raw_df, slim_chunk], ignore_index=True)
                                         last_timestep = raw_df[DEFAULT_X_NAME].max()
                                         print("[found a log] time_step ", last_timestep, target_reg)
-                                        if last_timestep < self.filter.timstep_bound:
-                                            self.small_timestep_regs.append(target_reg)
-                                            print("[delete] find an experiment with too small number of logs. ", file_list[0])
+                                        if timestep_lower_bound <= last_timestep <= timstep_upper_bound:
+                                            small_timestep_regs.append([target_reg, file_list[0]])
+                                            print("[delete] find an experiment satisfied timestep range. ", file_list[0])
                                         else:
                                             print("[valid]")
                                     except Exception as e:
@@ -101,8 +76,35 @@ def _find_small_timestep_log(self):
                             elif 'events' in file_list[0]: # in event dir
                                 pass
                             else: # empty dir
-                                self.small_timestep_regs.append(target_reg)
+                                if timestep_lower_bound <= 0:
+                                    small_timestep_regs.append([target_reg, file_list[0]])
                                 print("[delete] find an experiment without any files. ", file_list[0])
+        return small_timestep_regs
+
+class DownloadLogTool(BasicLogTool):
+    def __init__(self, rla_config_path, proj_root, task, regex, *args, **kwargs):
+        fs = open(rla_config_path, encoding="UTF-8")
+        self.private_config = yaml.load(fs)
+        self.proj_root = proj_root
+        self.task_table_name = task
+        self.regex = regex
+
+    def _download_log(self, show=False):
+        for log_type in self.log_types:
+            root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex)
+            empty = True
+            for root_dir in glob.glob(root_dir_regex):
+                pass
+
+class DeleteLogTool(BasicLogTool):
+    def __init__(self, proj_root, task_table_name, regex, filter, *args, **kwargs):
+        self.proj_root = proj_root
+        self.task_table_name = task_table_name
+        self.regex = regex
+        assert isinstance(filter, Filter)
+        self.filter = filter
+        self.small_timestep_regs = []
+        super(DeleteLogTool, self).__init__(*args, **kwargs)
 
     def _delete_related_log(self, regex, show=False):
         log_found = 0
@@ -155,7 +157,7 @@ def delete_related_log(self, skip_ask=False):
             return 0
 
     def delete_small_timestep_log(self, skip_ask=False):
-        self._find_small_timestep_log()
+        self.small_timestep_regs = self._find_small_timestep_log(self.proj_root, self.task_table_name, self.regex, timstep_upper_bound=self.filter.timstep_bound)
         print("complete searching.")
         if skip_ask:
             s = 'y'
@@ -164,17 +166,17 @@ def delete_small_timestep_log(self, skip_ask=False):
         log_found = 0
 
         if s == 'y' or skip_ask:
-            for reg in self.small_timestep_regs:
-                print("[delete small-timestep log] reg: ", reg)
-                self._delete_related_log(show=True, regex=reg + '*')
+            for res in self.small_timestep_regs:
+                print("[delete small-timestep log] reg: ", res[1])
+                self._delete_related_log(show=True, regex=res[0] + '*')
             if skip_ask:
                 s = 'y'
             else:
                 s = input("delete these files? (y/n)")
             if s == 'y' or skip_ask:
-                for reg in self.small_timestep_regs:
-                    print("do delete: ", reg)
-                    log_found += self._delete_related_log(show=False, regex=reg + '*')
+                for res in self.small_timestep_regs:
+                    print("do delete: ", res[1])
+                    log_found += self._delete_related_log(show=False, regex=res[0] + '*')
         return log_found
 
 class ArchiveLogTool(BasicLogTool):
@@ -231,6 +233,44 @@ def archive_log(self, skip_ask=False):
             print("do archive ...")
             self._archive_log(show=False)
 
+class ViewLogTool(BasicLogTool):
+    def __init__(self, proj_root, task_table_name, regex, *args, **kwargs):
+        self.proj_root = proj_root
+        self.task_table_name = task_table_name
+        self.regex = regex
+        super(ViewLogTool, self).__init__(*args, **kwargs)
+
+    def _view_log(self, regex):
+        root_dir_regex = osp.join(self.proj_root, LOG, self.task_table_name, regex)
+        for root_dir in glob.glob(root_dir_regex):
+            if os.path.exists(root_dir):
+                for file_list in os.walk(root_dir):
+                    if re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]):
+                        target_reg = re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]).group(0)
+                    else:
+                        target_reg = None
+                    if target_reg is not None:
+                        backup_file = file_list[0] + '/backup.txt'
+                        if file_list[1] == ['tb'] or os.path.exists(backup_file):  # in root of logdir
+                            with open(backup_file) as f:
+                                print(f.read())
+
+    def view_log(self, skip_ask=False):
+        found_regs = self._find_small_timestep_log(self.proj_root, self.task_table_name, self.regex, timestep_lower_bound=1)
+        for res in found_regs:
+            print("view experiments:", res[1])
+            if skip_ask:
+                s = 'y'
+            else:
+                s = input("press y to view \n ")
+            if s == 'y':
+                self._view_log(regex=res[0] + '*')
+
+
+
+
+
+
 # if __name__ == '__main__':
 #     dlt = DeleteLogTool("../", "var_seq_imitation", "self-transfer", "2019/11/29/01-11*")
 #     dlt.delete_related_log()

rla_scripts/view_expt.py

@@ -0,0 +1,19 @@
+"""
+A script to view data of experiments.
+"""
+
+from RLA.easy_log.log_tools import ViewLogTool
+import argparse
+from config import *
+
+def argsparser():
+    parser = argparse.ArgumentParser("Archive Log")
+    parser.add_argument('--task_table_name', type=str)
+    parser.add_argument('--regex', type=str)
+    args = parser.parse_args()
+    return args
+
+if __name__=='__main__':
+    args = argsparser()
+    dlt = ViewLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table_name, regex=args.regex)
+    dlt.view_log()

test/test_scripts.py

@@ -1,6 +1,7 @@
 from test._base import BaseTest
 from RLA.easy_log.log_tools import DeleteLogTool, Filter
-from RLA.easy_log.log_tools import ArchiveLogTool
+from RLA.easy_log.log_tools import ArchiveLogTool, ViewLogTool
+
 
 class ScriptTest(BaseTest):
 
@@ -51,4 +52,8 @@ def test_archive(self):
         dlt = DeleteLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name='archived', regex='2022/03/01/21-13*', filter=filter)
         log_found = dlt.delete_related_log(skip_ask=True)
         assert log_found == 10
-    
+
+    def test_view(self):
+        self.remove_and_copy_data()
+        dlt = ViewLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name=self.TASK_NAME, regex='2022/03/01/21-13*')
+        dlt.view_log(skip_ask=True)