Merge pull request #50 from polixir/dev

Dev (0.6.2)

Author: xionghuichen

README.md

@@ -122,7 +122,7 @@ pip install -e .
 We build an example project to include most of the features of RLA, which can be seen in ./example/simplest_code. Now we summarize the steps to use it.
 
 ### Step1: Configuration. 
-1. To configure the experiment "database", you need to create a YAML file rla_config.yaml. You can use the template provided in ./example/simplest_code/rla_config.yaml as a starting point.
+1. To configure the experiment "database", you need to create a YAML file rla_config.yaml. You can use the template provided in ./example/rla_config.yaml as a starting point.
 2. Before starting your experiment, you should configure the RLA.exp_manager object. Here's an example:
 
     ```python
@@ -256,7 +256,7 @@ from RLA import MatplotlibRecorder as mpr
 def plot_func():
    import matplotlib.pyplot as plt
    plt.plot([1,1,1], [2,2,2])
-mpr.pretty_plot_wrapper('func', plot_func, xlabel='x', ylabel='y', title='react test', )
+mpr.pretty_plot_wrapper('func', plot_func, pretty_plot=True, xlabel='x', ylabel='y', title='react test')
 ```
 
 This code plots a figure using Matplotlib and saves it in the "results" directory.

RLA/const.py

@@ -1,5 +1,5 @@
 DEFAULT_X_NAME = 'time-step'
-
+DEFAULT_TIMESTAMP = 'timestamp'
 class FRAMEWORK:
     tensorflow = 'tensorflow'
     torch = 'torch'
@@ -9,4 +9,11 @@ class FTP_PROTOCOL_NAME:
     SFTP = 'sftp'
 
 class LOG_NAME_FORMAT_VERSION:
-    V1 = 'v1'
+    V1 = 'v1'
+
+
+
+class PLATFORM_TYPE:
+    WIN = 'win'
+    LINUX = 'linux'
+    OTHER = 'other'

RLA/easy_log/complex_data_recorder.py

@@ -34,7 +34,7 @@ def save(cls, name=None, fig=None, cover=False, add_timestamp=True, **kwargs):
 
     @classmethod
     def pretty_plot_wrapper(cls, name:str, plot_func:Callable,
-                            cover=False, legend_outside=False, xlabel='', ylabel='', title='',
+                            cover=False, legend_outside=False, pretty_plot=False, xlabel='', ylabel='', title='',
                             add_timestamp=True, *args, **kwargs):
         """
         Save the customized plot figure to the RLA database.
@@ -47,6 +47,8 @@ def pretty_plot_wrapper(cls, name:str, plot_func:Callable,
         :type cover: bool
         :param legend_outside: let legend be outside of the figure.
         :type legend_outside: bool
+        :param pretty_plot: use predefined configurations for plotting.
+        :type pretty_plot: bool
         :param xlabel: name of xlabel
         :type xlabel: str
         :param ylabel: name of xlabel
@@ -66,12 +68,13 @@ def pretty_plot_wrapper(cls, name:str, plot_func:Callable,
         plot_func()
         lgd = plt.legend(prop={'size': 15}, loc=2 if legend_outside else None,
                          bbox_to_anchor=(1, 1) if legend_outside else None)
-        plt.xlabel(xlabel, fontsize=18)
-        plt.ylabel(ylabel, fontsize=18)
-        plt.xticks(fontsize=14)
-        plt.yticks(fontsize=14)
-        plt.title(title, fontsize=13)
-        plt.grid(True)
+        if pretty_plot:
+            plt.xlabel(xlabel, fontsize=18)
+            plt.ylabel(ylabel, fontsize=18)
+            plt.xticks(fontsize=14)
+            plt.yticks(fontsize=14)
+            plt.title(title, fontsize=13)
+            plt.grid(True)
         if lgd is not None:
             cls.save(name, cover=cover, add_timestamp=add_timestamp, bbox_extra_artists=tuple([lgd]),
                      bbox_inches='tight', *args, **kwargs)

RLA/easy_log/exp_loader.py

@@ -48,7 +48,7 @@ def is_valid_config(self):
         if self.load_date is not None and self.task_name is not None and self.data_root is not None:
             return True
         else:
-            logger.warn("meet invalid loader config when use it")
+            logger.warn("meet invalid loader config when using it")
             logger.warn("load_date", self.load_date)
             logger.warn("task_name", self.task_name)
             logger.warn("root", self.data_root)

RLA/easy_log/log_tools.py

@@ -16,7 +16,7 @@
 
 import json
 from RLA.easy_log.tester import Tester
-
+from RLA.utils.utils import get_dir_seperator
 class Filter(object):
     ALL = 'all'
     SMALL_TIMESTEP = 'small_ts'
@@ -214,8 +214,9 @@ def _archive_log(self, show=False):
                 empty = False
                 if os.path.exists(root_dir):
                     # remove the overlapped path.
+                    septor = get_dir_seperator()
                     archiving_target = osp.join(archive_root_dir, root_dir[prefix_len+1:])
-                    archiving_target_dir = '/'.join(archiving_target.split('/')[:-1])
+                    archiving_target_dir = septor.join(archiving_target.split(septor)[:-1])
                     os.makedirs(archiving_target_dir, exist_ok=True)
                     if os.path.isdir(root_dir):
                         if not show:
@@ -269,21 +270,23 @@ def _migrate_log(self, show=False):
                 if os.path.exists(root_dir):
                     # remove the overlapped path.
                     archiving_target = osp.join(target_root_dir, root_dir[prefix_len+1:])
-                    archiving_target_dir = '/'.join(archiving_target.split('/')[:-1])
+
+                    septor = get_dir_seperator()
+                    archiving_target_dir = septor.join(archiving_target.split(septor)[:-1])
+                    print("target dir", archiving_target_dir)
                     os.makedirs(archiving_target_dir, exist_ok=True)
                     if os.path.isdir(root_dir):
+                        print("copy dir {}, to {}".format(root_dir, archiving_target))
                         if not show:
                             # os.makedirs(archiving_target, exist_ok=True)
                             try:
                                 shutil.copytree(root_dir, archiving_target)
                             except FileExistsError as e:
                                 print(e)
-
-                        print("copy dir {}, to {}".format(root_dir, archiving_target))
                     else:
+                        print("copy file {}, to {}".format(root_dir, archiving_target))
                         if not show:
                             shutil.copy(root_dir, archiving_target)
-                        print("copy file {}, to {}".format(root_dir, archiving_target))
                 else:
                     print("no dir {}".format(root_dir))
             if empty: print("empty regex {}".format(root_dir_regex))

RLA/easy_log/logger.py

@@ -13,7 +13,7 @@
 
 from typing import Any, Dict, List, Optional, Sequence, TextIO, Tuple, Union
 from contextlib import contextmanager
-from RLA.const import DEFAULT_X_NAME, FRAMEWORK
+from RLA.const import DEFAULT_X_NAME, FRAMEWORK, DEFAULT_TIMESTAMP
 
 DEBUG = 10
 INFO = 20
@@ -510,6 +510,12 @@ def dumpkvs():
             print(e)
             for fmt in Logger.CURRENT.output_formats:
                 print(fmt)
+        try:
+            get_current().logkv(DEFAULT_TIMESTAMP, time.time())
+        except NotImplementedError as e:
+            print(e)
+            for fmt in Logger.CURRENT.output_formats:
+                print(fmt)
     return get_current().dumpkvs()
 
 def getkvs():

RLA/easy_log/tester.py

@@ -26,7 +26,7 @@
 import shutil
 import argparse
 from typing import Dict, List, Tuple, Type, Union, Optional
-from RLA.utils.utils import deprecated_alias, load_yaml
+from RLA.utils.utils import deprecated_alias, load_yaml, get_dir_seperator
 from RLA.const import DEFAULT_X_NAME, FRAMEWORK
 import pathspec
 
@@ -150,10 +150,11 @@ def configure(self, task_table_name: str, rla_config: Union[str, dict], data_roo
         logger.info("private_config: ")
         self.dl_framework = self.private_config["DL_FRAMEWORK"]
         self.is_master_node = is_master_node
-
+        self.septor = get_dir_seperator()
         if code_root is None:
             if isinstance(rla_config, str):
-                self.project_root = "/".join(rla_config.split("/")[:-1])
+
+                self.project_root = self.septor.join(rla_config.split(self.septor)[:-1])
             else:
                 raise NotImplementedError("If you pass the rla_config dict directly, "
                                           "you should define the root of your codebase (for backup) explicitly by pass the code_root.")
@@ -309,14 +310,19 @@ def load_tester(cls, record_date, task_table_name, log_root):
     def add_record_param(self, keys):
         for k in keys:
             if '.' in k:
+                sub_k = None
                 try:
                     sub_k_list = k.split('.')
-                    v = self.hyper_param[sub_k_list[0]]
+                    sub_k = sub_k_list[0]
+                    v = self.hyper_param[sub_k]
                     for sub_k in sub_k_list[1:]:
                         v = v[sub_k]
                     self.hyper_param_record.append(str(k) + '=' + str(v).replace('[', '{').replace(']', '}').replace('/', '_'))
                 except KeyError as e:
-                    print("do not include dot ('.') in your hyperparemeter name")
+                    print(f"the current key to parsed is: {k}. Can not find a matching key for {sub_k}."
+                          "\n Hint: do not include dot ('.') in your hyperparemeter name."
+                          "\n The recorded hyper parameters are")
+                    self.print_args()
             else:
                 self.hyper_param_record.append(str(k) + '=' + str(self.hyper_param[k]).replace('[', '{').replace(']', '}').replace('/', '_'))
 
@@ -620,7 +626,7 @@ def time_record_end(self, name:str):
         end_time = time.time()
         start_time = self._rc_start_time[name]
         logger.record_tabular("time_used/{}".format(name), end_time - start_time)
-        logger.info("[test] func {0} time used {1:.2f}".format(name, end_time - start_time))
+        # logger.info("[test] func {0} time used {1:.2f}".format(name, end_time - start_time))
         del self._rc_start_time[name]
 
     # Saver manger.

RLA/easy_log/time_step.py

@@ -1,5 +1,5 @@
 from RLA.easy_log import logger
-
+import time
 
 class TimeStepHolder(object):
 
@@ -8,20 +8,10 @@ def __init__(self, time, epoch, tf_log=False):
         self.__outer_epoch = epoch
         self.tf_log = tf_log
 
-    def config(self, time=0, epoch=0, tf_log=False):
+    def config(self, time=0, tf_log=False):
         self.__timesteps = time
-        self.__outer_epoch = epoch
         self.tf_log = tf_log
 
-    def set_outer_epoch(self, epoch):
-        self.__outer_epoch = epoch
-
-    def get_outer_epoch(self):
-        return self.__outer_epoch
-
-    def inc_outer_epoch(self):
-        self.__outer_epoch +=1
-
     def set_time(self, time):
         self.__timesteps = time
         self.__update_tf_times()

RLA/easy_log/time_used_recorder.py

@@ -103,5 +103,5 @@ def time_record_end(name):
     end_time = time.time()
     start_time = rc_start_time[name]
     logger.record_tabular("time_used/{}".format(name), end_time - start_time)
-    logger.info("[test] func {0} time used {1:.2f}".format(name, end_time - start_time))
+    # logger.info("[test] func {0} time used {1:.2f}".format(name, end_time - start_time))
     del rc_start_time[name]

RLA/easy_plot/__init__.py

@@ -0,0 +1,2 @@
+from RLA.easy_plot.plot_saved_images import plot_saved_images
+from RLA.easy_plot.plot_func_v2 import plot_func