updated gym to latest version. Gymnasium

Author: rubenlucas93

rl_studio/agents/pendulum/inference_ddpg.py

@@ -2,7 +2,7 @@
 import time
 import random
 
-import gym
+import gymnasium as gym
 import matplotlib.pyplot as plt
 from torch.utils import tensorboard
 from tqdm import tqdm
@@ -68,11 +68,8 @@ def __init__(self, params):
         #                                   # ,random_start_level=self.RANDOM_START_LEVEL, initial_pole_angle=self.INITIAL_POLE_ANGLE,
         #                                   # non_recoverable_angle=non_recoverable_angle
         #                                   ))
-        self.env = gym.make(self.env_name)
+        self.env = gym.make(self.env_name, render_mode="human")
         self.RUNS = self.environment_params["runs"]
-        self.SHOW_EVERY = self.environment_params[
-            "show_every"
-        ]
         self.UPDATE_EVERY = self.environment_params[
             "update_every"
         ]  # How often the current progress is recorded
@@ -129,7 +126,8 @@ def main(self):
         total_reward_in_epoch = 0
 
         for episode in tqdm(range(self.RUNS)):
-            state, done = self.env.reset(), False
+            state, _ = self.env.reset()
+            done = False
             episode_reward = 0
             step = 0
             while not done:
@@ -140,16 +138,13 @@ def main(self):
                 #     logging.debug("perturbated in step {} with action {}".format(episode_rew, perturbation_action))
 
                 action = self.inferencer.inference(state)
-                new_state, reward, done, _ = self.env.step(action)
+                new_state, reward, _, done, _ = self.env.step(action)
                 state = new_state
                 episode_reward += reward
                 total_reward_in_epoch += reward
 
                 w.add_scalar("reward/episode_reward", episode_reward, global_step=episode)
 
-                if episode % self.SHOW_EVERY == 0:
-                    self.env.render()
-
             self.gather_statistics(step, episode_reward)
 
             # monitor progress

rl_studio/agents/pendulum/requirements.txt

@@ -0,0 +1,7 @@
+gym==0.26.2
+gymnasium==0.27.0
+markdownTable==6.0.0
+matplotlib==3.3.2
+numpy==1.17.4
+torch==1.12.1
+tqdm==4.64.0

rl_studio/agents/pendulum/train_ddpg.py

@@ -2,7 +2,7 @@
 import time
 import random
 
-import gym
+import gymnasium as gym
 import matplotlib.pyplot as plt
 from torch.utils import tensorboard
 from tqdm import tqdm
@@ -69,9 +69,6 @@ def __init__(self, params):
         #                                   ))
         self.env = gym.make(self.env_name)
         self.RUNS = self.environment_params["runs"]
-        self.SHOW_EVERY = self.environment_params[
-            "show_every"
-        ]
         self.UPDATE_EVERY = self.environment_params[
             "update_every"
         ]  # How often the current progress is recorded
@@ -184,7 +181,8 @@ def main(self):
         total_reward_in_epoch = 0
 
         for episode in tqdm(range(self.RUNS)):
-            state, done = self.env.reset(), False
+            state, _ = self.env.reset()
+            done = False
             self.actor.reset_noise()
             episode_reward = 0
             step = 0
@@ -196,7 +194,7 @@ def main(self):
                 #     logging.debug("perturbated in step {} with action {}".format(episode_rew, perturbation_action))
 
                 action = self.actor.get_action(state, step)
-                new_state, reward, done, _ = self.env.step(action)
+                new_state, reward, _, done, _ = self.env.step(action)
                 self.memory.push(state, action, reward, new_state, done)
 
                 if len(self.memory) > self.batch_size:
@@ -210,9 +208,6 @@ def main(self):
                 w.add_scalar("loss/actor_loss", actor_loss, global_step=episode)
                 w.add_scalar("loss/critic_loss", critic_loss, global_step=episode)
 
-                if episode % self.SHOW_EVERY == 0:
-                    self.env.render()
-
             self.gather_statistics(actor_loss, step, episode_reward)
 
             # monitor progress
@@ -229,7 +224,7 @@ def main(self):
                 if self.config["save_model"] and last_average > self.max_avg:
                     self.max_avg = total_reward_in_epoch / self.UPDATE_EVERY
                     logging.info(f"Saving model . . .")
-                    utils.save_ddpg_model(self.actor, start_time_format, last_average, self.params)
+                    utils.save_ddpg_model(self.actor, start_time_format, last_average)
 
                 if last_average >= self.OBJECTIVE_REWARD:
                     logging.info("Training objective reached!!")

rl_studio/config/config_pendulum_ddpg.yaml

@@ -26,7 +26,7 @@ environments:
     full_experimentation_runs: 0
     update_every: 20
     show_every: 50
-    objective_reward: -400
+    objective_reward: -350
 #    block_experience_batch: False
     block_experience_batch: False
   #  random_start_level: 0.05