pre for other stable_baselines algorthems

Author: Sohojoe

.gitignore

@@ -94,3 +94,4 @@ dist/
 build/
 summaries.001/
 filename.monitor.csv
+logs/

UnityVecEnv.py

@@ -24,21 +24,21 @@ class UnityVecEnv(VecEnv):
     """
 
     @staticmethod
-    def GetFilePath(env_id, inference_mode=False):
+    def GetFilePath(env_id, inference_mode=False, n_agents=1):
         import psutil
         env_name = MarathonEnvs[env_id]
-        if not inference_mode:
-            env_name = env_name + '-x16'
-        else:
+        if inference_mode:
             env_name = env_name + '-run'
+        elif n_agents is 16:
+            env_name = env_name + '-x16'
         if psutil.MACOS:
             env_path = os.path.join('envs', env_name)
         elif psutil.WINDOWS:
             env_path = os.path.join('envs', env_name, 'Unity Environment.exe')
         return env_path
 
-    def __init__(self, env_id):
-        env_path = UnityVecEnv.GetFilePath(env_id)
+    def __init__(self, env_id, n_agents):
+        env_path = UnityVecEnv.GetFilePath(env_id, n_agents=n_agents)
         print ("**** ", env_path)
         env = UnityEnv(env_path, multiagent=True)
         self.env = env

hyperparams/a2c.yml

@@ -1,3 +1,12 @@
+MarathonEnvs:
+  n_agents: 16
+  n_timesteps: !!float 1e6
+  policy: 'MlpPolicy'
+  vf_coef: 0.25
+  learning_rate: !!float 3e-4
+  epsilon: !!float 1e-5
+  normalize: true
+
 atari:
   policy: 'CnnPolicy'
   n_envs: 16

hyperparams/acer.yml

@@ -1,3 +1,10 @@
+MarathonEnvs:
+  n_agents: 16
+  n_timesteps: !!float 1e6
+  policy: 'MlpPolicy'
+  learning_rate: 3e-4
+  normalize: true
+
 atari:
   policy: 'CnnPolicy'
   n_envs: 16

hyperparams/acktr.yml

@@ -1,3 +1,10 @@
+MarathonEnvs:
+  n_agents: 16
+  n_timesteps: !!float 1e6
+  policy: 'MlpPolicy'
+  learning_rate: 3e-4
+  normalize: true
+
 atari:
   policy: 'CnnPolicy'
   n_envs: 32

hyperparams/ddpg.yml

@@ -1,3 +1,10 @@
+MarathonEnvs:
+  n_agents: 1
+  n_timesteps: !!float 1e6
+  policy: 'MlpPolicy'
+  # learning_rate: 3e-4
+  # normalize: true
+
 MountainCarContinuous-v0:
   n_timesteps: 300000
   policy: 'MlpPolicy'

hyperparams/ppo2.yml

@@ -1,19 +1,5 @@
-MarathonHopperEnv-v0:  
-  n_envs: 16
-  n_timesteps: !!float 1e6
-  policy: 'MlpPolicy'
-  normalize: true
-  n_steps: 128 # 2048 / number of agents
-  nminibatches: 32
-  lam: 0.95
-  gamma: 0.99
-  noptepochs: 10
-  ent_coef: 0.0
-  learning_rate: lin_3e-4
-  cliprange: 0.2
-
-MarathonWalker2DEnv-v0:  
-  n_envs: 16
+MarathonEnvs:
+  n_agents: 16
   n_timesteps: !!float 1e6
   policy: 'MlpPolicy'
   normalize: true

sb_enjoy.py

@@ -43,6 +43,8 @@
 
 if algo in ['dqn', 'ddpg']:
     args.n_envs = 1
+if 'n_agents' not in args:
+    args.n_agents = 1 # 1 agent for playback
 
 set_global_seeds(args.seed)
 
@@ -54,7 +56,7 @@
 
 log_dir = args.reward_log if args.reward_log != '' else None
 
-env = create_test_env(env_id, n_envs=args.n_envs, is_atari=is_atari,
+env = create_test_env(env_id, n_envs=args.n_envs, n_agents=args.n_agents, is_atari=is_atari,
                       stats_path=stats_path, norm_reward=args.norm_reward,
                       seed=args.seed, log_dir=log_dir, should_render=not args.no_render)
 

sb_train.py

@@ -51,21 +51,26 @@
     is_atari = False
     if 'NoFrameskip' in env_id:
         is_atari = True
+    is_marathon_envs = False
+    if 'Marathon' in env_id:
+        is_marathon_envs = True
 
     print("=" * 10, env_id, "=" * 10)
 
     # Load hyperparameters from yaml file
     with open('hyperparams/{}.yml'.format(args.algo), 'r') as f:
         if is_atari:
             hyperparams = yaml.load(f)['atari']
+        elif is_marathon_envs:
+            hyperparams = yaml.load(f)['MarathonEnvs']
         else:
-            # hyperparams = yaml.load(f)['atari']
-            hyperparams = yaml.load(f)['MarathonHopperEnv-v0']
-            # hyperparams = yaml.load(f)[env_id]
+            hyperparams = yaml.load(f)[env_id]
 
     n_envs = hyperparams.get('n_envs', 1)
+    n_agents = hyperparams.get('n_agents', 1)
 
     print("Using {} environments".format(n_envs))
+    print("With {} agents per enviroment".format(n_agents))
 
     # Create learning rate schedules for ppo2
     if args.algo == "ppo2":
@@ -91,10 +96,14 @@
     if 'normalize' in hyperparams.keys():
         normalize = hyperparams['normalize']
         del hyperparams['normalize']
+        if args.algo in ['dqn', 'ddpg']:
+            print("WARNING: normalization not supported yet for DDPG/DQN")
 
     # Delete keys so the dict can be pass to the model constructor
     if 'n_envs' in hyperparams.keys():
         del hyperparams['n_envs']
+    if 'n_agents' in hyperparams.keys():
+        del hyperparams['n_agents']
     del hyperparams['n_timesteps']
 
     # Create the environment and wrap it if necessary
@@ -103,17 +112,21 @@
         env = make_atari_env(env_id, num_env=n_envs, seed=args.seed)
         # Frame-stacking with 4 frames
         env = VecFrameStack(env, n_stack=4)
-    elif args.algo in ['dqn', 'ddpg']:
-        if hyperparams.get('normalize', False):
-            print("WARNING: normalization not supported yet for DDPG/DQN")
-        env = gym.make(env_id)
-        env.seed(args.seed)
     elif 'Marathon' in env_id:
         from UnityVecEnv import UnityVecEnv
-        env = UnityVecEnv(env_id)
+        if n_agents is 1:
+            from gym_unity.envs import UnityEnv
+            env_path = UnityVecEnv.GetFilePath(env_id, n_agents=n_agents)
+            env = UnityEnv(env_path)
+            env = DummyVecEnv([lambda: env])  # The algorithms require a vectorized environment to run
+        else:
+            env = UnityVecEnv(env_id, n_agents=n_agents)
         if normalize:
             print("Normalizing input and return")
             env = VecNormalize(env)
+    elif args.algo in ['dqn', 'ddpg']:
+        env = gym.make(env_id)
+        env.seed(args.seed)
     else:
         if n_envs == 1:
             env = DummyVecEnv([make_env(env_id, 0, args.seed)])

utils/utils.py

@@ -63,14 +63,15 @@ def _init():
     return _init
 
 
-def create_test_env(env_id, n_envs=1, is_atari=False,
+def create_test_env(env_id, n_envs=1, n_agents=1, is_atari=False,
                     stats_path=None, norm_reward=False, seed=0,
                     log_dir='', should_render=True):
     """
     Create environment for testing a trained agent
 
     :param env_id: (str)
     :param n_envs: (int) number of processes
+    :param n_agents: (int) number of agents per enviroment
     :param is_atari: (bool)
     :param stats_path: (str) path to folder containing saved running averaged
     :param norm_reward: (bool) Whether to normalize rewards or not when using Vecnormalize