Merge branch 'master' into rainbow_scores

Author: prabhatnagarajan

.pfnci/config.pbtxt

@@ -10,6 +10,9 @@ configs {
       memory: 30
       gpu: 1
     }
+    time_limit {
+      seconds: 1200
+    }
     environment_variables { key: "GPU" value: "1" }
     command: "bash .pfnci/script.sh py3.gpu"
   }
@@ -24,6 +27,9 @@ configs {
       cpu: 10
       memory: 10
     }
+    time_limit {
+      seconds: 1200
+    }
     command: "bash .pfnci/script.sh py3.cpu"
   }
 }
@@ -53,6 +59,9 @@ configs {
       cpu: 10
       memory: 10
     }
+    time_limit {
+      seconds: 1200
+    }
     command: "bash .pfnci/script.sh py3.chainer4"
   }
 }

examples/mujoco/reproduction/trpo/README.md

@@ -0,0 +1,49 @@
+# TRPO on MuJoCo benchmarks
+
+This example trains a TRPO agent ([Trust Region Policy Optimization](https://arxiv.org/abs/1502.05477)) on MuJoCo benchmarks from OpenAI Gym.
+
+We follow the training and evaluation settings of [Deep Reinforcement Learning that Matters](https://arxiv.org/abs/1709.06560), which provides thorough, highly tuned benchmark results.
+
+## Requirements
+
+- MuJoCo Pro 1.5
+- mujoco_py>=1.50, <2.1
+
+## Running the Example
+
+```
+python train_trpo.py [options]
+```
+
+### Useful Options
+
+- `--gpu`. Specifies the GPU. If you do not have a GPU on your machine, run the example with the option `--gpu -1`. E.g. `python train_trpo.py --gpu -1`.
+- `--env`. Specifies the environment. E.g. `python train_trpo.py --env HalfCheetah-v2`.
+- `--render`. Add this option to render the states in a GUI window.
+- `--seed`. This option specifies the random seed used.
+- `--outdir` This option specifies the output directory to which the results are written.
+
+To view the full list of options, either view the code or run the example with the `--help` option.
+
+## Results
+
+These scores are evaluated by average return +/- standard error of 100 evaluation episodes after 2M training steps.
+
+Reported scores are taken from the row Table 1 of [Deep Reinforcement Learning that Matters](https://arxiv.org/abs/1709.06560).
+Here we try to reproduce TRPO (Schulman et al. 2017) of the (64, 64) column, which corresponds to the default settings.
+
+| Environment    | ChainerRL Score | Reported Score |
+| -------------- |:---------------:|:--------------:|
+| HalfCheetah-v2 |  **1474**+/-112 |      205+/-256 |
+| Hopper-v2      |   **3056**+/-44 |      2828+/-70 |
+| Walker2d-v2    |       3073+/-59 |            N/A |
+| Swimmer-v2     |        200+/-25 |            N/A |
+
+### Learning Curves
+
+The shaded region represents a standard deviation of the average evaluation over 20 trials.
+
+![HalfCheetah-v2](assets/HalfCheetah-v2.png)
+![Hopper-v2](assets/Hopper-v2.png)
+![Walker2d-v2](assets/Walker2d-v2.png)
+![Swimmer-v2](assets/Swimmer-v2.png)

examples/mujoco/reproduction/trpo/assets/HalfCheetah-v2.png

@@ -0,0 +1,190 @@
+"""A training script of TRPO on OpenAI Gym Mujoco environments.
+
+This script follows the settings of https://arxiv.org/abs/1709.06560 as much
+as possible.
+"""
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from builtins import *  # NOQA
+from future import standard_library
+standard_library.install_aliases()  # NOQA
+
+import argparse
+import logging
+import os
+
+import chainer
+from chainer import functions as F
+from chainer import links as L
+import gym
+import gym.spaces
+import gym.wrappers
+import numpy as np
+
+import chainerrl
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu', type=int, default=0,
+                        help='GPU device ID. Set to -1 to use CPUs only.')
+    parser.add_argument('--env', type=str, default='Hopper-v2',
+                        help='Gym Env ID')
+    parser.add_argument('--seed', type=int, default=0,
+                        help='Random seed [0, 2 ** 32)')
+    parser.add_argument('--outdir', type=str, default='results',
+                        help='Directory path to save output files.'
+                             ' If it does not exist, it will be created.')
+    parser.add_argument('--steps', type=int, default=2 * 10 ** 6,
+                        help='Total time steps for training.')
+    parser.add_argument('--eval-interval', type=int, default=100000,
+                        help='Interval between evaluation phases in steps.')
+    parser.add_argument('--eval-n-runs', type=int, default=100,
+                        help='Number of episodes ran in an evaluation phase')
+    parser.add_argument('--render', action='store_true', default=False,
+                        help='Render the env')
+    parser.add_argument('--demo', action='store_true', default=False,
+                        help='Run demo episodes, not training')
+    parser.add_argument('--load', type=str, default='',
+                        help='Directory path to load a saved agent data from'
+                             ' if it is a non-empty string.')
+    parser.add_argument('--trpo-update-interval', type=int, default=5000,
+                        help='Interval steps of TRPO iterations.')
+    parser.add_argument('--logger-level', type=int, default=logging.INFO,
+                        help='Level of the root logger.')
+    parser.add_argument('--monitor', action='store_true',
+                        help='Monitor the env by gym.wrappers.Monitor.'
+                             ' Videos and additional log will be saved.')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=args.logger_level)
+
+    # Set random seed
+    chainerrl.misc.set_random_seed(args.seed, gpus=(args.gpu,))
+
+    args.outdir = chainerrl.experiments.prepare_output_dir(args, args.outdir)
+
+    def make_env(test):
+        env = gym.make(args.env)
+        # Use different random seeds for train and test envs
+        env_seed = 2 ** 32 - 1 - args.seed if test else args.seed
+        env.seed(env_seed)
+        # Cast observations to float32 because our model uses float32
+        env = chainerrl.wrappers.CastObservationToFloat32(env)
+        if args.monitor:
+            env = gym.wrappers.Monitor(env, args.outdir)
+        if args.render:
+            env = chainerrl.wrappers.Render(env)
+        return env
+
+    env = make_env(test=False)
+    timestep_limit = env.spec.tags.get(
+        'wrapper_config.TimeLimit.max_episode_steps')
+    obs_space = env.observation_space
+    action_space = env.action_space
+    print('Observation space:', obs_space)
+    print('Action space:', action_space)
+
+    assert isinstance(obs_space, gym.spaces.Box)
+
+    # Normalize observations based on their empirical mean and variance
+    obs_normalizer = chainerrl.links.EmpiricalNormalization(
+        obs_space.low.size, clip_threshold=5)
+
+    # Orthogonal weight initialization is used as OpenAI Baselines does
+    winit = chainerrl.initializers.Orthogonal(1.)
+    winit_last = chainerrl.initializers.Orthogonal(1e-2)
+
+    action_size = action_space.low.size
+    policy = chainer.Sequential(
+        L.Linear(None, 64, initialW=winit),
+        F.tanh,
+        L.Linear(None, 64, initialW=winit),
+        F.tanh,
+        L.Linear(None, action_size, initialW=winit_last),
+        chainerrl.policies.GaussianHeadWithStateIndependentCovariance(
+            action_size=action_size,
+            var_type='diagonal',
+            var_func=lambda x: F.exp(2 * x),  # Parameterize log std
+            var_param_init=0,  # log std = 0 => std = 1
+        ),
+    )
+
+    vf = chainer.Sequential(
+        L.Linear(None, 64, initialW=winit),
+        F.tanh,
+        L.Linear(None, 64, initialW=winit),
+        F.tanh,
+        L.Linear(None, 1, initialW=winit),
+    )
+
+    if args.gpu >= 0:
+        chainer.cuda.get_device_from_id(args.gpu).use()
+        policy.to_gpu(args.gpu)
+        vf.to_gpu(args.gpu)
+        obs_normalizer.to_gpu(args.gpu)
+
+    # TRPO's policy is optimized via CG and line search, so it doesn't require
+    # a chainer.Optimizer. Only the value function needs it.
+    vf_opt = chainer.optimizers.Adam()
+    vf_opt.setup(vf)
+
+    # Draw the computational graph and save it in the output directory.
+    fake_obs = chainer.Variable(
+        policy.xp.zeros_like(obs_space.low, dtype=np.float32)[None],
+        name='observation')
+    chainerrl.misc.draw_computational_graph(
+        [policy(fake_obs)], os.path.join(args.outdir, 'policy'))
+    chainerrl.misc.draw_computational_graph(
+        [vf(fake_obs)], os.path.join(args.outdir, 'vf'))
+
+    # Hyperparameters in http://arxiv.org/abs/1709.06560
+    agent = chainerrl.agents.TRPO(
+        policy=policy,
+        vf=vf,
+        vf_optimizer=vf_opt,
+        obs_normalizer=obs_normalizer,
+        update_interval=args.trpo_update_interval,
+        max_kl=0.01,
+        conjugate_gradient_max_iter=20,
+        conjugate_gradient_damping=1e-1,
+        gamma=0.995,
+        lambd=0.97,
+        vf_epochs=5,
+        entropy_coef=0,
+    )
+
+    if args.load:
+        agent.load(args.load)
+
+    if args.demo:
+        env = make_env(test=True)
+        eval_stats = chainerrl.experiments.eval_performance(
+            env=env,
+            agent=agent,
+            n_steps=None,
+            n_episodes=args.eval_n_runs,
+            max_episode_len=timestep_limit)
+        print('n_runs: {} mean: {} median: {} stdev {}'.format(
+            args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
+            eval_stats['stdev']))
+    else:
+
+        chainerrl.experiments.train_agent_with_evaluation(
+            agent=agent,
+            env=env,
+            eval_env=make_env(test=True),
+            outdir=args.outdir,
+            steps=args.steps,
+            eval_n_steps=None,
+            eval_n_episodes=args.eval_n_runs,
+            eval_interval=args.eval_interval,
+            train_max_episode_len=timestep_limit,
+        )
+
+
+if __name__ == '__main__':
+    main()

examples/mujoco/reproduction/trpo/assets/Hopper-v2.png

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -Ceu
+
+outdir=$(mktemp -d)
+
+gpu="$1"
+
+# mujoco/reproduction/trpo (specify non-mujoco env to test without mujoco)
+python examples/mujoco/reproduction/trpo/train_trpo.py --steps 10 --trpo-update-interval 5 --outdir $outdir/mujoco/reproduction/trpo --env Pendulum-v0 --gpu $gpu
+model=$(find $outdir/mujoco/reproduction/trpo -name "*_finish")
+python examples/mujoco/reproduction/trpo/train_trpo.py --demo --load $model --eval-n-runs 1 --env Pendulum-v0 --outdir $outdir/temp --gpu $gpu