Use Natural Policy Gradient to Play Acrobot-v1¶

TensorFlow version

In [1]:
%matplotlib inline

import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import pandas as pd
import scipy.signal as signal
import gym
import matplotlib.pyplot as plt
import tensorflow.compat.v2 as tf
tf.random.set_seed(0)
from tensorflow import keras
from tensorflow import nn
from tensorflow import optimizers
from tensorflow import losses
from tensorflow.keras import layers

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')
In [2]:
env = gym.make('Acrobot-v1')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

14:00:56 [INFO] env: <AcrobotEnv<Acrobot-v1>>
14:00:56 [INFO] action_space: Discrete(3)
14:00:56 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
14:00:56 [INFO] reward_range: (-inf, inf)
14:00:56 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
14:00:56 [INFO] _max_episode_steps: 500
14:00:56 [INFO] _elapsed_steps: None
14:00:56 [INFO] id: Acrobot-v1
14:00:56 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
14:00:56 [INFO] reward_threshold: -100.0
14:00:56 [INFO] nondeterministic: False
14:00:56 [INFO] max_episode_steps: 500
14:00:56 [INFO] _kwargs: {}
14:00:56 [INFO] _env_name: Acrobot
In [3]:
class PPOReplayer:
    def __init__(self):
        self.fields = ['state', 'action', 'prob', 'advantage', 'return']
        self.memory = pd.DataFrame(columns=self.fields)

    def store(self, df):
        if self.memory.empty:
            self.memory = df[self.fields]
        else:
            self.memory = pd.concat([self.memory, df[self.fields]], ignore_index=True)

    def sample(self, size):
        indices = np.random.choice(self.memory.shape[0], size=size)
        return (np.stack(self.memory.loc[indices, field]) for field in
                self.fields)
In [4]:
def conjugate_gradient(f, b, iter_count=10, epsilon=1e-12, tol=1e-6):
    x = b * 0.
    r = tf.identity(b)
    p = tf.identity(b)
    rho = tf.reduce_sum(r * r)
    for i in range(iter_count):
        z = f(p)
        alpha = rho / (tf.reduce_sum(p * z) + epsilon)
        x += alpha * p
        r -= alpha * z
        rho_new = tf.reduce_sum(r * r)
        p = r + (rho_new / rho) * p
        rho = rho_new
        if rho < tol:
            break
    return x, f(x)
In [5]:
class NPGAgent:
    def __init__(self, env):
        self.action_n = env.action_space.n
        self.gamma = 0.99

        self.replayer = PPOReplayer()
        self.trajectory = []

        self.max_kl = 0.0005
        self.actor_net = self.build_net(hidden_sizes=[100,],
                output_size=self.action_n, output_activation=nn.softmax)
        self.critic_net = self.build_net(hidden_sizes=[100,],
                learning_rate=0.002)

    def build_net(self, input_size=None, hidden_sizes=None, output_size=1,
                activation=nn.relu, output_activation=None,
                loss=losses.mse, learning_rate=0.001):
        model = keras.Sequential()
        for hidden_size in hidden_sizes:
            model.add(layers.Dense(units=hidden_size,
                    activation=activation))
        model.add(layers.Dense(units=output_size,
                activation=output_activation))
        optimizer = optimizers.Adam(learning_rate)
        model.compile(optimizer=optimizer, loss=loss)
        return model

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []

    def step(self, observation, reward, terminated):
        probs = self.actor_net.predict(observation[np.newaxis], verbose=0)[0]
        action = np.random.choice(self.action_n, p=probs)
        if self.mode == 'train':
            self.trajectory += [observation, reward, terminated, action]
        return action

    def close(self):
        if self.mode == 'train':
            self.save_trajectory_to_replayer()
            if len(self.replayer.memory) >= 1000:
                for batch in range(5):  # learn multiple times
                    self.learn()
                self.replayer = PPOReplayer()
                        # reset replayer after the agent changes itself

    def save_trajectory_to_replayer(self):
        df = pd.DataFrame(
                np.array(self.trajectory, dtype=object).reshape(-1, 4),
                columns=['state', 'reward', 'terminated', 'action'], dtype=object)
        states = np.stack(df['state'])
        df['v'] = self.critic_net.predict(states, verbose=0)
        pis = self.actor_net.predict(states, verbose=0)
        df['prob'] = [pi[action] for pi, action in zip(pis, df['action'])]
        df['next_v'] = df['v'].shift(-1).fillna(0.)
        df['u'] = df['reward'] + self.gamma * df['next_v']
        df['delta'] = df['u'] - df['v']
        df['advantage'] = signal.lfilter([1.,], [1., -self.gamma],
                df['delta'][::-1])[::-1]
        df['return'] = signal.lfilter([1.,], [1., -self.gamma],
                df['reward'][::-1])[::-1]
        self.replayer.store(df)

    def learn(self):
        states, actions, old_pis, advantages, returns = \
                self.replayer.sample(size=64)
        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)
        action_tensor = tf.convert_to_tensor(actions, dtype=tf.int32)
        old_pi_tensor = tf.convert_to_tensor(old_pis, dtype=tf.float32)
        advantage_tensor = tf.convert_to_tensor(advantages, dtype=tf.float32)

        # update actor
        # ... calculate first order gradient of KL divergence
        with tf.GradientTape() as tape:
            all_pi_tensor = self.actor_net(state_tensor)
            pi_tensor = tf.gather(all_pi_tensor, action_tensor, batch_dims=1)
            surrogate_tensor = (pi_tensor / old_pi_tensor) * advantage_tensor
        actor_grads = tape.gradient(surrogate_tensor, self.actor_net.variables)
        loss_grad = tf.concat([tf.reshape(grad, (-1,)) for grad in actor_grads],
                axis=0)

        # ... calculate conjugate gradient: Fx = g
        def f(x):  # calculate Fx
            with tf.GradientTape() as tape2:  # tape for 2nd-order gradient
                with tf.GradientTape() as tape1:  # tape for 1st-order gradient
                    prob_tensor = self.actor_net(state_tensor)
                    prob_old_tensor = tf.stop_gradient(prob_tensor)
                    kld_tensor = tf.reduce_sum(prob_old_tensor * (tf.math.log(
                            prob_old_tensor) - tf.math.log(prob_tensor)), axis=1)
                    kld_loss_tensor = tf.reduce_mean(kld_tensor)
                grads = tape1.gradient(kld_loss_tensor, self.actor_net.variables)
                flatten_grad_tensor = tf.concat(
                        [tf.reshape(grad, (-1,)) for grad in grads], axis=-1)
                grad_matmul_x = tf.tensordot(flatten_grad_tensor, x,
                        axes=[[-1], [-1]])
            grad_grads = tape2.gradient(grad_matmul_x, self.actor_net.variables)
            flatten_grad_grad = tf.stop_gradient(tf.concat(
                    [tf.reshape(grad_grad, (-1,)) for grad_grad in grad_grads],
                    axis=-1))
            fx = flatten_grad_grad + x * 1e-2
            return fx
        x, fx = conjugate_gradient(f, loss_grad)

        # ... calculate natural gradient
        natural_gradient_tensor = tf.sqrt(2 * self.max_kl /
                tf.reduce_sum(fx * x)) * x
        # ....... refactor the flatten gradient into un-flatten version
        flatten_natural_gradient = natural_gradient_tensor.numpy()
        weights = []
        begin = 0
        for weight in self.actor_net.get_weights():
            end = begin + weight.size
            weight += flatten_natural_gradient[begin:end].reshape(weight.shape)
            weights.append(weight)
            begin = end
        self.actor_net.set_weights(weights)

        # update critic
        self.critic_net.fit(states, returns, verbose=0)


agent = NPGAgent(env)
In [6]:
def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
            mode='train')
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > -120:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

14:00:57 [INFO] ==== train ====
14:01:37 [INFO] NumExpr defaulting to 8 threads.
14:01:37 [INFO] train episode 0: reward = -500.00, steps = 500
14:02:18 [INFO] train episode 1: reward = -500.00, steps = 500
14:02:57 [INFO] train episode 2: reward = -500.00, steps = 500
14:03:37 [INFO] train episode 3: reward = -500.00, steps = 500
14:04:10 [INFO] train episode 4: reward = -422.00, steps = 423
14:04:40 [INFO] train episode 5: reward = -388.00, steps = 389
14:05:14 [INFO] train episode 6: reward = -411.00, steps = 412
14:05:47 [INFO] train episode 7: reward = -427.00, steps = 428
14:06:28 [INFO] train episode 8: reward = -500.00, steps = 500
14:07:12 [INFO] train episode 9: reward = -500.00, steps = 500
14:07:34 [INFO] train episode 10: reward = -279.00, steps = 280
14:08:13 [INFO] train episode 11: reward = -500.00, steps = 500
14:08:33 [INFO] train episode 12: reward = -237.00, steps = 238
14:08:55 [INFO] train episode 13: reward = -273.00, steps = 274
14:09:17 [INFO] train episode 14: reward = -282.00, steps = 283
14:09:49 [INFO] train episode 15: reward = -426.00, steps = 427
14:10:22 [INFO] train episode 16: reward = -395.00, steps = 396
14:10:51 [INFO] train episode 17: reward = -375.00, steps = 376
14:11:12 [INFO] train episode 18: reward = -252.00, steps = 253
14:11:52 [INFO] train episode 19: reward = -500.00, steps = 500
14:12:15 [INFO] train episode 20: reward = -289.00, steps = 290
14:12:48 [INFO] train episode 21: reward = -401.00, steps = 402
14:13:15 [INFO] train episode 22: reward = -310.00, steps = 311
14:13:51 [INFO] train episode 23: reward = -473.00, steps = 474
14:14:17 [INFO] train episode 24: reward = -341.00, steps = 342
14:14:57 [INFO] train episode 25: reward = -500.00, steps = 500
14:15:24 [INFO] train episode 26: reward = -340.00, steps = 341
14:16:03 [INFO] train episode 27: reward = -500.00, steps = 500
14:16:45 [INFO] train episode 28: reward = -487.00, steps = 488
14:17:17 [INFO] train episode 29: reward = -405.00, steps = 406
14:17:55 [INFO] train episode 30: reward = -500.00, steps = 500
14:18:24 [INFO] train episode 31: reward = -347.00, steps = 348
14:18:59 [INFO] train episode 32: reward = -413.00, steps = 414
14:19:32 [INFO] train episode 33: reward = -426.00, steps = 427
14:20:01 [INFO] train episode 34: reward = -367.00, steps = 368
14:20:41 [INFO] train episode 35: reward = -500.00, steps = 500
14:21:05 [INFO] train episode 36: reward = -295.00, steps = 296
14:21:24 [INFO] train episode 37: reward = -228.00, steps = 229
14:22:04 [INFO] train episode 38: reward = -500.00, steps = 500
14:22:47 [INFO] train episode 39: reward = -500.00, steps = 500
14:23:30 [INFO] train episode 40: reward = -500.00, steps = 500
14:24:10 [INFO] train episode 41: reward = -500.00, steps = 500
14:24:31 [INFO] train episode 42: reward = -266.00, steps = 267
14:25:06 [INFO] train episode 43: reward = -449.00, steps = 450
14:25:46 [INFO] train episode 44: reward = -500.00, steps = 500
14:26:19 [INFO] train episode 45: reward = -400.00, steps = 401
14:26:47 [INFO] train episode 46: reward = -339.00, steps = 340
14:27:11 [INFO] train episode 47: reward = -275.00, steps = 276
14:27:41 [INFO] train episode 48: reward = -383.00, steps = 384
14:27:57 [INFO] train episode 49: reward = -204.00, steps = 205
14:28:20 [INFO] train episode 50: reward = -273.00, steps = 274
14:28:38 [INFO] train episode 51: reward = -216.00, steps = 217
14:29:08 [INFO] train episode 52: reward = -371.00, steps = 372
14:29:46 [INFO] train episode 53: reward = -500.00, steps = 500
14:30:16 [INFO] train episode 54: reward = -355.00, steps = 356
14:30:44 [INFO] train episode 55: reward = -353.00, steps = 354
14:31:09 [INFO] train episode 56: reward = -308.00, steps = 309
14:31:45 [INFO] train episode 57: reward = -456.00, steps = 457
14:32:24 [INFO] train episode 58: reward = -500.00, steps = 500
14:32:41 [INFO] train episode 59: reward = -226.00, steps = 227
14:33:17 [INFO] train episode 60: reward = -449.00, steps = 450
14:33:42 [INFO] train episode 61: reward = -323.00, steps = 324
14:34:07 [INFO] train episode 62: reward = -318.00, steps = 319
14:34:34 [INFO] train episode 63: reward = -346.00, steps = 347
14:35:12 [INFO] train episode 64: reward = -467.00, steps = 468
14:35:24 [INFO] train episode 65: reward = -154.00, steps = 155
14:35:46 [INFO] train episode 66: reward = -281.00, steps = 282
14:36:27 [INFO] train episode 67: reward = -500.00, steps = 500
14:37:10 [INFO] train episode 68: reward = -500.00, steps = 500
14:37:29 [INFO] train episode 69: reward = -239.00, steps = 240
14:38:10 [INFO] train episode 70: reward = -500.00, steps = 500
14:38:47 [INFO] train episode 71: reward = -465.00, steps = 466
14:39:09 [INFO] train episode 72: reward = -277.00, steps = 278
14:39:30 [INFO] train episode 73: reward = -276.00, steps = 277
14:39:47 [INFO] train episode 74: reward = -211.00, steps = 212
14:40:10 [INFO] train episode 75: reward = -260.00, steps = 261
14:40:32 [INFO] train episode 76: reward = -263.00, steps = 264
14:41:14 [INFO] train episode 77: reward = -489.00, steps = 490
14:41:39 [INFO] train episode 78: reward = -293.00, steps = 294
14:41:57 [INFO] train episode 79: reward = -230.00, steps = 231
14:42:25 [INFO] train episode 80: reward = -368.00, steps = 369
14:42:55 [INFO] train episode 81: reward = -384.00, steps = 385
14:43:14 [INFO] train episode 82: reward = -229.00, steps = 230
14:43:52 [INFO] train episode 83: reward = -500.00, steps = 500
14:44:08 [INFO] train episode 84: reward = -198.00, steps = 199
14:44:33 [INFO] train episode 85: reward = -309.00, steps = 310
14:45:11 [INFO] train episode 86: reward = -485.00, steps = 486
14:45:41 [INFO] train episode 87: reward = -389.00, steps = 390
14:46:02 [INFO] train episode 88: reward = -249.00, steps = 250
14:46:21 [INFO] train episode 89: reward = -240.00, steps = 241
14:46:43 [INFO] train episode 90: reward = -285.00, steps = 286
14:47:04 [INFO] train episode 91: reward = -270.00, steps = 271
14:47:38 [INFO] train episode 92: reward = -418.00, steps = 419
14:48:16 [INFO] train episode 93: reward = -500.00, steps = 500
14:48:42 [INFO] train episode 94: reward = -337.00, steps = 338
14:49:07 [INFO] train episode 95: reward = -300.00, steps = 301
14:49:31 [INFO] train episode 96: reward = -324.00, steps = 325
14:49:51 [INFO] train episode 97: reward = -263.00, steps = 264
14:50:15 [INFO] train episode 98: reward = -330.00, steps = 331
14:50:37 [INFO] train episode 99: reward = -277.00, steps = 278
14:50:49 [INFO] train episode 100: reward = -159.00, steps = 160
14:51:08 [INFO] train episode 101: reward = -247.00, steps = 248
14:51:23 [INFO] train episode 102: reward = -212.00, steps = 213
14:51:34 [INFO] train episode 103: reward = -147.00, steps = 148
14:51:50 [INFO] train episode 104: reward = -206.00, steps = 207
14:52:13 [INFO] train episode 105: reward = -298.00, steps = 299
14:52:31 [INFO] train episode 106: reward = -245.00, steps = 246
14:52:43 [INFO] train episode 107: reward = -149.00, steps = 150
14:53:10 [INFO] train episode 108: reward = -372.00, steps = 373
14:53:23 [INFO] train episode 109: reward = -175.00, steps = 176
14:53:42 [INFO] train episode 110: reward = -246.00, steps = 247
14:54:08 [INFO] train episode 111: reward = -354.00, steps = 355
14:54:28 [INFO] train episode 112: reward = -274.00, steps = 275
14:55:04 [INFO] train episode 113: reward = -461.00, steps = 462
14:55:16 [INFO] train episode 114: reward = -160.00, steps = 161
14:55:34 [INFO] train episode 115: reward = -251.00, steps = 252
14:55:55 [INFO] train episode 116: reward = -284.00, steps = 285
14:56:08 [INFO] train episode 117: reward = -173.00, steps = 174
14:56:29 [INFO] train episode 118: reward = -274.00, steps = 275
14:56:49 [INFO] train episode 119: reward = -270.00, steps = 271
14:57:04 [INFO] train episode 120: reward = -198.00, steps = 199
14:57:24 [INFO] train episode 121: reward = -261.00, steps = 262
14:57:39 [INFO] train episode 122: reward = -198.00, steps = 199
14:57:50 [INFO] train episode 123: reward = -139.00, steps = 140
14:58:07 [INFO] train episode 124: reward = -225.00, steps = 226
14:58:20 [INFO] train episode 125: reward = -180.00, steps = 181
14:58:31 [INFO] train episode 126: reward = -143.00, steps = 144
14:59:01 [INFO] train episode 127: reward = -407.00, steps = 408
14:59:17 [INFO] train episode 128: reward = -197.00, steps = 198
14:59:29 [INFO] train episode 129: reward = -169.00, steps = 170
14:59:50 [INFO] train episode 130: reward = -281.00, steps = 282
15:00:03 [INFO] train episode 131: reward = -180.00, steps = 181
15:00:23 [INFO] train episode 132: reward = -273.00, steps = 274
15:00:37 [INFO] train episode 133: reward = -163.00, steps = 164
15:00:52 [INFO] train episode 134: reward = -216.00, steps = 217
15:01:07 [INFO] train episode 135: reward = -201.00, steps = 202
15:01:19 [INFO] train episode 136: reward = -150.00, steps = 151
15:01:30 [INFO] train episode 137: reward = -158.00, steps = 159
15:01:45 [INFO] train episode 138: reward = -192.00, steps = 193
15:01:59 [INFO] train episode 139: reward = -175.00, steps = 176
15:02:15 [INFO] train episode 140: reward = -221.00, steps = 222
15:02:29 [INFO] train episode 141: reward = -182.00, steps = 183
15:02:40 [INFO] train episode 142: reward = -150.00, steps = 151
15:02:53 [INFO] train episode 143: reward = -176.00, steps = 177
15:03:11 [INFO] train episode 144: reward = -239.00, steps = 240
15:03:28 [INFO] train episode 145: reward = -222.00, steps = 223
15:03:46 [INFO] train episode 146: reward = -234.00, steps = 235
15:03:59 [INFO] train episode 147: reward = -175.00, steps = 176
15:04:14 [INFO] train episode 148: reward = -205.00, steps = 206
15:04:41 [INFO] train episode 149: reward = -367.00, steps = 368
15:05:08 [INFO] train episode 150: reward = -341.00, steps = 342
15:05:21 [INFO] train episode 151: reward = -181.00, steps = 182
15:05:35 [INFO] train episode 152: reward = -180.00, steps = 181
15:05:46 [INFO] train episode 153: reward = -153.00, steps = 154
15:06:00 [INFO] train episode 154: reward = -190.00, steps = 191
15:06:13 [INFO] train episode 155: reward = -177.00, steps = 178
15:06:24 [INFO] train episode 156: reward = -132.00, steps = 133
15:06:40 [INFO] train episode 157: reward = -214.00, steps = 215
15:06:50 [INFO] train episode 158: reward = -130.00, steps = 131
15:07:06 [INFO] train episode 159: reward = -219.00, steps = 220
15:07:18 [INFO] train episode 160: reward = -151.00, steps = 152
15:07:28 [INFO] train episode 161: reward = -133.00, steps = 134
15:07:44 [INFO] train episode 162: reward = -202.00, steps = 203
15:07:58 [INFO] train episode 163: reward = -190.00, steps = 191
15:08:15 [INFO] train episode 164: reward = -218.00, steps = 219
15:08:27 [INFO] train episode 165: reward = -161.00, steps = 162
15:08:37 [INFO] train episode 166: reward = -136.00, steps = 137
15:08:50 [INFO] train episode 167: reward = -181.00, steps = 182
15:09:13 [INFO] train episode 168: reward = -288.00, steps = 289
15:09:25 [INFO] train episode 169: reward = -162.00, steps = 163
15:09:38 [INFO] train episode 170: reward = -178.00, steps = 179
15:09:49 [INFO] train episode 171: reward = -146.00, steps = 147
15:10:08 [INFO] train episode 172: reward = -243.00, steps = 244
15:10:17 [INFO] train episode 173: reward = -132.00, steps = 133
15:10:32 [INFO] train episode 174: reward = -184.00, steps = 185
15:10:45 [INFO] train episode 175: reward = -168.00, steps = 169
15:10:55 [INFO] train episode 176: reward = -134.00, steps = 135
15:11:06 [INFO] train episode 177: reward = -160.00, steps = 161
15:11:23 [INFO] train episode 178: reward = -216.00, steps = 217
15:11:35 [INFO] train episode 179: reward = -169.00, steps = 170
15:11:52 [INFO] train episode 180: reward = -216.00, steps = 217
15:12:01 [INFO] train episode 181: reward = -114.00, steps = 115
15:12:20 [INFO] train episode 182: reward = -253.00, steps = 254
15:12:31 [INFO] train episode 183: reward = -157.00, steps = 158
15:12:45 [INFO] train episode 184: reward = -189.00, steps = 190
15:12:57 [INFO] train episode 185: reward = -154.00, steps = 155
15:13:14 [INFO] train episode 186: reward = -210.00, steps = 211
15:13:27 [INFO] train episode 187: reward = -186.00, steps = 187
15:13:39 [INFO] train episode 188: reward = -150.00, steps = 151
15:13:53 [INFO] train episode 189: reward = -199.00, steps = 200
15:14:12 [INFO] train episode 190: reward = -250.00, steps = 251
15:14:26 [INFO] train episode 191: reward = -195.00, steps = 196
15:14:41 [INFO] train episode 192: reward = -173.00, steps = 174
15:14:54 [INFO] train episode 193: reward = -181.00, steps = 182
15:15:05 [INFO] train episode 194: reward = -147.00, steps = 148
15:15:19 [INFO] train episode 195: reward = -181.00, steps = 182
15:15:28 [INFO] train episode 196: reward = -124.00, steps = 125
15:16:07 [INFO] train episode 197: reward = -488.00, steps = 489
15:16:23 [INFO] train episode 198: reward = -220.00, steps = 221
15:16:32 [INFO] train episode 199: reward = -114.00, steps = 115
15:16:49 [INFO] train episode 200: reward = -245.00, steps = 246
15:17:05 [INFO] train episode 201: reward = -203.00, steps = 204
15:17:16 [INFO] train episode 202: reward = -153.00, steps = 154
15:17:31 [INFO] train episode 203: reward = -183.00, steps = 184
15:17:44 [INFO] train episode 204: reward = -173.00, steps = 174
15:18:03 [INFO] train episode 205: reward = -257.00, steps = 258
15:18:18 [INFO] train episode 206: reward = -203.00, steps = 204
15:18:32 [INFO] train episode 207: reward = -191.00, steps = 192
15:18:54 [INFO] train episode 208: reward = -282.00, steps = 283
15:19:12 [INFO] train episode 209: reward = -245.00, steps = 246
15:19:25 [INFO] train episode 210: reward = -172.00, steps = 173
15:19:43 [INFO] train episode 211: reward = -246.00, steps = 247
15:19:54 [INFO] train episode 212: reward = -142.00, steps = 143
15:20:06 [INFO] train episode 213: reward = -164.00, steps = 165
15:20:19 [INFO] train episode 214: reward = -158.00, steps = 159
15:20:30 [INFO] train episode 215: reward = -152.00, steps = 153
15:20:47 [INFO] train episode 216: reward = -222.00, steps = 223
15:21:00 [INFO] train episode 217: reward = -172.00, steps = 173
15:21:13 [INFO] train episode 218: reward = -177.00, steps = 178
15:21:24 [INFO] train episode 219: reward = -143.00, steps = 144
15:21:42 [INFO] train episode 220: reward = -226.00, steps = 227
15:21:59 [INFO] train episode 221: reward = -226.00, steps = 227
15:22:09 [INFO] train episode 222: reward = -128.00, steps = 129
15:22:20 [INFO] train episode 223: reward = -147.00, steps = 148
15:22:32 [INFO] train episode 224: reward = -164.00, steps = 165
15:22:44 [INFO] train episode 225: reward = -166.00, steps = 167
15:23:05 [INFO] train episode 226: reward = -261.00, steps = 262
15:23:17 [INFO] train episode 227: reward = -169.00, steps = 170
15:23:25 [INFO] train episode 228: reward = -103.00, steps = 104
15:23:37 [INFO] train episode 229: reward = -155.00, steps = 156
15:23:48 [INFO] train episode 230: reward = -137.00, steps = 138
15:24:07 [INFO] train episode 231: reward = -238.00, steps = 239
15:24:20 [INFO] train episode 232: reward = -159.00, steps = 160
15:24:39 [INFO] train episode 233: reward = -216.00, steps = 217
15:24:51 [INFO] train episode 234: reward = -144.00, steps = 145
15:25:07 [INFO] train episode 235: reward = -182.00, steps = 183
15:25:32 [INFO] train episode 236: reward = -294.00, steps = 295
15:25:46 [INFO] train episode 237: reward = -156.00, steps = 157
15:26:04 [INFO] train episode 238: reward = -243.00, steps = 244
15:26:12 [INFO] train episode 239: reward = -112.00, steps = 113
15:26:25 [INFO] train episode 240: reward = -219.00, steps = 220
15:26:34 [INFO] train episode 241: reward = -131.00, steps = 132
15:26:45 [INFO] train episode 242: reward = -173.00, steps = 174
15:26:54 [INFO] train episode 243: reward = -148.00, steps = 149
15:27:03 [INFO] train episode 244: reward = -130.00, steps = 131
15:27:21 [INFO] train episode 245: reward = -302.00, steps = 303
15:27:29 [INFO] train episode 246: reward = -148.00, steps = 149
15:27:41 [INFO] train episode 247: reward = -212.00, steps = 213
15:27:49 [INFO] train episode 248: reward = -144.00, steps = 145
15:27:59 [INFO] train episode 249: reward = -168.00, steps = 169
15:28:09 [INFO] train episode 250: reward = -169.00, steps = 170
15:28:22 [INFO] train episode 251: reward = -216.00, steps = 217
15:28:32 [INFO] train episode 252: reward = -175.00, steps = 176
15:28:45 [INFO] train episode 253: reward = -231.00, steps = 232
15:28:54 [INFO] train episode 254: reward = -158.00, steps = 159
15:29:08 [INFO] train episode 255: reward = -234.00, steps = 235
15:29:16 [INFO] train episode 256: reward = -152.00, steps = 153
15:29:27 [INFO] train episode 257: reward = -182.00, steps = 183
15:29:35 [INFO] train episode 258: reward = -143.00, steps = 144
15:29:43 [INFO] train episode 259: reward = -135.00, steps = 136
15:29:54 [INFO] train episode 260: reward = -198.00, steps = 199
15:30:11 [INFO] train episode 261: reward = -302.00, steps = 303
15:30:25 [INFO] train episode 262: reward = -217.00, steps = 218
15:30:40 [INFO] train episode 263: reward = -278.00, steps = 279
15:30:46 [INFO] train episode 264: reward = -99.00, steps = 100
15:30:54 [INFO] train episode 265: reward = -142.00, steps = 143
15:31:02 [INFO] train episode 266: reward = -121.00, steps = 122
15:31:11 [INFO] train episode 267: reward = -165.00, steps = 166
15:31:25 [INFO] train episode 268: reward = -229.00, steps = 230
15:31:32 [INFO] train episode 269: reward = -128.00, steps = 129
15:31:38 [INFO] train episode 270: reward = -100.00, steps = 101
15:31:47 [INFO] train episode 271: reward = -156.00, steps = 157
15:31:56 [INFO] train episode 272: reward = -162.00, steps = 163
15:32:07 [INFO] train episode 273: reward = -194.00, steps = 195
15:32:18 [INFO] train episode 274: reward = -190.00, steps = 191
15:32:26 [INFO] train episode 275: reward = -120.00, steps = 121
15:32:46 [INFO] train episode 276: reward = -366.00, steps = 367
15:32:59 [INFO] train episode 277: reward = -219.00, steps = 220
15:33:17 [INFO] train episode 278: reward = -314.00, steps = 315
15:33:26 [INFO] train episode 279: reward = -159.00, steps = 160
15:33:34 [INFO] train episode 280: reward = -138.00, steps = 139
15:33:42 [INFO] train episode 281: reward = -138.00, steps = 139
15:33:50 [INFO] train episode 282: reward = -135.00, steps = 136
15:33:57 [INFO] train episode 283: reward = -131.00, steps = 132
15:34:09 [INFO] train episode 284: reward = -192.00, steps = 193
15:34:21 [INFO] train episode 285: reward = -224.00, steps = 225
15:34:29 [INFO] train episode 286: reward = -126.00, steps = 127
15:34:39 [INFO] train episode 287: reward = -168.00, steps = 169
15:34:52 [INFO] train episode 288: reward = -237.00, steps = 238
15:35:01 [INFO] train episode 289: reward = -154.00, steps = 155
15:35:09 [INFO] train episode 290: reward = -139.00, steps = 140
15:35:20 [INFO] train episode 291: reward = -192.00, steps = 193
15:35:29 [INFO] train episode 292: reward = -148.00, steps = 149
15:35:38 [INFO] train episode 293: reward = -150.00, steps = 151
15:35:48 [INFO] train episode 294: reward = -176.00, steps = 177
15:35:55 [INFO] train episode 295: reward = -140.00, steps = 141
15:36:02 [INFO] train episode 296: reward = -119.00, steps = 120
15:36:11 [INFO] train episode 297: reward = -146.00, steps = 147
15:36:18 [INFO] train episode 298: reward = -128.00, steps = 129
15:36:34 [INFO] train episode 299: reward = -266.00, steps = 267
15:36:44 [INFO] train episode 300: reward = -170.00, steps = 171
15:36:51 [INFO] train episode 301: reward = -116.00, steps = 117
15:36:58 [INFO] train episode 302: reward = -113.00, steps = 114
15:37:09 [INFO] train episode 303: reward = -207.00, steps = 208
15:37:17 [INFO] train episode 304: reward = -129.00, steps = 130
15:37:24 [INFO] train episode 305: reward = -133.00, steps = 134
15:37:34 [INFO] train episode 306: reward = -150.00, steps = 151
15:37:46 [INFO] train episode 307: reward = -224.00, steps = 225
15:37:53 [INFO] train episode 308: reward = -119.00, steps = 120
15:38:00 [INFO] train episode 309: reward = -129.00, steps = 130
15:38:11 [INFO] train episode 310: reward = -180.00, steps = 181
15:38:18 [INFO] train episode 311: reward = -142.00, steps = 143
15:38:26 [INFO] train episode 312: reward = -125.00, steps = 126
15:38:35 [INFO] train episode 313: reward = -143.00, steps = 144
15:38:42 [INFO] train episode 314: reward = -118.00, steps = 119
15:38:51 [INFO] train episode 315: reward = -163.00, steps = 164
15:38:59 [INFO] train episode 316: reward = -144.00, steps = 145
15:39:08 [INFO] train episode 317: reward = -143.00, steps = 144
15:39:16 [INFO] train episode 318: reward = -153.00, steps = 154
15:39:27 [INFO] train episode 319: reward = -198.00, steps = 199
15:39:37 [INFO] train episode 320: reward = -154.00, steps = 155
15:39:45 [INFO] train episode 321: reward = -142.00, steps = 143
15:39:52 [INFO] train episode 322: reward = -119.00, steps = 120
15:40:01 [INFO] train episode 323: reward = -166.00, steps = 167
15:40:11 [INFO] train episode 324: reward = -179.00, steps = 180
15:40:19 [INFO] train episode 325: reward = -133.00, steps = 134
15:40:27 [INFO] train episode 326: reward = -143.00, steps = 144
15:40:35 [INFO] train episode 327: reward = -121.00, steps = 122
15:40:44 [INFO] train episode 328: reward = -157.00, steps = 158
15:40:51 [INFO] train episode 329: reward = -119.00, steps = 120
15:41:05 [INFO] train episode 330: reward = -253.00, steps = 254
15:41:14 [INFO] train episode 331: reward = -145.00, steps = 146
15:41:20 [INFO] train episode 332: reward = -120.00, steps = 121
15:41:29 [INFO] train episode 333: reward = -149.00, steps = 150
15:41:36 [INFO] train episode 334: reward = -108.00, steps = 109
15:41:46 [INFO] train episode 335: reward = -175.00, steps = 176
15:41:54 [INFO] train episode 336: reward = -140.00, steps = 141
15:42:04 [INFO] train episode 337: reward = -177.00, steps = 178
15:42:12 [INFO] train episode 338: reward = -141.00, steps = 142
15:42:20 [INFO] train episode 339: reward = -140.00, steps = 141
15:42:31 [INFO] train episode 340: reward = -193.00, steps = 194
15:42:40 [INFO] train episode 341: reward = -141.00, steps = 142
15:42:49 [INFO] train episode 342: reward = -163.00, steps = 164
15:42:57 [INFO] train episode 343: reward = -142.00, steps = 143
15:43:05 [INFO] train episode 344: reward = -135.00, steps = 136
15:43:13 [INFO] train episode 345: reward = -136.00, steps = 137
15:43:21 [INFO] train episode 346: reward = -148.00, steps = 149
15:43:29 [INFO] train episode 347: reward = -136.00, steps = 137
15:43:37 [INFO] train episode 348: reward = -127.00, steps = 128
15:43:45 [INFO] train episode 349: reward = -138.00, steps = 139
15:43:52 [INFO] train episode 350: reward = -127.00, steps = 128
15:44:00 [INFO] train episode 351: reward = -153.00, steps = 154
15:44:07 [INFO] train episode 352: reward = -110.00, steps = 111
15:44:15 [INFO] train episode 353: reward = -143.00, steps = 144
15:44:24 [INFO] train episode 354: reward = -162.00, steps = 163
15:44:34 [INFO] train episode 355: reward = -161.00, steps = 162
15:44:40 [INFO] train episode 356: reward = -106.00, steps = 107
15:44:51 [INFO] train episode 357: reward = -179.00, steps = 180
15:45:02 [INFO] train episode 358: reward = -182.00, steps = 183
15:45:10 [INFO] train episode 359: reward = -146.00, steps = 147
15:45:23 [INFO] train episode 360: reward = -235.00, steps = 236
15:45:33 [INFO] train episode 361: reward = -153.00, steps = 154
15:45:41 [INFO] train episode 362: reward = -138.00, steps = 139
15:45:48 [INFO] train episode 363: reward = -138.00, steps = 139
15:45:55 [INFO] train episode 364: reward = -126.00, steps = 127
15:46:03 [INFO] train episode 365: reward = -140.00, steps = 141
15:46:11 [INFO] train episode 366: reward = -134.00, steps = 135
15:46:17 [INFO] train episode 367: reward = -97.00, steps = 98
15:46:25 [INFO] train episode 368: reward = -144.00, steps = 145
15:46:34 [INFO] train episode 369: reward = -143.00, steps = 144
15:46:41 [INFO] train episode 370: reward = -112.00, steps = 113
15:46:49 [INFO] train episode 371: reward = -136.00, steps = 137
15:46:56 [INFO] train episode 372: reward = -124.00, steps = 125
15:47:03 [INFO] train episode 373: reward = -129.00, steps = 130
15:47:09 [INFO] train episode 374: reward = -102.00, steps = 103
15:47:15 [INFO] train episode 375: reward = -111.00, steps = 112
15:47:22 [INFO] train episode 376: reward = -104.00, steps = 105
15:47:30 [INFO] train episode 377: reward = -159.00, steps = 160
15:47:38 [INFO] train episode 378: reward = -113.00, steps = 114
15:47:46 [INFO] train episode 379: reward = -141.00, steps = 142
15:47:52 [INFO] train episode 380: reward = -110.00, steps = 111
15:47:59 [INFO] train episode 381: reward = -122.00, steps = 123
15:48:06 [INFO] train episode 382: reward = -114.00, steps = 115
15:48:15 [INFO] train episode 383: reward = -167.00, steps = 168
15:48:23 [INFO] train episode 384: reward = -130.00, steps = 131
15:48:29 [INFO] train episode 385: reward = -116.00, steps = 117
15:48:40 [INFO] train episode 386: reward = -179.00, steps = 180
15:48:47 [INFO] train episode 387: reward = -106.00, steps = 107
15:48:54 [INFO] train episode 388: reward = -125.00, steps = 126
15:49:00 [INFO] train episode 389: reward = -111.00, steps = 112
15:49:07 [INFO] train episode 390: reward = -117.00, steps = 118
15:49:12 [INFO] train episode 391: reward = -89.00, steps = 90
15:49:21 [INFO] train episode 392: reward = -160.00, steps = 161
15:49:26 [INFO] train episode 393: reward = -96.00, steps = 97
15:49:35 [INFO] train episode 394: reward = -152.00, steps = 153
15:49:42 [INFO] train episode 395: reward = -110.00, steps = 111
15:49:49 [INFO] train episode 396: reward = -124.00, steps = 125
15:49:49 [INFO] ==== test ====
15:49:57 [INFO] test episode 0: reward = -146.00, steps = 147
15:50:04 [INFO] test episode 1: reward = -105.00, steps = 106
15:50:10 [INFO] test episode 2: reward = -122.00, steps = 123
15:50:16 [INFO] test episode 3: reward = -102.00, steps = 103
15:50:24 [INFO] test episode 4: reward = -148.00, steps = 149
15:50:34 [INFO] test episode 5: reward = -180.00, steps = 181
15:50:40 [INFO] test episode 6: reward = -89.00, steps = 90
15:50:45 [INFO] test episode 7: reward = -101.00, steps = 102
15:50:51 [INFO] test episode 8: reward = -104.00, steps = 105
15:50:58 [INFO] test episode 9: reward = -119.00, steps = 120
15:51:08 [INFO] test episode 10: reward = -178.00, steps = 179
15:51:16 [INFO] test episode 11: reward = -146.00, steps = 147
15:51:22 [INFO] test episode 12: reward = -103.00, steps = 104
15:51:29 [INFO] test episode 13: reward = -127.00, steps = 128
15:51:37 [INFO] test episode 14: reward = -143.00, steps = 144
15:51:43 [INFO] test episode 15: reward = -112.00, steps = 113
15:51:52 [INFO] test episode 16: reward = -158.00, steps = 159
15:51:59 [INFO] test episode 17: reward = -123.00, steps = 124
15:52:08 [INFO] test episode 18: reward = -158.00, steps = 159
15:52:14 [INFO] test episode 19: reward = -115.00, steps = 116
15:52:21 [INFO] test episode 20: reward = -127.00, steps = 128
15:52:29 [INFO] test episode 21: reward = -143.00, steps = 144
15:52:37 [INFO] test episode 22: reward = -133.00, steps = 134
15:52:43 [INFO] test episode 23: reward = -113.00, steps = 114
15:52:51 [INFO] test episode 24: reward = -132.00, steps = 133
15:52:59 [INFO] test episode 25: reward = -137.00, steps = 138
15:53:06 [INFO] test episode 26: reward = -133.00, steps = 134
15:53:14 [INFO] test episode 27: reward = -134.00, steps = 135
15:53:20 [INFO] test episode 28: reward = -120.00, steps = 121
15:53:27 [INFO] test episode 29: reward = -116.00, steps = 117
15:53:33 [INFO] test episode 30: reward = -105.00, steps = 106
15:53:39 [INFO] test episode 31: reward = -115.00, steps = 116
15:53:46 [INFO] test episode 32: reward = -128.00, steps = 129
15:53:54 [INFO] test episode 33: reward = -128.00, steps = 129
15:54:00 [INFO] test episode 34: reward = -125.00, steps = 126
15:54:09 [INFO] test episode 35: reward = -140.00, steps = 141
15:54:17 [INFO] test episode 36: reward = -157.00, steps = 158
15:54:26 [INFO] test episode 37: reward = -156.00, steps = 157
15:54:32 [INFO] test episode 38: reward = -103.00, steps = 104
15:54:38 [INFO] test episode 39: reward = -96.00, steps = 97
15:54:44 [INFO] test episode 40: reward = -123.00, steps = 124
15:54:52 [INFO] test episode 41: reward = -123.00, steps = 124
15:55:00 [INFO] test episode 42: reward = -144.00, steps = 145
15:55:08 [INFO] test episode 43: reward = -139.00, steps = 140
15:55:14 [INFO] test episode 44: reward = -120.00, steps = 121
15:55:29 [INFO] test episode 45: reward = -258.00, steps = 259
15:55:35 [INFO] test episode 46: reward = -121.00, steps = 122
15:55:42 [INFO] test episode 47: reward = -115.00, steps = 116
15:55:48 [INFO] test episode 48: reward = -105.00, steps = 106
15:55:56 [INFO] test episode 49: reward = -140.00, steps = 141
15:56:04 [INFO] test episode 50: reward = -144.00, steps = 145
15:56:11 [INFO] test episode 51: reward = -130.00, steps = 131
15:56:18 [INFO] test episode 52: reward = -131.00, steps = 132
15:56:25 [INFO] test episode 53: reward = -123.00, steps = 124
15:56:33 [INFO] test episode 54: reward = -137.00, steps = 138
15:56:42 [INFO] test episode 55: reward = -151.00, steps = 152
15:56:48 [INFO] test episode 56: reward = -113.00, steps = 114
15:57:01 [INFO] test episode 57: reward = -234.00, steps = 235
15:57:09 [INFO] test episode 58: reward = -139.00, steps = 140
15:57:22 [INFO] test episode 59: reward = -229.00, steps = 230
15:57:29 [INFO] test episode 60: reward = -115.00, steps = 116
15:57:35 [INFO] test episode 61: reward = -113.00, steps = 114
15:57:41 [INFO] test episode 62: reward = -103.00, steps = 104
15:57:55 [INFO] test episode 63: reward = -249.00, steps = 250
15:58:00 [INFO] test episode 64: reward = -100.00, steps = 101
15:58:07 [INFO] test episode 65: reward = -122.00, steps = 123
15:58:15 [INFO] test episode 66: reward = -133.00, steps = 134
15:58:24 [INFO] test episode 67: reward = -168.00, steps = 169
15:58:35 [INFO] test episode 68: reward = -204.00, steps = 205
15:58:43 [INFO] test episode 69: reward = -141.00, steps = 142
15:58:50 [INFO] test episode 70: reward = -112.00, steps = 113
15:58:57 [INFO] test episode 71: reward = -115.00, steps = 116
15:59:02 [INFO] test episode 72: reward = -104.00, steps = 105
15:59:09 [INFO] test episode 73: reward = -110.00, steps = 111
15:59:14 [INFO] test episode 74: reward = -90.00, steps = 91
15:59:20 [INFO] test episode 75: reward = -113.00, steps = 114
15:59:25 [INFO] test episode 76: reward = -95.00, steps = 96
15:59:32 [INFO] test episode 77: reward = -114.00, steps = 115
15:59:39 [INFO] test episode 78: reward = -128.00, steps = 129
15:59:46 [INFO] test episode 79: reward = -120.00, steps = 121
15:59:52 [INFO] test episode 80: reward = -121.00, steps = 122
15:59:58 [INFO] test episode 81: reward = -106.00, steps = 107
16:00:08 [INFO] test episode 82: reward = -171.00, steps = 172
16:00:18 [INFO] test episode 83: reward = -172.00, steps = 173
16:00:25 [INFO] test episode 84: reward = -132.00, steps = 133
16:00:32 [INFO] test episode 85: reward = -119.00, steps = 120
16:00:40 [INFO] test episode 86: reward = -135.00, steps = 136
16:00:48 [INFO] test episode 87: reward = -155.00, steps = 156
16:00:56 [INFO] test episode 88: reward = -131.00, steps = 132
16:01:03 [INFO] test episode 89: reward = -135.00, steps = 136
16:01:11 [INFO] test episode 90: reward = -130.00, steps = 131
16:01:19 [INFO] test episode 91: reward = -151.00, steps = 152
16:01:26 [INFO] test episode 92: reward = -133.00, steps = 134
16:01:33 [INFO] test episode 93: reward = -115.00, steps = 116
16:01:39 [INFO] test episode 94: reward = -98.00, steps = 99
16:01:47 [INFO] test episode 95: reward = -150.00, steps = 151
16:01:58 [INFO] test episode 96: reward = -203.00, steps = 204
16:02:08 [INFO] test episode 97: reward = -166.00, steps = 167
16:02:15 [INFO] test episode 98: reward = -120.00, steps = 121
16:02:22 [INFO] test episode 99: reward = -126.00, steps = 127
16:02:22 [INFO] average episode reward = -133.57 ± 31.50
In [7]:
env.close()