Use TRPO to Play Acrobot-v1¶

TensorFlow version

In [1]:
%matplotlib inline

import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import pandas as pd
import scipy.signal as signal
import gym
import matplotlib.pyplot as plt
import tensorflow.compat.v2 as tf
tf.random.set_seed(0)
from tensorflow import keras
from tensorflow import nn
from tensorflow import optimizers
from tensorflow import losses
from tensorflow.keras import layers

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')
In [2]:
env = gym.make('Acrobot-v1')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

11:56:31 [INFO] env: <AcrobotEnv<Acrobot-v1>>
11:56:31 [INFO] action_space: Discrete(3)
11:56:31 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
11:56:31 [INFO] reward_range: (-inf, inf)
11:56:31 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
11:56:31 [INFO] _max_episode_steps: 500
11:56:31 [INFO] _elapsed_steps: None
11:56:31 [INFO] id: Acrobot-v1
11:56:31 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
11:56:31 [INFO] reward_threshold: -100.0
11:56:31 [INFO] nondeterministic: False
11:56:31 [INFO] max_episode_steps: 500
11:56:31 [INFO] _kwargs: {}
11:56:31 [INFO] _env_name: Acrobot
In [3]:
class PPOReplayer:
    def __init__(self):
        self.fields = ['state', 'action', 'prob', 'advantage', 'return']
        self.memory = pd.DataFrame(columns=self.fields)

    def store(self, df):
        if self.memory.empty:
            self.memory = df[self.fields]
        else:
            self.memory = pd.concat([self.memory, df[self.fields]], ignore_index=True)

    def sample(self, size):
        indices = np.random.choice(self.memory.shape[0], size=size)
        return (np.stack(self.memory.loc[indices, field]) for field in
                self.fields)
In [4]:
def conjugate_gradient(f, b, iter_count=10, epsilon=1e-12, tol=1e-6):
    x = b * 0.
    r = tf.identity(b)
    p = tf.identity(b)
    rho = tf.reduce_sum(r * r)
    for i in range(iter_count):
        z = f(p)
        alpha = rho / (tf.reduce_sum(p * z) + epsilon)
        x += alpha * p
        r -= alpha * z
        rho_new = tf.reduce_sum(r * r)
        p = r + (rho_new / rho) * p
        rho = rho_new
        if rho < tol:
            break
    return x, f(x)
In [5]:
class TRPOAgent:
    def __init__(self, env):
        self.action_n = env.action_space.n
        self.gamma = 0.99

        self.replayer = PPOReplayer()
        self.trajectory = []

        self.max_kl = 0.01
        self.actor_net = self.build_net(hidden_sizes=[100,],
                output_size=self.action_n, output_activation=nn.softmax)
        self.critic_net = self.build_net(hidden_sizes=[100,],
                learning_rate=0.002)

    def build_net(self, input_size=None, hidden_sizes=None, output_size=1,
                activation=nn.relu, output_activation=None,
                loss=losses.mse, learning_rate=0.001):
        model = keras.Sequential()
        for hidden_size in hidden_sizes:
            model.add(layers.Dense(units=hidden_size,
                    activation=activation))
        model.add(layers.Dense(units=output_size,
                activation=output_activation))
        optimizer = optimizers.Adam(learning_rate)
        model.compile(optimizer=optimizer, loss=loss)
        return model

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []

    def step(self, observation, reward, terminated):
        probs = self.actor_net.predict(observation[np.newaxis], verbose=0)[0]
        action = np.random.choice(self.action_n, p=probs)
        if self.mode == 'train':
            self.trajectory += [observation, reward, terminated, action]
        return action

    def close(self):
        if self.mode == 'train':
            self.save_trajectory_to_replayer()
            if len(self.replayer.memory) >= 1000:
                for batch in range(5):  # learn multiple times
                    self.learn()
                self.replayer = PPOReplayer()
                        # reset replayer after the agent changes itself

    def save_trajectory_to_replayer(self):
        df = pd.DataFrame(
                np.array(self.trajectory, dtype=object).reshape(-1, 4),
                columns=['state', 'reward', 'terminated', 'action'], dtype=object)
        states = np.stack(df['state'])
        df['v'] = self.critic_net.predict(states, verbose=0)
        pis = self.actor_net.predict(states, verbose=0)
        df['prob'] = [pi[action] for pi, action in zip(pis, df['action'])]
        df['next_v'] = df['v'].shift(-1).fillna(0.)
        df['u'] = df['reward'] + self.gamma * df['next_v']
        df['delta'] = df['u'] - df['v']
        df['advantage'] = signal.lfilter([1.,], [1., -self.gamma],
                df['delta'][::-1])[::-1]
        df['return'] = signal.lfilter([1.,], [1., -self.gamma],
                df['reward'][::-1])[::-1]
        self.replayer.store(df)

    def learn(self):
        states, actions, old_pis, advantages, returns = \
                self.replayer.sample(size=64)
        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)
        action_tensor = tf.convert_to_tensor(actions, dtype=tf.int32)
        old_pi_tensor = tf.convert_to_tensor(old_pis, dtype=tf.float32)
        advantage_tensor = tf.convert_to_tensor(advantages, dtype=tf.float32)

        # update actor
        # ... calculate first order gradient of KL divergence
        with tf.GradientTape() as tape:
            all_pi_tensor = self.actor_net(state_tensor)
            pi_tensor = tf.gather(all_pi_tensor, action_tensor, batch_dims=1)
            surrogate_tensor = (pi_tensor / old_pi_tensor) * advantage_tensor
        actor_grads = tape.gradient(surrogate_tensor, self.actor_net.variables)
        loss_grad = tf.concat([tf.reshape(grad, (-1,)) for grad in actor_grads],
                axis=0)

        # ... calculate conjugate gradient: Fx = g
        def f(x):  # calculate Fx
            with tf.GradientTape() as tape2:  # tape for 2nd-order gradient
                with tf.GradientTape() as tape1:  # tape for 1st-order gradient
                    prob_tensor = self.actor_net(state_tensor)
                    prob_old_tensor = tf.stop_gradient(prob_tensor)
                    kld_tensor = tf.reduce_sum(prob_old_tensor * (tf.math.log(
                            prob_old_tensor) - tf.math.log(prob_tensor)), axis=1)
                    kld_loss_tensor = tf.reduce_mean(kld_tensor)
                grads = tape1.gradient(kld_loss_tensor, self.actor_net.variables)
                flatten_grad_tensor = tf.concat(
                        [tf.reshape(grad, (-1,)) for grad in grads], axis=-1)
                grad_matmul_x = tf.tensordot(flatten_grad_tensor, x,
                        axes=[[-1], [-1]])
            grad_grads = tape2.gradient(grad_matmul_x, self.actor_net.variables)
            flatten_grad_grad = tf.stop_gradient(tf.concat(
                    [tf.reshape(grad_grad, (-1,)) for grad_grad in grad_grads],
                    axis=-1))
            fx = flatten_grad_grad + x * 1e-2
            return fx
        x, fx = conjugate_gradient(f, loss_grad)

        # ... calculate natural gradient
        natural_gradient_tensor = tf.sqrt(2 * self.max_kl /
                tf.reduce_sum(fx * x)) * x
        # ....... refactor the flatten gradient into un-flatten version
        flatten_natural_gradient = natural_gradient_tensor.numpy()
        natural_grads = []
        begin = 0
        for weight in self.actor_net.get_weights():
            end = begin + weight.size
            natural_grad = flatten_natural_gradient[begin:end].reshape(
                    weight.shape)
            natural_grads.append(natural_grad)
            begin = end

        # ... line search
        old_weights = self.actor_net.get_weights()
        expected_improve = tf.reduce_sum(loss_grad *
                natural_gradient_tensor).numpy()
        for learning_step in [0.,] + [.5 ** j for j in range(10)]:
            self.actor_net.set_weights([weight + learning_step * grad
                    for weight, grad in zip(old_weights, natural_grads)])
            all_pi_tensor = self.actor_net(state_tensor)
            new_pi_tensor = tf.gather(all_pi_tensor,
                    action_tensor[:, np.newaxis], axis=1)[:, 0]
            new_pi_tensor = tf.stop_gradient(new_pi_tensor)
            surrogate_tensor = (new_pi_tensor / pi_tensor) * advantage_tensor
            objective = tf.reduce_sum(surrogate_tensor).numpy()
            if np.isclose(learning_step, 0.):
                old_objective = objective
            else:
                if objective - old_objective > 0.1 * expected_improve * \
                        learning_step:
                    break # success, keep the weight
        else:
            self.actor_net.set_weights(old_weights)

        # update critic
        self.critic_net.fit(states, returns, verbose=0)


agent = TRPOAgent(env)
In [6]:
def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
            mode='train')
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > -120:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

11:56:32 [INFO] ==== train ====
11:57:06 [INFO] NumExpr defaulting to 8 threads.
11:57:06 [INFO] train episode 0: reward = -500.00, steps = 500
11:57:41 [INFO] train episode 1: reward = -500.00, steps = 500
11:58:14 [INFO] train episode 2: reward = -500.00, steps = 500
11:58:39 [INFO] train episode 3: reward = -386.00, steps = 387
11:59:14 [INFO] train episode 4: reward = -500.00, steps = 500
11:59:42 [INFO] train episode 5: reward = -425.00, steps = 426
12:00:09 [INFO] train episode 6: reward = -399.00, steps = 400
12:00:39 [INFO] train episode 7: reward = -427.00, steps = 428
12:00:59 [INFO] train episode 8: reward = -299.00, steps = 300
12:01:23 [INFO] train episode 9: reward = -369.00, steps = 370
12:01:58 [INFO] train episode 10: reward = -500.00, steps = 500
12:02:21 [INFO] train episode 11: reward = -341.00, steps = 342
12:02:48 [INFO] train episode 12: reward = -405.00, steps = 406
12:03:15 [INFO] train episode 13: reward = -394.00, steps = 395
12:03:36 [INFO] train episode 14: reward = -306.00, steps = 307
12:04:09 [INFO] train episode 15: reward = -500.00, steps = 500
12:04:30 [INFO] train episode 16: reward = -310.00, steps = 311
12:04:54 [INFO] train episode 17: reward = -356.00, steps = 357
12:05:11 [INFO] train episode 18: reward = -259.00, steps = 260
12:05:32 [INFO] train episode 19: reward = -315.00, steps = 316
12:06:02 [INFO] train episode 20: reward = -420.00, steps = 421
12:06:34 [INFO] train episode 21: reward = -500.00, steps = 500
12:06:58 [INFO] train episode 22: reward = -351.00, steps = 352
12:07:33 [INFO] train episode 23: reward = -500.00, steps = 500
12:07:53 [INFO] train episode 24: reward = -308.00, steps = 309
12:08:25 [INFO] train episode 25: reward = -486.00, steps = 487
12:08:38 [INFO] train episode 26: reward = -195.00, steps = 196
12:09:11 [INFO] train episode 27: reward = -479.00, steps = 480
12:09:44 [INFO] train episode 28: reward = -500.00, steps = 500
12:10:04 [INFO] train episode 29: reward = -300.00, steps = 301
12:10:22 [INFO] train episode 30: reward = -265.00, steps = 266
12:10:43 [INFO] train episode 31: reward = -307.00, steps = 308
12:11:11 [INFO] train episode 32: reward = -427.00, steps = 428
12:11:31 [INFO] train episode 33: reward = -272.00, steps = 273
12:11:55 [INFO] train episode 34: reward = -375.00, steps = 376
12:12:18 [INFO] train episode 35: reward = -337.00, steps = 338
12:12:42 [INFO] train episode 36: reward = -339.00, steps = 340
12:13:09 [INFO] train episode 37: reward = -419.00, steps = 420
12:13:24 [INFO] train episode 38: reward = -215.00, steps = 216
12:13:37 [INFO] train episode 39: reward = -200.00, steps = 201
12:13:53 [INFO] train episode 40: reward = -215.00, steps = 216
12:14:10 [INFO] train episode 41: reward = -252.00, steps = 253
12:14:24 [INFO] train episode 42: reward = -223.00, steps = 224
12:14:33 [INFO] train episode 43: reward = -123.00, steps = 124
12:14:52 [INFO] train episode 44: reward = -298.00, steps = 299
12:15:07 [INFO] train episode 45: reward = -196.00, steps = 197
12:15:20 [INFO] train episode 46: reward = -198.00, steps = 199
12:15:41 [INFO] train episode 47: reward = -315.00, steps = 316
12:15:57 [INFO] train episode 48: reward = -244.00, steps = 245
12:16:15 [INFO] train episode 49: reward = -249.00, steps = 250
12:16:28 [INFO] train episode 50: reward = -193.00, steps = 194
12:16:41 [INFO] train episode 51: reward = -199.00, steps = 200
12:16:51 [INFO] train episode 52: reward = -141.00, steps = 142
12:17:09 [INFO] train episode 53: reward = -286.00, steps = 287
12:17:20 [INFO] train episode 54: reward = -163.00, steps = 164
12:17:38 [INFO] train episode 55: reward = -250.00, steps = 251
12:17:51 [INFO] train episode 56: reward = -187.00, steps = 188
12:18:03 [INFO] train episode 57: reward = -178.00, steps = 179
12:18:10 [INFO] train episode 58: reward = -104.00, steps = 105
12:18:20 [INFO] train episode 59: reward = -161.00, steps = 162
12:18:30 [INFO] train episode 60: reward = -145.00, steps = 146
12:18:43 [INFO] train episode 61: reward = -193.00, steps = 194
12:19:00 [INFO] train episode 62: reward = -236.00, steps = 237
12:19:11 [INFO] train episode 63: reward = -165.00, steps = 166
12:19:27 [INFO] train episode 64: reward = -244.00, steps = 245
12:19:41 [INFO] train episode 65: reward = -202.00, steps = 203
12:19:50 [INFO] train episode 66: reward = -133.00, steps = 134
12:19:58 [INFO] train episode 67: reward = -124.00, steps = 125
12:20:09 [INFO] train episode 68: reward = -140.00, steps = 141
12:20:19 [INFO] train episode 69: reward = -142.00, steps = 143
12:20:30 [INFO] train episode 70: reward = -164.00, steps = 165
12:20:45 [INFO] train episode 71: reward = -223.00, steps = 224
12:21:05 [INFO] train episode 72: reward = -295.00, steps = 296
12:21:16 [INFO] train episode 73: reward = -154.00, steps = 155
12:21:27 [INFO] train episode 74: reward = -142.00, steps = 143
12:21:39 [INFO] train episode 75: reward = -177.00, steps = 178
12:21:53 [INFO] train episode 76: reward = -184.00, steps = 185
12:22:03 [INFO] train episode 77: reward = -139.00, steps = 140
12:22:18 [INFO] train episode 78: reward = -236.00, steps = 237
12:22:34 [INFO] train episode 79: reward = -229.00, steps = 230
12:22:45 [INFO] train episode 80: reward = -146.00, steps = 147
12:23:01 [INFO] train episode 81: reward = -243.00, steps = 244
12:23:12 [INFO] train episode 82: reward = -163.00, steps = 164
12:23:25 [INFO] train episode 83: reward = -195.00, steps = 196
12:23:34 [INFO] train episode 84: reward = -119.00, steps = 120
12:23:49 [INFO] train episode 85: reward = -235.00, steps = 236
12:24:11 [INFO] train episode 86: reward = -303.00, steps = 304
12:24:22 [INFO] train episode 87: reward = -164.00, steps = 165
12:24:33 [INFO] train episode 88: reward = -173.00, steps = 174
12:24:43 [INFO] train episode 89: reward = -137.00, steps = 138
12:24:53 [INFO] train episode 90: reward = -148.00, steps = 149
12:25:05 [INFO] train episode 91: reward = -187.00, steps = 188
12:25:13 [INFO] train episode 92: reward = -106.00, steps = 107
12:25:26 [INFO] train episode 93: reward = -185.00, steps = 186
12:25:41 [INFO] train episode 94: reward = -227.00, steps = 228
12:25:51 [INFO] train episode 95: reward = -143.00, steps = 144
12:26:02 [INFO] train episode 96: reward = -162.00, steps = 163
12:26:13 [INFO] train episode 97: reward = -163.00, steps = 164
12:26:23 [INFO] train episode 98: reward = -156.00, steps = 157
12:26:34 [INFO] train episode 99: reward = -143.00, steps = 144
12:26:42 [INFO] train episode 100: reward = -120.00, steps = 121
12:26:51 [INFO] train episode 101: reward = -128.00, steps = 129
12:27:03 [INFO] train episode 102: reward = -177.00, steps = 178
12:27:11 [INFO] train episode 103: reward = -125.00, steps = 126
12:27:27 [INFO] train episode 104: reward = -242.00, steps = 243
12:27:37 [INFO] train episode 105: reward = -155.00, steps = 156
12:27:50 [INFO] train episode 106: reward = -177.00, steps = 178
12:27:56 [INFO] train episode 107: reward = -83.00, steps = 84
12:28:08 [INFO] train episode 108: reward = -181.00, steps = 182
12:28:18 [INFO] train episode 109: reward = -148.00, steps = 149
12:28:27 [INFO] train episode 110: reward = -133.00, steps = 134
12:28:35 [INFO] train episode 111: reward = -128.00, steps = 129
12:28:42 [INFO] train episode 112: reward = -97.00, steps = 98
12:28:51 [INFO] train episode 113: reward = -132.00, steps = 133
12:29:01 [INFO] train episode 114: reward = -130.00, steps = 131
12:29:13 [INFO] train episode 115: reward = -171.00, steps = 172
12:29:22 [INFO] train episode 116: reward = -140.00, steps = 141
12:29:32 [INFO] train episode 117: reward = -148.00, steps = 149
12:29:42 [INFO] train episode 118: reward = -153.00, steps = 154
12:29:53 [INFO] train episode 119: reward = -151.00, steps = 152
12:30:05 [INFO] train episode 120: reward = -186.00, steps = 187
12:30:18 [INFO] train episode 121: reward = -169.00, steps = 170
12:30:30 [INFO] train episode 122: reward = -184.00, steps = 185
12:30:41 [INFO] train episode 123: reward = -157.00, steps = 158
12:30:50 [INFO] train episode 124: reward = -143.00, steps = 144
12:31:05 [INFO] train episode 125: reward = -213.00, steps = 214
12:31:16 [INFO] train episode 126: reward = -167.00, steps = 168
12:31:32 [INFO] train episode 127: reward = -212.00, steps = 213
12:31:39 [INFO] train episode 128: reward = -115.00, steps = 116
12:31:49 [INFO] train episode 129: reward = -145.00, steps = 146
12:32:00 [INFO] train episode 130: reward = -161.00, steps = 162
12:32:10 [INFO] train episode 131: reward = -153.00, steps = 154
12:32:33 [INFO] train episode 132: reward = -343.00, steps = 344
12:32:47 [INFO] train episode 133: reward = -185.00, steps = 186
12:33:00 [INFO] train episode 134: reward = -194.00, steps = 195
12:33:11 [INFO] train episode 135: reward = -160.00, steps = 161
12:33:25 [INFO] train episode 136: reward = -223.00, steps = 224
12:33:40 [INFO] train episode 137: reward = -211.00, steps = 212
12:33:56 [INFO] train episode 138: reward = -237.00, steps = 238
12:34:07 [INFO] train episode 139: reward = -163.00, steps = 164
12:34:22 [INFO] train episode 140: reward = -213.00, steps = 214
12:34:37 [INFO] train episode 141: reward = -226.00, steps = 227
12:34:46 [INFO] train episode 142: reward = -140.00, steps = 141
12:34:57 [INFO] train episode 143: reward = -169.00, steps = 170
12:35:09 [INFO] train episode 144: reward = -149.00, steps = 150
12:35:17 [INFO] train episode 145: reward = -128.00, steps = 129
12:35:26 [INFO] train episode 146: reward = -130.00, steps = 131
12:35:37 [INFO] train episode 147: reward = -159.00, steps = 160
12:35:45 [INFO] train episode 148: reward = -125.00, steps = 126
12:35:56 [INFO] train episode 149: reward = -156.00, steps = 157
12:36:03 [INFO] train episode 150: reward = -102.00, steps = 103
12:36:09 [INFO] train episode 151: reward = -88.00, steps = 89
12:36:21 [INFO] train episode 152: reward = -151.00, steps = 152
12:36:28 [INFO] train episode 153: reward = -108.00, steps = 109
12:36:38 [INFO] train episode 154: reward = -156.00, steps = 157
12:36:48 [INFO] train episode 155: reward = -135.00, steps = 136
12:36:57 [INFO] train episode 156: reward = -137.00, steps = 138
12:37:05 [INFO] train episode 157: reward = -120.00, steps = 121
12:37:12 [INFO] train episode 158: reward = -102.00, steps = 103
12:37:21 [INFO] train episode 159: reward = -135.00, steps = 136
12:37:29 [INFO] train episode 160: reward = -94.00, steps = 95
12:37:42 [INFO] train episode 161: reward = -195.00, steps = 196
12:37:50 [INFO] train episode 162: reward = -119.00, steps = 120
12:37:59 [INFO] train episode 163: reward = -130.00, steps = 131
12:38:07 [INFO] train episode 164: reward = -118.00, steps = 119
12:38:18 [INFO] train episode 165: reward = -162.00, steps = 163
12:38:26 [INFO] train episode 166: reward = -129.00, steps = 130
12:38:34 [INFO] train episode 167: reward = -122.00, steps = 123
12:38:44 [INFO] train episode 168: reward = -117.00, steps = 118
12:38:53 [INFO] train episode 169: reward = -137.00, steps = 138
12:39:03 [INFO] train episode 170: reward = -149.00, steps = 150
12:39:17 [INFO] train episode 171: reward = -215.00, steps = 216
12:39:24 [INFO] train episode 172: reward = -105.00, steps = 106
12:39:34 [INFO] train episode 173: reward = -139.00, steps = 140
12:39:41 [INFO] train episode 174: reward = -98.00, steps = 99
12:39:48 [INFO] train episode 175: reward = -113.00, steps = 114
12:39:58 [INFO] train episode 176: reward = -134.00, steps = 135
12:40:07 [INFO] train episode 177: reward = -127.00, steps = 128
12:40:15 [INFO] train episode 178: reward = -124.00, steps = 125
12:40:32 [INFO] train episode 179: reward = -255.00, steps = 256
12:40:42 [INFO] train episode 180: reward = -146.00, steps = 147
12:40:49 [INFO] train episode 181: reward = -108.00, steps = 109
12:41:00 [INFO] train episode 182: reward = -163.00, steps = 164
12:41:11 [INFO] train episode 183: reward = -148.00, steps = 149
12:41:20 [INFO] train episode 184: reward = -134.00, steps = 135
12:41:27 [INFO] train episode 185: reward = -98.00, steps = 99
12:41:36 [INFO] train episode 186: reward = -120.00, steps = 121
12:41:43 [INFO] train episode 187: reward = -117.00, steps = 118
12:41:51 [INFO] train episode 188: reward = -119.00, steps = 120
12:41:58 [INFO] train episode 189: reward = -97.00, steps = 98
12:42:08 [INFO] train episode 190: reward = -147.00, steps = 148
12:42:18 [INFO] train episode 191: reward = -137.00, steps = 138
12:42:26 [INFO] train episode 192: reward = -105.00, steps = 106
12:42:32 [INFO] train episode 193: reward = -94.00, steps = 95
12:42:32 [INFO] ==== test ====
12:42:40 [INFO] test episode 0: reward = -112.00, steps = 113
12:42:47 [INFO] test episode 1: reward = -110.00, steps = 111
12:42:55 [INFO] test episode 2: reward = -120.00, steps = 121
12:43:02 [INFO] test episode 3: reward = -105.00, steps = 106
12:43:09 [INFO] test episode 4: reward = -105.00, steps = 106
12:43:15 [INFO] test episode 5: reward = -95.00, steps = 96
12:43:22 [INFO] test episode 6: reward = -118.00, steps = 119
12:43:32 [INFO] test episode 7: reward = -140.00, steps = 141
12:43:39 [INFO] test episode 8: reward = -114.00, steps = 115
12:43:48 [INFO] test episode 9: reward = -127.00, steps = 128
12:43:55 [INFO] test episode 10: reward = -107.00, steps = 108
12:44:02 [INFO] test episode 11: reward = -104.00, steps = 105
12:44:11 [INFO] test episode 12: reward = -141.00, steps = 142
12:44:18 [INFO] test episode 13: reward = -98.00, steps = 99
12:44:25 [INFO] test episode 14: reward = -115.00, steps = 116
12:44:31 [INFO] test episode 15: reward = -91.00, steps = 92
12:44:42 [INFO] test episode 16: reward = -156.00, steps = 157
12:44:48 [INFO] test episode 17: reward = -97.00, steps = 98
12:44:57 [INFO] test episode 18: reward = -131.00, steps = 132
12:45:07 [INFO] test episode 19: reward = -155.00, steps = 156
12:45:13 [INFO] test episode 20: reward = -97.00, steps = 98
12:45:21 [INFO] test episode 21: reward = -112.00, steps = 113
12:45:28 [INFO] test episode 22: reward = -115.00, steps = 116
12:45:37 [INFO] test episode 23: reward = -135.00, steps = 136
12:45:47 [INFO] test episode 24: reward = -143.00, steps = 144
12:45:57 [INFO] test episode 25: reward = -158.00, steps = 159
12:46:05 [INFO] test episode 26: reward = -111.00, steps = 112
12:46:11 [INFO] test episode 27: reward = -98.00, steps = 99
12:46:19 [INFO] test episode 28: reward = -121.00, steps = 122
12:46:26 [INFO] test episode 29: reward = -106.00, steps = 107
12:46:34 [INFO] test episode 30: reward = -118.00, steps = 119
12:46:42 [INFO] test episode 31: reward = -118.00, steps = 119
12:46:52 [INFO] test episode 32: reward = -168.00, steps = 169
12:46:59 [INFO] test episode 33: reward = -100.00, steps = 101
12:47:07 [INFO] test episode 34: reward = -117.00, steps = 118
12:47:14 [INFO] test episode 35: reward = -104.00, steps = 105
12:47:21 [INFO] test episode 36: reward = -110.00, steps = 111
12:47:28 [INFO] test episode 37: reward = -109.00, steps = 110
12:47:43 [INFO] test episode 38: reward = -225.00, steps = 226
12:48:08 [INFO] test episode 39: reward = -372.00, steps = 373
12:48:17 [INFO] test episode 40: reward = -144.00, steps = 145
12:48:25 [INFO] test episode 41: reward = -120.00, steps = 121
12:48:31 [INFO] test episode 42: reward = -95.00, steps = 96
12:48:41 [INFO] test episode 43: reward = -152.00, steps = 153
12:48:51 [INFO] test episode 44: reward = -142.00, steps = 143
12:48:59 [INFO] test episode 45: reward = -122.00, steps = 123
12:49:07 [INFO] test episode 46: reward = -117.00, steps = 118
12:49:14 [INFO] test episode 47: reward = -122.00, steps = 123
12:49:21 [INFO] test episode 48: reward = -102.00, steps = 103
12:49:29 [INFO] test episode 49: reward = -115.00, steps = 116
12:49:38 [INFO] test episode 50: reward = -131.00, steps = 132
12:49:45 [INFO] test episode 51: reward = -99.00, steps = 100
12:49:52 [INFO] test episode 52: reward = -114.00, steps = 115
12:50:01 [INFO] test episode 53: reward = -128.00, steps = 129
12:50:10 [INFO] test episode 54: reward = -149.00, steps = 150
12:50:18 [INFO] test episode 55: reward = -110.00, steps = 111
12:50:27 [INFO] test episode 56: reward = -135.00, steps = 136
12:50:33 [INFO] test episode 57: reward = -98.00, steps = 99
12:50:42 [INFO] test episode 58: reward = -137.00, steps = 138
12:50:49 [INFO] test episode 59: reward = -102.00, steps = 103
12:50:57 [INFO] test episode 60: reward = -116.00, steps = 117
12:51:03 [INFO] test episode 61: reward = -82.00, steps = 83
12:51:11 [INFO] test episode 62: reward = -124.00, steps = 125
12:51:18 [INFO] test episode 63: reward = -98.00, steps = 99
12:51:24 [INFO] test episode 64: reward = -92.00, steps = 93
12:51:32 [INFO] test episode 65: reward = -118.00, steps = 119
12:51:47 [INFO] test episode 66: reward = -228.00, steps = 229
12:51:54 [INFO] test episode 67: reward = -119.00, steps = 120
12:52:04 [INFO] test episode 68: reward = -146.00, steps = 147
12:52:14 [INFO] test episode 69: reward = -146.00, steps = 147
12:52:21 [INFO] test episode 70: reward = -117.00, steps = 118
12:52:30 [INFO] test episode 71: reward = -124.00, steps = 125
12:52:39 [INFO] test episode 72: reward = -143.00, steps = 144
12:52:50 [INFO] test episode 73: reward = -163.00, steps = 164
12:53:04 [INFO] test episode 74: reward = -218.00, steps = 219
12:53:14 [INFO] test episode 75: reward = -147.00, steps = 148
12:53:21 [INFO] test episode 76: reward = -110.00, steps = 111
12:53:29 [INFO] test episode 77: reward = -109.00, steps = 110
12:53:37 [INFO] test episode 78: reward = -123.00, steps = 124
12:53:47 [INFO] test episode 79: reward = -150.00, steps = 151
12:53:55 [INFO] test episode 80: reward = -122.00, steps = 123
12:54:04 [INFO] test episode 81: reward = -135.00, steps = 136
12:54:14 [INFO] test episode 82: reward = -156.00, steps = 157
12:54:25 [INFO] test episode 83: reward = -164.00, steps = 165
12:54:32 [INFO] test episode 84: reward = -96.00, steps = 97
12:54:42 [INFO] test episode 85: reward = -158.00, steps = 159
12:54:49 [INFO] test episode 86: reward = -106.00, steps = 107
12:54:55 [INFO] test episode 87: reward = -97.00, steps = 98
12:55:02 [INFO] test episode 88: reward = -98.00, steps = 99
12:55:10 [INFO] test episode 89: reward = -118.00, steps = 119
12:55:17 [INFO] test episode 90: reward = -117.00, steps = 118
12:55:26 [INFO] test episode 91: reward = -132.00, steps = 133
12:55:34 [INFO] test episode 92: reward = -125.00, steps = 126
12:55:41 [INFO] test episode 93: reward = -109.00, steps = 110
12:55:49 [INFO] test episode 94: reward = -107.00, steps = 108
12:56:02 [INFO] test episode 95: reward = -209.00, steps = 210
12:56:11 [INFO] test episode 96: reward = -139.00, steps = 140
12:56:32 [INFO] test episode 97: reward = -310.00, steps = 311
12:56:44 [INFO] test episode 98: reward = -184.00, steps = 185
12:56:53 [INFO] test episode 99: reward = -144.00, steps = 145
12:56:53 [INFO] average episode reward = -130.11 ± 41.24
In [7]:
env.close()