Use Action-Actor-Critic to Play Acrobot-v1¶

TensorFlow version

In [1]:
%matplotlib inline

import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import pandas as pd
import gym
import matplotlib.pyplot as plt
import tensorflow.compat.v2 as tf
tf.random.set_seed(0)
from tensorflow import keras
from tensorflow import nn
from tensorflow import optimizers
from tensorflow import losses
from tensorflow.keras import layers
from tensorflow.keras import models

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')
In [2]:
env = gym.make('Acrobot-v1')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

21:54:57 [INFO] env: <AcrobotEnv<Acrobot-v1>>
21:54:57 [INFO] action_space: Discrete(3)
21:54:57 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
21:54:57 [INFO] reward_range: (-inf, inf)
21:54:57 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
21:54:57 [INFO] _max_episode_steps: 500
21:54:57 [INFO] _elapsed_steps: None
21:54:57 [INFO] id: Acrobot-v1
21:54:57 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
21:54:57 [INFO] reward_threshold: -100.0
21:54:57 [INFO] nondeterministic: False
21:54:57 [INFO] max_episode_steps: 500
21:54:57 [INFO] _kwargs: {}
21:54:57 [INFO] _env_name: Acrobot
In [3]:
class QActorCriticAgent:
    def __init__(self, env):
        self.action_n = env.action_space.n
        self.gamma = 0.99

        self.actor_net = self.build_net(hidden_sizes=[100,],
                output_size=self.action_n, output_activation=nn.softmax,
                loss=losses.categorical_crossentropy,
                learning_rate=0.0001)
        self.critic_net = self.build_net(hidden_sizes=[100,],
                output_size=self.action_n,
                learning_rate=0.0002)

    def build_net(self, hidden_sizes, output_size, input_size=None,
                activation=nn.relu, output_activation=None,
                loss=losses.mse, learning_rate=0.01):
        model = keras.Sequential()
        for hidden_size in hidden_sizes:
            model.add(layers.Dense(units=hidden_size,
                    activation=activation))
        model.add(layers.Dense(units=output_size,
                activation=output_activation))
        optimizer = optimizers.Adam(learning_rate)
        model.compile(optimizer=optimizer, loss=loss)
        return model

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []
            self.discount = 1.

    def step(self, observation, reward, terminated):
        probs = self.actor_net.predict(observation[np.newaxis], verbose=0)[0]
        action = np.random.choice(self.action_n, p=probs)
        if self.mode == 'train':
            self.trajectory += [observation, reward, terminated, action]
            if len(self.trajectory) >= 8:
                self.learn()
            self.discount *= self.gamma
        return action

    def close(self):
        pass

    def learn(self):
        state, _, _, action, next_state, reward, terminated, next_action \
                = self.trajectory[-8:]

        # update actor
        states = state[np.newaxis]
        preds = self.critic_net.predict(states, verbose=0)
        q = preds[0, action]
        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)
        with tf.GradientTape() as tape:
            pi_tensor = self.actor_net(state_tensor)[0, action]
            log_pi_tensor = tf.math.log(tf.clip_by_value(pi_tensor, 1e-6, 1.))
            loss_tensor = -self.discount * q * log_pi_tensor
        grad_tensors = tape.gradient(loss_tensor, self.actor_net.variables)
        self.actor_net.optimizer.apply_gradients(zip(
                grad_tensors, self.actor_net.variables))

        # update critic
        next_q = self.critic_net.predict(
                next_state[np.newaxis], verbose=0)[0, next_action]
        preds[0, action] = reward + (1. - terminated) * self.gamma * next_q
        self.critic_net.fit(states, preds, verbose=0)


agent = QActorCriticAgent(env)
In [4]:
def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
            mode='train')
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > -120:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

21:54:58 [INFO] ==== train ====
21:57:40 [INFO] train episode 0: reward = -429.00, steps = 430
22:00:44 [INFO] train episode 1: reward = -500.00, steps = 500
22:03:49 [INFO] train episode 2: reward = -500.00, steps = 500
22:06:54 [INFO] train episode 3: reward = -500.00, steps = 500
22:10:00 [INFO] train episode 4: reward = -500.00, steps = 500
22:13:06 [INFO] train episode 5: reward = -500.00, steps = 500
22:16:12 [INFO] train episode 6: reward = -500.00, steps = 500
22:19:18 [INFO] train episode 7: reward = -500.00, steps = 500
22:22:32 [INFO] train episode 8: reward = -500.00, steps = 500
22:25:37 [INFO] train episode 9: reward = -464.00, steps = 465
22:28:08 [INFO] train episode 10: reward = -388.00, steps = 389
22:31:13 [INFO] train episode 11: reward = -500.00, steps = 500
22:34:15 [INFO] train episode 12: reward = -500.00, steps = 500
22:37:14 [INFO] train episode 13: reward = -500.00, steps = 500
22:39:47 [INFO] train episode 14: reward = -408.00, steps = 409
22:42:28 [INFO] train episode 15: reward = -427.00, steps = 428
22:44:46 [INFO] train episode 16: reward = -366.00, steps = 367
22:46:49 [INFO] train episode 17: reward = -320.00, steps = 321
22:49:33 [INFO] train episode 18: reward = -411.00, steps = 412
22:51:31 [INFO] train episode 19: reward = -306.00, steps = 307
22:54:42 [INFO] train episode 20: reward = -500.00, steps = 500
22:57:26 [INFO] train episode 21: reward = -441.00, steps = 442
22:59:18 [INFO] train episode 22: reward = -308.00, steps = 309
23:00:34 [INFO] train episode 23: reward = -204.00, steps = 205
23:03:22 [INFO] train episode 24: reward = -455.00, steps = 456
23:06:30 [INFO] train episode 25: reward = -500.00, steps = 500
23:09:14 [INFO] train episode 26: reward = -429.00, steps = 430
23:11:34 [INFO] train episode 27: reward = -383.00, steps = 384
23:14:36 [INFO] train episode 28: reward = -500.00, steps = 500
23:17:40 [INFO] train episode 29: reward = -499.00, steps = 500
23:20:27 [INFO] train episode 30: reward = -458.00, steps = 459
23:23:29 [INFO] train episode 31: reward = -500.00, steps = 500
23:25:18 [INFO] train episode 32: reward = -300.00, steps = 301
23:26:44 [INFO] train episode 33: reward = -232.00, steps = 233
23:29:05 [INFO] train episode 34: reward = -369.00, steps = 370
23:31:31 [INFO] train episode 35: reward = -379.00, steps = 380
23:33:09 [INFO] train episode 36: reward = -260.00, steps = 261
23:34:55 [INFO] train episode 37: reward = -283.00, steps = 284
23:37:31 [INFO] train episode 38: reward = -415.00, steps = 416
23:39:08 [INFO] train episode 39: reward = -261.00, steps = 262
23:40:46 [INFO] train episode 40: reward = -252.00, steps = 253
23:43:34 [INFO] train episode 41: reward = -413.00, steps = 414
23:46:04 [INFO] train episode 42: reward = -361.00, steps = 362
23:47:20 [INFO] train episode 43: reward = -206.00, steps = 207
23:48:58 [INFO] train episode 44: reward = -264.00, steps = 265
23:50:55 [INFO] train episode 45: reward = -315.00, steps = 316
23:52:09 [INFO] train episode 46: reward = -186.00, steps = 187
23:53:58 [INFO] train episode 47: reward = -265.00, steps = 266
23:55:55 [INFO] train episode 48: reward = -276.00, steps = 277
23:57:30 [INFO] train episode 49: reward = -217.00, steps = 218
23:59:21 [INFO] train episode 50: reward = -329.00, steps = 330
00:00:16 [INFO] train episode 51: reward = -170.00, steps = 171
00:01:17 [INFO] train episode 52: reward = -190.00, steps = 191
00:02:16 [INFO] train episode 53: reward = -186.00, steps = 187
00:03:16 [INFO] train episode 54: reward = -185.00, steps = 186
00:04:02 [INFO] train episode 55: reward = -139.00, steps = 140
00:05:15 [INFO] train episode 56: reward = -208.00, steps = 209
00:06:43 [INFO] train episode 57: reward = -269.00, steps = 270
00:07:48 [INFO] train episode 58: reward = -204.00, steps = 205
00:08:57 [INFO] train episode 59: reward = -217.00, steps = 218
00:10:12 [INFO] train episode 60: reward = -230.00, steps = 231
00:11:02 [INFO] train episode 61: reward = -158.00, steps = 159
00:11:48 [INFO] train episode 62: reward = -141.00, steps = 142
00:13:23 [INFO] train episode 63: reward = -299.00, steps = 300
00:14:19 [INFO] train episode 64: reward = -174.00, steps = 175
00:15:13 [INFO] train episode 65: reward = -167.00, steps = 168
00:16:15 [INFO] train episode 66: reward = -195.00, steps = 196
00:17:51 [INFO] train episode 67: reward = -299.00, steps = 300
00:19:00 [INFO] train episode 68: reward = -205.00, steps = 206
00:19:56 [INFO] train episode 69: reward = -156.00, steps = 157
00:20:56 [INFO] train episode 70: reward = -182.00, steps = 183
00:21:53 [INFO] train episode 71: reward = -178.00, steps = 179
00:22:41 [INFO] train episode 72: reward = -148.00, steps = 149
00:23:53 [INFO] train episode 73: reward = -206.00, steps = 207
00:24:48 [INFO] train episode 74: reward = -162.00, steps = 163
00:25:35 [INFO] train episode 75: reward = -138.00, steps = 139
00:26:29 [INFO] train episode 76: reward = -150.00, steps = 151
00:27:29 [INFO] train episode 77: reward = -164.00, steps = 165
00:28:43 [INFO] train episode 78: reward = -192.00, steps = 193
00:29:37 [INFO] train episode 79: reward = -143.00, steps = 144
00:30:45 [INFO] train episode 80: reward = -176.00, steps = 177
00:31:44 [INFO] train episode 81: reward = -163.00, steps = 164
00:32:40 [INFO] train episode 82: reward = -148.00, steps = 149
00:33:21 [INFO] train episode 83: reward = -121.00, steps = 122
00:34:20 [INFO] train episode 84: reward = -178.00, steps = 179
00:35:13 [INFO] train episode 85: reward = -162.00, steps = 163
00:36:05 [INFO] train episode 86: reward = -145.00, steps = 146
08:41:02 [INFO] train episode 87: reward = -153.00, steps = 154
08:42:01 [INFO] train episode 88: reward = -168.00, steps = 169
08:42:53 [INFO] train episode 89: reward = -163.00, steps = 164
08:43:37 [INFO] train episode 90: reward = -137.00, steps = 138
08:44:26 [INFO] train episode 91: reward = -138.00, steps = 139
08:45:19 [INFO] train episode 92: reward = -149.00, steps = 150
08:46:25 [INFO] train episode 93: reward = -187.00, steps = 188
08:47:04 [INFO] train episode 94: reward = -111.00, steps = 112
08:48:03 [INFO] train episode 95: reward = -172.00, steps = 173
08:49:17 [INFO] train episode 96: reward = -232.00, steps = 233
08:49:53 [INFO] train episode 97: reward = -112.00, steps = 113
08:50:51 [INFO] train episode 98: reward = -182.00, steps = 183
08:51:37 [INFO] train episode 99: reward = -146.00, steps = 147
08:52:33 [INFO] train episode 100: reward = -177.00, steps = 178
08:53:07 [INFO] train episode 101: reward = -104.00, steps = 105
08:54:01 [INFO] train episode 102: reward = -169.00, steps = 170
08:54:33 [INFO] train episode 103: reward = -102.00, steps = 103
08:55:33 [INFO] train episode 104: reward = -191.00, steps = 192
08:56:22 [INFO] train episode 105: reward = -153.00, steps = 154
08:57:27 [INFO] train episode 106: reward = -204.00, steps = 205
08:58:12 [INFO] train episode 107: reward = -142.00, steps = 143
08:59:00 [INFO] train episode 108: reward = -152.00, steps = 153
08:59:36 [INFO] train episode 109: reward = -111.00, steps = 112
09:00:16 [INFO] train episode 110: reward = -127.00, steps = 128
09:00:53 [INFO] train episode 111: reward = -116.00, steps = 117
09:01:34 [INFO] train episode 112: reward = -129.00, steps = 130
09:02:17 [INFO] train episode 113: reward = -133.00, steps = 134
09:03:18 [INFO] train episode 114: reward = -193.00, steps = 194
09:04:01 [INFO] train episode 115: reward = -117.00, steps = 118
09:05:04 [INFO] train episode 116: reward = -176.00, steps = 177
09:05:55 [INFO] train episode 117: reward = -142.00, steps = 143
09:06:44 [INFO] train episode 118: reward = -140.00, steps = 141
09:07:29 [INFO] train episode 119: reward = -121.00, steps = 122
09:08:20 [INFO] train episode 120: reward = -141.00, steps = 142
09:08:58 [INFO] train episode 121: reward = -110.00, steps = 111
09:09:48 [INFO] train episode 122: reward = -151.00, steps = 152
09:10:46 [INFO] train episode 123: reward = -164.00, steps = 165
09:11:20 [INFO] train episode 124: reward = -98.00, steps = 99
09:12:00 [INFO] train episode 125: reward = -121.00, steps = 122
09:12:40 [INFO] train episode 126: reward = -123.00, steps = 124
09:13:27 [INFO] train episode 127: reward = -143.00, steps = 144
09:14:10 [INFO] train episode 128: reward = -127.00, steps = 128
09:15:03 [INFO] train episode 129: reward = -157.00, steps = 158
09:16:02 [INFO] train episode 130: reward = -181.00, steps = 182
09:16:46 [INFO] train episode 131: reward = -122.00, steps = 123
09:17:32 [INFO] train episode 132: reward = -137.00, steps = 138
09:18:20 [INFO] train episode 133: reward = -132.00, steps = 133
09:19:03 [INFO] train episode 134: reward = -118.00, steps = 119
09:19:44 [INFO] train episode 135: reward = -117.00, steps = 118
09:20:36 [INFO] train episode 136: reward = -164.00, steps = 165
09:21:14 [INFO] train episode 137: reward = -118.00, steps = 119
09:22:56 [INFO] train episode 138: reward = -300.00, steps = 301
09:23:57 [INFO] train episode 139: reward = -169.00, steps = 170
09:24:44 [INFO] train episode 140: reward = -138.00, steps = 139
09:27:10 [INFO] train episode 141: reward = -417.00, steps = 418
09:28:05 [INFO] train episode 142: reward = -153.00, steps = 154
09:28:52 [INFO] train episode 143: reward = -137.00, steps = 138
09:29:40 [INFO] train episode 144: reward = -140.00, steps = 141
09:30:24 [INFO] train episode 145: reward = -124.00, steps = 125
09:31:18 [INFO] train episode 146: reward = -159.00, steps = 160
09:32:14 [INFO] train episode 147: reward = -155.00, steps = 156
09:33:12 [INFO] train episode 148: reward = -170.00, steps = 171
09:33:58 [INFO] train episode 149: reward = -132.00, steps = 133
09:34:33 [INFO] train episode 150: reward = -102.00, steps = 103
09:35:18 [INFO] train episode 151: reward = -129.00, steps = 130
09:35:56 [INFO] train episode 152: reward = -106.00, steps = 107
09:36:42 [INFO] train episode 153: reward = -122.00, steps = 123
09:37:41 [INFO] train episode 154: reward = -160.00, steps = 161
09:38:38 [INFO] train episode 155: reward = -163.00, steps = 164
09:39:21 [INFO] train episode 156: reward = -126.00, steps = 127
09:40:09 [INFO] train episode 157: reward = -139.00, steps = 140
09:40:48 [INFO] train episode 158: reward = -111.00, steps = 112
09:41:43 [INFO] train episode 159: reward = -159.00, steps = 160
09:42:29 [INFO] train episode 160: reward = -136.00, steps = 137
09:43:24 [INFO] train episode 161: reward = -154.00, steps = 155
09:44:06 [INFO] train episode 162: reward = -109.00, steps = 110
09:44:54 [INFO] train episode 163: reward = -129.00, steps = 130
09:45:59 [INFO] train episode 164: reward = -173.00, steps = 174
09:47:14 [INFO] train episode 165: reward = -203.00, steps = 204
09:47:56 [INFO] train episode 166: reward = -111.00, steps = 112
09:48:47 [INFO] train episode 167: reward = -135.00, steps = 136
09:49:41 [INFO] train episode 168: reward = -146.00, steps = 147
09:50:25 [INFO] train episode 169: reward = -119.00, steps = 120
09:51:20 [INFO] train episode 170: reward = -146.00, steps = 147
09:52:19 [INFO] train episode 171: reward = -160.00, steps = 161
09:53:06 [INFO] train episode 172: reward = -129.00, steps = 130
09:54:04 [INFO] train episode 173: reward = -157.00, steps = 158
09:54:51 [INFO] train episode 174: reward = -125.00, steps = 126
09:55:30 [INFO] train episode 175: reward = -104.00, steps = 105
09:56:06 [INFO] train episode 176: reward = -97.00, steps = 98
09:56:53 [INFO] train episode 177: reward = -124.00, steps = 125
09:57:43 [INFO] train episode 178: reward = -137.00, steps = 138
09:58:39 [INFO] train episode 179: reward = -148.00, steps = 149
09:59:17 [INFO] train episode 180: reward = -106.00, steps = 107
10:00:09 [INFO] train episode 181: reward = -139.00, steps = 140
10:00:47 [INFO] train episode 182: reward = -103.00, steps = 104
10:01:26 [INFO] train episode 183: reward = -107.00, steps = 108
10:01:26 [INFO] ==== test ====
10:01:39 [INFO] test episode 0: reward = -136.00, steps = 137
10:01:51 [INFO] test episode 1: reward = -129.00, steps = 130
10:02:00 [INFO] test episode 2: reward = -102.00, steps = 103
10:02:10 [INFO] test episode 3: reward = -104.00, steps = 105
10:02:20 [INFO] test episode 4: reward = -113.00, steps = 114
10:02:29 [INFO] test episode 5: reward = -104.00, steps = 105
10:02:42 [INFO] test episode 6: reward = -136.00, steps = 137
10:02:56 [INFO] test episode 7: reward = -164.00, steps = 165
10:03:08 [INFO] test episode 8: reward = -128.00, steps = 129
10:03:21 [INFO] test episode 9: reward = -144.00, steps = 145
10:03:32 [INFO] test episode 10: reward = -127.00, steps = 128
10:03:43 [INFO] test episode 11: reward = -117.00, steps = 118
10:03:55 [INFO] test episode 12: reward = -131.00, steps = 132
10:04:15 [INFO] test episode 13: reward = -231.00, steps = 232
10:04:30 [INFO] test episode 14: reward = -163.00, steps = 164
10:04:42 [INFO] test episode 15: reward = -126.00, steps = 127
10:04:52 [INFO] test episode 16: reward = -120.00, steps = 121
10:05:01 [INFO] test episode 17: reward = -101.00, steps = 102
10:05:11 [INFO] test episode 18: reward = -110.00, steps = 111
10:05:24 [INFO] test episode 19: reward = -142.00, steps = 143
10:05:34 [INFO] test episode 20: reward = -121.00, steps = 122
10:05:49 [INFO] test episode 21: reward = -171.00, steps = 172
10:06:01 [INFO] test episode 22: reward = -140.00, steps = 141
10:06:14 [INFO] test episode 23: reward = -141.00, steps = 142
10:06:25 [INFO] test episode 24: reward = -125.00, steps = 126
10:06:36 [INFO] test episode 25: reward = -126.00, steps = 127
10:06:49 [INFO] test episode 26: reward = -144.00, steps = 145
10:06:58 [INFO] test episode 27: reward = -99.00, steps = 100
10:07:07 [INFO] test episode 28: reward = -101.00, steps = 102
10:07:18 [INFO] test episode 29: reward = -122.00, steps = 123
10:07:30 [INFO] test episode 30: reward = -129.00, steps = 130
10:07:40 [INFO] test episode 31: reward = -109.00, steps = 110
10:07:54 [INFO] test episode 32: reward = -160.00, steps = 161
10:08:05 [INFO] test episode 33: reward = -124.00, steps = 125
10:08:15 [INFO] test episode 34: reward = -102.00, steps = 103
10:08:24 [INFO] test episode 35: reward = -107.00, steps = 108
10:08:34 [INFO] test episode 36: reward = -107.00, steps = 108
10:08:44 [INFO] test episode 37: reward = -107.00, steps = 108
10:08:57 [INFO] test episode 38: reward = -133.00, steps = 134
10:09:08 [INFO] test episode 39: reward = -127.00, steps = 128
10:09:19 [INFO] test episode 40: reward = -121.00, steps = 122
10:09:30 [INFO] test episode 41: reward = -124.00, steps = 125
10:09:41 [INFO] test episode 42: reward = -126.00, steps = 127
10:09:53 [INFO] test episode 43: reward = -128.00, steps = 129
10:10:01 [INFO] test episode 44: reward = -89.00, steps = 90
10:10:11 [INFO] test episode 45: reward = -116.00, steps = 117
10:10:23 [INFO] test episode 46: reward = -134.00, steps = 135
10:10:36 [INFO] test episode 47: reward = -142.00, steps = 143
10:10:47 [INFO] test episode 48: reward = -123.00, steps = 124
10:11:03 [INFO] test episode 49: reward = -165.00, steps = 166
10:11:12 [INFO] test episode 50: reward = -105.00, steps = 106
10:11:30 [INFO] test episode 51: reward = -194.00, steps = 195
10:11:39 [INFO] test episode 52: reward = -108.00, steps = 109
10:11:53 [INFO] test episode 53: reward = -140.00, steps = 141
10:12:02 [INFO] test episode 54: reward = -105.00, steps = 106
10:12:12 [INFO] test episode 55: reward = -108.00, steps = 109
10:12:24 [INFO] test episode 56: reward = -133.00, steps = 134
10:12:36 [INFO] test episode 57: reward = -132.00, steps = 133
10:12:49 [INFO] test episode 58: reward = -148.00, steps = 149
10:13:08 [INFO] test episode 59: reward = -222.00, steps = 223
10:13:18 [INFO] test episode 60: reward = -108.00, steps = 109
10:13:31 [INFO] test episode 61: reward = -143.00, steps = 144
10:13:43 [INFO] test episode 62: reward = -130.00, steps = 131
10:13:52 [INFO] test episode 63: reward = -95.00, steps = 96
10:14:05 [INFO] test episode 64: reward = -147.00, steps = 148
10:14:17 [INFO] test episode 65: reward = -122.00, steps = 123
10:14:26 [INFO] test episode 66: reward = -96.00, steps = 97
10:14:37 [INFO] test episode 67: reward = -123.00, steps = 124
10:14:48 [INFO] test episode 68: reward = -121.00, steps = 122
10:14:58 [INFO] test episode 69: reward = -109.00, steps = 110
10:15:07 [INFO] test episode 70: reward = -100.00, steps = 101
10:15:17 [INFO] test episode 71: reward = -109.00, steps = 110
10:15:28 [INFO] test episode 72: reward = -115.00, steps = 116
10:15:39 [INFO] test episode 73: reward = -114.00, steps = 115
10:15:50 [INFO] test episode 74: reward = -111.00, steps = 112
10:16:00 [INFO] test episode 75: reward = -108.00, steps = 109
10:16:13 [INFO] test episode 76: reward = -138.00, steps = 139
10:16:23 [INFO] test episode 77: reward = -106.00, steps = 107
10:16:36 [INFO] test episode 78: reward = -143.00, steps = 144
10:16:54 [INFO] test episode 79: reward = -202.00, steps = 203
10:17:06 [INFO] test episode 80: reward = -125.00, steps = 126
10:17:28 [INFO] test episode 81: reward = -238.00, steps = 239
10:17:41 [INFO] test episode 82: reward = -149.00, steps = 150
10:17:53 [INFO] test episode 83: reward = -131.00, steps = 132
10:18:06 [INFO] test episode 84: reward = -143.00, steps = 144
10:18:17 [INFO] test episode 85: reward = -123.00, steps = 124
10:18:30 [INFO] test episode 86: reward = -142.00, steps = 143
10:18:40 [INFO] test episode 87: reward = -104.00, steps = 105
10:18:50 [INFO] test episode 88: reward = -106.00, steps = 107
10:19:02 [INFO] test episode 89: reward = -133.00, steps = 134
10:19:11 [INFO] test episode 90: reward = -107.00, steps = 108
10:19:26 [INFO] test episode 91: reward = -159.00, steps = 160
10:19:37 [INFO] test episode 92: reward = -114.00, steps = 115
10:19:49 [INFO] test episode 93: reward = -141.00, steps = 142
10:20:03 [INFO] test episode 94: reward = -167.00, steps = 168
10:20:48 [INFO] test episode 95: reward = -500.00, steps = 500
10:21:02 [INFO] test episode 96: reward = -154.00, steps = 155
10:21:14 [INFO] test episode 97: reward = -127.00, steps = 128
10:21:23 [INFO] test episode 98: reward = -102.00, steps = 103
10:21:31 [INFO] test episode 99: reward = -94.00, steps = 95
10:21:31 [INFO] average episode reward = -132.85 ± 45.99
In [9]:
env.close()